diff --git a/.gitignore b/.gitignore
index 20c4f52cd3786..0e7c6c7900133 100644
--- a/.gitignore
+++ b/.gitignore
@@ -51,6 +51,7 @@ autoconf/autom4te.cache
 /CMakeSettings.json
 # CLion project configuration
 /.idea
+/cmake-build*
 
 #==============================================================================#
 # Directories to ignore (do not add trailing '/'s, they skip symlinks).
diff --git a/clang-tools-extra/clang-tidy/ClangTidy.cpp b/clang-tools-extra/clang-tidy/ClangTidy.cpp
index 1cd7cdd10bc25..62f9d19b2a362 100644
--- a/clang-tools-extra/clang-tidy/ClangTidy.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidy.cpp
@@ -672,6 +672,18 @@ getAllChecksAndOptions(bool AllowEnablingAnalyzerAlphaCheckers) {
     Buffer.append(AnalyzerCheck);
     Result.Names.insert(Buffer);
   }
+  for (std::string OptionName : {
+#define GET_CHECKER_OPTIONS
+#define CHECKER_OPTION(TYPE, CHECKER, OPTION_NAME, DESCRIPTION, DEFAULT,       \
+                       RELEASE, HIDDEN)                                        \
+  Twine(AnalyzerCheckNamePrefix).concat(CHECKER ":" OPTION_NAME).str(),
+
+#include "clang/StaticAnalyzer/Checkers/Checkers.inc"
+#undef CHECKER_OPTION
+#undef GET_CHECKER_OPTIONS
+       }) {
+    Result.Options.insert(OptionName);
+  }
 #endif // CLANG_TIDY_ENABLE_STATIC_ANALYZER
 
   Context.setOptionsCollector(&Result.Options);
diff --git a/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.cpp b/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.cpp
index c0bf4903ec391..90c16bfddd84f 100644
--- a/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.cpp
@@ -777,7 +777,7 @@ bool ForLoopIndexUseVisitor::TraverseLambdaCapture(LambdaExpr *LE,
                                                    const LambdaCapture *C,
                                                    Expr *Init) {
   if (C->capturesVariable()) {
-    const ValueDecl *VDecl = C->getCapturedVar();
+    ValueDecl *VDecl = C->getCapturedVar();
     if (areSameVariable(IndexVar, VDecl)) {
       // FIXME: if the index is captured, it will count as an usage and the
       // alias (if any) won't work, because it is only used in case of having
@@ -787,6 +787,8 @@ bool ForLoopIndexUseVisitor::TraverseLambdaCapture(LambdaExpr *LE,
                                                        : Usage::UK_CaptureByRef,
                      C->getLocation()));
     }
+    if (VDecl->isInitCapture())
+      TraverseStmtImpl(cast<VarDecl>(VDecl)->getInit());
   }
   return VisitorBase::TraverseLambdaCapture(LE, C, Init);
 }
@@ -816,6 +818,17 @@ bool ForLoopIndexUseVisitor::VisitDeclStmt(DeclStmt *S) {
   return true;
 }
 
+bool ForLoopIndexUseVisitor::TraverseStmtImpl(Stmt *S) {
+  // All this pointer swapping is a mechanism for tracking immediate parentage
+  // of Stmts.
+  const Stmt *OldNextParent = NextStmtParent;
+  CurrStmtParent = NextStmtParent;
+  NextStmtParent = S;
+  bool Result = VisitorBase::TraverseStmt(S);
+  NextStmtParent = OldNextParent;
+  return Result;
+}
+
 bool ForLoopIndexUseVisitor::TraverseStmt(Stmt *S) {
   // If this is an initialization expression for a lambda capture, prune the
   // traversal so that we don't end up diagnosing the contained DeclRefExpr as
@@ -828,15 +841,7 @@ bool ForLoopIndexUseVisitor::TraverseStmt(Stmt *S) {
       return true;
     }
   }
-
-  // All this pointer swapping is a mechanism for tracking immediate parentage
-  // of Stmts.
-  const Stmt *OldNextParent = NextStmtParent;
-  CurrStmtParent = NextStmtParent;
-  NextStmtParent = S;
-  bool Result = VisitorBase::TraverseStmt(S);
-  NextStmtParent = OldNextParent;
-  return Result;
+  return TraverseStmtImpl(S);
 }
 
 std::string VariableNamer::createIndexName() {
diff --git a/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.h b/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.h
index cfceb3b586842..ca9c1855038b5 100644
--- a/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.h
+++ b/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.h
@@ -354,6 +354,8 @@ class ForLoopIndexUseVisitor
   bool VisitDeclStmt(DeclStmt *S);
   bool TraverseStmt(Stmt *S);
 
+  bool TraverseStmtImpl(Stmt *S);
+
   /// Add an expression to the list of expressions on which the container
   /// expression depends.
   void addComponent(const Expr *E);
diff --git a/clang-tools-extra/clangd/refactor/tweaks/CMakeLists.txt b/clang-tools-extra/clangd/refactor/tweaks/CMakeLists.txt
index 2e948c23569f6..59475b0dfd3d2 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/CMakeLists.txt
+++ b/clang-tools-extra/clangd/refactor/tweaks/CMakeLists.txt
@@ -29,6 +29,7 @@ add_clang_library(clangDaemonTweaks OBJECT
   RemoveUsingNamespace.cpp
   ScopifyEnum.cpp
   SpecialMembers.cpp
+  SwapBinaryOperands.cpp
   SwapIfBranches.cpp
 
   LINK_LIBS
diff --git a/clang-tools-extra/clangd/refactor/tweaks/SwapBinaryOperands.cpp b/clang-tools-extra/clangd/refactor/tweaks/SwapBinaryOperands.cpp
new file mode 100644
index 0000000000000..9ad0089a5d035
--- /dev/null
+++ b/clang-tools-extra/clangd/refactor/tweaks/SwapBinaryOperands.cpp
@@ -0,0 +1,217 @@
+//===--- SwapBinaryOperands.cpp ----------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "ParsedAST.h"
+#include "Protocol.h"
+#include "Selection.h"
+#include "SourceCode.h"
+#include "refactor/Tweak.h"
+#include "support/Logger.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/OperationKinds.h"
+#include "clang/AST/Stmt.h"
+#include "clang/Basic/LLVM.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Tooling/Core/Replacement.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/FormatVariadic.h"
+#include <string>
+#include <utility>
+
+namespace clang {
+namespace clangd {
+namespace {
+/// Check whether it makes logical sense to swap operands to an operator.
+/// Assignment or member access operators are rarely swappable
+/// while keeping the meaning intact, whereas comparison operators, mathematical
+/// operators, etc. are often desired to be swappable for readability, avoiding
+/// bugs by assigning to nullptr when comparison was desired, etc.
+bool isOpSwappable(const BinaryOperatorKind Opcode) {
+  switch (Opcode) {
+  case BinaryOperatorKind::BO_Mul:
+  case BinaryOperatorKind::BO_Add:
+  case BinaryOperatorKind::BO_LT:
+  case BinaryOperatorKind::BO_GT:
+  case BinaryOperatorKind::BO_LE:
+  case BinaryOperatorKind::BO_GE:
+  case BinaryOperatorKind::BO_EQ:
+  case BinaryOperatorKind::BO_NE:
+  case BinaryOperatorKind::BO_And:
+  case BinaryOperatorKind::BO_Xor:
+  case BinaryOperatorKind::BO_Or:
+  case BinaryOperatorKind::BO_LAnd:
+  case BinaryOperatorKind::BO_LOr:
+  case BinaryOperatorKind::BO_Comma:
+    return true;
+  // Noncommutative operators:
+  case BinaryOperatorKind::BO_Div:
+  case BinaryOperatorKind::BO_Sub:
+  case BinaryOperatorKind::BO_Shl:
+  case BinaryOperatorKind::BO_Shr:
+  case BinaryOperatorKind::BO_Rem:
+  // <=> is noncommutative
+  case BinaryOperatorKind::BO_Cmp:
+  // Member access:
+  case BinaryOperatorKind::BO_PtrMemD:
+  case BinaryOperatorKind::BO_PtrMemI:
+  // Assignment:
+  case BinaryOperatorKind::BO_Assign:
+  case BinaryOperatorKind::BO_MulAssign:
+  case BinaryOperatorKind::BO_DivAssign:
+  case BinaryOperatorKind::BO_RemAssign:
+  case BinaryOperatorKind::BO_AddAssign:
+  case BinaryOperatorKind::BO_SubAssign:
+  case BinaryOperatorKind::BO_ShlAssign:
+  case BinaryOperatorKind::BO_ShrAssign:
+  case BinaryOperatorKind::BO_AndAssign:
+  case BinaryOperatorKind::BO_XorAssign:
+  case BinaryOperatorKind::BO_OrAssign:
+    return false;
+  }
+  return false;
+}
+
+/// Some operators are asymmetric and need to be flipped when swapping their
+/// operands
+/// @param[out] Opcode the opcode to potentially swap
+/// If the opcode does not need to be swapped or is not swappable, does nothing
+BinaryOperatorKind swapOperator(const BinaryOperatorKind Opcode) {
+  switch (Opcode) {
+  case BinaryOperatorKind::BO_LT:
+    return BinaryOperatorKind::BO_GT;
+
+  case BinaryOperatorKind::BO_GT:
+    return BinaryOperatorKind::BO_LT;
+
+  case BinaryOperatorKind::BO_LE:
+    return BinaryOperatorKind::BO_GE;
+
+  case BinaryOperatorKind::BO_GE:
+    return BinaryOperatorKind::BO_LE;
+
+  case BinaryOperatorKind::BO_Mul:
+  case BinaryOperatorKind::BO_Add:
+  case BinaryOperatorKind::BO_Cmp:
+  case BinaryOperatorKind::BO_EQ:
+  case BinaryOperatorKind::BO_NE:
+  case BinaryOperatorKind::BO_And:
+  case BinaryOperatorKind::BO_Xor:
+  case BinaryOperatorKind::BO_Or:
+  case BinaryOperatorKind::BO_LAnd:
+  case BinaryOperatorKind::BO_LOr:
+  case BinaryOperatorKind::BO_Comma:
+  case BinaryOperatorKind::BO_Div:
+  case BinaryOperatorKind::BO_Sub:
+  case BinaryOperatorKind::BO_Shl:
+  case BinaryOperatorKind::BO_Shr:
+  case BinaryOperatorKind::BO_Rem:
+  case BinaryOperatorKind::BO_PtrMemD:
+  case BinaryOperatorKind::BO_PtrMemI:
+  case BinaryOperatorKind::BO_Assign:
+  case BinaryOperatorKind::BO_MulAssign:
+  case BinaryOperatorKind::BO_DivAssign:
+  case BinaryOperatorKind::BO_RemAssign:
+  case BinaryOperatorKind::BO_AddAssign:
+  case BinaryOperatorKind::BO_SubAssign:
+  case BinaryOperatorKind::BO_ShlAssign:
+  case BinaryOperatorKind::BO_ShrAssign:
+  case BinaryOperatorKind::BO_AndAssign:
+  case BinaryOperatorKind::BO_XorAssign:
+  case BinaryOperatorKind::BO_OrAssign:
+    return Opcode;
+  }
+  llvm_unreachable("Unknown BinaryOperatorKind enum");
+}
+
+/// Swaps the operands to a binary operator
+/// Before:
+///   x != nullptr
+///   ^    ^^^^^^^
+/// After:
+///   nullptr != x
+class SwapBinaryOperands : public Tweak {
+public:
+  const char *id() const final;
+
+  bool prepare(const Selection &Inputs) override;
+  Expected<Effect> apply(const Selection &Inputs) override;
+  std::string title() const override {
+    return llvm::formatv("Swap operands to {0}",
+                         Op ? Op->getOpcodeStr() : "binary operator");
+  }
+  llvm::StringLiteral kind() const override {
+    return CodeAction::REFACTOR_KIND;
+  }
+  bool hidden() const override { return false; }
+
+private:
+  const BinaryOperator *Op;
+};
+
+REGISTER_TWEAK(SwapBinaryOperands)
+
+bool SwapBinaryOperands::prepare(const Selection &Inputs) {
+  for (const SelectionTree::Node *N = Inputs.ASTSelection.commonAncestor();
+       N && !Op; N = N->Parent) {
+    // Stop once we hit a block, e.g. a lambda in one of the operands.
+    // This makes sure that the selection point is in the 'scope' of the binary
+    // operator, not from somewhere inside a lambda for example
+    // (5 < [](){ ^return 1; })
+    if (llvm::isa_and_nonnull<CompoundStmt>(N->ASTNode.get<Stmt>()))
+      return false;
+    Op = dyn_cast_or_null<BinaryOperator>(N->ASTNode.get<Stmt>());
+    // If we hit upon a nonswappable binary operator, ignore and keep going
+    if (Op && !isOpSwappable(Op->getOpcode())) {
+      Op = nullptr;
+    }
+  }
+  return Op != nullptr;
+}
+
+Expected<Tweak::Effect> SwapBinaryOperands::apply(const Selection &Inputs) {
+  const auto &Ctx = Inputs.AST->getASTContext();
+  const auto &SrcMgr = Inputs.AST->getSourceManager();
+
+  const auto LHSRng = toHalfOpenFileRange(SrcMgr, Ctx.getLangOpts(),
+                                          Op->getLHS()->getSourceRange());
+  if (!LHSRng)
+    return error(
+        "Could not obtain range of the 'lhs' of the operator. Macros?");
+  const auto RHSRng = toHalfOpenFileRange(SrcMgr, Ctx.getLangOpts(),
+                                          Op->getRHS()->getSourceRange());
+  if (!RHSRng)
+    return error(
+        "Could not obtain range of the 'rhs' of the operator. Macros?");
+  const auto OpRng =
+      toHalfOpenFileRange(SrcMgr, Ctx.getLangOpts(), Op->getOperatorLoc());
+  if (!OpRng)
+    return error("Could not obtain range of the operator itself. Macros?");
+
+  const auto LHSCode = toSourceCode(SrcMgr, *LHSRng);
+  const auto RHSCode = toSourceCode(SrcMgr, *RHSRng);
+  const auto OperatorCode = toSourceCode(SrcMgr, *OpRng);
+
+  tooling::Replacements Result;
+  if (auto Err = Result.add(tooling::Replacement(
+          Ctx.getSourceManager(), LHSRng->getBegin(), LHSCode.size(), RHSCode)))
+    return std::move(Err);
+  if (auto Err = Result.add(tooling::Replacement(
+          Ctx.getSourceManager(), RHSRng->getBegin(), RHSCode.size(), LHSCode)))
+    return std::move(Err);
+  const auto SwappedOperator = swapOperator(Op->getOpcode());
+  if (auto Err = Result.add(tooling::Replacement(
+          Ctx.getSourceManager(), OpRng->getBegin(), OperatorCode.size(),
+          Op->getOpcodeStr(SwappedOperator))))
+    return std::move(Err);
+  return Effect::mainFileEdit(SrcMgr, std::move(Result));
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt
index 4fa9f18407ae9..dffdcd5d014ca 100644
--- a/clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -137,6 +137,7 @@ add_unittest(ClangdUnitTests ClangdTests
   tweaks/ScopifyEnumTests.cpp
   tweaks/ShowSelectionTreeTests.cpp
   tweaks/SpecialMembersTests.cpp
+  tweaks/SwapBinaryOperandsTests.cpp
   tweaks/SwapIfBranchesTests.cpp
   tweaks/TweakTesting.cpp
   tweaks/TweakTests.cpp
diff --git a/clang-tools-extra/clangd/unittests/tweaks/SwapBinaryOperandsTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/SwapBinaryOperandsTests.cpp
new file mode 100644
index 0000000000000..e157bbefbdaaf
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/tweaks/SwapBinaryOperandsTests.cpp
@@ -0,0 +1,92 @@
+//===-- SwapBinaryOperandsTests.cpp -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TweakTesting.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TWEAK_TEST(SwapBinaryOperands);
+
+TEST_F(SwapBinaryOperandsTest, Test) {
+  Context = Function;
+  EXPECT_EQ(apply("int *p = nullptr; bool c = ^p == nullptr;"),
+            "int *p = nullptr; bool c = nullptr == p;");
+  EXPECT_EQ(apply("int *p = nullptr; bool c = p ^== nullptr;"),
+            "int *p = nullptr; bool c = nullptr == p;");
+  EXPECT_EQ(apply("int x = 3; bool c = ^x >= 5;"),
+            "int x = 3; bool c = 5 <= x;");
+  EXPECT_EQ(apply("int x = 3; bool c = x >^= 5;"),
+            "int x = 3; bool c = 5 <= x;");
+  EXPECT_EQ(apply("int x = 3; bool c = x >=^ 5;"),
+            "int x = 3; bool c = 5 <= x;");
+  EXPECT_EQ(apply("int x = 3; bool c = x >=^ 5;"),
+            "int x = 3; bool c = 5 <= x;");
+  EXPECT_EQ(apply("int f(); int x = 3; bool c = x >=^ f();"),
+            "int f(); int x = 3; bool c = f() <= x;");
+  EXPECT_EQ(apply(R"cpp(
+            int f();
+            #define F f
+            int x = 3; bool c = x >=^ F();
+            )cpp"),
+            R"cpp(
+            int f();
+            #define F f
+            int x = 3; bool c = F() <= x;
+            )cpp");
+  EXPECT_EQ(apply(R"cpp(
+            int f();
+            #define F f()
+            int x = 3; bool c = x >=^ F;
+            )cpp"),
+            R"cpp(
+            int f();
+            #define F f()
+            int x = 3; bool c = F <= x;
+            )cpp");
+  EXPECT_EQ(apply(R"cpp(
+            int f(bool);
+            #define F(v) f(v)
+            int x = 0;
+            bool c = F(x^ < 5);
+            )cpp"),
+            R"cpp(
+            int f(bool);
+            #define F(v) f(v)
+            int x = 0;
+            bool c = F(5 > x);
+            )cpp");
+  ExtraArgs = {"-std=c++20"};
+  Context = CodeContext::File;
+  EXPECT_UNAVAILABLE(R"cpp(
+            namespace std {
+                struct strong_ordering {
+                    int val; 
+                    static const strong_ordering less;
+                    static const strong_ordering equivalent;
+                    static const strong_ordering equal;
+                    static const strong_ordering greater;
+                }; 
+                    inline constexpr strong_ordering strong_ordering::less {-1};
+                    inline constexpr strong_ordering strong_ordering::equivalent {0};
+                    inline constexpr strong_ordering strong_ordering::equal {0};
+                    inline constexpr strong_ordering strong_ordering::greater {1};
+            };
+            #define F(v) v
+            int x = 0;
+            auto c = F(5^ <=> x);
+            )cpp");
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index e34e296b5a096..b4792d749a86c 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -74,6 +74,8 @@ Code completion
 Code actions
 ^^^^^^^^^^^^
 
+- Added `Swap operands` tweak for certain binary operators.
+
 Signature help
 ^^^^^^^^^^^^^^
 
@@ -97,6 +99,10 @@ The improvements are...
 Improvements to clang-tidy
 --------------------------
 
+- Improved :program:`clang-tidy`'s `--verify-config` flag by adding support for
+  the configuration options of the `Clang Static Analyzer Checks
+  <https://clang.llvm.org/docs/analyzer/checkers.html>`_.
+
 - Improved :program:`run-clang-tidy.py` script. Fixed minor shutdown noise
   happening on certain platforms when interrupting the script.
 
@@ -162,18 +168,18 @@ Changes in existing checks
   <clang-tidy/checks/misc/definitions-in-headers>` check by rewording the
   diagnostic note that suggests adding ``inline``.
 
+- Improved :doc:`misc-unconventional-assign-operator
+  <clang-tidy/checks/misc/unconventional-assign-operator>` check to avoid
+  false positive for C++23 deducing this.
+
 - Improved :doc:`modernize-avoid-c-arrays
   <clang-tidy/checks/modernize/avoid-c-arrays>` check to suggest using ``std::span``
   as a replacement for parameters of incomplete C array type in C++20 and 
   ``std::array`` or ``std::vector`` before C++20.
 
-- Improved :doc:`modernize-use-std-format
-  <clang-tidy/checks/modernize/use-std-format>` check to support replacing
-  member function calls too.
-
-- Improved :doc:`misc-unconventional-assign-operator
-  <clang-tidy/checks/misc/unconventional-assign-operator>` check to avoid
-  false positive for C++23 deducing this.
+- Improved :doc:`modernize-loop-convert
+  <clang-tidy/checks/modernize/loop-convert>` check to fix false positive when
+  using loop variable in initializer of lambda capture.
 
 - Improved :doc:`modernize-min-max-use-initializer-list
   <clang-tidy/checks/modernize/min-max-use-initializer-list>` check by fixing
@@ -184,15 +190,14 @@ Changes in existing checks
   <clang-tidy/checks/modernize/use-nullptr>` check to also recognize
   ``NULL``/``__null`` (but not ``0``) when used with a templated type.
 
+- Improved :doc:`modernize-use-std-format
+  <clang-tidy/checks/modernize/use-std-format>` check to support replacing
+  member function calls too.
+
 - Improved :doc:`modernize-use-std-print
   <clang-tidy/checks/modernize/use-std-print>` check to support replacing
   member function calls too.
 
-- Improved :doc:`readability-enum-initial-value
-  <clang-tidy/checks/readability/enum-initial-value>` check by only issuing
-  diagnostics for the definition of an ``enum``, and by fixing a typo in the
-  diagnostic.
-
 - Improved :doc:`performance-avoid-endl
   <clang-tidy/checks/performance/avoid-endl>` check to use ``std::endl`` as
   placeholder when lexer cannot get source text.
@@ -201,6 +206,11 @@ Changes in existing checks
   <clang-tidy/checks/readability/container-contains>` check to let it work on
   any class that has a ``contains`` method.
 
+- Improved :doc:`readability-enum-initial-value
+  <clang-tidy/checks/readability/enum-initial-value>` check by only issuing
+  diagnostics for the definition of an ``enum``, and by fixing a typo in the
+  diagnostic.
+
 - Improved :doc:`readability-implicit-bool-conversion
   <clang-tidy/checks/readability/implicit-bool-conversion>` check
   by adding the option `UseUpperCaseLiteralSuffix` to select the
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/loop-convert-basic.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/loop-convert-basic.cpp
index 695925a5d877c..df2a2c1af1f54 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/loop-convert-basic.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/loop-convert-basic.cpp
@@ -980,3 +980,30 @@ namespace PR78381 {
     }
   }
 }
+
+namespace GH109083 {
+void test() {
+  const int N = 6;
+  int Arr[N] = {1, 2, 3, 4, 5, 6};
+
+  for (int I = 0; I < N; ++I) {
+    auto V = [T = Arr[I]]() {};
+  }
+  // CHECK-MESSAGES: :[[@LINE-3]]:3: warning: use range-based for loop instead [modernize-loop-convert]
+  // CHECK-FIXES: for (int I : Arr)
+  // CHECK-FIXES-NEXT: auto V = [T = I]() {};
+  for (int I = 0; I < N; ++I) {
+    auto V = [T = 10 + Arr[I]]() {};
+  }
+  // CHECK-MESSAGES: :[[@LINE-3]]:3: warning: use range-based for loop instead [modernize-loop-convert]
+  // CHECK-FIXES: for (int I : Arr)
+  // CHECK-FIXES-NEXT: auto V = [T = 10 + I]() {};
+
+  for (int I = 0; I < N; ++I) {
+    auto V = [T = I]() {};
+  }
+  for (int I = 0; I < N; ++I) {
+    auto V = [T = I + 10]() {};
+  }
+}
+} // namespace GH109083
diff --git a/clang-tools-extra/test/clang-tidy/infrastructure/verify-config.cpp b/clang-tools-extra/test/clang-tidy/infrastructure/verify-config.cpp
index 3659285986482..93f6f9fc156a4 100644
--- a/clang-tools-extra/test/clang-tidy/infrastructure/verify-config.cpp
+++ b/clang-tools-extra/test/clang-tidy/infrastructure/verify-config.cpp
@@ -30,3 +30,11 @@
 // CHECK-VERIFY-BLOCK-BAD: command-line option '-config': warning: check glob 'bugprone-arguments-*' doesn't match any known check [-verify-config]
 // CHECK-VERIFY-BLOCK-BAD: command-line option '-config': warning: unknown check 'bugprone-assert-side-effects'; did you mean 'bugprone-assert-side-effect' [-verify-config]
 
+// RUN: echo -e 'Checks: "-*,clang-analyzer-optin.cplusplus.UninitializedObject"\nCheckOptions:\n clang-analyzer-optin.cplusplus.UninitializedObject:Pedantic: true' > %T/MyClangTidyConfigCSA
+// RUN: clang-tidy --verify-config --config-file=%T/MyClangTidyConfigCSA 2>&1 | FileCheck %s -check-prefix=CHECK-VERIFY-CSA-OK -implicit-check-not='{{warnings|error}}'
+// CHECK-VERIFY-CSA-OK: No config errors detected.
+
+// RUN: echo -e 'Checks: "-*,clang-analyzer-optin.cplusplus.UninitializedObject"\nCheckOptions:\n clang-analyzer-optin.cplusplus.UninitializedObject.Pedantic: true' > %T/MyClangTidyConfigCSABad
+// RUN: not clang-tidy --verify-config --config-file=%T/MyClangTidyConfigCSABad 2>&1 | FileCheck %s -check-prefix=CHECK-VERIFY-CSA-BAD -implicit-check-not='{{warnings|error}}'
+// CHECK-VERIFY-CSA-BAD: command-line option '-config': warning: unknown check option 'clang-analyzer-optin.cplusplus.UninitializedObject.Pedantic'; did you mean 'clang-analyzer-optin.cplusplus.UninitializedObject:Pedantic' [-verify-config]
+
diff --git a/clang/cmake/caches/Release.cmake b/clang/cmake/caches/Release.cmake
index c93ff40ff3ee4..23e99493087ff 100644
--- a/clang/cmake/caches/Release.cmake
+++ b/clang/cmake/caches/Release.cmake
@@ -109,3 +109,6 @@ set_final_stage_var(LLVM_ENABLE_PROJECTS "${LLVM_RELEASE_ENABLE_PROJECTS}" STRIN
 set_final_stage_var(CPACK_GENERATOR "TXZ" STRING)
 set_final_stage_var(CPACK_ARCHIVE_THREADS "0" STRING)
 
+if(${CMAKE_HOST_SYSTEM_NAME} MATCHES "Darwin")
+  set_final_stage_var(LLVM_USE_STATIC_ZSTD "ON" BOOL)
+endif()
diff --git a/clang/cmake/caches/hexagon-unknown-linux-musl-clang-cross.cmake b/clang/cmake/caches/hexagon-unknown-linux-musl-clang-cross.cmake
new file mode 100644
index 0000000000000..91bbe26b62105
--- /dev/null
+++ b/clang/cmake/caches/hexagon-unknown-linux-musl-clang-cross.cmake
@@ -0,0 +1,15 @@
+# This file is for the llvm+clang options that are specific to building
+# a cross-toolchain targeting hexagon linux.
+set(DEFAULT_SYSROOT "../target/hexagon-unknown-linux-musl/" CACHE STRING "")
+set(CLANG_LINKS_TO_CREATE
+            hexagon-linux-musl-clang++
+            hexagon-linux-musl-clang
+            hexagon-unknown-linux-musl-clang++
+            hexagon-unknown-linux-musl-clang
+            hexagon-none-elf-clang++
+            hexagon-none-elf-clang
+            hexagon-unknown-none-elf-clang++
+            hexagon-unknown-none-elf-clang
+            CACHE STRING "")
+
+set(LLVM_INSTALL_TOOLCHAIN_ONLY ON CACHE BOOL "")
diff --git a/clang/cmake/caches/hexagon-unknown-linux-musl-clang.cmake b/clang/cmake/caches/hexagon-unknown-linux-musl-clang.cmake
new file mode 100644
index 0000000000000..9f3eb4678fd17
--- /dev/null
+++ b/clang/cmake/caches/hexagon-unknown-linux-musl-clang.cmake
@@ -0,0 +1,15 @@
+
+set(LLVM_TARGETS_TO_BUILD "Hexagon" CACHE STRING "")
+set(LLVM_DEFAULT_TARGET_TRIPLE "hexagon-unknown-linux-musl" CACHE STRING "")
+set(CLANG_DEFAULT_CXX_STDLIB "libc++" CACHE STRING "")
+set(CLANG_DEFAULT_OBJCOPY "llvm-objcopy" CACHE STRING "")
+set(CLANG_DEFAULT_RTLIB "compiler-rt" CACHE STRING "")
+set(CLANG_DEFAULT_UNWINDLIB "libunwind" CACHE STRING "")
+set(CLANG_DEFAULT_LINKER "lld" CACHE STRING "")
+set(LLVM_ENABLE_PROJECTS "clang;lld" CACHE STRING "")
+
+set(LLVM_INCLUDE_TESTS OFF CACHE BOOL "")
+set(LLVM_INCLUDE_DOCS OFF CACHE BOOL "")
+# Enabling toolchain-only causes problems when doing some of the
+# subsequent builds, will need to investigate:
+set(LLVM_INSTALL_TOOLCHAIN_ONLY OFF CACHE BOOL "")
diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst
index a427d7cd40fcd..a16edb0989b05 100644
--- a/clang/docs/ClangFormatStyleOptions.rst
+++ b/clang/docs/ClangFormatStyleOptions.rst
@@ -6554,6 +6554,15 @@ the configuration (without a prefix: ``Auto``).
     let DAGArgOtherID = (other i32:$other1, i32:$other2);
     let DAGArgBang = (!cast<SomeType>("Some") i32:$src1, i32:$src2)
 
+.. _TemplateNames:
+
+**TemplateNames** (``List of Strings``) :versionbadge:`clang-format 20` :ref:`¶ <TemplateNames>`
+  A vector of non-keyword identifiers that should be interpreted as
+  template names.
+
+  A ``<`` after a template name is annotated as a template opener instead of
+  a binary operator.
+
 .. _TypeNames:
 
 **TypeNames** (``List of Strings``) :versionbadge:`clang-format 17` :ref:`¶ <TypeNames>`
diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index ea4b4bcec55e7..fe41742a25c64 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -647,68 +647,77 @@ elementwise to the input.
 
 Unless specified otherwise operation(±0) = ±0 and operation(±infinity) = ±infinity
 
-=========================================== ================================================================ =========================================
-         Name                                Operation                                                        Supported element types
-=========================================== ================================================================ =========================================
- T __builtin_elementwise_abs(T x)            return the absolute value of a number x; the absolute value of   signed integer and floating point types
-                                             the most negative integer remains the most negative integer
- T __builtin_elementwise_fma(T x, T y, T z)  fused multiply add, (x * y) +  z.                                floating point types
- T __builtin_elementwise_ceil(T x)           return the smallest integral value greater than or equal to x    floating point types
- T __builtin_elementwise_sin(T x)            return the sine of x interpreted as an angle in radians          floating point types
- T __builtin_elementwise_cos(T x)            return the cosine of x interpreted as an angle in radians        floating point types
- T __builtin_elementwise_tan(T x)            return the tangent of x interpreted as an angle in radians       floating point types
- T __builtin_elementwise_asin(T x)           return the arcsine of x interpreted as an angle in radians       floating point types
- T __builtin_elementwise_acos(T x)           return the arccosine of x interpreted as an angle in radians     floating point types
- T __builtin_elementwise_atan(T x)           return the arctangent of x interpreted as an angle in radians    floating point types
- T __builtin_elementwise_sinh(T x)           return the hyperbolic sine of angle x in radians                 floating point types
- T __builtin_elementwise_cosh(T x)           return the hyperbolic cosine of angle x in radians               floating point types
- T __builtin_elementwise_tanh(T x)           return the hyperbolic tangent of angle x in radians              floating point types
- T __builtin_elementwise_floor(T x)          return the largest integral value less than or equal to x        floating point types
- T __builtin_elementwise_log(T x)            return the natural logarithm of x                                floating point types
- T __builtin_elementwise_log2(T x)           return the base 2 logarithm of x                                 floating point types
- T __builtin_elementwise_log10(T x)          return the base 10 logarithm of x                                floating point types
- T __builtin_elementwise_popcount(T x)       return the number of 1 bits in x                                 integer types 
- T __builtin_elementwise_pow(T x, T y)       return x raised to the power of y                                floating point types
- T __builtin_elementwise_bitreverse(T x)     return the integer represented after reversing the bits of x     integer types
- T __builtin_elementwise_exp(T x)            returns the base-e exponential, e^x, of the specified value      floating point types
- T __builtin_elementwise_exp2(T x)           returns the base-2 exponential, 2^x, of the specified value      floating point types
-
- T __builtin_elementwise_sqrt(T x)           return the square root of a floating-point number                floating point types
- T __builtin_elementwise_roundeven(T x)      round x to the nearest integer value in floating point format,   floating point types
-                                             rounding halfway cases to even (that is, to the nearest value
-                                             that is an even integer), regardless of the current rounding
-                                             direction.
- T __builtin_elementwise_round(T x)          round x to the nearest  integer value in floating point format,      floating point types
-                                             rounding halfway cases away from zero, regardless of the
-                                             current rounding direction. May raise floating-point
-                                             exceptions.
- T __builtin_elementwise_trunc(T x)          return the integral value nearest to but no larger in            floating point types
-                                             magnitude than x
-
-  T __builtin_elementwise_nearbyint(T x)     round x to the nearest  integer value in floating point format,      floating point types
-                                             rounding according to the current rounding direction.
-                                             May not raise the inexact floating-point exception. This is
-                                             treated the same as ``__builtin_elementwise_rint`` unless
-                                             :ref:`FENV_ACCESS is enabled <floating-point-environment>`.
-
- T __builtin_elementwise_rint(T x)           round x to the nearest  integer value in floating point format,      floating point types
-                                             rounding according to the current rounding
-                                             direction. May raise floating-point exceptions. This is treated
-                                             the same as ``__builtin_elementwise_nearbyint`` unless
-                                             :ref:`FENV_ACCESS is enabled <floating-point-environment>`.
-
- T __builtin_elementwise_canonicalize(T x)   return the platform specific canonical encoding                  floating point types
-                                             of a floating-point number
- T __builtin_elementwise_copysign(T x, T y)  return the magnitude of x with the sign of y.                    floating point types
- T __builtin_elementwise_fmod(T x, T y)      return The floating-point remainder of (x/y) whose sign          floating point types
-                                             matches the sign of x.
- T __builtin_elementwise_max(T x, T y)       return x or y, whichever is larger                               integer and floating point types
- T __builtin_elementwise_min(T x, T y)       return x or y, whichever is smaller                              integer and floating point types
- T __builtin_elementwise_add_sat(T x, T y)   return the sum of x and y, clamped to the range of               integer types
-                                             representable values for the signed/unsigned integer type.
- T __builtin_elementwise_sub_sat(T x, T y)   return the difference of x and y, clamped to the range of        integer types
-                                             representable values for the signed/unsigned integer type.
-=========================================== ================================================================ =========================================
+============================================== ====================================================================== =========================================
+         Name                                   Operation                                                             Supported element types
+============================================== ====================================================================== =========================================
+ T __builtin_elementwise_abs(T x)               return the absolute value of a number x; the absolute value of         signed integer and floating point types
+                                                the most negative integer remains the most negative integer
+ T __builtin_elementwise_fma(T x, T y, T z)     fused multiply add, (x * y) +  z.                                      floating point types
+ T __builtin_elementwise_ceil(T x)              return the smallest integral value greater than or equal to x          floating point types
+ T __builtin_elementwise_sin(T x)               return the sine of x interpreted as an angle in radians                floating point types
+ T __builtin_elementwise_cos(T x)               return the cosine of x interpreted as an angle in radians              floating point types
+ T __builtin_elementwise_tan(T x)               return the tangent of x interpreted as an angle in radians             floating point types
+ T __builtin_elementwise_asin(T x)              return the arcsine of x interpreted as an angle in radians             floating point types
+ T __builtin_elementwise_acos(T x)              return the arccosine of x interpreted as an angle in radians           floating point types
+ T __builtin_elementwise_atan(T x)              return the arctangent of x interpreted as an angle in radians          floating point types
+ T __builtin_elementwise_atan2(T y, T x)        return the arctangent of y/x                                           floating point types
+ T __builtin_elementwise_sinh(T x)              return the hyperbolic sine of angle x in radians                       floating point types
+ T __builtin_elementwise_cosh(T x)              return the hyperbolic cosine of angle x in radians                     floating point types
+ T __builtin_elementwise_tanh(T x)              return the hyperbolic tangent of angle x in radians                    floating point types
+ T __builtin_elementwise_floor(T x)             return the largest integral value less than or equal to x              floating point types
+ T __builtin_elementwise_log(T x)               return the natural logarithm of x                                      floating point types
+ T __builtin_elementwise_log2(T x)              return the base 2 logarithm of x                                       floating point types
+ T __builtin_elementwise_log10(T x)             return the base 10 logarithm of x                                      floating point types
+ T __builtin_elementwise_popcount(T x)          return the number of 1 bits in x                                       integer types
+ T __builtin_elementwise_pow(T x, T y)          return x raised to the power of y                                      floating point types
+ T __builtin_elementwise_bitreverse(T x)        return the integer represented after reversing the bits of x           integer types
+ T __builtin_elementwise_exp(T x)               returns the base-e exponential, e^x, of the specified value            floating point types
+ T __builtin_elementwise_exp2(T x)              returns the base-2 exponential, 2^x, of the specified value            floating point types
+
+ T __builtin_elementwise_sqrt(T x)              return the square root of a floating-point number                      floating point types
+ T __builtin_elementwise_roundeven(T x)         round x to the nearest integer value in floating point format,         floating point types
+                                                rounding halfway cases to even (that is, to the nearest value
+                                                that is an even integer), regardless of the current rounding
+                                                direction.
+ T __builtin_elementwise_round(T x)             round x to the nearest  integer value in floating point format,        floating point types
+                                                rounding halfway cases away from zero, regardless of the
+                                                current rounding direction. May raise floating-point
+                                                exceptions.
+ T __builtin_elementwise_trunc(T x)             return the integral value nearest to but no larger in                  floating point types
+                                                magnitude than x
+
+  T __builtin_elementwise_nearbyint(T x)        round x to the nearest  integer value in floating point format,        floating point types
+                                                rounding according to the current rounding direction.
+                                                May not raise the inexact floating-point exception. This is
+                                                treated the same as ``__builtin_elementwise_rint`` unless
+                                                :ref:`FENV_ACCESS is enabled <floating-point-environment>`.
+
+ T __builtin_elementwise_rint(T x)              round x to the nearest  integer value in floating point format,        floating point types
+                                                rounding according to the current rounding
+                                                direction. May raise floating-point exceptions. This is treated
+                                                the same as ``__builtin_elementwise_nearbyint`` unless
+                                                :ref:`FENV_ACCESS is enabled <floating-point-environment>`.
+
+ T __builtin_elementwise_canonicalize(T x)      return the platform specific canonical encoding                        floating point types
+                                                of a floating-point number
+ T __builtin_elementwise_copysign(T x, T y)     return the magnitude of x with the sign of y.                          floating point types
+ T __builtin_elementwise_fmod(T x, T y)         return The floating-point remainder of (x/y) whose sign                floating point types
+                                                matches the sign of x.
+ T __builtin_elementwise_max(T x, T y)          return x or y, whichever is larger                                     integer and floating point types
+ T __builtin_elementwise_min(T x, T y)          return x or y, whichever is smaller                                    integer and floating point types
+ T __builtin_elementwise_add_sat(T x, T y)      return the sum of x and y, clamped to the range of                     integer types
+                                                representable values for the signed/unsigned integer type.
+ T __builtin_elementwise_sub_sat(T x, T y)      return the difference of x and y, clamped to the range of              integer types
+                                                representable values for the signed/unsigned integer type.
+ T __builtin_elementwise_maximum(T x, T y)      return x or y, whichever is larger. Follows IEEE 754-2019              floating point types
+                                                semantics, see `LangRef
+                                                <http://llvm.org/docs/LangRef.html#llvm-min-intrinsics-comparation>`_
+                                                for the comparison.
+ T __builtin_elementwise_minimum(T x, T y)      return x or y, whichever is smaller. Follows IEEE 754-2019             floating point types
+                                                semantics, see `LangRef
+                                                <http://llvm.org/docs/LangRef.html#llvm-min-intrinsics-comparation>`_
+                                                for the comparison.
+============================================== ====================================================================== =========================================
 
 
 *Reduction Builtins*
@@ -733,21 +742,29 @@ Example:
 
 Let ``VT`` be a vector type and ``ET`` the element type of ``VT``.
 
-======================================= ================================================================ ==================================
-         Name                            Operation                                                        Supported element types
-======================================= ================================================================ ==================================
- ET __builtin_reduce_max(VT a)           return x or y, whichever is larger; If exactly one argument is   integer and floating point types
+======================================= ====================================================================== ==================================
+         Name                            Operation                                                              Supported element types
+======================================= ====================================================================== ==================================
+ ET __builtin_reduce_max(VT a)           return x or y, whichever is larger; If exactly one argument is         integer and floating point types
                                          a NaN, return the other argument. If both arguments are NaNs,
                                          fmax() return a NaN.
- ET __builtin_reduce_min(VT a)           return x or y, whichever is smaller; If exactly one argument     integer and floating point types
+ ET __builtin_reduce_min(VT a)           return x or y, whichever is smaller; If exactly one argument           integer and floating point types
                                          is a NaN, return the other argument. If both arguments are
                                          NaNs, fmax() return a NaN.
- ET __builtin_reduce_add(VT a)           \+                                                               integer types
- ET __builtin_reduce_mul(VT a)           \*                                                               integer types
- ET __builtin_reduce_and(VT a)           &                                                                integer types
- ET __builtin_reduce_or(VT a)            \|                                                               integer types
- ET __builtin_reduce_xor(VT a)           ^                                                                integer types
-======================================= ================================================================ ==================================
+ ET __builtin_reduce_add(VT a)           \+                                                                     integer types
+ ET __builtin_reduce_mul(VT a)           \*                                                                     integer types
+ ET __builtin_reduce_and(VT a)           &                                                                      integer types
+ ET __builtin_reduce_or(VT a)            \|                                                                     integer types
+ ET __builtin_reduce_xor(VT a)           ^                                                                      integer types
+ ET __builtin_reduce_maximum(VT a)       return the largest element of the vector. Follows IEEE 754-2019        floating point types
+                                         semantics, see `LangRef
+                                         <http://llvm.org/docs/LangRef.html#llvm-min-intrinsics-comparation>`_
+                                         for the comparison.
+ ET __builtin_reduce_minimum(VT a)       return the smallest element of the vector. Follows IEEE 754-2019       floating point types
+                                         semantics, see `LangRef
+                                         <http://llvm.org/docs/LangRef.html#llvm-min-intrinsics-comparation>`_
+                                         for the comparison.
+======================================= ====================================================================== ==================================
 
 Matrix Types
 ============
@@ -5759,6 +5776,8 @@ The following builtin intrinsics can be used in constant expressions:
 
 The following x86-specific intrinsics can be used in constant expressions:
 
+* ``_addcarry_u32``
+* ``_addcarry_u64``
 * ``_bit_scan_forward``
 * ``_bit_scan_reverse``
 * ``__bsfd``
@@ -5799,6 +5818,8 @@ The following x86-specific intrinsics can be used in constant expressions:
 * ``_rotwr``
 * ``_lrotl``
 * ``_lrotr``
+* ``_subborrow_u32``
+* ``_subborrow_u64``
 
 Debugging the Compiler
 ======================
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 34d2b584274a5..44d5f348ed2d5 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -104,6 +104,7 @@ ABI Changes in This Version
 
 - Fixed Microsoft name mangling of placeholder, auto and decltype(auto), return types for MSVC 1920+. This change resolves incompatibilities with code compiled by MSVC 1920+ but will introduce incompatibilities with code compiled by earlier versions of Clang unless such code is built with the compiler option -fms-compatibility-version=19.14 to imitate the MSVC 1914 mangling behavior.
 - Fixed the Itanium mangling of the construction vtable name. This change will introduce incompatibilities with code compiled by Clang 19 and earlier versions, unless the -fclang-abi-compat=19 option is used. (#GH108015)
+- Mangle member-like friend function templates as members of the enclosing class. (#GH110247, #GH110503)
 
 AST Dumping Potentially Breaking Changes
 ----------------------------------------
@@ -144,6 +145,9 @@ C++ Language Changes
 
 - Add ``__builtin_elementwise_fmod`` builtin for floating point types only.
 
+- Add ``__builtin_elementwise_minimum`` and ``__builtin_elementwise_maximum``
+  builtin for floating point types only.
+
 - The builtin type alias ``__builtin_common_type`` has been added to improve the
   performance of ``std::common_type``.
 
@@ -170,6 +174,10 @@ C++23 Feature Support
 C++20 Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
 
+C++17 Feature Support
+^^^^^^^^^^^^^^^^^^^^^
+- The implementation of the relaxed template template argument matching rules is
+  more complete and reliable, and should provide more accurate diagnostics.
 
 Resolutions to C++ Defect Reports
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -327,6 +335,10 @@ Improvements to Clang's diagnostics
 
 - Clang now diagnoses when the result of a [[nodiscard]] function is discarded after being cast in C. Fixes #GH104391.
 
+- Clang now properly explains the reason a template template argument failed to
+  match a template template parameter, in terms of the C++17 relaxed matching rules
+  instead of the old ones.
+
 - Don't emit duplicated dangling diagnostics. (#GH93386).
 
 - Improved diagnostic when trying to befriend a concept. (#GH45182).
@@ -364,6 +376,8 @@ Improvements to Clang's diagnostics
 
 - Clang now diagnoses when a ``requires`` expression has a local parameter of void type, aligning with the function parameter (#GH109831).
 
+- Clang now emits a diagnostic note at the class declaration when the method definition does not match any declaration (#GH110638).
+
 Improvements to Clang's time-trace
 ----------------------------------
 
@@ -424,6 +438,8 @@ Bug Fixes to C++ Support
 - Correctly check constraints of explicit instantiations of member functions. (#GH46029)
 - When performing partial ordering of function templates, clang now checks that
   the deduction was consistent. Fixes (#GH18291).
+- Fixes to several issues in partial ordering of template template parameters, which
+  were documented in the test suite.
 - Fixed an assertion failure about a constraint of a friend function template references to a value with greater
   template depth than the friend function template. (#GH98258)
 - Clang now rebuilds the template parameters of out-of-line declarations and specializations in the context
@@ -452,7 +468,7 @@ Bug Fixes to C++ Support
 - Fixed an assertion failure in debug mode, and potential crashes in release mode, when
   diagnosing a failed cast caused indirectly by a failed implicit conversion to the type of the constructor parameter.
 - Fixed an assertion failure by adjusting integral to boolean vector conversions (#GH108326)
-- Mangle friend function templates with a constraint that depends on a template parameter from an enclosing template as members of the enclosing class. (#GH110247)
+- Fixed an issue deducing non-type template arguments of reference type. (#GH73460)
 - Fixed an issue in constraint evaluation, where type constraints on the lambda expression
   containing outer unexpanded parameters were not correctly expanded. (#GH101754)
 - Fixed a bug in constraint expression comparison where the ``sizeof...`` expression was not handled properly
@@ -520,6 +536,10 @@ X86 Support
   * Supported MINMAX intrinsics of ``*_(mask(z)))_minmax(ne)_p[s|d|h|bh]`` and
   ``*_(mask(z)))_minmax_s[s|d|h]``.
 
+- All intrinsics in adcintrin.h can now be used in constant expressions.
+
+- All intrinsics in adxintrin.h can now be used in constant expressions.
+
 - All intrinsics in lzcntintrin.h can now be used in constant expressions.
 
 - All intrinsics in bmiintrin.h can now be used in constant expressions.
@@ -575,6 +595,8 @@ DWARF Support in Clang
 Floating Point Support in Clang
 -------------------------------
 
+- Add ``__builtin_elementwise_atan2`` builtin for floating point types only.
+
 Fixed Point Support in Clang
 ----------------------------
 
@@ -593,6 +615,7 @@ clang-format
 ------------
 
 - Adds ``BreakBinaryOperations`` option.
+- Adds ``TemplateNames`` option.
 
 libclang
 --------
@@ -605,11 +628,18 @@ Static Analyzer
 New features
 ^^^^^^^^^^^^
 
+- Now CSA models `__builtin_*_overflow` functions. (#GH102602)
+
 - MallocChecker now checks for ``ownership_returns(class, idx)`` and ``ownership_takes(class, idx)``
   attributes with class names different from "malloc". Clang static analyzer now reports an error
   if class of allocation and deallocation function mismatches.
   `Documentation <https://clang.llvm.org/docs/analyzer/checkers.html#unix-mismatcheddeallocator-c-c>`__.
 
+- Function effects, e.g. the ``nonblocking`` and ``nonallocating`` "performance constraint" 
+  attributes, are now verified. For example, for functions declared with the ``nonblocking`` 
+  attribute, the compiler can generate warnings about the use of any language features, or calls to
+  other functions, which may block.
+
 Crash and bug fixes
 ^^^^^^^^^^^^^^^^^^^
 
diff --git a/clang/include/clang/AST/APValue.h b/clang/include/clang/AST/APValue.h
index c4206b73b1156..7869ee386689d 100644
--- a/clang/include/clang/AST/APValue.h
+++ b/clang/include/clang/AST/APValue.h
@@ -314,51 +314,95 @@ class APValue {
   DataType Data;
 
 public:
+  /// Creates an empty APValue of type None.
   APValue() : Kind(None) {}
+  /// Creates an integer APValue holding the given value.
   explicit APValue(APSInt I) : Kind(None) {
     MakeInt(); setInt(std::move(I));
   }
+  /// Creates a float APValue holding the given value.
   explicit APValue(APFloat F) : Kind(None) {
     MakeFloat(); setFloat(std::move(F));
   }
+  /// Creates a fixed-point APValue holding the given value.
   explicit APValue(APFixedPoint FX) : Kind(None) {
     MakeFixedPoint(std::move(FX));
   }
+  /// Creates a vector APValue with \p N elements. The elements
+  /// are read from \p E.
   explicit APValue(const APValue *E, unsigned N) : Kind(None) {
     MakeVector(); setVector(E, N);
   }
+  /// Creates an integer complex APValue with the given real and imaginary
+  /// values.
   APValue(APSInt R, APSInt I) : Kind(None) {
     MakeComplexInt(); setComplexInt(std::move(R), std::move(I));
   }
+  /// Creates a float complex APValue with the given real and imaginary values.
   APValue(APFloat R, APFloat I) : Kind(None) {
     MakeComplexFloat(); setComplexFloat(std::move(R), std::move(I));
   }
   APValue(const APValue &RHS);
   APValue(APValue &&RHS);
-  APValue(LValueBase B, const CharUnits &O, NoLValuePath N,
+  /// Creates an lvalue APValue without an lvalue path.
+  /// \param Base The base of the lvalue.
+  /// \param Offset The offset of the lvalue.
+  /// \param IsNullPtr Whether this lvalue is a null pointer.
+  APValue(LValueBase Base, const CharUnits &Offset, NoLValuePath,
           bool IsNullPtr = false)
       : Kind(None) {
-    MakeLValue(); setLValue(B, O, N, IsNullPtr);
-  }
-  APValue(LValueBase B, const CharUnits &O, ArrayRef<LValuePathEntry> Path,
-          bool OnePastTheEnd, bool IsNullPtr = false)
+    MakeLValue();
+    setLValue(Base, Offset, NoLValuePath{}, IsNullPtr);
+  }
+  /// Creates an lvalue APValue with an lvalue path.
+  /// \param Base The base of the lvalue.
+  /// \param Offset The offset of the lvalue.
+  /// \param Path The lvalue path.
+  /// \param OnePastTheEnd Whether this lvalue is one-past-the-end of the
+  /// subobject it points to.
+  /// \param IsNullPtr Whether this lvalue is a null pointer.
+  APValue(LValueBase Base, const CharUnits &Offset,
+          ArrayRef<LValuePathEntry> Path, bool OnePastTheEnd,
+          bool IsNullPtr = false)
       : Kind(None) {
-    MakeLValue(); setLValue(B, O, Path, OnePastTheEnd, IsNullPtr);
-  }
+    MakeLValue();
+    setLValue(Base, Offset, Path, OnePastTheEnd, IsNullPtr);
+  }
+  /// Creates a new array APValue.
+  /// \param UninitArray Marker. Pass an empty UninitArray.
+  /// \param InitElts Number of elements you're going to initialize in the
+  /// array.
+  /// \param Size Full size of the array.
   APValue(UninitArray, unsigned InitElts, unsigned Size) : Kind(None) {
     MakeArray(InitElts, Size);
   }
-  APValue(UninitStruct, unsigned B, unsigned M) : Kind(None) {
-    MakeStruct(B, M);
-  }
-  explicit APValue(const FieldDecl *D, const APValue &V = APValue())
+  /// Creates a new struct APValue.
+  /// \param UninitStruct Marker. Pass an empty UninitStruct.
+  /// \param NumBases Number of bases.
+  /// \param NumMembers Number of members.
+  APValue(UninitStruct, unsigned NumBases, unsigned NumMembers) : Kind(None) {
+    MakeStruct(NumBases, NumMembers);
+  }
+  /// Creates a new union APValue.
+  /// \param ActiveDecl The FieldDecl of the active union member.
+  /// \param ActiveValue The value of the active union member.
+  explicit APValue(const FieldDecl *ActiveDecl,
+                   const APValue &ActiveValue = APValue())
       : Kind(None) {
-    MakeUnion(); setUnion(D, V);
+    MakeUnion();
+    setUnion(ActiveDecl, ActiveValue);
   }
+  /// Creates a new member pointer APValue.
+  /// \param Member Declaration of the member
+  /// \param IsDerivedMember Whether member is a derived one.
+  /// \param Path The path of the member.
   APValue(const ValueDecl *Member, bool IsDerivedMember,
           ArrayRef<const CXXRecordDecl*> Path) : Kind(None) {
     MakeMemberPointer(Member, IsDerivedMember, Path);
   }
+  /// Creates a new address label diff APValue.
+  /// \param LHSExpr The left-hand side of the difference.
+  /// \param RHSExpr The right-hand side of the difference.
   APValue(const AddrLabelExpr* LHSExpr, const AddrLabelExpr* RHSExpr)
       : Kind(None) {
     MakeAddrLabelDiff(); setAddrLabelDiff(LHSExpr, RHSExpr);
diff --git a/clang/include/clang/AST/CanonicalType.h b/clang/include/clang/AST/CanonicalType.h
index dde08f0394c98..6102eb0179353 100644
--- a/clang/include/clang/AST/CanonicalType.h
+++ b/clang/include/clang/AST/CanonicalType.h
@@ -299,6 +299,7 @@ class CanProxyBase {
   LLVM_CLANG_CANPROXY_SIMPLE_ACCESSOR(bool, isDependentType)
   LLVM_CLANG_CANPROXY_SIMPLE_ACCESSOR(bool, isOverloadableType)
   LLVM_CLANG_CANPROXY_SIMPLE_ACCESSOR(bool, isArrayType)
+  LLVM_CLANG_CANPROXY_SIMPLE_ACCESSOR(bool, isConstantArrayType)
   LLVM_CLANG_CANPROXY_SIMPLE_ACCESSOR(bool, hasPointerRepresentation)
   LLVM_CLANG_CANPROXY_SIMPLE_ACCESSOR(bool, hasObjCPointerRepresentation)
   LLVM_CLANG_CANPROXY_SIMPLE_ACCESSOR(bool, hasIntegerRepresentation)
diff --git a/clang/include/clang/AST/ComputeDependence.h b/clang/include/clang/AST/ComputeDependence.h
index 6d3a51c379f9d..1a8507cfbf987 100644
--- a/clang/include/clang/AST/ComputeDependence.h
+++ b/clang/include/clang/AST/ComputeDependence.h
@@ -107,6 +107,7 @@ class ObjCSubscriptRefExpr;
 class ObjCIsaExpr;
 class ObjCIndirectCopyRestoreExpr;
 class ObjCMessageExpr;
+class OpenACCAsteriskSizeExpr;
 
 // The following functions are called from constructors of `Expr`, so they
 // should not access anything beyond basic
@@ -203,6 +204,7 @@ ExprDependence computeDependence(ObjCSubscriptRefExpr *E);
 ExprDependence computeDependence(ObjCIsaExpr *E);
 ExprDependence computeDependence(ObjCIndirectCopyRestoreExpr *E);
 ExprDependence computeDependence(ObjCMessageExpr *E);
+ExprDependence computeDependence(OpenACCAsteriskSizeExpr *E);
 
 } // namespace clang
 #endif
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index 66c746cc25040..57353855c51e7 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -2072,6 +2072,41 @@ class PredefinedExpr final
   }
 };
 
+/// This expression type represents an asterisk in an OpenACC Size-Expr, used in
+/// the 'tile' and 'gang' clauses. It is of 'int' type, but should not be
+/// evaluated.
+class OpenACCAsteriskSizeExpr final : public Expr {
+  friend class ASTStmtReader;
+  SourceLocation AsteriskLoc;
+
+  OpenACCAsteriskSizeExpr(SourceLocation AsteriskLoc, QualType IntTy)
+      : Expr(OpenACCAsteriskSizeExprClass, IntTy, VK_PRValue, OK_Ordinary),
+        AsteriskLoc(AsteriskLoc) {}
+
+  void setAsteriskLocation(SourceLocation Loc) { AsteriskLoc = Loc; }
+
+public:
+  static OpenACCAsteriskSizeExpr *Create(const ASTContext &C,
+                                         SourceLocation Loc);
+  static OpenACCAsteriskSizeExpr *CreateEmpty(const ASTContext &C);
+
+  SourceLocation getBeginLoc() const { return AsteriskLoc; }
+  SourceLocation getEndLoc() const { return AsteriskLoc; }
+  SourceLocation getLocation() const { return AsteriskLoc; }
+
+  static bool classof(const Stmt *T) {
+    return T->getStmtClass() == OpenACCAsteriskSizeExprClass;
+  }
+  // Iterators
+  child_range children() {
+    return child_range(child_iterator(), child_iterator());
+  }
+
+  const_child_range children() const {
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+};
+
 // This represents a use of the __builtin_sycl_unique_stable_name, which takes a
 // type-id, and at CodeGen time emits a unique string representation of the
 // type in a way that permits us to properly encode information about the SYCL
diff --git a/clang/include/clang/AST/JSONNodeDumper.h b/clang/include/clang/AST/JSONNodeDumper.h
index 55bd583e304e8..9422c8fceccfb 100644
--- a/clang/include/clang/AST/JSONNodeDumper.h
+++ b/clang/include/clang/AST/JSONNodeDumper.h
@@ -283,6 +283,7 @@ class JSONNodeDumper
 
   void VisitDeclRefExpr(const DeclRefExpr *DRE);
   void VisitSYCLUniqueStableNameExpr(const SYCLUniqueStableNameExpr *E);
+  void VisitOpenACCAsteriskSizeExpr(const OpenACCAsteriskSizeExpr *E);
   void VisitPredefinedExpr(const PredefinedExpr *PE);
   void VisitUnaryOperator(const UnaryOperator *UO);
   void VisitBinaryOperator(const BinaryOperator *BO);
diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index 90f5b7fc9ab6f..e4f2e07222a33 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -481,6 +481,35 @@ class OpenACCNumGangsClause final
   }
 };
 
+class OpenACCTileClause final
+    : public OpenACCClauseWithExprs,
+      public llvm::TrailingObjects<OpenACCTileClause, Expr *> {
+  OpenACCTileClause(SourceLocation BeginLoc, SourceLocation LParenLoc,
+                    ArrayRef<Expr *> SizeExprs, SourceLocation EndLoc)
+      : OpenACCClauseWithExprs(OpenACCClauseKind::Tile, BeginLoc, LParenLoc,
+                               EndLoc) {
+    std::uninitialized_copy(SizeExprs.begin(), SizeExprs.end(),
+                            getTrailingObjects<Expr *>());
+    setExprs(MutableArrayRef(getTrailingObjects<Expr *>(), SizeExprs.size()));
+  }
+
+public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::Tile;
+  }
+  static OpenACCTileClause *Create(const ASTContext &C, SourceLocation BeginLoc,
+                                   SourceLocation LParenLoc,
+                                   ArrayRef<Expr *> SizeExprs,
+                                   SourceLocation EndLoc);
+  llvm::ArrayRef<Expr *> getSizeExprs() {
+    return OpenACCClauseWithExprs::getExprs();
+  }
+
+  llvm::ArrayRef<Expr *> getSizeExprs() const {
+    return OpenACCClauseWithExprs::getExprs();
+  }
+};
+
 /// Represents one of a handful of clauses that have a single integer
 /// expression.
 class OpenACCClauseWithSingleIntExpr : public OpenACCClauseWithExprs {
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index cd9947f7ab980..cbbba9e88b7f5 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -2867,6 +2867,7 @@ DEF_TRAVERSE_STMT(ParenListExpr, {})
 DEF_TRAVERSE_STMT(SYCLUniqueStableNameExpr, {
   TRY_TO(TraverseTypeLoc(S->getTypeSourceInfo()->getTypeLoc()));
 })
+DEF_TRAVERSE_STMT(OpenACCAsteriskSizeExpr, {})
 DEF_TRAVERSE_STMT(PredefinedExpr, {})
 DEF_TRAVERSE_STMT(ShuffleVectorExpr, {})
 DEF_TRAVERSE_STMT(ConvertVectorExpr, {})
diff --git a/clang/include/clang/AST/TextNodeDumper.h b/clang/include/clang/AST/TextNodeDumper.h
index 57100e7ede171..9c320c8ae3e54 100644
--- a/clang/include/clang/AST/TextNodeDumper.h
+++ b/clang/include/clang/AST/TextNodeDumper.h
@@ -410,6 +410,7 @@ class TextNodeDumper
   void VisitHLSLOutArgExpr(const HLSLOutArgExpr *E);
   void VisitOpenACCConstructStmt(const OpenACCConstructStmt *S);
   void VisitOpenACCLoopConstruct(const OpenACCLoopConstruct *S);
+  void VisitOpenACCAsteriskSizeExpr(const OpenACCAsteriskSizeExpr *S);
   void VisitEmbedExpr(const EmbedExpr *S);
   void VisitAtomicExpr(const AtomicExpr *AE);
 };
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index 7126940058bae..8ff04cf89a6b9 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -49,6 +49,7 @@
 #include "llvm/Support/PointerLikeTypeTraits.h"
 #include "llvm/Support/TrailingObjects.h"
 #include "llvm/Support/type_traits.h"
+#include <bitset>
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
@@ -119,6 +120,8 @@ class EnumDecl;
 class Expr;
 class ExtQualsTypeCommonBase;
 class FunctionDecl;
+class FunctionEffectsRef;
+class FunctionEffectKindSet;
 class FunctionEffectSet;
 class IdentifierInfo;
 class NamedDecl;
@@ -4712,12 +4715,13 @@ class FunctionEffect {
 public:
   /// Identifies the particular effect.
   enum class Kind : uint8_t {
-    None = 0,
-    NonBlocking = 1,
-    NonAllocating = 2,
-    Blocking = 3,
-    Allocating = 4
+    NonBlocking,
+    NonAllocating,
+    Blocking,
+    Allocating,
+    Last = Allocating
   };
+  constexpr static size_t KindCount = static_cast<size_t>(Kind::Last) + 1;
 
   /// Flags describing some behaviors of the effect.
   using Flags = unsigned;
@@ -4743,8 +4747,6 @@ class FunctionEffect {
   // be considered for uniqueness.
 
 public:
-  FunctionEffect() : FKind(Kind::None) {}
-
   explicit FunctionEffect(Kind K) : FKind(K) {}
 
   /// The kind of the effect.
@@ -4773,8 +4775,6 @@ class FunctionEffect {
     case Kind::Blocking:
     case Kind::Allocating:
       return 0;
-    case Kind::None:
-      break;
     }
     llvm_unreachable("unknown effect kind");
   }
@@ -4782,26 +4782,36 @@ class FunctionEffect {
   /// The description printed in diagnostics, e.g. 'nonblocking'.
   StringRef name() const;
 
-  /// Return true if the effect is allowed to be inferred on the callee,
-  /// which is either a FunctionDecl or BlockDecl.
+  friend raw_ostream &operator<<(raw_ostream &OS,
+                                 const FunctionEffect &Effect) {
+    OS << Effect.name();
+    return OS;
+  }
+
+  /// Determine whether the effect is allowed to be inferred on the callee,
+  /// which is either a FunctionDecl or BlockDecl. If the returned optional
+  /// is empty, inference is permitted; otherwise it holds the effect which
+  /// blocked inference.
   /// Example: This allows nonblocking(false) to prevent inference for the
   /// function.
-  bool canInferOnFunction(const Decl &Callee) const;
+  std::optional<FunctionEffect>
+  effectProhibitingInference(const Decl &Callee,
+                             FunctionEffectKindSet CalleeFX) const;
 
   // Return false for success. When true is returned for a direct call, then the
   // FE_InferrableOnCallees flag may trigger inference rather than an immediate
   // diagnostic. Caller should be assumed to have the effect (it may not have it
   // explicitly when inferring).
   bool shouldDiagnoseFunctionCall(bool Direct,
-                                  ArrayRef<FunctionEffect> CalleeFX) const;
+                                  FunctionEffectKindSet CalleeFX) const;
 
-  friend bool operator==(const FunctionEffect &LHS, const FunctionEffect &RHS) {
+  friend bool operator==(FunctionEffect LHS, FunctionEffect RHS) {
     return LHS.FKind == RHS.FKind;
   }
-  friend bool operator!=(const FunctionEffect &LHS, const FunctionEffect &RHS) {
+  friend bool operator!=(FunctionEffect LHS, FunctionEffect RHS) {
     return !(LHS == RHS);
   }
-  friend bool operator<(const FunctionEffect &LHS, const FunctionEffect &RHS) {
+  friend bool operator<(FunctionEffect LHS, FunctionEffect RHS) {
     return LHS.FKind < RHS.FKind;
   }
 };
@@ -4829,13 +4839,14 @@ struct FunctionEffectWithCondition {
   FunctionEffect Effect;
   EffectConditionExpr Cond;
 
-  FunctionEffectWithCondition() = default;
-  FunctionEffectWithCondition(const FunctionEffect &E,
-                              const EffectConditionExpr &C)
+  FunctionEffectWithCondition(FunctionEffect E, const EffectConditionExpr &C)
       : Effect(E), Cond(C) {}
 
   /// Return a textual description of the effect, and its condition, if any.
   std::string description() const;
+
+  friend raw_ostream &operator<<(raw_ostream &OS,
+                                 const FunctionEffectWithCondition &CFE);
 };
 
 /// Support iteration in parallel through a pair of FunctionEffect and
@@ -4940,6 +4951,85 @@ class FunctionEffectsRef {
   void dump(llvm::raw_ostream &OS) const;
 };
 
+/// A mutable set of FunctionEffect::Kind.
+class FunctionEffectKindSet {
+  // For now this only needs to be a bitmap.
+  constexpr static size_t EndBitPos = FunctionEffect::KindCount;
+  using KindBitsT = std::bitset<EndBitPos>;
+
+  KindBitsT KindBits{};
+
+  explicit FunctionEffectKindSet(KindBitsT KB) : KindBits(KB) {}
+
+  // Functions to translate between an effect kind, starting at 1, and a
+  // position in the bitset.
+
+  constexpr static size_t kindToPos(FunctionEffect::Kind K) {
+    return static_cast<size_t>(K);
+  }
+
+  constexpr static FunctionEffect::Kind posToKind(size_t Pos) {
+    return static_cast<FunctionEffect::Kind>(Pos);
+  }
+
+  // Iterates through the bits which are set.
+  class iterator {
+    const FunctionEffectKindSet *Outer = nullptr;
+    size_t Idx = 0;
+
+    // If Idx does not reference a set bit, advance it until it does,
+    // or until it reaches EndBitPos.
+    void advanceToNextSetBit() {
+      while (Idx < EndBitPos && !Outer->KindBits.test(Idx))
+        ++Idx;
+    }
+
+  public:
+    iterator();
+    iterator(const FunctionEffectKindSet &O, size_t I) : Outer(&O), Idx(I) {
+      advanceToNextSetBit();
+    }
+    bool operator==(const iterator &Other) const { return Idx == Other.Idx; }
+    bool operator!=(const iterator &Other) const { return Idx != Other.Idx; }
+
+    iterator operator++() {
+      ++Idx;
+      advanceToNextSetBit();
+      return *this;
+    }
+
+    FunctionEffect operator*() const {
+      assert(Idx < EndBitPos && "Dereference of end iterator");
+      return FunctionEffect(posToKind(Idx));
+    }
+  };
+
+public:
+  FunctionEffectKindSet() = default;
+  explicit FunctionEffectKindSet(FunctionEffectsRef FX) { insert(FX); }
+
+  iterator begin() const { return iterator(*this, 0); }
+  iterator end() const { return iterator(*this, EndBitPos); }
+
+  void insert(FunctionEffect Effect) { KindBits.set(kindToPos(Effect.kind())); }
+  void insert(FunctionEffectsRef FX) {
+    for (FunctionEffect Item : FX.effects())
+      insert(Item);
+  }
+  void insert(FunctionEffectKindSet Set) { KindBits |= Set.KindBits; }
+
+  bool empty() const { return KindBits.none(); }
+  bool contains(const FunctionEffect::Kind EK) const {
+    return KindBits.test(kindToPos(EK));
+  }
+  void dump(llvm::raw_ostream &OS) const;
+
+  static FunctionEffectKindSet difference(FunctionEffectKindSet LHS,
+                                          FunctionEffectKindSet RHS) {
+    return FunctionEffectKindSet(LHS.KindBits & ~RHS.KindBits);
+  }
+};
+
 /// A mutable set of FunctionEffects and possibly conditions attached to them.
 /// Used to compare and merge effects on declarations.
 ///
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 53d88482698f0..b1512e22ee2dd 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -8482,9 +8482,9 @@ compiler warnings:
 - A redeclaration of a ``nonblocking`` or ``nonallocating`` function must also be declared with
   the same attribute (or a stronger one). A redeclaration may add an attribute.
 
-The warnings are controlled by ``-Wfunction-effects``, which is enabled by default.
+The warnings are controlled by ``-Wfunction-effects``, which is disabled by default.
 
-In a future commit, the compiler will diagnose function calls from ``nonblocking`` and ``nonallocating``
+The compiler also diagnoses function calls from ``nonblocking`` and ``nonallocating``
 functions to other functions which lack the appropriate attribute.
   }];
 }
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 8090119e512fb..a726a0ef4b4bd 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -1250,6 +1250,12 @@ def ElementwiseATan : Builtin {
   let Prototype = "void(...)";
 }
 
+def ElementwiseATan2 : Builtin {
+  let Spellings = ["__builtin_elementwise_atan2"];
+  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Prototype = "void(...)";
+}
+
 def ElementwiseBitreverse : Builtin {
   let Spellings = ["__builtin_elementwise_bitreverse"];
   let Attributes = [NoThrow, Const, CustomTypeChecking];
@@ -1268,6 +1274,18 @@ def ElementwiseMin : Builtin {
   let Prototype = "void(...)";
 }
 
+def ElementwiseMaximum : Builtin {
+  let Spellings = ["__builtin_elementwise_maximum"];
+  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Prototype = "void(...)";
+}
+
+def ElementwiseMinimum : Builtin {
+  let Spellings = ["__builtin_elementwise_minimum"];
+  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Prototype = "void(...)";
+}
+
 def ElementwiseCeil : Builtin {
   let Spellings = ["__builtin_elementwise_ceil"];
   let Attributes = [NoThrow, Const, CustomTypeChecking];
@@ -1442,6 +1460,18 @@ def ReduceMin : Builtin {
   let Prototype = "void(...)";
 }
 
+def ReduceMaximum : Builtin {
+  let Spellings = ["__builtin_reduce_maximum"];
+  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Prototype = "void(...)";
+}
+
+def ReduceMinimum : Builtin {
+  let Spellings = ["__builtin_reduce_minimum"];
+  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Prototype = "void(...)";
+}
+
 def ReduceXor : Builtin {
   let Spellings = ["__builtin_reduce_xor"];
   let Attributes = [NoThrow, Const, CustomTypeChecking];
@@ -4709,6 +4739,12 @@ def HLSLClamp : LangBuiltin<"HLSL_LANG"> {
   let Prototype = "void(...)";
 }
 
+def HLSLCross: LangBuiltin<"HLSL_LANG"> {
+  let Spellings = ["__builtin_hlsl_cross"];
+  let Attributes = [NoThrow, Const];
+  let Prototype = "void(...)";
+}
+
 def HLSLDotProduct : LangBuiltin<"HLSL_LANG"> {
   let Spellings = ["__builtin_hlsl_dot"];
   let Attributes = [NoThrow, Const];
diff --git a/clang/include/clang/Basic/BuiltinsX86.def b/clang/include/clang/Basic/BuiltinsX86.def
index 2a987abcf9a35..4c6b22cca421c 100644
--- a/clang/include/clang/Basic/BuiltinsX86.def
+++ b/clang/include/clang/Basic/BuiltinsX86.def
@@ -543,8 +543,8 @@ TARGET_BUILTIN(__builtin_ia32_wbinvd, "v", "n", "")
 TARGET_BUILTIN(__builtin_ia32_wbnoinvd, "v", "n", "wbnoinvd")
 
 // ADX
-TARGET_BUILTIN(__builtin_ia32_addcarryx_u32, "UcUcUiUiUi*", "n", "")
-TARGET_BUILTIN(__builtin_ia32_subborrow_u32, "UcUcUiUiUi*", "n", "")
+TARGET_BUILTIN(__builtin_ia32_addcarryx_u32, "UcUcUiUiUi*", "nE", "")
+TARGET_BUILTIN(__builtin_ia32_subborrow_u32, "UcUcUiUiUi*", "nE", "")
 
 // RDSEED
 TARGET_BUILTIN(__builtin_ia32_rdseed16_step, "UiUs*", "n", "rdseed")
diff --git a/clang/include/clang/Basic/BuiltinsX86_64.def b/clang/include/clang/Basic/BuiltinsX86_64.def
index d5fdb272d92d1..2c591edb2835c 100644
--- a/clang/include/clang/Basic/BuiltinsX86_64.def
+++ b/clang/include/clang/Basic/BuiltinsX86_64.def
@@ -66,8 +66,8 @@ TARGET_BUILTIN(__builtin_ia32_incsspq, "vUOi", "n", "shstk")
 TARGET_BUILTIN(__builtin_ia32_rdsspq, "UOiUOi", "n", "shstk")
 TARGET_BUILTIN(__builtin_ia32_wrssq, "vUOiv*", "n", "shstk")
 TARGET_BUILTIN(__builtin_ia32_wrussq, "vUOiv*", "n", "shstk")
-TARGET_BUILTIN(__builtin_ia32_addcarryx_u64, "UcUcUOiUOiUOi*", "n", "")
-TARGET_BUILTIN(__builtin_ia32_subborrow_u64, "UcUcUOiUOiUOi*", "n", "")
+TARGET_BUILTIN(__builtin_ia32_addcarryx_u64, "UcUcUOiUOiUOi*", "nE", "")
+TARGET_BUILTIN(__builtin_ia32_subborrow_u64, "UcUcUOiUOiUOi*", "nE", "")
 TARGET_BUILTIN(__builtin_ia32_rdrand64_step, "UiUOi*", "n", "rdrnd")
 TARGET_BUILTIN(__builtin_ia32_rdseed64_step, "UiUOi*", "n", "rdseed")
 TARGET_BUILTIN(__builtin_ia32_lzcnt_u64, "UOiUOi", "ncE", "lzcnt")
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index 7d81bdf827ea0..41e719d4d5781 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -1125,6 +1125,11 @@ def ThreadSafety : DiagGroup<"thread-safety",
 def ThreadSafetyVerbose : DiagGroup<"thread-safety-verbose">;
 def ThreadSafetyBeta : DiagGroup<"thread-safety-beta">;
 
+// Warnings and notes related to the function effects system which underlies
+// the nonblocking and nonallocating attributes.
+def FunctionEffects : DiagGroup<"function-effects">;
+def PerfConstraintImpliesNoexcept : DiagGroup<"perf-constraint-implies-noexcept">;
+
 // Uniqueness Analysis warnings
 def Consumed       : DiagGroup<"consumed">;
 
@@ -1133,7 +1138,7 @@ def Consumed       : DiagGroup<"consumed">;
 // DefaultIgnore in addition to putting it here.
 def All : DiagGroup<"all", [Most, Parentheses, Switch, SwitchBool,
                             MisleadingIndentation, PackedNonPod,
-                            VLACxxExtension]>;
+                            VLACxxExtension, PerfConstraintImpliesNoexcept]>;
 
 // Warnings that should be in clang-cl /w4.
 def : DiagGroup<"CL4", [All, Extra]>;
@@ -1566,10 +1571,6 @@ def UnsafeBufferUsageInContainer : DiagGroup<"unsafe-buffer-usage-in-container">
 def UnsafeBufferUsageInLibcCall : DiagGroup<"unsafe-buffer-usage-in-libc-call">;
 def UnsafeBufferUsage : DiagGroup<"unsafe-buffer-usage", [UnsafeBufferUsageInContainer, UnsafeBufferUsageInLibcCall]>;
 
-// Warnings and notes related to the function effects system underlying
-// the nonblocking and nonallocating attributes.
-def FunctionEffects : DiagGroup<"function-effects">;
-
 // Warnings and notes InstallAPI verification.
 def InstallAPIViolation : DiagGroup<"installapi-violation">;
 
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 64e6d0407b0ce..583475327c522 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -5262,6 +5262,13 @@ def note_template_arg_refers_here_func : Note<
 def err_template_arg_template_params_mismatch : Error<
   "template template argument has different template parameters than its "
   "corresponding template template parameter">;
+def note_template_arg_template_params_mismatch : Note<
+  "template template argument has different template parameters than its "
+  "corresponding template template parameter">;
+def err_non_deduced_mismatch : Error<
+  "could not match %diff{$ against $|types}0,1">;
+def err_inconsistent_deduction : Error<
+  "conflicting deduction %diff{$ against $|types}0,1 for parameter">;
 def err_template_arg_not_integral_or_enumeral : Error<
   "non-type template argument of type %0 must have an integral or enumeration"
   " type">;
@@ -10503,8 +10510,8 @@ def err_first_argument_to_cwsc_pdtor_call : Error<
 def err_second_argument_to_cwsc_not_pointer : Error<
   "second argument to __builtin_call_with_static_chain must be of pointer type">;
 
-def err_vector_incorrect_num_initializers : Error<
-  "%select{too many|too few}0 elements in vector initialization (expected %1 elements, have %2)">;
+def err_vector_incorrect_num_elements : Error<
+  "%select{too many|too few}0 elements in vector %select{initialization|operand}3 (expected %1 elements, have %2)">;
 def err_altivec_empty_initializer : Error<"expected initializer">;
 
 def err_invalid_neon_type_code : Error<
@@ -10965,19 +10972,68 @@ def warn_imp_cast_drops_unaligned : Warning<
   InGroup<DiagGroup<"unaligned-qualifier-implicit-cast">>;
 
 // Function effects
+def warn_func_effect_violation : Warning<
+  "%select{function|constructor|destructor|lambda|block|member initializer of constructor}0 "
+  "with '%1' attribute "
+  "must not %select{allocate or deallocate memory|throw or catch exceptions|"
+  "have static local variables|use thread-local variables|access ObjC methods or properties}2">,
+  InGroup<FunctionEffects>, DefaultIgnore;
+def note_func_effect_violation : Note<
+  "%select{function|constructor|destructor|lambda|block|member initializer}0 "
+  "cannot be inferred '%1' because it "
+  "%select{allocates or deallocates memory|throws or catches exceptions|"
+  "has a static local variable|uses a thread-local variable|"
+  "accesses an ObjC method or property}2">;
+def warn_func_effect_calls_func_without_effect : Warning<
+  "%select{function|constructor|destructor|lambda|block|member initializer of constructor}0 "
+  "with '%1' attribute "
+  "must not call non-'%1' "
+  "%select{function|constructor|destructor|lambda|block}2 "
+  "'%3'">,
+  InGroup<FunctionEffects>, DefaultIgnore;
+def note_func_effect_calls_func_without_effect : Note<
+  "%select{function|constructor|destructor|lambda|block|member initializer}0 "
+  "cannot be inferred '%1' because it calls non-'%1' "
+  "%select{function|constructor|destructor|lambda|block}2 "
+  "'%3'">;
+def warn_func_effect_calls_expr_without_effect : Warning<
+  "%select{function|constructor|destructor|lambda|block|member initializer of constructor}0 "
+  "with '%1' attribute "
+  "must not call non-'%1' expression">,
+  InGroup<FunctionEffects>, DefaultIgnore;
+def note_func_effect_call_extern : Note<
+  "declaration cannot be inferred '%0' because it has no definition in this translation unit">;
+def note_func_effect_call_disallows_inference : Note<
+  "%select{function|constructor|destructor|lambda|block}0 "
+  "does not permit inference of '%1' because it is declared '%2'">;
+def note_func_effect_call_indirect : Note<
+  "%select{virtual method|function pointer}0 cannot be inferred '%1'">;
+def warn_perf_constraint_implies_noexcept : Warning<
+  "%select{function|constructor|destructor|lambda|block}0 "
+  "with '%1' attribute should be declared noexcept">,
+  InGroup<PerfConstraintImpliesNoexcept>, DefaultIgnore;
+
+// FIXME: It would be nice if we could provide fuller template expansion notes.
+def note_func_effect_from_template : Note<
+  "in template expansion here">;
+def note_func_effect_in_constructor : Note<
+  "in%select{| implicit}0 constructor here">;
+def note_in_evaluating_default_argument : Note<
+  "in evaluating default argument here">;
+
 // spoofing nonblocking/nonallocating
 def warn_invalid_add_func_effects : Warning<
   "attribute '%0' should not be added via type conversion">,
-  InGroup<FunctionEffects>;
+  InGroup<FunctionEffects>, DefaultIgnore;
 def warn_mismatched_func_effect_override : Warning<
   "attribute '%0' on overriding function does not match base declaration">,
-  InGroup<FunctionEffects>;
+  InGroup<FunctionEffects>, DefaultIgnore;
 def warn_mismatched_func_effect_redeclaration : Warning<
   "attribute '%0' on function does not match previous declaration">,
-  InGroup<FunctionEffects>;
+  InGroup<FunctionEffects>, DefaultIgnore;
 def warn_conflicting_func_effects : Warning<
   "effects conflict when merging declarations; kept '%0', discarded '%1'">,
-  InGroup<FunctionEffects>;
+  InGroup<FunctionEffects>, DefaultIgnore;
 def err_func_with_effects_no_prototype : Error<
   "'%0' function must have a prototype">;
 
@@ -12214,7 +12270,8 @@ def err_builtin_invalid_arg_type: Error <
   "a floating point type|"
   "a vector of integers|"
   "an unsigned integer|"
-  "an 'int'}1 (was %2)">;
+  "an 'int'|"
+  "a vector of floating points}1 (was %2)">;
 
 def err_builtin_matrix_disabled: Error<
   "matrix types extension is disabled. Pass -fenable-matrix to enable it">;
@@ -12609,6 +12666,24 @@ def err_acc_loop_spec_conflict
 def err_acc_collapse_loop_count
     : Error<"OpenACC 'collapse' clause loop count must be a %select{constant "
             "expression|positive integer value, evaluated to %1}0">;
+def err_acc_size_expr_value
+    : Error<
+          "OpenACC 'tile' clause size expression must be %select{an asterisk "
+          "or a constant expression|positive integer value, evaluated to %1}0">;
+def err_acc_invalid_in_loop
+    : Error<"%select{OpenACC '%2' construct|while loop|do loop}0 cannot appear "
+            "in intervening code of a 'loop' with a '%1' clause">;
+def note_acc_active_clause_here
+    : Note<"active '%0' clause defined here">;
+def err_acc_clause_multiple_loops
+    : Error<"more than one for-loop in a loop associated with OpenACC 'loop' "
+            "construct with a '%select{collapse|tile}0' clause">;
+def err_acc_insufficient_loops
+    : Error<"'%0' clause specifies a loop count greater than the number "
+            "of available loops">;
+def err_acc_intervening_code
+    : Error<"inner loops must be tightly nested inside a '%0' clause on "
+            "a 'loop' construct">;
 
 // AMDGCN builtins diagnostics
 def err_amdgcn_global_load_lds_size_invalid_value : Error<"invalid size value">;
diff --git a/clang/include/clang/Basic/DiagnosticSerializationKinds.td b/clang/include/clang/Basic/DiagnosticSerializationKinds.td
index 253a955431997..3914d3930bec7 100644
--- a/clang/include/clang/Basic/DiagnosticSerializationKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSerializationKinds.td
@@ -51,9 +51,9 @@ def note_pch_vfsoverlay_files : Note<"%select{PCH|current translation unit}0 has
 def note_pch_vfsoverlay_empty : Note<"%select{PCH|current translation unit}0 has no VFS overlays">;
 
 def err_ast_file_version_too_old : Error<
-    "%select{PCH|module|AST}0 file '%1' uses an older PCH format that is no longer supported">;
+    "%select{PCH|module|AST}0 file '%1' uses an older format that is no longer supported">;
 def err_ast_file_version_too_new : Error<
-    "%select{PCH|module|AST}0 file '%1' uses a newer PCH format that cannot be read">;
+    "%select{PCH|module|AST}0 file '%1' uses a newer format that cannot be read">;
 def err_ast_file_different_branch : Error<
     "%select{PCH|module|AST}0 file '%1' built from a different branch (%2) than the compiler (%3)">;
 def err_ast_file_with_compiler_errors : Error<
diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h
index 8c605f6852016..e2d4206c72cc9 100644
--- a/clang/include/clang/Basic/LangOptions.h
+++ b/clang/include/clang/Basic/LangOptions.h
@@ -241,8 +241,10 @@ class LangOptionsBase {
 
     /// Attempt to be ABI-compatible with code generated by Clang 19.0.x.
     /// This causes clang to:
-    ///   - Incorrectly mangles the 'base type' substitutions of the CXX
+    ///   - Incorrectly mangle the 'base type' substitutions of the CXX
     ///   construction vtable because it hasn't added 'type' as a substitution.
+    ///   - Skip mangling enclosing class templates of member-like friend
+    ///   function templates.
     Ver19,
 
     /// Conform to the underlying platform's C and C++ ABIs as closely
diff --git a/clang/include/clang/Basic/OpenACCClauses.def b/clang/include/clang/Basic/OpenACCClauses.def
index 19cdfe7672133..a380e5ae69c41 100644
--- a/clang/include/clang/Basic/OpenACCClauses.def
+++ b/clang/include/clang/Basic/OpenACCClauses.def
@@ -52,6 +52,7 @@ VISIT_CLAUSE(Private)
 VISIT_CLAUSE(Reduction)
 VISIT_CLAUSE(Self)
 VISIT_CLAUSE(Seq)
+VISIT_CLAUSE(Tile)
 VISIT_CLAUSE(VectorLength)
 VISIT_CLAUSE(Wait)
 
diff --git a/clang/include/clang/Basic/StmtNodes.td b/clang/include/clang/Basic/StmtNodes.td
index 30f2c8f1dbfde..dc82a4b6d1f77 100644
--- a/clang/include/clang/Basic/StmtNodes.td
+++ b/clang/include/clang/Basic/StmtNodes.td
@@ -308,5 +308,8 @@ def OpenACCAssociatedStmtConstruct
 def OpenACCComputeConstruct : StmtNode<OpenACCAssociatedStmtConstruct>;
 def OpenACCLoopConstruct : StmtNode<OpenACCAssociatedStmtConstruct>;
 
+// OpenACC Additional Expressions.
+def OpenACCAsteriskSizeExpr : StmtNode<Expr>;
+
 // HLSL Constructs.
 def HLSLOutArgExpr : StmtNode<Expr>;
diff --git a/clang/include/clang/Basic/arm_sme.td b/clang/include/clang/Basic/arm_sme.td
index 45673cb915c5e..0f689e82bdb74 100644
--- a/clang/include/clang/Basic/arm_sme.td
+++ b/clang/include/clang/Basic/arm_sme.td
@@ -818,7 +818,7 @@ multiclass ZAReadzArray<string vg_num>{
 defm SVREADZ_VG2 :  ZAReadzArray<"2">;
 defm SVREADZ_VG4 :  ZAReadzArray<"4">;
 
-let SMETargetGuard = "sme2,sme-lutv2" in {
+let SMETargetGuard = "sme-lutv2" in {
   def SVWRITE_LANE_ZT : SInst<"svwrite_lane_zt[_{d}]", "vidi", "cUcsUsiUilUlfhdb", MergeNone, "aarch64_sme_write_lane_zt", [IsStreaming, IsInOutZT0], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck1_3>]>;
   def SVWRITE_ZT : SInst<"svwrite_zt[_{d}]", "vid", "cUcsUsiUilUlfhdb", MergeNone, "aarch64_sme_write_zt", [IsStreaming, IsOutZT0], [ImmCheck<0, ImmCheck0_0>]>;
   def SVLUTI4_ZT_X4 : SInst<"svluti4_zt_{d}_x4", "4i2.u", "cUc", MergeNone, "aarch64_sme_luti4_zt_x4", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>]>;
diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td
index da496e30fbb52..90b1ec242e6ba 100644
--- a/clang/include/clang/Basic/arm_sve.td
+++ b/clang/include/clang/Basic/arm_sve.td
@@ -943,11 +943,6 @@ defm SVFCVTZS_S64_F16 : SInstCvtMXZ<"svcvt_s64[_f16]", "ddPO", "dPO", "l",  "aar
 defm SVFCVTZS_S32_F32 : SInstCvtMXZ<"svcvt_s32[_f32]", "ddPM", "dPM", "i",  "aarch64_sve_fcvtzs", [IsOverloadCvt]>;
 defm SVFCVTZS_S64_F32 : SInstCvtMXZ<"svcvt_s64[_f32]", "ddPM", "dPM", "l",  "aarch64_sve_fcvtzs_i64f32">;
 
-let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in {
-  defm SVCVT_BF16_F32   : SInstCvtMXZ<"svcvt_bf16[_f32]",  "ddPM", "dPM", "b",  "aarch64_sve_fcvt_bf16f32">;
-  def SVCVTNT_BF16_F32 : SInst<"svcvtnt_bf16[_f32]", "ddPM", "b",  MergeOp1, "aarch64_sve_fcvtnt_bf16f32", [IsOverloadNone, VerifyRuntimeMode]>;
-}
-
 // svcvt_s##_f64
 defm SVFCVTZS_S32_F64 : SInstCvtMXZ<"svcvt_s32[_f64]", "ttPd", "tPd", "d",  "aarch64_sve_fcvtzs_i32f64">;
 defm SVFCVTZS_S64_F64 : SInstCvtMXZ<"svcvt_s64[_f64]", "ddPN", "dPN", "l",  "aarch64_sve_fcvtzs", [IsOverloadCvt]>;
@@ -1003,19 +998,26 @@ defm SVFCVT_F32_F64   : SInstCvtMXZ<"svcvt_f32[_f64]", "MMPd", "MPd", "d", "aarc
 defm SVFCVT_F64_F16   : SInstCvtMXZ<"svcvt_f64[_f16]", "ddPO", "dPO", "d", "aarch64_sve_fcvt_f64f16">;
 defm SVFCVT_F64_F32   : SInstCvtMXZ<"svcvt_f64[_f32]", "ddPM", "dPM", "d", "aarch64_sve_fcvt_f64f32">;
 
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
-defm SVCVTLT_F32    : SInstCvtMX<"svcvtlt_f32[_f16]",  "ddPh", "dPh", "f", "aarch64_sve_fcvtlt_f32f16">;
-defm SVCVTLT_F64    : SInstCvtMX<"svcvtlt_f64[_f32]",  "ddPh", "dPh", "d", "aarch64_sve_fcvtlt_f64f32">;
+let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in {
+defm SVCVT_BF16_F32    : SInstCvtMXZ<"svcvt_bf16[_f32]", "$$Pd", "$Pd", "f", "aarch64_sve_fcvt_bf16f32_v2">;
+
+def SVCVTNT_BF16_F32   : SInst<"svcvtnt_bf16[_f32]", "$$Pd", "f", MergeOp1, "aarch64_sve_fcvtnt_bf16f32_v2", [IsOverloadNone, VerifyRuntimeMode]>;
+//  SVCVTNT_X_BF16_F32 : Implemented as macro by SveEmitter.cpp
+}
 
-defm SVCVTX_F32     : SInstCvtMXZ<"svcvtx_f32[_f64]",  "MMPd", "MPd", "d", "aarch64_sve_fcvtx_f32f64">;
+let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+defm SVCVTLT_F32_F16   : SInstCvtMX<"svcvtlt_f32[_f16]",  "ddPh", "dPh", "f", "aarch64_sve_fcvtlt_f32f16">;
+defm SVCVTLT_F64_F32   : SInstCvtMX<"svcvtlt_f64[_f32]",  "ddPh", "dPh", "d", "aarch64_sve_fcvtlt_f64f32">;
 
-def SVCVTNT_F32     : SInst<"svcvtnt_f16[_f32]",  "hhPd", "f", MergeOp1, "aarch64_sve_fcvtnt_f16f32", [IsOverloadNone, VerifyRuntimeMode]>;
-def SVCVTNT_F64     : SInst<"svcvtnt_f32[_f64]",  "hhPd", "d", MergeOp1, "aarch64_sve_fcvtnt_f32f64", [IsOverloadNone, VerifyRuntimeMode]>;
-//  SVCVTNT_X       : Implemented as macro by SveEmitter.cpp
+defm SVCVTX_F32_F64    : SInstCvtMXZ<"svcvtx_f32[_f64]",  "MMPd", "MPd", "d", "aarch64_sve_fcvtx_f32f64">;
 
-def SVCVTXNT_F32    : SInst<"svcvtxnt_f32[_f64]", "MMPd", "d", MergeOp1, "aarch64_sve_fcvtxnt_f32f64", [IsOverloadNone, VerifyRuntimeMode]>;
-//  SVCVTXNT_X_F32  : Implemented as macro by SveEmitter.cpp
+def SVCVTNT_F16_F32    : SInst<"svcvtnt_f16[_f32]",  "hhPd", "f", MergeOp1, "aarch64_sve_fcvtnt_f16f32", [IsOverloadNone, VerifyRuntimeMode]>;
+def SVCVTNT_F32_F64    : SInst<"svcvtnt_f32[_f64]",  "hhPd", "d", MergeOp1, "aarch64_sve_fcvtnt_f32f64", [IsOverloadNone, VerifyRuntimeMode]>;
+//  SVCVTNT_X_F16_F32  : Implemented as macro by SveEmitter.cpp
+//  SVCVTNT_X_F32_F64  : Implemented as macro by SveEmitter.cpp
 
+def SVCVTXNT_F32_F64   : SInst<"svcvtxnt_f32[_f64]", "MMPd", "d", MergeOp1, "aarch64_sve_fcvtxnt_f32f64", [IsOverloadNone, VerifyRuntimeMode]>;
+//  SVCVTXNT_X_F32_F64 : Implemented as macro by SveEmitter.cpp
 }
 
 ////////////////////////////////////////////////////////////////////////////////
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 9d183ff2d69b3..445e681313eba 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1195,10 +1195,11 @@ def cxx_isystem : JoinedOrSeparate<["-"], "cxx-isystem">, Group<clang_i_Group>,
 def c : Flag<["-"], "c">, Flags<[NoXarchOption]>,
   Visibility<[ClangOption, FlangOption]>, Group<Action_Group>,
   HelpText<"Only run preprocess, compile, and assemble steps">;
-defm convergent_functions : BoolFOption<"convergent-functions",
-  LangOpts<"ConvergentFunctions">, DefaultFalse,
-  NegFlag<SetFalse, [], [ClangOption], "Assume all functions may be convergent.">,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option]>>;
+def fconvergent_functions : Flag<["-"], "fconvergent-functions">,
+  Visibility<[ClangOption, CC1Option]>,
+  HelpText< "Assume all functions may be convergent.">;
+def fno_convergent_functions : Flag<["-"], "fno-convergent-functions">,
+  Visibility<[ClangOption, CC1Option]>;
 
 // Common offloading options
 let Group = offload_Group in {
diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h
index d8b62c7652a0f..53a9577e0f72e 100644
--- a/clang/include/clang/Format/Format.h
+++ b/clang/include/clang/Format/Format.h
@@ -4974,6 +4974,15 @@ struct FormatStyle {
   /// \version 3.7
   unsigned TabWidth;
 
+  /// A vector of non-keyword identifiers that should be interpreted as
+  /// template names.
+  ///
+  /// A ``<`` after a template name is annotated as a template opener instead of
+  /// a binary operator.
+  ///
+  /// \version 20
+  std::vector<std::string> TemplateNames;
+
   /// A vector of non-keyword identifiers that should be interpreted as type
   /// names.
   ///
@@ -5230,6 +5239,7 @@ struct FormatStyle {
            TableGenBreakingDAGArgOperators ==
                R.TableGenBreakingDAGArgOperators &&
            TableGenBreakInsideDAGArg == R.TableGenBreakInsideDAGArg &&
+           TabWidth == R.TabWidth && TemplateNames == R.TemplateNames &&
            TabWidth == R.TabWidth && TypeNames == R.TypeNames &&
            TypenameMacros == R.TypenameMacros && UseTab == R.UseTab &&
            VerilogBreakBetweenInstancePorts ==
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index eb8a851da7e04..93e49d395388a 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -3786,10 +3786,13 @@ class Parser : public CodeCompletionHandler {
   OpenACCIntExprParseResult ParseOpenACCAsyncArgument(OpenACCDirectiveKind DK,
                                                       OpenACCClauseKind CK,
                                                       SourceLocation Loc);
+
   /// Parses the 'size-expr', which is an integral value, or an asterisk.
-  bool ParseOpenACCSizeExpr();
+  /// Asterisk is represented by a OpenACCAsteriskSizeExpr
+  ExprResult ParseOpenACCSizeExpr(OpenACCClauseKind CK);
   /// Parses a comma delimited list of 'size-expr's.
-  bool ParseOpenACCSizeExprList();
+  bool ParseOpenACCSizeExprList(OpenACCClauseKind CK,
+                                llvm::SmallVectorImpl<Expr *> &SizeExprs);
   /// Parses a 'gang-arg-list', used for the 'gang' clause.
   bool ParseOpenACCGangArgList(SourceLocation GangLoc);
   /// Parses a 'gang-arg', used for the 'gang' clause.
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index d616c3834c429..bede971ce0191 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -457,55 +457,6 @@ enum class FunctionEffectMode : uint8_t {
   Dependent // effect(expr) where expr is dependent.
 };
 
-struct FunctionEffectDiff {
-  enum class Kind { Added, Removed, ConditionMismatch };
-
-  FunctionEffect::Kind EffectKind;
-  Kind DiffKind;
-  FunctionEffectWithCondition Old; // invalid when Added.
-  FunctionEffectWithCondition New; // invalid when Removed.
-
-  StringRef effectName() const {
-    if (Old.Effect.kind() != FunctionEffect::Kind::None)
-      return Old.Effect.name();
-    return New.Effect.name();
-  }
-
-  /// Describes the result of effects differing between a base class's virtual
-  /// method and an overriding method in a subclass.
-  enum class OverrideResult {
-    NoAction,
-    Warn,
-    Merge // Merge missing effect from base to derived.
-  };
-
-  /// Return true if adding or removing the effect as part of a type conversion
-  /// should generate a diagnostic.
-  bool shouldDiagnoseConversion(QualType SrcType,
-                                const FunctionEffectsRef &SrcFX,
-                                QualType DstType,
-                                const FunctionEffectsRef &DstFX) const;
-
-  /// Return true if adding or removing the effect in a redeclaration should
-  /// generate a diagnostic.
-  bool shouldDiagnoseRedeclaration(const FunctionDecl &OldFunction,
-                                   const FunctionEffectsRef &OldFX,
-                                   const FunctionDecl &NewFunction,
-                                   const FunctionEffectsRef &NewFX) const;
-
-  /// Return true if adding or removing the effect in a C++ virtual method
-  /// override should generate a diagnostic.
-  OverrideResult shouldDiagnoseMethodOverride(
-      const CXXMethodDecl &OldMethod, const FunctionEffectsRef &OldFX,
-      const CXXMethodDecl &NewMethod, const FunctionEffectsRef &NewFX) const;
-};
-
-struct FunctionEffectDifferences : public SmallVector<FunctionEffectDiff> {
-  /// Caller should short-circuit by checking for equality first.
-  FunctionEffectDifferences(const FunctionEffectsRef &Old,
-                            const FunctionEffectsRef &New);
-};
-
 /// Sema - This implements semantic analysis and AST building for C.
 /// \nosubgrouping
 class Sema final : public SemaBase {
@@ -546,6 +497,7 @@ class Sema final : public SemaBase {
   // 32. Constraints and Concepts (SemaConcept.cpp)
   // 33. Types (SemaType.cpp)
   // 34. FixIt Helpers (SemaFixItUtils.cpp)
+  // 35. Function Effects (SemaFunctionEffects.cpp)
 
   /// \name Semantic Analysis
   /// Implementations are in Sema.cpp
@@ -851,30 +803,10 @@ class Sema final : public SemaBase {
   /// Warn when implicitly casting 0 to nullptr.
   void diagnoseZeroToNullptrConversion(CastKind Kind, const Expr *E);
 
-  // ----- function effects ---
-
   /// Warn when implicitly changing function effects.
   void diagnoseFunctionEffectConversion(QualType DstType, QualType SrcType,
                                         SourceLocation Loc);
 
-  /// Warn and return true if adding an effect to a set would create a conflict.
-  bool diagnoseConflictingFunctionEffect(const FunctionEffectsRef &FX,
-                                         const FunctionEffectWithCondition &EC,
-                                         SourceLocation NewAttrLoc);
-
-  // Report a failure to merge function effects between declarations due to a
-  // conflict.
-  void
-  diagnoseFunctionEffectMergeConflicts(const FunctionEffectSet::Conflicts &Errs,
-                                       SourceLocation NewLoc,
-                                       SourceLocation OldLoc);
-
-  /// Try to parse the conditional expression attached to an effect attribute
-  /// (e.g. 'nonblocking'). (c.f. Sema::ActOnNoexceptSpec). Return an empty
-  /// optional on error.
-  std::optional<FunctionEffectMode>
-  ActOnEffectExpression(Expr *CondExpr, StringRef AttributeName);
-
   /// makeUnavailableInSystemHeader - There is an error in the current
   /// context.  If we're still in a system header, and we can plausibly
   /// make the relevant declaration unavailable instead of erroring, do
@@ -2381,7 +2313,8 @@ class Sema final : public SemaBase {
   bool CheckFunctionCall(FunctionDecl *FDecl, CallExpr *TheCall,
                          const FunctionProtoType *Proto);
 
-  bool BuiltinVectorMath(CallExpr *TheCall, QualType &Res);
+  /// \param FPOnly restricts the arguments to floating-point types.
+  bool BuiltinVectorMath(CallExpr *TheCall, QualType &Res, bool FPOnly = false);
   bool BuiltinVectorToScalarMath(CallExpr *TheCall);
 
   /// Handles the checks for format strings, non-POD arguments to vararg
@@ -2573,7 +2506,8 @@ class Sema final : public SemaBase {
   ExprResult AtomicOpsOverloaded(ExprResult TheCallResult,
                                  AtomicExpr::AtomicOp Op);
 
-  bool BuiltinElementwiseMath(CallExpr *TheCall);
+  /// \param FPOnly restricts the arguments to floating-point types.
+  bool BuiltinElementwiseMath(CallExpr *TheCall, bool FPOnly = false);
   bool PrepareBuiltinReduceMathOneArgCall(CallExpr *TheCall);
 
   bool BuiltinNonDeterministicValue(CallExpr *TheCall);
@@ -12482,8 +12416,9 @@ class Sema final : public SemaBase {
                                     sema::TemplateDeductionInfo &Info);
 
   bool isTemplateTemplateParameterAtLeastAsSpecializedAs(
-      TemplateParameterList *PParam, TemplateDecl *AArg,
-      const DefaultArguments &DefaultArgs, SourceLocation Loc, bool IsDeduced);
+      TemplateParameterList *PParam, TemplateDecl *PArg, TemplateDecl *AArg,
+      const DefaultArguments &DefaultArgs, SourceLocation ArgLoc,
+      bool IsDeduced);
 
   /// Mark which template parameters are used in a given expression.
   ///
@@ -12792,6 +12727,9 @@ class Sema final : public SemaBase {
 
       /// We are instantiating a type alias template declaration.
       TypeAliasTemplateInstantiation,
+
+      /// We are performing partial ordering for template template parameters.
+      PartialOrderingTTP,
     } Kind;
 
     /// Was the enclosing context a non-instantiation SFINAE context?
@@ -13013,6 +12951,12 @@ class Sema final : public SemaBase {
                           TemplateDecl *Entity, BuildingDeductionGuidesTag,
                           SourceRange InstantiationRange = SourceRange());
 
+    struct PartialOrderingTTP {};
+    /// \brief Note that we are partial ordering template template parameters.
+    InstantiatingTemplate(Sema &SemaRef, SourceLocation ArgLoc,
+                          PartialOrderingTTP, TemplateDecl *PArg,
+                          SourceRange InstantiationRange = SourceRange());
+
     /// Note that we have finished instantiating this template.
     void Clear();
 
@@ -15050,6 +14994,12 @@ class Sema final : public SemaBase {
     return hasAcceptableDefinition(D, &Hidden, Kind);
   }
 
+  /// Try to parse the conditional expression attached to an effect attribute
+  /// (e.g. 'nonblocking'). (c.f. Sema::ActOnNoexceptSpec). Return an empty
+  /// optional on error.
+  std::optional<FunctionEffectMode>
+  ActOnEffectExpression(Expr *CondExpr, StringRef AttributeName);
+
 private:
   /// The implementation of RequireCompleteType
   bool RequireCompleteTypeImpl(SourceLocation Loc, QualType T,
@@ -15080,6 +15030,108 @@ class Sema final : public SemaBase {
   std::string getFixItZeroLiteralForType(QualType T, SourceLocation Loc) const;
 
   ///@}
+
+  //
+  //
+  // -------------------------------------------------------------------------
+  //
+  //
+
+  /// \name Function Effects
+  /// Implementations are in SemaFunctionEffects.cpp
+  ///@{
+public:
+  struct FunctionEffectDiff {
+    enum class Kind { Added, Removed, ConditionMismatch };
+
+    FunctionEffect::Kind EffectKind;
+    Kind DiffKind;
+    std::optional<FunctionEffectWithCondition>
+        Old; // Invalid when 'Kind' is 'Added'.
+    std::optional<FunctionEffectWithCondition>
+        New; // Invalid when 'Kind' is 'Removed'.
+
+    StringRef effectName() const {
+      if (Old)
+        return Old.value().Effect.name();
+      return New.value().Effect.name();
+    }
+
+    /// Describes the result of effects differing between a base class's virtual
+    /// method and an overriding method in a subclass.
+    enum class OverrideResult {
+      NoAction,
+      Warn,
+      Merge // Merge missing effect from base to derived.
+    };
+
+    /// Return true if adding or removing the effect as part of a type
+    /// conversion should generate a diagnostic.
+    bool shouldDiagnoseConversion(QualType SrcType,
+                                  const FunctionEffectsRef &SrcFX,
+                                  QualType DstType,
+                                  const FunctionEffectsRef &DstFX) const;
+
+    /// Return true if adding or removing the effect in a redeclaration should
+    /// generate a diagnostic.
+    bool shouldDiagnoseRedeclaration(const FunctionDecl &OldFunction,
+                                     const FunctionEffectsRef &OldFX,
+                                     const FunctionDecl &NewFunction,
+                                     const FunctionEffectsRef &NewFX) const;
+
+    /// Return true if adding or removing the effect in a C++ virtual method
+    /// override should generate a diagnostic.
+    OverrideResult shouldDiagnoseMethodOverride(
+        const CXXMethodDecl &OldMethod, const FunctionEffectsRef &OldFX,
+        const CXXMethodDecl &NewMethod, const FunctionEffectsRef &NewFX) const;
+  };
+
+  struct FunctionEffectDiffVector : public SmallVector<FunctionEffectDiff> {
+    /// Caller should short-circuit by checking for equality first.
+    FunctionEffectDiffVector(const FunctionEffectsRef &Old,
+                             const FunctionEffectsRef &New);
+  };
+
+  /// All functions/lambdas/blocks which have bodies and which have a non-empty
+  /// FunctionEffectsRef to be verified.
+  SmallVector<const Decl *> DeclsWithEffectsToVerify;
+
+  /// The union of all effects present on DeclsWithEffectsToVerify. Conditions
+  /// are all null.
+  FunctionEffectKindSet AllEffectsToVerify;
+
+public:
+  /// Warn and return true if adding a function effect to a set would create a
+  /// conflict.
+  bool diagnoseConflictingFunctionEffect(const FunctionEffectsRef &FX,
+                                         const FunctionEffectWithCondition &EC,
+                                         SourceLocation NewAttrLoc);
+
+  // Report a failure to merge function effects between declarations due to a
+  // conflict.
+  void
+  diagnoseFunctionEffectMergeConflicts(const FunctionEffectSet::Conflicts &Errs,
+                                       SourceLocation NewLoc,
+                                       SourceLocation OldLoc);
+
+  /// Inline checks from the start of maybeAddDeclWithEffects, to
+  /// minimize performance impact on code not using effects.
+  template <class FuncOrBlockDecl>
+  void maybeAddDeclWithEffects(FuncOrBlockDecl *D) {
+    if (Context.hasAnyFunctionEffects())
+      if (FunctionEffectsRef FX = D->getFunctionEffects(); !FX.empty())
+        maybeAddDeclWithEffects(D, FX);
+  }
+
+  /// Potentially add a FunctionDecl or BlockDecl to DeclsWithEffectsToVerify.
+  void maybeAddDeclWithEffects(const Decl *D, const FunctionEffectsRef &FX);
+
+  /// Unconditionally add a Decl to DeclsWithEfffectsToVerify.
+  void addDeclWithEffects(const Decl *D, const FunctionEffectsRef &FX);
+
+  void performFunctionEffectAnalysis(TranslationUnitDecl *TU);
+
+  ///@}
 };
 
 DeductionFailureInfo
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index 839fdb79cd0ac..97386d2378b75 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -22,6 +22,7 @@
 #include "clang/Sema/Ownership.h"
 #include "clang/Sema/SemaBase.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 #include <optional>
 #include <utility>
@@ -41,6 +42,62 @@ class SemaOpenACC : public SemaBase {
   /// above collection.
   bool InsideComputeConstruct = false;
 
+  /// Certain clauses care about the same things that aren't specific to the
+  /// individual clause, but can be shared by a few, so store them here. All
+  /// require a 'no intervening constructs' rule, so we know they are all from
+  /// the same 'place'.
+  struct LoopCheckingInfo {
+    /// Records whether we've seen the top level 'for'. We already diagnose
+    /// later that the 'top level' is a for loop, so we use this to suppress the
+    /// 'collapse inner loop not a 'for' loop' diagnostic.
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned TopLevelLoopSeen : 1;
+
+    /// Records whether this 'tier' of the loop has already seen a 'for' loop,
+    /// used to diagnose if there are multiple 'for' loops at any one level.
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned CurLevelHasLoopAlready : 1;
+  } LoopInfo{/*TopLevelLoopSeen=*/false, /*CurLevelHasLoopAlready=*/false};
+
+  /// The 'collapse' clause requires quite a bit of checking while
+  /// parsing/instantiating its body, so this structure/object keeps all of the
+  /// necessary information as we do checking.  This should rarely be directly
+  /// modified, and typically should be controlled by the RAII objects.
+  ///
+  /// Collapse has an 'N' count that makes it apply to a number of loops 'below'
+  /// it.
+  struct CollapseCheckingInfo {
+    OpenACCCollapseClause *ActiveCollapse = nullptr;
+
+    /// This is a value that maintains the current value of the 'N' on the
+    /// current collapse, minus the depth that has already been traversed. When
+    /// there is not an active collapse, or a collapse whose depth we don't know
+    /// (for example, if it is a dependent value), this should be `nullopt`,
+    /// else it should be 'N' minus the current depth traversed.
+    std::optional<llvm::APSInt> CurCollapseCount;
+
+    /// Records whether we've hit a CurCollapseCount of '0' on the way down,
+    /// which allows us to diagnose if the value of 'N' is too large for the
+    /// current number of 'for' loops.
+    bool CollapseDepthSatisfied = true;
+  } CollapseInfo;
+
+  /// The 'tile' clause requires a bit of additional checking as well, so like
+  /// the `CollapseCheckingInfo`, ensure we maintain information here too.
+  struct TileCheckingInfo {
+    OpenACCTileClause *ActiveTile = nullptr;
+
+    /// This is the number of expressions on a 'tile' clause.  This doesn't have
+    /// to be an APSInt because it isn't the result of a constexpr, just by our
+    /// own counting of elements.
+    std::optional<unsigned> CurTileCount;
+
+    /// Records whether we've hit a 'CurTileCount' of '0' on the wya down,
+    /// which allows us to diagnose if the number of arguments is too large for
+    /// the current number of 'for' loops.
+    bool TileDepthSatisfied = true;
+  } TileInfo;
+
 public:
   // Redeclaration of the version in OpenACCClause.h.
   using DeviceTypeArgument = std::pair<IdentifierInfo *, SourceLocation>;
@@ -141,6 +198,7 @@ class SemaOpenACC : public SemaBase {
       assert((ClauseKind == OpenACCClauseKind::NumGangs ||
               ClauseKind == OpenACCClauseKind::NumWorkers ||
               ClauseKind == OpenACCClauseKind::Async ||
+              ClauseKind == OpenACCClauseKind::Tile ||
               ClauseKind == OpenACCClauseKind::VectorLength) &&
              "Parsed clause kind does not have a int exprs");
 
@@ -186,6 +244,7 @@ class SemaOpenACC : public SemaBase {
       assert((ClauseKind == OpenACCClauseKind::NumGangs ||
               ClauseKind == OpenACCClauseKind::NumWorkers ||
               ClauseKind == OpenACCClauseKind::Async ||
+              ClauseKind == OpenACCClauseKind::Tile ||
               ClauseKind == OpenACCClauseKind::VectorLength) &&
              "Parsed clause kind does not have a int exprs");
 
@@ -297,6 +356,7 @@ class SemaOpenACC : public SemaBase {
       assert((ClauseKind == OpenACCClauseKind::NumGangs ||
               ClauseKind == OpenACCClauseKind::NumWorkers ||
               ClauseKind == OpenACCClauseKind::Async ||
+              ClauseKind == OpenACCClauseKind::Tile ||
               ClauseKind == OpenACCClauseKind::VectorLength) &&
              "Parsed clause kind does not have a int exprs");
       Details = IntExprDetails{{IntExprs.begin(), IntExprs.end()}};
@@ -305,6 +365,7 @@ class SemaOpenACC : public SemaBase {
       assert((ClauseKind == OpenACCClauseKind::NumGangs ||
               ClauseKind == OpenACCClauseKind::NumWorkers ||
               ClauseKind == OpenACCClauseKind::Async ||
+              ClauseKind == OpenACCClauseKind::Tile ||
               ClauseKind == OpenACCClauseKind::VectorLength) &&
              "Parsed clause kind does not have a int exprs");
       Details = IntExprDetails{std::move(IntExprs)};
@@ -411,6 +472,17 @@ class SemaOpenACC : public SemaBase {
 
   SemaOpenACC(Sema &S);
 
+  // Called when we encounter a 'while' statement, before looking at its 'body'.
+  void ActOnWhileStmt(SourceLocation WhileLoc);
+  // Called when we encounter a 'do' statement, before looking at its 'body'.
+  void ActOnDoStmt(SourceLocation DoLoc);
+  // Called when we encounter a 'for' statement, before looking at its 'body'.
+  void ActOnForStmtBegin(SourceLocation ForLoc);
+  // Called when we encounter a 'for' statement, after we've consumed/checked
+  // the body. This is necessary for a number of checks on the contents of the
+  // 'for' statement.
+  void ActOnForStmtEnd(SourceLocation ForLoc, StmtResult Body);
+
   /// Called after parsing an OpenACC Clause so that it can be checked.
   OpenACCClause *ActOnClause(ArrayRef<const OpenACCClause *> ExistingClauses,
                              OpenACCParsedClause &Clause);
@@ -473,6 +545,50 @@ class SemaOpenACC : public SemaBase {
                                    SourceLocation RBLoc);
   /// Checks the loop depth value for a collapse clause.
   ExprResult CheckCollapseLoopCount(Expr *LoopCount);
+  /// Checks a single size expr for a tile clause. 'gang' could possibly call
+  /// this, but has slightly stricter rules as to valid values.
+  ExprResult CheckTileSizeExpr(Expr *SizeExpr);
+
+  ExprResult BuildOpenACCAsteriskSizeExpr(SourceLocation AsteriskLoc);
+  ExprResult ActOnOpenACCAsteriskSizeExpr(SourceLocation AsteriskLoc);
+
+  /// Helper type to restore the state of various 'loop' constructs when we run
+  /// into a loop (for, etc) inside the construct.
+  class LoopInConstructRAII {
+    SemaOpenACC &SemaRef;
+    LoopCheckingInfo OldLoopInfo;
+    CollapseCheckingInfo OldCollapseInfo;
+    TileCheckingInfo OldTileInfo;
+    bool PreserveDepth;
+
+  public:
+    LoopInConstructRAII(SemaOpenACC &SemaRef, bool PreserveDepth = true)
+        : SemaRef(SemaRef), OldLoopInfo(SemaRef.LoopInfo),
+          OldCollapseInfo(SemaRef.CollapseInfo), OldTileInfo(SemaRef.TileInfo),
+          PreserveDepth(PreserveDepth) {}
+    ~LoopInConstructRAII() {
+      // The associated-statement level of this should NOT preserve this, as it
+      // is a new construct, but other loop uses need to preserve the depth so
+      // it makes it to the 'top level' for diagnostics.
+      bool CollapseDepthSatisified =
+          PreserveDepth ? SemaRef.CollapseInfo.CollapseDepthSatisfied
+                        : OldCollapseInfo.CollapseDepthSatisfied;
+      bool TileDepthSatisfied = PreserveDepth
+                                    ? SemaRef.TileInfo.TileDepthSatisfied
+                                    : OldTileInfo.TileDepthSatisfied;
+      bool CurLevelHasLoopAlready =
+          PreserveDepth ? SemaRef.LoopInfo.CurLevelHasLoopAlready
+                        : OldLoopInfo.CurLevelHasLoopAlready;
+
+      SemaRef.LoopInfo = OldLoopInfo;
+      SemaRef.CollapseInfo = OldCollapseInfo;
+      SemaRef.TileInfo = OldTileInfo;
+
+      SemaRef.CollapseInfo.CollapseDepthSatisfied = CollapseDepthSatisified;
+      SemaRef.TileInfo.TileDepthSatisfied = TileDepthSatisfied;
+      SemaRef.LoopInfo.CurLevelHasLoopAlready = CurLevelHasLoopAlready;
+    }
+  };
 
   /// Helper type for the registration/assignment of constructs that need to
   /// 'know' about their parent constructs and hold a reference to them, such as
@@ -482,9 +598,18 @@ class SemaOpenACC : public SemaBase {
     bool WasInsideComputeConstruct;
     OpenACCDirectiveKind DirKind;
     llvm::SmallVector<OpenACCLoopConstruct *> ParentlessLoopConstructs;
+    LoopInConstructRAII LoopRAII;
 
   public:
-    AssociatedStmtRAII(SemaOpenACC &, OpenACCDirectiveKind);
+    AssociatedStmtRAII(SemaOpenACC &, OpenACCDirectiveKind,
+                       ArrayRef<const OpenACCClause *>,
+                       ArrayRef<OpenACCClause *>);
+    void SetCollapseInfoBeforeAssociatedStmt(
+        ArrayRef<const OpenACCClause *> UnInstClauses,
+        ArrayRef<OpenACCClause *> Clauses);
+    void SetTileInfoBeforeAssociatedStmt(
+        ArrayRef<const OpenACCClause *> UnInstClauses,
+        ArrayRef<OpenACCClause *> Clauses);
     ~AssociatedStmtRAII();
   };
 };
diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index bde19a09d6ae0..4b79d4b771190 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -730,6 +730,10 @@ enum ASTRecordTypes {
   /// canonical declaration for the lambda class from the same module as
   /// enclosing function.
   FUNCTION_DECL_TO_LAMBDAS_MAP = 71,
+
+  /// Record code for Sema's vector of functions/blocks with effects to
+  /// be verified.
+  DECLS_WITH_EFFECTS_TO_VERIFY = 72,
 };
 
 /// Record types used within a source manager block.
@@ -1998,12 +2002,14 @@ enum StmtCode {
   // SYCLUniqueStableNameExpr
   EXPR_SYCL_UNIQUE_STABLE_NAME,
 
-  // OpenACC Constructs
+  // OpenACC Constructs/Exprs
   STMT_OPENACC_COMPUTE_CONSTRUCT,
   STMT_OPENACC_LOOP_CONSTRUCT,
+  EXPR_OPENACC_ASTERISK_SIZE,
 
   // HLSL Constructs
   EXPR_HLSL_OUT_ARG,
+
 };
 
 /// The kinds of designators that can occur in a
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index c1843218a4b8b..aa88560b259a3 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -984,6 +984,9 @@ class ASTReader
   /// Sema tracks these to emit deferred diags.
   llvm::SmallSetVector<GlobalDeclID, 4> DeclsToCheckForDeferredDiags;
 
+  /// The IDs of all decls with function effects to be checked.
+  SmallVector<GlobalDeclID> DeclsWithEffectsToVerify;
+
 private:
   struct ImportedSubmodule {
     serialization::SubmoduleID ID;
diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h
index e21d41c867314..198dd01b8d07a 100644
--- a/clang/include/clang/Serialization/ASTWriter.h
+++ b/clang/include/clang/Serialization/ASTWriter.h
@@ -604,6 +604,7 @@ class ASTWriter : public ASTDeserializationListener,
   void WriteMSPointersToMembersPragmaOptions(Sema &SemaRef);
   void WritePackPragmaOptions(Sema &SemaRef);
   void WriteFloatControlPragmaOptions(Sema &SemaRef);
+  void WriteDeclsWithEffectsToVerify(Sema &SemaRef);
   void WriteModuleFileExtension(Sema &SemaRef,
                                 ModuleFileExtensionWriter &Writer);
 
diff --git a/clang/include/module.modulemap b/clang/include/module.modulemap
index b6ab99bb85d8a..6f18c0a49641c 100644
--- a/clang/include/module.modulemap
+++ b/clang/include/module.modulemap
@@ -65,6 +65,7 @@ module Clang_Basic {
   textual header "clang/Basic/BuiltinsX86.def"
   textual header "clang/Basic/BuiltinsX86_64.def"
   textual header "clang/Basic/BuiltinsXCore.def"
+  textual header "clang/Basic/CFProtectionOptions.def"
   textual header "clang/Basic/CodeGenOptions.def"
   textual header "clang/Basic/DebugOptions.def"
   textual header "clang/Basic/DiagnosticOptions.def"
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 735def67f7840..a81429ad6a238 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -14439,8 +14439,6 @@ bool ASTContext::useAbbreviatedThunkName(GlobalDecl VirtualMethodDecl,
         Mangler->mangleThunk(Method, Thunk, /* elideOverrideInfo */ false,
                              mangledNameStream);
 
-      if (Thunks.find(ElidedName) == Thunks.end())
-        Thunks[ElidedName] = {};
       Thunks[ElidedName].push_back(std::string(MangledName));
     }
   }
diff --git a/clang/lib/AST/ByteCode/Descriptor.cpp b/clang/lib/AST/ByteCode/Descriptor.cpp
index 65ac7a3129aba..5a8a2b64d5582 100644
--- a/clang/lib/AST/ByteCode/Descriptor.cpp
+++ b/clang/lib/AST/ByteCode/Descriptor.cpp
@@ -103,6 +103,7 @@ static void ctorArrayDesc(Block *B, std::byte *Ptr, bool IsConst,
     Desc->IsConst = IsConst || D->IsConst;
     Desc->IsFieldMutable = IsMutable || D->IsMutable;
     Desc->InUnion = InUnion;
+    Desc->IsArrayElement = true;
 
     if (auto Fn = D->ElemDesc->CtorFn)
       Fn(B, ElemLoc, Desc->IsConst, Desc->IsFieldMutable, IsActive,
@@ -408,6 +409,8 @@ QualType Descriptor::getType() const {
 QualType Descriptor::getElemQualType() const {
   assert(isArray());
   QualType T = getType();
+  if (T->isPointerOrReferenceType())
+    return T->getPointeeType();
   if (const auto *AT = T->getAsArrayTypeUnsafe())
     return AT->getElementType();
   if (const auto *CT = T->getAs<ComplexType>())
diff --git a/clang/lib/AST/ByteCode/Descriptor.h b/clang/lib/AST/ByteCode/Descriptor.h
index 5460199e0e991..bcb635e157f00 100644
--- a/clang/lib/AST/ByteCode/Descriptor.h
+++ b/clang/lib/AST/ByteCode/Descriptor.h
@@ -96,12 +96,16 @@ struct InlineDescriptor {
   /// Flag indicating if the field is mutable (if in a record).
   LLVM_PREFERRED_TYPE(bool)
   unsigned IsFieldMutable : 1;
+  /// Flag indicating if the field is an element of a composite array.
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned IsArrayElement : 1;
 
   const Descriptor *Desc;
 
   InlineDescriptor(const Descriptor *D)
       : Offset(sizeof(InlineDescriptor)), IsConst(false), IsInitialized(false),
-        IsBase(false), IsActive(false), IsFieldMutable(false), Desc(D) {}
+        IsBase(false), IsActive(false), IsFieldMutable(false),
+        IsArrayElement(false), Desc(D) {}
 
   void dump() const { dump(llvm::errs()); }
   void dump(llvm::raw_ostream &OS) const;
diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index fd9a256843a0e..b3ba81f04ff1f 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -1406,6 +1406,10 @@ bool handleFixedPointOverflow(InterpState &S, CodePtr OpPC,
   return S.noteUndefinedBehavior();
 }
 
+// https://github.com/llvm/llvm-project/issues/102513
+#if defined(_WIN32) && !defined(__clang__) && !defined(NDEBUG)
+#pragma optimize("", off)
+#endif
 bool Interpret(InterpState &S, APValue &Result) {
   // The current stack frame when we started Interpret().
   // This is being used by the ops to determine wheter
@@ -1430,6 +1434,10 @@ bool Interpret(InterpState &S, APValue &Result) {
     }
   }
 }
+// https://github.com/llvm/llvm-project/issues/102513
+#if defined(_WIN32) && !defined(__clang__) && !defined(NDEBUG)
+#pragma optimize("", on)
+#endif
 
 } // namespace interp
 } // namespace clang
diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index ebc800623f0d4..72c94e6fad3e0 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -1205,6 +1205,10 @@ static bool interp__builtin_ia32_lzcnt(InterpState &S, CodePtr OpPC,
                                        const InterpFrame *Frame,
                                        const Function *Func,
                                        const CallExpr *Call) {
+  QualType CallType = Call->getType();
+  if (!CallType->isIntegerType())
+    return false;
+
   APSInt Val = peekToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(0)));
   pushInteger(S, Val.countLeadingZeros(), Call->getType());
   return true;
@@ -1214,6 +1218,10 @@ static bool interp__builtin_ia32_tzcnt(InterpState &S, CodePtr OpPC,
                                        const InterpFrame *Frame,
                                        const Function *Func,
                                        const CallExpr *Call) {
+  QualType CallType = Call->getType();
+  if (!CallType->isIntegerType())
+    return false;
+
   APSInt Val = peekToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(0)));
   pushInteger(S, Val.countTrailingZeros(), Call->getType());
   return true;
diff --git a/clang/lib/AST/ByteCode/Pointer.cpp b/clang/lib/AST/ByteCode/Pointer.cpp
index 387cad9b137c0..a52f0e336ef29 100644
--- a/clang/lib/AST/ByteCode/Pointer.cpp
+++ b/clang/lib/AST/ByteCode/Pointer.cpp
@@ -204,18 +204,22 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
       Path.push_back(APValue::LValuePathEntry(
           {Ptr.getFieldDesc()->asDecl(), /*IsVirtual=*/false}));
 
-      if (const auto *FD = dyn_cast<FieldDecl>(Ptr.getFieldDesc()->asDecl()))
+      if (const auto *FD =
+              dyn_cast_if_present<FieldDecl>(Ptr.getFieldDesc()->asDecl()))
         Offset += getFieldOffset(FD);
 
       Ptr = Ptr.getBase();
     } else if (Ptr.isArrayElement()) {
+      Ptr = Ptr.expand();
       unsigned Index;
       if (Ptr.isOnePastEnd())
         Index = Ptr.getArray().getNumElems();
       else
         Index = Ptr.getIndex();
 
-      Offset += (Index * ASTCtx.getTypeSizeInChars(Ptr.getType()));
+      QualType ElemType = Ptr.getFieldDesc()->getElemQualType();
+      Offset += (Index * ASTCtx.getTypeSizeInChars(ElemType));
+
       Path.push_back(APValue::LValuePathEntry::ArrayIndex(Index));
       Ptr = Ptr.getArray();
     } else {
diff --git a/clang/lib/AST/ByteCode/Pointer.h b/clang/lib/AST/ByteCode/Pointer.h
index ac9b9ed4091b6..08bc4b7e40b63 100644
--- a/clang/lib/AST/ByteCode/Pointer.h
+++ b/clang/lib/AST/ByteCode/Pointer.h
@@ -420,8 +420,18 @@ class Pointer {
   }
   /// Checks if the pointer points to an array.
   bool isArrayElement() const {
-    if (isBlockPointer())
-      return inArray() && asBlockPointer().Base != Offset;
+    if (!isBlockPointer())
+      return false;
+
+    const BlockPointer &BP = asBlockPointer();
+    if (inArray() && BP.Base != Offset)
+      return true;
+
+    // Might be a narrow()'ed element in a composite array.
+    // Check the inline descriptor.
+    if (BP.Base >= sizeof(InlineDescriptor) && getInlineDesc()->IsArrayElement)
+      return true;
+
     return false;
   }
   /// Pointer points directly to a block.
diff --git a/clang/lib/AST/ComputeDependence.cpp b/clang/lib/AST/ComputeDependence.cpp
index 6ef49790481ac..8c79df8317a2e 100644
--- a/clang/lib/AST/ComputeDependence.cpp
+++ b/clang/lib/AST/ComputeDependence.cpp
@@ -958,3 +958,9 @@ ExprDependence clang::computeDependence(ObjCMessageExpr *E) {
     D |= A->getDependence();
   return D;
 }
+
+ExprDependence clang::computeDependence(OpenACCAsteriskSizeExpr *E) {
+  // This represents a simple asterisk as typed, so cannot be dependent in any
+  // way.
+  return ExprDependence::None;
+}
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 2e463fc00c6b6..9ecbf121e3fc0 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -3640,6 +3640,7 @@ bool Expr::HasSideEffects(const ASTContext &Ctx,
   case SYCLUniqueStableNameExprClass:
   case PackIndexingExprClass:
   case HLSLOutArgExprClass:
+  case OpenACCAsteriskSizeExprClass:
     // These never have a side-effect.
     return false;
 
@@ -5408,3 +5409,13 @@ HLSLOutArgExpr *HLSLOutArgExpr::Create(const ASTContext &C, QualType Ty,
 HLSLOutArgExpr *HLSLOutArgExpr::CreateEmpty(const ASTContext &C) {
   return new (C) HLSLOutArgExpr(EmptyShell());
 }
+
+OpenACCAsteriskSizeExpr *OpenACCAsteriskSizeExpr::Create(const ASTContext &C,
+                                                         SourceLocation Loc) {
+  return new (C) OpenACCAsteriskSizeExpr(Loc, C.IntTy);
+}
+
+OpenACCAsteriskSizeExpr *
+OpenACCAsteriskSizeExpr::CreateEmpty(const ASTContext &C) {
+  return new (C) OpenACCAsteriskSizeExpr({}, C.IntTy);
+}
diff --git a/clang/lib/AST/ExprClassification.cpp b/clang/lib/AST/ExprClassification.cpp
index 5dde923312698..3f37d06cc8f3a 100644
--- a/clang/lib/AST/ExprClassification.cpp
+++ b/clang/lib/AST/ExprClassification.cpp
@@ -471,6 +471,7 @@ static Cl::Kinds ClassifyInternal(ASTContext &Ctx, const Expr *E) {
   case Expr::CoyieldExprClass:
     return ClassifyInternal(Ctx, cast<CoroutineSuspendExpr>(E)->getResumeExpr());
   case Expr::SYCLUniqueStableNameExprClass:
+  case Expr::OpenACCAsteriskSizeExprClass:
     return Cl::CL_PRValue;
     break;
 
@@ -704,7 +705,8 @@ static Cl::ModifiableType IsModifiable(ASTContext &Ctx, const Expr *E,
     return Cl::CM_ConstAddrSpace;
 
   // Arrays are not modifiable, only their elements are.
-  if (CT->isArrayType())
+  if (CT->isArrayType() &&
+      !(Ctx.getLangOpts().HLSL && CT->isConstantArrayType()))
     return Cl::CM_ArrayType;
   // Incomplete types are not modifiable.
   if (CT->isIncompleteType())
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 48816d3078826..4d5af96093cfe 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -11873,6 +11873,13 @@ class IntExprEvaluator
     return Success(E->getValue(), E);
   }
 
+  bool VisitOpenACCAsteriskSizeExpr(const OpenACCAsteriskSizeExpr *E) {
+    // This should not be evaluated during constant expr evaluation, as it
+    // should always be in an unevaluated context (the args list of a 'gang' or
+    // 'tile' clause).
+    return Error(E);
+  }
+
   bool VisitUnaryReal(const UnaryOperator *E);
   bool VisitUnaryImag(const UnaryOperator *E);
 
@@ -13464,6 +13471,38 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
     return Success(DidOverflow, E);
   }
 
+  case clang::X86::BI__builtin_ia32_addcarryx_u32:
+  case clang::X86::BI__builtin_ia32_addcarryx_u64:
+  case clang::X86::BI__builtin_ia32_subborrow_u32:
+  case clang::X86::BI__builtin_ia32_subborrow_u64: {
+    LValue ResultLValue;
+    APSInt CarryIn, LHS, RHS;
+    QualType ResultType = E->getArg(3)->getType()->getPointeeType();
+    if (!EvaluateInteger(E->getArg(0), CarryIn, Info) ||
+        !EvaluateInteger(E->getArg(1), LHS, Info) ||
+        !EvaluateInteger(E->getArg(2), RHS, Info) ||
+        !EvaluatePointer(E->getArg(3), ResultLValue, Info))
+      return false;
+
+    bool IsAdd = BuiltinOp == clang::X86::BI__builtin_ia32_addcarryx_u32 ||
+                 BuiltinOp == clang::X86::BI__builtin_ia32_addcarryx_u64;
+
+    unsigned BitWidth = LHS.getBitWidth();
+    unsigned CarryInBit = CarryIn.ugt(0) ? 1 : 0;
+    APInt ExResult =
+        IsAdd
+            ? (LHS.zext(BitWidth + 1) + (RHS.zext(BitWidth + 1) + CarryInBit))
+            : (LHS.zext(BitWidth + 1) - (RHS.zext(BitWidth + 1) + CarryInBit));
+
+    APInt Result = ExResult.extractBits(BitWidth, 0);
+    uint64_t CarryOut = ExResult.extractBitsAsZExtValue(1, BitWidth);
+
+    APValue APV{APSInt(Result, /*isUnsigned=*/true)};
+    if (!handleAssignment(Info, E, ResultLValue, ResultType, APV))
+      return false;
+    return Success(CarryOut, E);
+  }
+
   case clang::X86::BI__builtin_ia32_bextr_u32:
   case clang::X86::BI__builtin_ia32_bextr_u64:
   case clang::X86::BI__builtin_ia32_bextri_u32:
@@ -16876,6 +16915,7 @@ static ICEDiag CheckICE(const Expr* E, const ASTContext &Ctx) {
   case Expr::GNUNullExprClass:
   case Expr::SourceLocExprClass:
   case Expr::EmbedExprClass:
+  case Expr::OpenACCAsteriskSizeExprClass:
     return NoDiag();
 
   case Expr::PackIndexingExprClass:
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index 1feec13f9deef..769a863c2b676 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -693,7 +693,10 @@ ItaniumMangleContextImpl::getEffectiveDeclContext(const Decl *D) {
     if (VD->isExternC())
       return getASTContext().getTranslationUnitDecl();
 
-  if (const auto *FD = D->getAsFunction()) {
+  if (const auto *FD = getASTContext().getLangOpts().getClangABICompat() >
+                               LangOptions::ClangABI::Ver19
+                           ? D->getAsFunction()
+                           : dyn_cast<FunctionDecl>(D)) {
     if (FD->isExternC())
       return getASTContext().getTranslationUnitDecl();
     // Member-like constrained friends are mangled as if they were members of
@@ -5742,6 +5745,15 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity,
   case Expr::HLSLOutArgExprClass:
     llvm_unreachable(
         "cannot mangle hlsl temporary value; mangling wrong thing?");
+  case Expr::OpenACCAsteriskSizeExprClass: {
+    // We shouldn't ever be able to get here, but diagnose anyway.
+    DiagnosticsEngine &Diags = Context.getDiags();
+    unsigned DiagID = Diags.getCustomDiagID(
+        DiagnosticsEngine::Error,
+        "cannot yet mangle OpenACC Asterisk Size expression");
+    Diags.Report(DiagID);
+    return;
+  }
   }
 
   if (AsTemplateArg && !IsPrimaryExpr)
diff --git a/clang/lib/AST/JSONNodeDumper.cpp b/clang/lib/AST/JSONNodeDumper.cpp
index 565f1e05710c8..ddbe2136a671f 100644
--- a/clang/lib/AST/JSONNodeDumper.cpp
+++ b/clang/lib/AST/JSONNodeDumper.cpp
@@ -1354,6 +1354,9 @@ void JSONNodeDumper::VisitSYCLUniqueStableNameExpr(
                 createQualType(E->getTypeSourceInfo()->getType()));
 }
 
+void JSONNodeDumper::VisitOpenACCAsteriskSizeExpr(
+    const OpenACCAsteriskSizeExpr *E) {}
+
 void JSONNodeDumper::VisitPredefinedExpr(const PredefinedExpr *PE) {
   JOS.attribute("name", PredefinedExpr::getIdentKindName(PE->getIdentKind()));
 }
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index e4c8663c134fd..4ccf3f76bf0ce 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -1024,6 +1024,7 @@ void MicrosoftCXXNameMangler::mangleFloat(llvm::APFloat Number) {
   case APFloat::S_Float8E4M3B11FNUZ:
   case APFloat::S_Float8E3M4:
   case APFloat::S_FloatTF32:
+  case APFloat::S_Float8E8M0FNU:
   case APFloat::S_Float6E3M2FN:
   case APFloat::S_Float6E2M3FN:
   case APFloat::S_Float4E2M1FN:
diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp
index d864ded33e8d1..0b34ed6189593 100644
--- a/clang/lib/AST/OpenACCClause.cpp
+++ b/clang/lib/AST/OpenACCClause.cpp
@@ -24,6 +24,7 @@ bool OpenACCClauseWithParams::classof(const OpenACCClause *C) {
 }
 bool OpenACCClauseWithExprs::classof(const OpenACCClause *C) {
   return OpenACCWaitClause::classof(C) || OpenACCNumGangsClause::classof(C) ||
+         OpenACCTileClause::classof(C) ||
          OpenACCClauseWithSingleIntExpr::classof(C) ||
          OpenACCClauseWithVarList::classof(C);
 }
@@ -221,6 +222,16 @@ OpenACCNumGangsClause *OpenACCNumGangsClause::Create(const ASTContext &C,
   return new (Mem) OpenACCNumGangsClause(BeginLoc, LParenLoc, IntExprs, EndLoc);
 }
 
+OpenACCTileClause *OpenACCTileClause::Create(const ASTContext &C,
+                                             SourceLocation BeginLoc,
+                                             SourceLocation LParenLoc,
+                                             ArrayRef<Expr *> SizeExprs,
+                                             SourceLocation EndLoc) {
+  void *Mem =
+      C.Allocate(OpenACCTileClause::totalSizeToAlloc<Expr *>(SizeExprs.size()));
+  return new (Mem) OpenACCTileClause(BeginLoc, LParenLoc, SizeExprs, EndLoc);
+}
+
 OpenACCPrivateClause *OpenACCPrivateClause::Create(const ASTContext &C,
                                                    SourceLocation BeginLoc,
                                                    SourceLocation LParenLoc,
@@ -420,6 +431,13 @@ void OpenACCClausePrinter::VisitNumGangsClause(const OpenACCNumGangsClause &C) {
   OS << ")";
 }
 
+void OpenACCClausePrinter::VisitTileClause(const OpenACCTileClause &C) {
+  OS << "tile(";
+  llvm::interleaveComma(C.getSizeExprs(), OS,
+                        [&](const Expr *E) { printExpr(E); });
+  OS << ")";
+}
+
 void OpenACCClausePrinter::VisitNumWorkersClause(
     const OpenACCNumWorkersClause &C) {
   OS << "num_workers(";
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index e1b5bec7a50d0..641f7b52de6df 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -1309,6 +1309,10 @@ void StmtPrinter::VisitPredefinedExpr(PredefinedExpr *Node) {
   OS << PredefinedExpr::getIdentKindName(Node->getIdentKind());
 }
 
+void StmtPrinter::VisitOpenACCAsteriskSizeExpr(OpenACCAsteriskSizeExpr *Node) {
+  OS << '*';
+}
+
 void StmtPrinter::VisitCharacterLiteral(CharacterLiteral *Node) {
   CharacterLiteral::print(Node->getValue(), Node->getKind(), OS);
 }
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index c3812844ab8a3..299ac005c7fdb 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -1360,6 +1360,11 @@ void StmtProfiler::VisitPredefinedExpr(const PredefinedExpr *S) {
   ID.AddInteger(llvm::to_underlying(S->getIdentKind()));
 }
 
+void StmtProfiler::VisitOpenACCAsteriskSizeExpr(
+    const OpenACCAsteriskSizeExpr *S) {
+  VisitExpr(S);
+}
+
 void StmtProfiler::VisitIntegerLiteral(const IntegerLiteral *S) {
   VisitExpr(S);
   S->getValue().Profile(ID);
@@ -2552,6 +2557,11 @@ void OpenACCClauseProfiler::VisitNumGangsClause(
     Profiler.VisitStmt(E);
 }
 
+void OpenACCClauseProfiler::VisitTileClause(const OpenACCTileClause &Clause) {
+  for (auto *E : Clause.getSizeExprs())
+    Profiler.VisitStmt(E);
+}
+
 void OpenACCClauseProfiler::VisitNumWorkersClause(
     const OpenACCNumWorkersClause &Clause) {
   assert(Clause.hasIntExpr() && "num_workers clause requires a valid int expr");
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index 8a74159c7c93e..15b23d60c3ffa 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -381,6 +381,11 @@ void TextNodeDumper::Visit(const OMPClause *C) {
     OS << " <implicit>";
 }
 
+void TextNodeDumper::VisitOpenACCAsteriskSizeExpr(
+    const OpenACCAsteriskSizeExpr *E) {
+  // Nothing to do here, only location exists, and that is printed elsewhere.
+}
+
 void TextNodeDumper::Visit(const OpenACCClause *C) {
   if (!C) {
     ColorScope Color(OS, ShowColors, NullColor);
@@ -414,6 +419,7 @@ void TextNodeDumper::Visit(const OpenACCClause *C) {
     case OpenACCClauseKind::Private:
     case OpenACCClauseKind::Self:
     case OpenACCClauseKind::Seq:
+    case OpenACCClauseKind::Tile:
     case OpenACCClauseKind::VectorLength:
       // The condition expression will be printed as a part of the 'children',
       // but print 'clause' here so it is clear what is happening from the dump.
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index c703e43f12a9a..6f4958801cfe8 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -5117,8 +5117,6 @@ FunctionEffect::Kind FunctionEffect::oppositeKind() const {
     return Kind::Allocating;
   case Kind::Allocating:
     return Kind::NonAllocating;
-  case Kind::None:
-    return Kind::None;
   }
   llvm_unreachable("unknown effect kind");
 }
@@ -5133,53 +5131,41 @@ StringRef FunctionEffect::name() const {
     return "blocking";
   case Kind::Allocating:
     return "allocating";
-  case Kind::None:
-    return "(none)";
   }
   llvm_unreachable("unknown effect kind");
 }
 
-bool FunctionEffect::canInferOnFunction(const Decl &Callee) const {
+std::optional<FunctionEffect> FunctionEffect::effectProhibitingInference(
+    const Decl &Callee, FunctionEffectKindSet CalleeFX) const {
   switch (kind()) {
   case Kind::NonAllocating:
   case Kind::NonBlocking: {
-    FunctionEffectsRef CalleeFX;
-    if (auto *FD = Callee.getAsFunction())
-      CalleeFX = FD->getFunctionEffects();
-    else if (auto *BD = dyn_cast<BlockDecl>(&Callee))
-      CalleeFX = BD->getFunctionEffects();
-    else
-      return false;
-    for (const FunctionEffectWithCondition &CalleeEC : CalleeFX) {
+    for (FunctionEffect Effect : CalleeFX) {
       // nonblocking/nonallocating cannot call allocating.
-      if (CalleeEC.Effect.kind() == Kind::Allocating)
-        return false;
+      if (Effect.kind() == Kind::Allocating)
+        return Effect;
       // nonblocking cannot call blocking.
-      if (kind() == Kind::NonBlocking &&
-          CalleeEC.Effect.kind() == Kind::Blocking)
-        return false;
+      if (kind() == Kind::NonBlocking && Effect.kind() == Kind::Blocking)
+        return Effect;
     }
-    return true;
+    return std::nullopt;
   }
 
   case Kind::Allocating:
   case Kind::Blocking:
-    return false;
-
-  case Kind::None:
-    assert(0 && "canInferOnFunction with None");
+    assert(0 && "effectProhibitingInference with non-inferable effect kind");
     break;
   }
   llvm_unreachable("unknown effect kind");
 }
 
 bool FunctionEffect::shouldDiagnoseFunctionCall(
-    bool Direct, ArrayRef<FunctionEffect> CalleeFX) const {
+    bool Direct, FunctionEffectKindSet CalleeFX) const {
   switch (kind()) {
   case Kind::NonAllocating:
   case Kind::NonBlocking: {
     const Kind CallerKind = kind();
-    for (const auto &Effect : CalleeFX) {
+    for (FunctionEffect Effect : CalleeFX) {
       const Kind EK = Effect.kind();
       // Does callee have same or stronger constraint?
       if (EK == CallerKind ||
@@ -5192,9 +5178,6 @@ bool FunctionEffect::shouldDiagnoseFunctionCall(
   case Kind::Allocating:
   case Kind::Blocking:
     return false;
-  case Kind::None:
-    assert(0 && "shouldDiagnoseFunctionCall with None");
-    break;
   }
   llvm_unreachable("unknown effect kind");
 }
@@ -5300,21 +5283,24 @@ FunctionEffectSet FunctionEffectSet::getUnion(FunctionEffectsRef LHS,
   return Combined;
 }
 
+namespace clang {
+
+raw_ostream &operator<<(raw_ostream &OS,
+                        const FunctionEffectWithCondition &CFE) {
+  OS << CFE.Effect.name();
+  if (Expr *E = CFE.Cond.getCondition()) {
+    OS << '(';
+    E->dump();
+    OS << ')';
+  }
+  return OS;
+}
+
+} // namespace clang
+
 LLVM_DUMP_METHOD void FunctionEffectsRef::dump(llvm::raw_ostream &OS) const {
   OS << "Effects{";
-  bool First = true;
-  for (const auto &CFE : *this) {
-    if (!First)
-      OS << ", ";
-    else
-      First = false;
-    OS << CFE.Effect.name();
-    if (Expr *E = CFE.Cond.getCondition()) {
-      OS << '(';
-      E->dump();
-      OS << ')';
-    }
-  }
+  llvm::interleaveComma(*this, OS);
   OS << "}";
 }
 
@@ -5322,6 +5308,12 @@ LLVM_DUMP_METHOD void FunctionEffectSet::dump(llvm::raw_ostream &OS) const {
   FunctionEffectsRef(*this).dump(OS);
 }
 
+LLVM_DUMP_METHOD void FunctionEffectKindSet::dump(llvm::raw_ostream &OS) const {
+  OS << "Effects{";
+  llvm::interleaveComma(*this, OS);
+  OS << "}";
+}
+
 FunctionEffectsRef
 FunctionEffectsRef::create(ArrayRef<FunctionEffect> FX,
                            ArrayRef<EffectConditionExpr> Conds) {
diff --git a/clang/lib/Analysis/ExprMutationAnalyzer.cpp b/clang/lib/Analysis/ExprMutationAnalyzer.cpp
index 6d726ae44104e..5a95ef36d0502 100644
--- a/clang/lib/Analysis/ExprMutationAnalyzer.cpp
+++ b/clang/lib/Analysis/ExprMutationAnalyzer.cpp
@@ -231,12 +231,11 @@ ExprMutationAnalyzer::Analyzer::findPointeeMutation(const Decl *Dec) {
 const Stmt *ExprMutationAnalyzer::Analyzer::findMutationMemoized(
     const Expr *Exp, llvm::ArrayRef<MutationFinder> Finders,
     Memoized::ResultMap &MemoizedResults) {
-  const auto Memoized = MemoizedResults.find(Exp);
-  if (Memoized != MemoizedResults.end())
+  auto [Memoized, Inserted] = MemoizedResults.try_emplace(Exp);
+  if (!Inserted)
     return Memoized->second;
 
   // Assume Exp is not mutated before analyzing Exp.
-  MemoizedResults[Exp] = nullptr;
   if (isUnevaluated(Exp))
     return nullptr;
 
diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 5f5dfcb722f9d..61889861c9c80 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -373,6 +373,12 @@ void AArch64TargetInfo::getTargetDefinesARMV95A(const LangOptions &Opts,
   getTargetDefinesARMV94A(Opts, Builder);
 }
 
+void AArch64TargetInfo::getTargetDefinesARMV96A(const LangOptions &Opts,
+                                                MacroBuilder &Builder) const {
+  // Armv9.6-A does not have a v8.* equivalent, but is a superset of v9.5-A.
+  getTargetDefinesARMV95A(Opts, Builder);
+}
+
 void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
                                          MacroBuilder &Builder) const {
   // Target identification.
@@ -657,6 +663,8 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
     getTargetDefinesARMV94A(Opts, Builder);
   else if (*ArchInfo == llvm::AArch64::ARMV9_5A)
     getTargetDefinesARMV95A(Opts, Builder);
+  else if (*ArchInfo == llvm::AArch64::ARMV9_6A)
+    getTargetDefinesARMV96A(Opts, Builder);
 
   // All of the __sync_(bool|val)_compare_and_swap_(1|2|4|8|16) builtins work.
   Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_1");
@@ -1044,6 +1052,9 @@ bool AArch64TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
     if (Feature == "+v9.5a" &&
         ArchInfo->Version < llvm::AArch64::ARMV9_5A.Version)
       ArchInfo = &llvm::AArch64::ARMV9_5A;
+    if (Feature == "+v9.6a" &&
+        ArchInfo->Version < llvm::AArch64::ARMV9_6A.Version)
+      ArchInfo = &llvm::AArch64::ARMV9_6A;
     if (Feature == "+v8r")
       ArchInfo = &llvm::AArch64::ARMV8R;
     if (Feature == "+fullfp16") {
diff --git a/clang/lib/Basic/Targets/AArch64.h b/clang/lib/Basic/Targets/AArch64.h
index 526f7f30a3861..1226ce4d4355c 100644
--- a/clang/lib/Basic/Targets/AArch64.h
+++ b/clang/lib/Basic/Targets/AArch64.h
@@ -148,6 +148,8 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo {
                                MacroBuilder &Builder) const;
   void getTargetDefinesARMV95A(const LangOptions &Opts,
                                MacroBuilder &Builder) const;
+  void getTargetDefinesARMV96A(const LangOptions &Opts,
+                               MacroBuilder &Builder) const;
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
diff --git a/clang/lib/Basic/Targets/ARM.cpp b/clang/lib/Basic/Targets/ARM.cpp
index 7423626d7c3cb..c56b8d9a44850 100644
--- a/clang/lib/Basic/Targets/ARM.cpp
+++ b/clang/lib/Basic/Targets/ARM.cpp
@@ -228,6 +228,8 @@ StringRef ARMTargetInfo::getCPUAttr() const {
     return "9_4A";
   case llvm::ARM::ArchKind::ARMV9_5A:
     return "9_5A";
+  case llvm::ARM::ArchKind::ARMV9_6A:
+    return "9_6A";
   case llvm::ARM::ArchKind::ARMV8MBaseline:
     return "8M_BASE";
   case llvm::ARM::ArchKind::ARMV8MMainline:
@@ -891,6 +893,7 @@ void ARMTargetInfo::getTargetDefines(const LangOptions &Opts,
   case llvm::ARM::ArchKind::ARMV9_3A:
   case llvm::ARM::ArchKind::ARMV9_4A:
   case llvm::ARM::ArchKind::ARMV9_5A:
+  case llvm::ARM::ArchKind::ARMV9_6A:
     // Filter __arm_cdp, __arm_ldcl, __arm_stcl in arm_acle.h
     FeatureCoprocBF = FEATURE_COPROC_B1 | FEATURE_COPROC_B3;
     break;
@@ -1060,6 +1063,7 @@ void ARMTargetInfo::getTargetDefines(const LangOptions &Opts,
   case llvm::ARM::ArchKind::ARMV9_3A:
   case llvm::ARM::ArchKind::ARMV9_4A:
   case llvm::ARM::ArchKind::ARMV9_5A:
+  case llvm::ARM::ArchKind::ARMV9_6A:
     getTargetDefinesARMV83A(Opts, Builder);
     break;
   }
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 62c6a57e8b7c8..abc936f2c686d 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -1321,10 +1321,10 @@ static void runThinLTOBackend(
     Conf.CGFileType = getCodeGenFileType(Action);
     break;
   }
-  if (Error E =
-          thinBackend(Conf, -1, AddStream, *M, *CombinedIndex, ImportList,
-                      ModuleToDefinedGVSummaries[M->getModuleIdentifier()],
-                      /* ModuleMap */ nullptr, CGOpts.CmdArgs)) {
+  if (Error E = thinBackend(
+          Conf, -1, AddStream, *M, *CombinedIndex, ImportList,
+          ModuleToDefinedGVSummaries[M->getModuleIdentifier()],
+          /* ModuleMap */ nullptr, Conf.CodeGenOnly, CGOpts.CmdArgs)) {
     handleAllErrors(std::move(E), [&](ErrorInfoBase &EIB) {
       errs() << "Error running ThinLTO backend: " << EIB.message() << '\n';
     });
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index da3eca73bfb57..43700ea9dd3cf 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -3836,6 +3836,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   case Builtin::BI__builtin_elementwise_atan:
     return RValue::get(emitBuiltinWithOneOverloadedType<1>(
         *this, E, llvm::Intrinsic::atan, "elt.atan"));
+  case Builtin::BI__builtin_elementwise_atan2:
+    return RValue::get(emitBuiltinWithOneOverloadedType<2>(
+        *this, E, llvm::Intrinsic::atan2, "elt.atan2"));
   case Builtin::BI__builtin_elementwise_ceil:
     return RValue::get(emitBuiltinWithOneOverloadedType<1>(
         *this, E, llvm::Intrinsic::ceil, "elt.ceil"));
@@ -3961,6 +3964,22 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     return RValue::get(Result);
   }
 
+  case Builtin::BI__builtin_elementwise_maximum: {
+    Value *Op0 = EmitScalarExpr(E->getArg(0));
+    Value *Op1 = EmitScalarExpr(E->getArg(1));
+    Value *Result = Builder.CreateBinaryIntrinsic(llvm::Intrinsic::maximum, Op0,
+                                                  Op1, nullptr, "elt.maximum");
+    return RValue::get(Result);
+  }
+
+  case Builtin::BI__builtin_elementwise_minimum: {
+    Value *Op0 = EmitScalarExpr(E->getArg(0));
+    Value *Op1 = EmitScalarExpr(E->getArg(1));
+    Value *Result = Builder.CreateBinaryIntrinsic(llvm::Intrinsic::minimum, Op0,
+                                                  Op1, nullptr, "elt.minimum");
+    return RValue::get(Result);
+  }
+
   case Builtin::BI__builtin_reduce_max: {
     auto GetIntrinsicID = [this](QualType QT) {
       if (auto *VecTy = QT->getAs<VectorType>())
@@ -4013,6 +4032,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   case Builtin::BI__builtin_reduce_and:
     return RValue::get(emitBuiltinWithOneOverloadedType<1>(
         *this, E, llvm::Intrinsic::vector_reduce_and, "rdx.and"));
+  case Builtin::BI__builtin_reduce_maximum:
+    return RValue::get(emitBuiltinWithOneOverloadedType<1>(
+        *this, E, llvm::Intrinsic::vector_reduce_fmaximum, "rdx.maximum"));
+  case Builtin::BI__builtin_reduce_minimum:
+    return RValue::get(emitBuiltinWithOneOverloadedType<1>(
+        *this, E, llvm::Intrinsic::vector_reduce_fminimum, "rdx.minimum"));
 
   case Builtin::BI__builtin_matrix_transpose: {
     auto *MatrixTy = E->getArg(0)->getType()->castAs<ConstantMatrixType>();
@@ -18642,6 +18667,21 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
         IsUnsigned ? Intrinsic::dx_uclamp : Intrinsic::dx_clamp,
         ArrayRef<Value *>{OpX, OpMin, OpMax}, nullptr, "dx.clamp");
   }
+  case Builtin::BI__builtin_hlsl_cross: {
+    Value *Op0 = EmitScalarExpr(E->getArg(0));
+    Value *Op1 = EmitScalarExpr(E->getArg(1));
+    assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
+           E->getArg(1)->getType()->hasFloatingRepresentation() &&
+           "cross operands must have a float representation");
+    // make sure each vector has exactly 3 elements
+    assert(
+        E->getArg(0)->getType()->getAs<VectorType>()->getNumElements() == 3 &&
+        E->getArg(1)->getType()->getAs<VectorType>()->getNumElements() == 3 &&
+        "input vectors must have 3 elements each");
+    return Builder.CreateIntrinsic(
+        /*ReturnType=*/Op0->getType(), CGM.getHLSLRuntime().getCrossIntrinsic(),
+        ArrayRef<Value *>{Op0, Op1}, nullptr, "hlsl.cross");
+  }
   case Builtin::BI__builtin_hlsl_dot: {
     Value *Op0 = EmitScalarExpr(E->getArg(0));
     Value *Op1 = EmitScalarExpr(E->getArg(1));
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 8887c4de7c4c8..609957b75d6e7 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -1472,15 +1472,6 @@ llvm::DIType *CGDebugInfo::CreateType(const TemplateSpecializationType *Ty,
     return AliasTy;
   }
 
-  // Disable PrintCanonicalTypes here because we want
-  // the DW_AT_name to benefit from the TypePrinter's ability
-  // to skip defaulted template arguments.
-  //
-  // FIXME: Once -gsimple-template-names is enabled by default
-  // and we attach template parameters to alias template DIEs
-  // we don't need to worry about customizing the PrintingPolicy
-  // here anymore.
-  PP.PrintCanonicalTypes = false;
   printTemplateArgumentList(OS, Ty->template_arguments(), PP,
                             TD->getTemplateParameters());
   return DBuilder.createTypedef(Src, OS.str(), getOrCreateFile(Loc),
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index df4994ba9af6e..52d2f6d52abf9 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -1095,6 +1095,8 @@ class StructAccessBase
     return Visit(E->getBase());
   }
   const Expr *VisitCastExpr(const CastExpr *E) {
+    if (E->getCastKind() == CK_LValueToRValue)
+      return E;
     return Visit(E->getSubExpr());
   }
   const Expr *VisitParenExpr(const ParenExpr *E) {
@@ -1161,19 +1163,15 @@ llvm::Value *CodeGenFunction::EmitLoadOfCountedByField(
     return nullptr;
 
   llvm::Value *Res = nullptr;
-  if (const auto *DRE = dyn_cast<DeclRefExpr>(StructBase)) {
-    Res = EmitDeclRefLValue(DRE).getPointer(*this);
-    Res = Builder.CreateAlignedLoad(ConvertType(DRE->getType()), Res,
-                                    getPointerAlign(), "dre.load");
-  } else if (const MemberExpr *ME = dyn_cast<MemberExpr>(StructBase)) {
-    LValue LV = EmitMemberExpr(ME);
-    Address Addr = LV.getAddress();
-    Res = Addr.emitRawPointer(*this);
-  } else if (StructBase->getType()->isPointerType()) {
+  if (StructBase->getType()->isPointerType()) {
     LValueBaseInfo BaseInfo;
     TBAAAccessInfo TBAAInfo;
     Address Addr = EmitPointerWithAlignment(StructBase, &BaseInfo, &TBAAInfo);
     Res = Addr.emitRawPointer(*this);
+  } else if (StructBase->isLValue()) {
+    LValue LV = EmitLValue(StructBase);
+    Address Addr = LV.getAddress();
+    Res = Addr.emitRawPointer(*this);
   } else {
     return nullptr;
   }
@@ -5797,11 +5795,26 @@ LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
     return EmitComplexAssignmentLValue(E);
 
   case TEK_Aggregate:
+    // If the lang opt is HLSL and the LHS is a constant array
+    // then we are performing a copy assignment and call a special
+    // function because EmitAggExprToLValue emits to a temporary LValue
+    if (getLangOpts().HLSL && E->getLHS()->getType()->isConstantArrayType())
+      return EmitHLSLArrayAssignLValue(E);
+
     return EmitAggExprToLValue(E);
   }
   llvm_unreachable("bad evaluation kind");
 }
 
+// This function implements trivial copy assignment for HLSL's
+// assignable constant arrays.
+LValue CodeGenFunction::EmitHLSLArrayAssignLValue(const BinaryOperator *E) {
+  LValue TrivialAssignmentRHS = EmitLValue(E->getRHS());
+  LValue LHS = EmitLValue(E->getLHS());
+  EmitAggregateAssign(LHS, TrivialAssignmentRHS, E->getLHS()->getType());
+  return LHS;
+}
+
 LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E,
                                            llvm::CallBase **CallOrInvoke) {
   RValue RV = EmitCallExpr(E, ReturnValueSlot(), CallOrInvoke);
diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp
index bbfc6672ecc25..74d4c5ea7bcaa 100644
--- a/clang/lib/CodeGen/CGExprAgg.cpp
+++ b/clang/lib/CodeGen/CGExprAgg.cpp
@@ -39,7 +39,7 @@ namespace llvm {
 extern cl::opt<bool> EnableSingleByteCoverage;
 } // namespace llvm
 
-namespace  {
+namespace {
 class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
   CodeGenFunction &CGF;
   CGBuilderTy &Builder;
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index b7f5b932c56b6..7529d4465f2c3 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -531,6 +531,10 @@ class ScalarExprEmitter
     return CGF.getOrCreateOpaqueRValueMapping(E).getScalarVal();
   }
 
+  Value *VisitOpenACCAsteriskSizeExpr(OpenACCAsteriskSizeExpr *E) {
+    llvm_unreachable("Codegen for this isn't defined/implemented");
+  }
+
   // l-values.
   Value *VisitDeclRefExpr(DeclRefExpr *E) {
     if (CodeGenFunction::ConstantEmission Constant = CGF.tryEmitAsConstant(E))
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h
index a8aabca7348ff..6722d2c7c50a2 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.h
+++ b/clang/lib/CodeGen/CGHLSLRuntime.h
@@ -74,6 +74,7 @@ class CGHLSLRuntime {
 
   GENERATE_HLSL_INTRINSIC_FUNCTION(All, all)
   GENERATE_HLSL_INTRINSIC_FUNCTION(Any, any)
+  GENERATE_HLSL_INTRINSIC_FUNCTION(Cross, cross)
   GENERATE_HLSL_INTRINSIC_FUNCTION(Frac, frac)
   GENERATE_HLSL_INTRINSIC_FUNCTION(Length, length)
   GENERATE_HLSL_INTRINSIC_FUNCTION(Lerp, lerp)
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 9bf15fca0de48..41dc91c578c80 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -801,10 +801,12 @@ void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) {
 }
 
 void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
+  const Stmt *Else = S.getElse();
+
   // The else branch of a consteval if statement is always the only branch that
   // can be runtime evaluated.
   if (S.isConsteval()) {
-    const Stmt *Executed = S.isNegatedConsteval() ? S.getThen() : S.getElse();
+    const Stmt *Executed = S.isNegatedConsteval() ? S.getThen() : Else;
     if (Executed) {
       RunCleanupsScope ExecutedScope(*this);
       EmitStmt(Executed);
@@ -830,8 +832,8 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
                                    S.isConstexpr())) {
     // Figure out which block (then or else) is executed.
     const Stmt *Executed = S.getThen();
-    const Stmt *Skipped  = S.getElse();
-    if (!CondConstant)  // Condition false?
+    const Stmt *Skipped = Else;
+    if (!CondConstant) // Condition false?
       std::swap(Executed, Skipped);
 
     // If the skipped block has no labels in it, just emit the executed block.
@@ -852,7 +854,7 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
   llvm::BasicBlock *ThenBlock = createBasicBlock("if.then");
   llvm::BasicBlock *ContBlock = createBasicBlock("if.end");
   llvm::BasicBlock *ElseBlock = ContBlock;
-  if (S.getElse())
+  if (Else)
     ElseBlock = createBasicBlock("if.else");
 
   // Prefer the PGO based weights over the likelihood attribute.
@@ -870,7 +872,7 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
   uint64_t ThenCount = getProfileCount(S.getThen());
   if (!ThenCount && !getCurrentProfileCount() &&
       CGM.getCodeGenOpts().OptimizationLevel)
-    LH = Stmt::getLikelihood(S.getThen(), S.getElse());
+    LH = Stmt::getLikelihood(S.getThen(), Else);
 
   // When measuring MC/DC, always fully evaluate the condition up front using
   // EvaluateExprAsBool() so that the test vector bitmap can be updated prior to
@@ -898,7 +900,7 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
   EmitBranch(ContBlock);
 
   // Emit the 'else' code if present.
-  if (const Stmt *Else = S.getElse()) {
+  if (Else) {
     {
       // There is no need to emit line number for an unconditional branch.
       auto NL = ApplyDebugLocation::CreateEmpty(*this);
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index eda96f3e352ce..e1fd9b72b8d7b 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -850,6 +850,8 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
       for (const FunctionEffectWithCondition &Fe : FD->getFunctionEffects()) {
         if (Fe.Effect.kind() == FunctionEffect::Kind::NonBlocking)
           Fn->addFnAttr(llvm::Attribute::SanitizeRealtime);
+        else if (Fe.Effect.kind() == FunctionEffect::Kind::Blocking)
+          Fn->addFnAttr(llvm::Attribute::SanitizeRealtimeUnsafe);
       }
 
   // Apply fuzzing attribute to the function.
@@ -1762,6 +1764,8 @@ void CodeGenFunction::EmitBranchToCounterBlock(
   if (!InstrumentRegions || !isInstrumentedCondition(Cond))
     return EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount, LH);
 
+  const Stmt *CntrStmt = (CntrIdx ? CntrIdx : Cond);
+
   llvm::BasicBlock *ThenBlock = nullptr;
   llvm::BasicBlock *ElseBlock = nullptr;
   llvm::BasicBlock *NextBlock = nullptr;
@@ -1814,7 +1818,7 @@ void CodeGenFunction::EmitBranchToCounterBlock(
   EmitBlock(CounterIncrBlock);
 
   // Increment corresponding counter; if index not provided, use Cond as index.
-  incrementProfileCounter(CntrIdx ? CntrIdx : Cond);
+  incrementProfileCounter(CntrStmt);
 
   // Go to the next block.
   EmitBranch(NextBlock);
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 3e2abbd9bc109..9ba0ed02a564d 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -4296,6 +4296,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   LValue EmitCastLValue(const CastExpr *E);
   LValue EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E);
   LValue EmitOpaqueValueLValue(const OpaqueValueExpr *e);
+  LValue EmitHLSLArrayAssignLValue(const BinaryOperator *E);
   void EmitHLSLOutArgExpr(const HLSLOutArgExpr *E, CallArgList &Args,
                           QualType Ty);
 
diff --git a/clang/lib/CodeGen/CodeGenPGO.cpp b/clang/lib/CodeGen/CodeGenPGO.cpp
index b745ad37fc96b..820bb521ccf85 100644
--- a/clang/lib/CodeGen/CodeGenPGO.cpp
+++ b/clang/lib/CodeGen/CodeGenPGO.cpp
@@ -1206,14 +1206,12 @@ void CodeGenPGO::emitCounterSetOrIncrement(CGBuilderTy &Builder, const Stmt *S,
   if (llvm::EnableSingleByteCoverage)
     Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::instrprof_cover),
                        ArrayRef(Args, 4));
-  else {
-    if (!StepV)
-      Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::instrprof_increment),
-                         ArrayRef(Args, 4));
-    else
-      Builder.CreateCall(
-          CGM.getIntrinsic(llvm::Intrinsic::instrprof_increment_step), Args);
-  }
+  else if (!StepV)
+    Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::instrprof_increment),
+                       ArrayRef(Args, 4));
+  else
+    Builder.CreateCall(
+        CGM.getIntrinsic(llvm::Intrinsic::instrprof_increment_step), Args);
 }
 
 bool CodeGenPGO::canEmitMCDCCoverage(const CGBuilderTy &Builder) {
diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
index 79dcdc04b0996..0b0b45ffead92 100644
--- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp
+++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
@@ -518,7 +518,7 @@ class MicrosoftCXXABI : public CGCXXABI {
         CGM.IntTy,
         CGM.IntTy,
         CGM.IntTy,
-        getImageRelativeType(getClassHierarchyDescriptorType()->getPointerTo()),
+        getImageRelativeType(CGM.UnqualPtrTy),
     };
     BaseClassDescriptorType = llvm::StructType::create(
         CGM.getLLVMContext(), FieldTypes, "rtti.BaseClassDescriptor");
@@ -531,13 +531,8 @@ class MicrosoftCXXABI : public CGCXXABI {
     // Forward-declare RTTIClassHierarchyDescriptor to break a cycle.
     ClassHierarchyDescriptorType = llvm::StructType::create(
         CGM.getLLVMContext(), "rtti.ClassHierarchyDescriptor");
-    llvm::Type *FieldTypes[] = {
-        CGM.IntTy,
-        CGM.IntTy,
-        CGM.IntTy,
-        getImageRelativeType(
-            getBaseClassDescriptorType()->getPointerTo()->getPointerTo()),
-    };
+    llvm::Type *FieldTypes[] = {CGM.IntTy, CGM.IntTy, CGM.IntTy,
+                                getImageRelativeType(CGM.UnqualPtrTy)};
     ClassHierarchyDescriptorType->setBody(FieldTypes);
     return ClassHierarchyDescriptorType;
   }
@@ -552,7 +547,7 @@ class MicrosoftCXXABI : public CGCXXABI {
         CGM.IntTy,
         CGM.IntTy,
         getImageRelativeType(CGM.Int8PtrTy),
-        getImageRelativeType(getClassHierarchyDescriptorType()->getPointerTo()),
+        getImageRelativeType(CGM.UnqualPtrTy),
         getImageRelativeType(CompleteObjectLocatorType),
     };
     llvm::ArrayRef<llvm::Type *> FieldTypesRef(FieldTypes);
@@ -749,8 +744,7 @@ class MicrosoftCXXABI : public CGCXXABI {
 
     llvm::SmallString<23> CTATypeName("eh.CatchableTypeArray.");
     CTATypeName += llvm::utostr(NumEntries);
-    llvm::Type *CTType =
-        getImageRelativeType(getCatchableTypeType()->getPointerTo());
+    llvm::Type *CTType = getImageRelativeType(CGM.UnqualPtrTy);
     llvm::Type *FieldTypes[] = {
         CGM.IntTy,                               // NumEntries
         llvm::ArrayType::get(CTType, NumEntries) // CatchableTypes
@@ -777,7 +771,7 @@ class MicrosoftCXXABI : public CGCXXABI {
   llvm::FunctionCallee getThrowFn() {
     // _CxxThrowException is passed an exception object and a ThrowInfo object
     // which describes the exception.
-    llvm::Type *Args[] = {CGM.Int8PtrTy, getThrowInfoType()->getPointerTo()};
+    llvm::Type *Args[] = {CGM.Int8PtrTy, CGM.UnqualPtrTy};
     llvm::FunctionType *FTy =
         llvm::FunctionType::get(CGM.VoidTy, Args, /*isVarArg=*/false);
     llvm::FunctionCallee Throw =
@@ -909,9 +903,8 @@ void MicrosoftCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF,
 }
 
 void MicrosoftCXXABI::emitRethrow(CodeGenFunction &CGF, bool isNoReturn) {
-  llvm::Value *Args[] = {
-      llvm::ConstantPointerNull::get(CGM.Int8PtrTy),
-      llvm::ConstantPointerNull::get(getThrowInfoType()->getPointerTo())};
+  llvm::Value *Args[] = {llvm::ConstantPointerNull::get(CGM.Int8PtrTy),
+                         llvm::ConstantPointerNull::get(CGM.UnqualPtrTy)};
   llvm::FunctionCallee Fn = getThrowFn();
   if (isNoReturn)
     CGF.EmitNoreturnRuntimeCallOrInvoke(Fn, Args);
@@ -1958,13 +1951,13 @@ CGCallee MicrosoftCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF,
                                                     SourceLocation Loc) {
   CGBuilderTy &Builder = CGF.Builder;
 
-  Ty = Ty->getPointerTo();
+  Ty = CGF.UnqualPtrTy;
   Address VPtr =
       adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, true);
 
   auto *MethodDecl = cast<CXXMethodDecl>(GD.getDecl());
-  llvm::Value *VTable = CGF.GetVTablePtr(VPtr, Ty->getPointerTo(),
-                                         MethodDecl->getParent());
+  llvm::Value *VTable =
+      CGF.GetVTablePtr(VPtr, CGF.UnqualPtrTy, MethodDecl->getParent());
 
   MicrosoftVTableContext &VFTContext = CGM.getMicrosoftVTableContext();
   MethodVFTableLocation ML = VFTContext.getMethodVFTableLocation(GD);
@@ -2125,9 +2118,9 @@ MicrosoftCXXABI::EmitVirtualMemPtrThunk(const CXXMethodDecl *MD,
 
   // Load the vfptr and then callee from the vftable.  The callee should have
   // adjusted 'this' so that the vfptr is at offset zero.
-  llvm::Type *ThunkPtrTy = ThunkTy->getPointerTo();
-  llvm::Value *VTable = CGF.GetVTablePtr(
-      getThisAddress(CGF), ThunkPtrTy->getPointerTo(), MD->getParent());
+  llvm::Type *ThunkPtrTy = CGF.UnqualPtrTy;
+  llvm::Value *VTable =
+      CGF.GetVTablePtr(getThisAddress(CGF), CGF.UnqualPtrTy, MD->getParent());
 
   llvm::Value *VFuncPtr = CGF.Builder.CreateConstInBoundsGEP1_64(
       ThunkPtrTy, VTable, ML.Index, "vfn");
@@ -2551,7 +2544,7 @@ static ConstantAddress getInitThreadEpochPtr(CodeGenModule &CGM) {
 static llvm::FunctionCallee getInitThreadHeaderFn(CodeGenModule &CGM) {
   llvm::FunctionType *FTy =
       llvm::FunctionType::get(llvm::Type::getVoidTy(CGM.getLLVMContext()),
-                              CGM.IntTy->getPointerTo(), /*isVarArg=*/false);
+                              CGM.UnqualPtrTy, /*isVarArg=*/false);
   return CGM.CreateRuntimeFunction(
       FTy, "_Init_thread_header",
       llvm::AttributeList::get(CGM.getLLVMContext(),
@@ -2563,7 +2556,7 @@ static llvm::FunctionCallee getInitThreadHeaderFn(CodeGenModule &CGM) {
 static llvm::FunctionCallee getInitThreadFooterFn(CodeGenModule &CGM) {
   llvm::FunctionType *FTy =
       llvm::FunctionType::get(llvm::Type::getVoidTy(CGM.getLLVMContext()),
-                              CGM.IntTy->getPointerTo(), /*isVarArg=*/false);
+                              CGM.UnqualPtrTy, /*isVarArg=*/false);
   return CGM.CreateRuntimeFunction(
       FTy, "_Init_thread_footer",
       llvm::AttributeList::get(CGM.getLLVMContext(),
@@ -2575,7 +2568,7 @@ static llvm::FunctionCallee getInitThreadFooterFn(CodeGenModule &CGM) {
 static llvm::FunctionCallee getInitThreadAbortFn(CodeGenModule &CGM) {
   llvm::FunctionType *FTy =
       llvm::FunctionType::get(llvm::Type::getVoidTy(CGM.getLLVMContext()),
-                              CGM.IntTy->getPointerTo(), /*isVarArg=*/false);
+                              CGM.UnqualPtrTy, /*isVarArg=*/false);
   return CGM.CreateRuntimeFunction(
       FTy, "_Init_thread_abort",
       llvm::AttributeList::get(CGM.getLLVMContext(),
@@ -3157,8 +3150,8 @@ MicrosoftCXXABI::GetVBaseOffsetFromVBPtr(CodeGenFunction &CGF,
     VBPtrAlign = CGF.getPointerAlign();
   }
 
-  llvm::Value *VBTable = Builder.CreateAlignedLoad(
-      CGM.Int32Ty->getPointerTo(0), VBPtr, VBPtrAlign, "vbtable");
+  llvm::Value *VBTable =
+      Builder.CreateAlignedLoad(CGM.UnqualPtrTy, VBPtr, VBPtrAlign, "vbtable");
 
   // Translate from byte offset to table index. It improves analyzability.
   llvm::Value *VBTableIndex = Builder.CreateAShr(
@@ -3813,8 +3806,7 @@ MSRTTIBuilder::getBaseClassArray(SmallVectorImpl<MSRTTIClass> &Classes) {
   // mode) bytes of padding.  We provide a pointer sized amount of padding by
   // adding +1 to Classes.size().  The sections have pointer alignment and are
   // marked pick-any so it shouldn't matter.
-  llvm::Type *PtrType = ABI.getImageRelativeType(
-      ABI.getBaseClassDescriptorType()->getPointerTo());
+  llvm::Type *PtrType = ABI.getImageRelativeType(CGM.UnqualPtrTy);
   auto *ArrType = llvm::ArrayType::get(PtrType, Classes.size() + 1);
   auto *BCA =
       new llvm::GlobalVariable(Module, ArrType,
@@ -4360,8 +4352,7 @@ llvm::GlobalVariable *MicrosoftCXXABI::getCatchableTypeArray(QualType T) {
     CatchableTypes.insert(getCatchableType(getContext().VoidPtrTy));
 
   uint32_t NumEntries = CatchableTypes.size();
-  llvm::Type *CTType =
-      getImageRelativeType(getCatchableTypeType()->getPointerTo());
+  llvm::Type *CTType = getImageRelativeType(CGM.UnqualPtrTy);
   llvm::ArrayType *AT = llvm::ArrayType::get(CTType, NumEntries);
   llvm::StructType *CTAType = getCatchableTypeArrayType(NumEntries);
   llvm::Constant *Fields[] = {
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index fba6a8853c396..e9bf60d5e2ee4 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -229,7 +229,14 @@ Driver::Driver(StringRef ClangExecutable, StringRef TargetTriple,
   }
 
 #if defined(CLANG_CONFIG_FILE_SYSTEM_DIR)
-  SystemConfigDir = CLANG_CONFIG_FILE_SYSTEM_DIR;
+  if (llvm::sys::path::is_absolute(CLANG_CONFIG_FILE_SYSTEM_DIR)) {
+    SystemConfigDir = CLANG_CONFIG_FILE_SYSTEM_DIR;
+  } else {
+    SmallString<128> configFileDir(Dir);
+    llvm::sys::path::append(configFileDir, CLANG_CONFIG_FILE_SYSTEM_DIR);
+    llvm::sys::path::remove_dots(configFileDir, true);
+    SystemConfigDir = static_cast<std::string>(configFileDir);
+  }
 #endif
 #if defined(CLANG_CONFIG_FILE_USER_DIR)
   {
@@ -4022,7 +4029,8 @@ void Driver::handleArguments(Compilation &C, DerivedArgList &Args,
     // Emitting LLVM while linking disabled except in HIPAMD Toolchain
     if (Args.hasArg(options::OPT_emit_llvm) && !Args.hasArg(options::OPT_hip_link))
       Diag(clang::diag::err_drv_emit_llvm_link);
-    if (IsCLMode() && LTOMode != LTOK_None &&
+    if (C.getDefaultToolChain().getTriple().isWindowsMSVCEnvironment() &&
+        LTOMode != LTOK_None &&
         !Args.getLastArgValue(options::OPT_fuse_ld_EQ)
              .equals_insensitive("lld"))
       Diag(clang::diag::err_drv_lto_without_lld);
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 16f9b629fc538..de250322b3b34 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -227,6 +227,11 @@ static void getAArch64MultilibFlags(const Driver &D,
   if (BranchProtectionArg) {
     Result.push_back(BranchProtectionArg->getAsString(Args));
   }
+
+  const Arg *ABIArg = Args.getLastArgNoClaim(options::OPT_mabi_EQ);
+  if (ABIArg) {
+    Result.push_back(ABIArg->getAsString(Args));
+  }
 }
 
 static void getARMMultilibFlags(const Driver &D,
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index b9987288d82d1..66ec0a7fd32f9 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -3604,7 +3604,8 @@ static void RenderSSPOptions(const Driver &D, const ToolChain &TC,
   if (Arg *A = Args.getLastArg(options::OPT_mstack_protector_guard_EQ)) {
     StringRef Value = A->getValue();
     if (!EffectiveTriple.isX86() && !EffectiveTriple.isAArch64() &&
-        !EffectiveTriple.isARM() && !EffectiveTriple.isThumb())
+        !EffectiveTriple.isARM() && !EffectiveTriple.isThumb() &&
+        !EffectiveTriple.isRISCV())
       D.Diag(diag::err_drv_unsupported_opt_for_target)
           << A->getAsString(Args) << TripleStr;
     if ((EffectiveTriple.isX86() || EffectiveTriple.isARM() ||
@@ -3644,13 +3645,28 @@ static void RenderSSPOptions(const Driver &D, const ToolChain &TC,
           << A->getOption().getName() << Value << "sysreg global";
       return;
     }
+    if (EffectiveTriple.isRISCV()) {
+      if (Value != "tls" && Value != "global") {
+        D.Diag(diag::err_drv_invalid_value_with_suggestion)
+            << A->getOption().getName() << Value << "tls global";
+        return;
+      }
+      if (Value == "tls") {
+        if (!Args.hasArg(options::OPT_mstack_protector_guard_offset_EQ)) {
+          D.Diag(diag::err_drv_ssp_missing_offset_argument)
+              << A->getAsString(Args);
+          return;
+        }
+      }
+    }
     A->render(Args, CmdArgs);
   }
 
   if (Arg *A = Args.getLastArg(options::OPT_mstack_protector_guard_offset_EQ)) {
     StringRef Value = A->getValue();
     if (!EffectiveTriple.isX86() && !EffectiveTriple.isAArch64() &&
-        !EffectiveTriple.isARM() && !EffectiveTriple.isThumb())
+        !EffectiveTriple.isARM() && !EffectiveTriple.isThumb() &&
+        !EffectiveTriple.isRISCV())
       D.Diag(diag::err_drv_unsupported_opt_for_target)
           << A->getAsString(Args) << TripleStr;
     int Offset;
@@ -3669,7 +3685,8 @@ static void RenderSSPOptions(const Driver &D, const ToolChain &TC,
 
   if (Arg *A = Args.getLastArg(options::OPT_mstack_protector_guard_reg_EQ)) {
     StringRef Value = A->getValue();
-    if (!EffectiveTriple.isX86() && !EffectiveTriple.isAArch64())
+    if (!EffectiveTriple.isX86() && !EffectiveTriple.isAArch64() &&
+        !EffectiveTriple.isRISCV())
       D.Diag(diag::err_drv_unsupported_opt_for_target)
           << A->getAsString(Args) << TripleStr;
     if (EffectiveTriple.isX86() && (Value != "fs" && Value != "gs")) {
@@ -3681,6 +3698,11 @@ static void RenderSSPOptions(const Driver &D, const ToolChain &TC,
       D.Diag(diag::err_drv_invalid_value) << A->getOption().getName() << Value;
       return;
     }
+    if (EffectiveTriple.isRISCV() && Value != "tp") {
+      D.Diag(diag::err_drv_invalid_value_with_suggestion)
+          << A->getOption().getName() << Value << "tp";
+      return;
+    }
     A->render(Args, CmdArgs);
   }
 
@@ -6258,8 +6280,6 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                     options::OPT_fno_unique_internal_linkage_names);
   Args.addOptInFlag(CmdArgs, options::OPT_funique_basic_block_section_names,
                     options::OPT_fno_unique_basic_block_section_names);
-  Args.addOptInFlag(CmdArgs, options::OPT_fconvergent_functions,
-                    options::OPT_fno_convergent_functions);
 
   if (Arg *A = Args.getLastArg(options::OPT_fsplit_machine_functions,
                                options::OPT_fno_split_machine_functions)) {
@@ -6276,6 +6296,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddLastArg(CmdArgs, options::OPT_finstrument_functions,
                   options::OPT_finstrument_functions_after_inlining,
                   options::OPT_finstrument_function_entry_bare);
+  Args.AddLastArg(CmdArgs, options::OPT_fconvergent_functions,
+                  options::OPT_fno_convergent_functions);
 
   // NVPTX/AMDGCN doesn't support PGO or coverage. There's no runtime support
   // for sampling, overhead of call arc collection is way too high and there's
diff --git a/clang/lib/Driver/ToolChains/ZOS.cpp b/clang/lib/Driver/ToolChains/ZOS.cpp
index 074e0556ecd2a..c5ad3ef1b00f1 100644
--- a/clang/lib/Driver/ToolChains/ZOS.cpp
+++ b/clang/lib/Driver/ToolChains/ZOS.cpp
@@ -37,6 +37,10 @@ void ZOS::addClangTargetOptions(const ArgList &DriverArgs,
                                 options::OPT_fno_aligned_allocation))
     CC1Args.push_back("-faligned-alloc-unavailable");
 
+  if (DriverArgs.hasFlag(options::OPT_fxl_pragma_pack,
+                         options::OPT_fno_xl_pragma_pack, true))
+    CC1Args.push_back("-fxl-pragma-pack");
+
   // Pass "-fno-sized-deallocation" only when the user hasn't manually enabled
   // or disabled sized deallocations.
   if (!DriverArgs.hasArgNoClaim(options::OPT_fsized_deallocation,
diff --git a/clang/lib/ExtractAPI/DeclarationFragments.cpp b/clang/lib/ExtractAPI/DeclarationFragments.cpp
index 9cb45c8fbf9cb..66c03863293c2 100644
--- a/clang/lib/ExtractAPI/DeclarationFragments.cpp
+++ b/clang/lib/ExtractAPI/DeclarationFragments.cpp
@@ -1621,6 +1621,9 @@ DeclarationFragmentsBuilder::getSubHeading(const NamedDecl *Decl) {
              cast<CXXMethodDecl>(Decl)->isOverloadedOperator()) {
     Fragments.append(Decl->getNameAsString(),
                      DeclarationFragments::FragmentKind::Identifier);
+  } else if (isa<TagDecl>(Decl) &&
+             cast<TagDecl>(Decl)->getTypedefNameForAnonDecl()) {
+    return getSubHeading(cast<TagDecl>(Decl)->getTypedefNameForAnonDecl());
   } else if (Decl->getIdentifier()) {
     Fragments.append(Decl->getName(),
                      DeclarationFragments::FragmentKind::Identifier);
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index d2463b892fbb9..5350c66ea5132 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -1147,6 +1147,7 @@ template <> struct MappingTraits<FormatStyle> {
     IO.mapOptional("TableGenBreakInsideDAGArg",
                    Style.TableGenBreakInsideDAGArg);
     IO.mapOptional("TabWidth", Style.TabWidth);
+    IO.mapOptional("TemplateNames", Style.TemplateNames);
     IO.mapOptional("TypeNames", Style.TypeNames);
     IO.mapOptional("TypenameMacros", Style.TypenameMacros);
     IO.mapOptional("UseTab", Style.UseTab);
diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h
index 03c0cbd60961a..7d342a7dcca01 100644
--- a/clang/lib/Format/FormatToken.h
+++ b/clang/lib/Format/FormatToken.h
@@ -179,6 +179,7 @@ namespace format {
   TYPE(TrailingReturnArrow)                                                    \
   TYPE(TrailingUnaryOperator)                                                  \
   TYPE(TypeDeclarationParen)                                                   \
+  TYPE(TemplateName)                                                           \
   TYPE(TypeName)                                                               \
   TYPE(TypenameMacro)                                                          \
   TYPE(UnaryOperator)                                                          \
diff --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp
index 63949b2e26bdc..2cdf6cd286b28 100644
--- a/clang/lib/Format/FormatTokenLexer.cpp
+++ b/clang/lib/Format/FormatTokenLexer.cpp
@@ -72,6 +72,8 @@ FormatTokenLexer::FormatTokenLexer(
     Macros.insert({Identifier, TT_StatementAttributeLikeMacro});
   }
 
+  for (const auto &TemplateName : Style.TemplateNames)
+    TemplateNames.insert(&IdentTable.get(TemplateName));
   for (const auto &TypeName : Style.TypeNames)
     TypeNames.insert(&IdentTable.get(TypeName));
 }
@@ -1368,6 +1370,8 @@ FormatToken *FormatTokenLexer::getNextToken() {
         FormatTok->setType(TT_MacroBlockBegin);
       else if (MacroBlockEndRegex.match(Text))
         FormatTok->setType(TT_MacroBlockEnd);
+      else if (TemplateNames.contains(Identifier))
+        FormatTok->setFinalizedType(TT_TemplateName);
       else if (TypeNames.contains(Identifier))
         FormatTok->setFinalizedType(TT_TypeName);
     }
diff --git a/clang/lib/Format/FormatTokenLexer.h b/clang/lib/Format/FormatTokenLexer.h
index 277cc0a2dfde6..71389d2ade2b7 100644
--- a/clang/lib/Format/FormatTokenLexer.h
+++ b/clang/lib/Format/FormatTokenLexer.h
@@ -129,7 +129,7 @@ class FormatTokenLexer {
 
   llvm::SmallMapVector<IdentifierInfo *, TokenType, 8> Macros;
 
-  llvm::SmallPtrSet<IdentifierInfo *, 8> TypeNames;
+  llvm::SmallPtrSet<IdentifierInfo *, 8> TemplateNames, TypeNames;
 
   bool FormattingDisabled;
 
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index f665ce2ad81eb..e2068e557732a 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -149,36 +149,36 @@ class AnnotatingParser {
   }
 
   bool parseAngle() {
-    if (!CurrentToken || !CurrentToken->Previous)
+    if (!CurrentToken)
       return false;
-    if (NonTemplateLess.count(CurrentToken->Previous) > 0)
+
+    auto *Left = CurrentToken->Previous; // The '<'.
+    if (!Left)
       return false;
 
-    if (const auto &Previous = *CurrentToken->Previous; // The '<'.
-        Previous.Previous) {
-      if (Previous.Previous->Tok.isLiteral())
+    if (NonTemplateLess.count(Left) > 0)
+      return false;
+
+    const auto *BeforeLess = Left->Previous;
+
+    if (BeforeLess) {
+      if (BeforeLess->Tok.isLiteral())
         return false;
-      if (Previous.Previous->is(tok::r_brace))
+      if (BeforeLess->is(tok::r_brace))
         return false;
-      if (Previous.Previous->is(tok::r_paren) && Contexts.size() > 1 &&
-          (!Previous.Previous->MatchingParen ||
-           Previous.Previous->MatchingParen->isNot(
-               TT_OverloadedOperatorLParen))) {
+      if (BeforeLess->is(tok::r_paren) && Contexts.size() > 1 &&
+          !(BeforeLess->MatchingParen &&
+            BeforeLess->MatchingParen->is(TT_OverloadedOperatorLParen))) {
         return false;
       }
-      if (Previous.Previous->is(tok::kw_operator) &&
-          CurrentToken->is(tok::l_paren)) {
+      if (BeforeLess->is(tok::kw_operator) && CurrentToken->is(tok::l_paren))
         return false;
-      }
     }
 
-    FormatToken *Left = CurrentToken->Previous;
     Left->ParentBracket = Contexts.back().ContextKind;
     ScopedContextCreator ContextCreator(*this, tok::less, 12);
     Contexts.back().IsExpression = false;
 
-    const auto *BeforeLess = Left->Previous;
-
     // If there's a template keyword before the opening angle bracket, this is a
     // template parameter, not an argument.
     if (BeforeLess && BeforeLess->isNot(tok::kw_template))
@@ -189,25 +189,29 @@ class AnnotatingParser {
       next();
     }
 
-    for (bool SeenTernaryOperator = false; CurrentToken;) {
+    for (bool SeenTernaryOperator = false, MaybeAngles = true; CurrentToken;) {
       const bool InExpr = Contexts[Contexts.size() - 2].IsExpression;
       if (CurrentToken->is(tok::greater)) {
         const auto *Next = CurrentToken->Next;
-        // Try to do a better job at looking for ">>" within the condition of
-        // a statement. Conservatively insert spaces between consecutive ">"
-        // tokens to prevent splitting right bitshift operators and potentially
-        // altering program semantics. This check is overly conservative and
-        // will prevent spaces from being inserted in select nested template
-        // parameter cases, but should not alter program semantics.
-        if (Next && Next->is(tok::greater) &&
-            Left->ParentBracket != tok::less &&
-            CurrentToken->getStartOfNonWhitespace() ==
-                Next->getStartOfNonWhitespace().getLocWithOffset(-1)) {
-          return false;
-        }
-        if (InExpr && SeenTernaryOperator &&
-            (!Next || !Next->isOneOf(tok::l_paren, tok::l_brace))) {
-          return false;
+        if (CurrentToken->isNot(TT_TemplateCloser)) {
+          // Try to do a better job at looking for ">>" within the condition of
+          // a statement. Conservatively insert spaces between consecutive ">"
+          // tokens to prevent splitting right shift operators and potentially
+          // altering program semantics. This check is overly conservative and
+          // will prevent spaces from being inserted in select nested template
+          // parameter cases, but should not alter program semantics.
+          if (Next && Next->is(tok::greater) &&
+              Left->ParentBracket != tok::less &&
+              CurrentToken->getStartOfNonWhitespace() ==
+                  Next->getStartOfNonWhitespace().getLocWithOffset(-1)) {
+            return false;
+          }
+          if (InExpr && SeenTernaryOperator &&
+              (!Next || !Next->isOneOf(tok::l_paren, tok::l_brace))) {
+            return false;
+          }
+          if (!MaybeAngles)
+            return false;
         }
         Left->MatchingParen = CurrentToken;
         CurrentToken->MatchingParen = Left;
@@ -229,6 +233,10 @@ class AnnotatingParser {
         next();
         return true;
       }
+      if (BeforeLess && BeforeLess->is(TT_TemplateName)) {
+        next();
+        continue;
+      }
       if (CurrentToken->is(tok::question) &&
           Style.Language == FormatStyle::LK_Java) {
         next();
@@ -243,11 +251,11 @@ class AnnotatingParser {
       // operator that was misinterpreted because we are parsing template
       // parameters.
       // FIXME: This is getting out of hand, write a decent parser.
-      if (InExpr && !Line.startsWith(tok::kw_template) &&
+      if (MaybeAngles && InExpr && !Line.startsWith(tok::kw_template) &&
           Prev.is(TT_BinaryOperator)) {
         const auto Precedence = Prev.getPrecedence();
         if (Precedence > prec::Conditional && Precedence < prec::Relational)
-          return false;
+          MaybeAngles = false;
       }
       if (Prev.isOneOf(tok::question, tok::colon) && !Style.isProto())
         SeenTernaryOperator = true;
@@ -358,7 +366,7 @@ class AnnotatingParser {
     } else if (OpeningParen.Previous &&
                (OpeningParen.Previous->isOneOf(
                     tok::kw_static_assert, tok::kw_noexcept, tok::kw_explicit,
-                    tok::kw_while, tok::l_paren, tok::comma,
+                    tok::kw_while, tok::l_paren, tok::comma, TT_CastRParen,
                     TT_BinaryOperator) ||
                 OpeningParen.Previous->isIf())) {
       // static_assert, if and while usually contain expressions.
@@ -468,16 +476,18 @@ class AnnotatingParser {
         OpeningParen.Previous && OpeningParen.Previous->is(tok::kw_for);
     FormatToken *PossibleObjCForInToken = nullptr;
     while (CurrentToken) {
-      if (CurrentToken->Previous->is(TT_PointerOrReference) &&
-          CurrentToken->Previous->Previous->isOneOf(tok::l_paren,
-                                                    tok::coloncolon)) {
+      const auto &Prev = *CurrentToken->Previous;
+      if (Prev.is(TT_PointerOrReference) &&
+          Prev.Previous->isOneOf(tok::l_paren, tok::coloncolon)) {
         ProbablyFunctionType = true;
       }
       if (CurrentToken->is(tok::comma))
         MightBeFunctionType = false;
-      if (CurrentToken->Previous->is(TT_BinaryOperator))
+      if (Prev.is(TT_BinaryOperator))
         Contexts.back().IsExpression = true;
       if (CurrentToken->is(tok::r_paren)) {
+        if (Prev.is(TT_PointerOrReference) && Prev.Previous == &OpeningParen)
+          MightBeFunctionType = true;
         if (OpeningParen.isNot(TT_CppCastLParen) && MightBeFunctionType &&
             ProbablyFunctionType && CurrentToken->Next &&
             (CurrentToken->Next->is(tok::l_paren) ||
@@ -550,8 +560,8 @@ class AnnotatingParser {
       bool ProbablyFunctionTypeLParen =
           (CurrentToken->is(tok::l_paren) && CurrentToken->Next &&
            CurrentToken->Next->isOneOf(tok::star, tok::amp, tok::caret));
-      if ((CurrentToken->Previous->isOneOf(tok::kw_const, tok::kw_auto) ||
-           CurrentToken->Previous->isTypeName(LangOpts)) &&
+      if ((Prev.isOneOf(tok::kw_const, tok::kw_auto) ||
+           Prev.isTypeName(LangOpts)) &&
           !(CurrentToken->is(tok::l_brace) ||
             (CurrentToken->is(tok::l_paren) && !ProbablyFunctionTypeLParen))) {
         Contexts.back().IsExpression = false;
@@ -1165,19 +1175,26 @@ class AnnotatingParser {
 
     ScopedContextCreator ContextCreator(*this, tok::l_brace, 1);
     Contexts.back().ColonIsDictLiteral = true;
-    if (OpeningBrace.is(BK_BracedInit))
+
+    const auto *Prev = OpeningBrace.getPreviousNonComment();
+
+    if (OpeningBrace.is(BK_BracedInit)) {
       Contexts.back().IsExpression = true;
-    if (Style.isJavaScript() && OpeningBrace.Previous &&
-        OpeningBrace.Previous->is(TT_JsTypeColon)) {
-      Contexts.back().IsExpression = false;
-    }
-    if (Style.isVerilog() &&
-        (!OpeningBrace.getPreviousNonComment() ||
-         OpeningBrace.getPreviousNonComment()->isNot(Keywords.kw_apostrophe))) {
-      Contexts.back().VerilogMayBeConcatenation = true;
+      if (Prev) {
+        for (auto *Tok = Prev->Previous; Tok && Tok->isPointerOrReference();
+             Tok = Tok->Previous) {
+          Tok->setFinalizedType(TT_PointerOrReference);
+        }
+      }
     }
+
+    if (Style.isJavaScript() && Prev && Prev->is(TT_JsTypeColon))
+      Contexts.back().IsExpression = false;
+
     if (Style.isTableGen())
       Contexts.back().ColonIsDictLiteral = false;
+    else if (Style.isVerilog() && !(Prev && Prev->is(Keywords.kw_apostrophe)))
+      Contexts.back().VerilogMayBeConcatenation = true;
 
     unsigned CommaCount = 0;
     while (CurrentToken) {
@@ -1614,7 +1631,7 @@ class AnnotatingParser {
         return false;
       break;
     case tok::greater:
-      if (Style.Language != FormatStyle::LK_TextProto)
+      if (Style.Language != FormatStyle::LK_TextProto && Tok->is(TT_Unknown))
         Tok->setType(TT_BinaryOperator);
       if (Tok->Previous && Tok->Previous->is(TT_TemplateCloser))
         Tok->SpacesRequiredBefore = 1;
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 40f77266fabdc..a38a86764d68c 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -2556,7 +2556,7 @@ bool UnwrappedLineParser::parseParens(TokenType AmpAmpTokenType) {
         parseChildBlock();
       break;
     case tok::r_paren: {
-      const auto *Prev = LeftParen->Previous;
+      auto *Prev = LeftParen->Previous;
       if (!MightBeStmtExpr && !MightBeFoldExpr && !Line->InMacroBody &&
           Style.RemoveParentheses > FormatStyle::RPS_Leave) {
         const auto *Next = Tokens->peekNextToken();
@@ -2585,9 +2585,13 @@ bool UnwrappedLineParser::parseParens(TokenType AmpAmpTokenType) {
           FormatTok->Optional = true;
         }
       }
-      if (Prev && Prev->is(TT_TypenameMacro)) {
-        LeftParen->setFinalizedType(TT_TypeDeclarationParen);
-        FormatTok->setFinalizedType(TT_TypeDeclarationParen);
+      if (Prev) {
+        if (Prev->is(TT_TypenameMacro)) {
+          LeftParen->setFinalizedType(TT_TypeDeclarationParen);
+          FormatTok->setFinalizedType(TT_TypeDeclarationParen);
+        } else if (Prev->is(tok::greater) && FormatTok->Previous == LeftParen) {
+          Prev->setFinalizedType(TT_TemplateCloser);
+        }
       }
       nextToken();
       return SeenEqual;
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index a0291ccfea245..db7c791059a32 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -3687,10 +3687,10 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts,
   if (Opts.Blocks && !(Opts.OpenCL && Opts.OpenCLVersion == 200))
     GenerateArg(Consumer, OPT_fblocks);
 
-  if (Opts.ConvergentFunctions &&
-      !(Opts.OpenCL || (Opts.CUDA && Opts.CUDAIsDevice) || Opts.SYCLIsDevice ||
-        Opts.HLSL))
+  if (Opts.ConvergentFunctions)
     GenerateArg(Consumer, OPT_fconvergent_functions);
+  else
+    GenerateArg(Consumer, OPT_fno_convergent_functions);
 
   if (Opts.NoBuiltin && !Opts.Freestanding)
     GenerateArg(Consumer, OPT_fno_builtin);
@@ -4106,9 +4106,12 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   Opts.Blocks = Args.hasArg(OPT_fblocks) || (Opts.OpenCL
     && Opts.OpenCLVersion == 200);
 
-  Opts.ConvergentFunctions = Args.hasArg(OPT_fconvergent_functions) ||
-                             Opts.OpenCL || (Opts.CUDA && Opts.CUDAIsDevice) ||
-                             Opts.SYCLIsDevice || Opts.HLSL;
+  bool HasConvergentOperations = Opts.OpenMPIsTargetDevice || Opts.OpenCL ||
+                                 Opts.CUDAIsDevice || Opts.SYCLIsDevice ||
+                                 Opts.HLSL || T.isAMDGPU() || T.isNVPTX();
+  Opts.ConvergentFunctions =
+      Args.hasFlag(OPT_fconvergent_functions, OPT_fno_convergent_functions,
+                   HasConvergentOperations);
 
   Opts.NoBuiltin = Args.hasArg(OPT_fno_builtin) || Opts.Freestanding;
   if (!Opts.NoBuiltin)
@@ -4164,9 +4167,6 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   bool IsTargetSpecified =
       Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_fopenmp_targets_EQ);
 
-  Opts.ConvergentFunctions =
-      Opts.ConvergentFunctions || Opts.OpenMPIsTargetDevice;
-
   if (Opts.OpenMP || Opts.OpenMPSimd) {
     if (int Version = getLastArgIntValue(
             Args, OPT_fopenmp_version_EQ,
diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp
index 64f90c493c105..e4b462b9b0fd8 100644
--- a/clang/lib/Frontend/FrontendActions.cpp
+++ b/clang/lib/Frontend/FrontendActions.cpp
@@ -457,6 +457,8 @@ class DefaultTemplateInstCallback : public TemplateInstantiationCallback {
       return "BuildingDeductionGuides";
     case CodeSynthesisContext::TypeAliasTemplateInstantiation:
       return "TypeAliasTemplateInstantiation";
+    case CodeSynthesisContext::PartialOrderingTTP:
+      return "PartialOrderingTTP";
     }
     return "";
   }
diff --git a/clang/lib/Headers/adcintrin.h b/clang/lib/Headers/adcintrin.h
index 0065a1b543f81..5c68fce9370b2 100644
--- a/clang/lib/Headers/adcintrin.h
+++ b/clang/lib/Headers/adcintrin.h
@@ -15,7 +15,12 @@
 #endif
 
 /* Define the default attributes for the functions in this file. */
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS                                                     \
+  __attribute__((__always_inline__, __nodebug__)) constexpr
+#else
 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
+#endif
 
 /* Use C++ inline semantics in C++, GNU inline for C mode. */
 #if defined(__cplusplus)
diff --git a/clang/lib/Headers/adxintrin.h b/clang/lib/Headers/adxintrin.h
index bc6a4caf35337..055e91f8e2b30 100644
--- a/clang/lib/Headers/adxintrin.h
+++ b/clang/lib/Headers/adxintrin.h
@@ -15,8 +15,13 @@
 #define __ADXINTRIN_H
 
 /* Define the default attributes for the functions in this file. */
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS                                                     \
+  __attribute__((__always_inline__, __nodebug__, __target__("adx"))) constexpr
+#else
 #define __DEFAULT_FN_ATTRS                                                     \
   __attribute__((__always_inline__, __nodebug__, __target__("adx")))
+#endif
 
 /* Use C++ inline semantics in C++, GNU inline for C mode. */
 #if defined(__cplusplus)
diff --git a/clang/lib/Headers/bmiintrin.h b/clang/lib/Headers/bmiintrin.h
index 673f043ccfb30..59c5ece3977f3 100644
--- a/clang/lib/Headers/bmiintrin.h
+++ b/clang/lib/Headers/bmiintrin.h
@@ -36,8 +36,7 @@
 ///    bits in the operand.
 /// \see _tzcnt_u16
 static __inline__ unsigned short __RELAXED_FN_ATTRS
-__tzcnt_u16(unsigned short __X)
-{
+__tzcnt_u16(unsigned short __X) {
   return __builtin_ia32_tzcnt_u16(__X);
 }
 
@@ -70,8 +69,7 @@ __tzcnt_u16(unsigned short __X)
 ///    bits in the operand.
 /// \see { _mm_tzcnt_32 _tzcnt_u32 }
 static __inline__ unsigned int __RELAXED_FN_ATTRS
-__tzcnt_u32(unsigned int __X)
-{
+__tzcnt_u32(unsigned int __X) {
   return __builtin_ia32_tzcnt_u32(__X);
 }
 
@@ -87,8 +85,7 @@ __tzcnt_u32(unsigned int __X)
 ///    the operand.
 /// \see { __tzcnt_u32 _tzcnt_u32 }
 static __inline__ int __RELAXED_FN_ATTRS
-_mm_tzcnt_32(unsigned int __X)
-{
+_mm_tzcnt_32(unsigned int __X) {
   return (int)__builtin_ia32_tzcnt_u32(__X);
 }
 
@@ -123,8 +120,7 @@ _mm_tzcnt_32(unsigned int __X)
 ///    bits in the operand.
 /// \see { _mm_tzcnt_64 _tzcnt_u64 }
 static __inline__ unsigned long long __RELAXED_FN_ATTRS
-__tzcnt_u64(unsigned long long __X)
-{
+__tzcnt_u64(unsigned long long __X) {
   return __builtin_ia32_tzcnt_u64(__X);
 }
 
@@ -140,8 +136,7 @@ __tzcnt_u64(unsigned long long __X)
 ///    the operand.
 /// \see { __tzcnt_u64 _tzcnt_u64 }
 static __inline__ long long __RELAXED_FN_ATTRS
-_mm_tzcnt_64(unsigned long long __X)
-{
+_mm_tzcnt_64(unsigned long long __X) {
   return (long long)__builtin_ia32_tzcnt_u64(__X);
 }
 
@@ -192,8 +187,7 @@ _mm_tzcnt_64(unsigned long long __X)
 ///    operand with the one's complement of the first operand.
 /// \see _andn_u32
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
-__andn_u32(unsigned int __X, unsigned int __Y)
-{
+__andn_u32(unsigned int __X, unsigned int __Y) {
   return ~__X & __Y;
 }
 
@@ -298,8 +292,7 @@ _bextr2_u32(unsigned int __X, unsigned int __Y) {
 ///    the source operand.
 /// \see _blsi_u32
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
-__blsi_u32(unsigned int __X)
-{
+__blsi_u32(unsigned int __X) {
   return __X & -__X;
 }
 
@@ -334,8 +327,7 @@ __blsi_u32(unsigned int __X)
 /// \returns An unsigned integer containing the newly created mask.
 /// \see _blsmsk_u32
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
-__blsmsk_u32(unsigned int __X)
-{
+__blsmsk_u32(unsigned int __X) {
   return __X ^ (__X - 1);
 }
 
@@ -370,8 +362,7 @@ __blsmsk_u32(unsigned int __X)
 ///    operand.
 /// \see _blsr_u32
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
-__blsr_u32(unsigned int __X)
-{
+__blsr_u32(unsigned int __X) {
   return __X & (__X - 1);
 }
 
@@ -410,8 +401,7 @@ __blsr_u32(unsigned int __X)
 ///    operand with the one's complement of the first operand.
 /// \see _andn_u64
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
-__andn_u64 (unsigned long long __X, unsigned long long __Y)
-{
+__andn_u64 (unsigned long long __X, unsigned long long __Y) {
   return ~__X & __Y;
 }
 
@@ -517,8 +507,7 @@ _bextr2_u64(unsigned long long __X, unsigned long long __Y) {
 ///    bits from the source operand.
 /// \see _blsi_u64
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
-__blsi_u64(unsigned long long __X)
-{
+__blsi_u64(unsigned long long __X) {
   return __X & -__X;
 }
 
@@ -553,8 +542,7 @@ __blsi_u64(unsigned long long __X)
 /// \returns An unsigned 64-bit integer containing the newly created mask.
 /// \see _blsmsk_u64
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
-__blsmsk_u64(unsigned long long __X)
-{
+__blsmsk_u64(unsigned long long __X) {
   return __X ^ (__X - 1);
 }
 
@@ -589,8 +577,7 @@ __blsmsk_u64(unsigned long long __X)
 ///    source operand.
 /// \see _blsr_u64
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
-__blsr_u64(unsigned long long __X)
-{
+__blsr_u64(unsigned long long __X) {
   return __X & (__X - 1);
 }
 
diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
index 810a16d75f022..c3ecfc7c90d43 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
@@ -466,6 +466,36 @@ float3 atan(float3);
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_atan)
 float4 atan(float4);
 
+//===----------------------------------------------------------------------===//
+// atan2 builtins
+//===----------------------------------------------------------------------===//
+
+/// \fn T atan2(T y, T x)
+/// \brief Returns the arctangent of y/x, using the signs of the arguments to
+/// determine the correct quadrant.
+/// \param y The y-coordinate.
+/// \param x The x-coordinate.
+
+#ifdef __HLSL_ENABLE_16_BIT
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_atan2)
+half atan2(half y, half x);
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_atan2)
+half2 atan2(half2 y, half2 x);
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_atan2)
+half3 atan2(half3 y, half3 x);
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_atan2)
+half4 atan2(half4 y, half4 x);
+#endif
+
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_atan2)
+float atan2(float y, float x);
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_atan2)
+float2 atan2(float2 y, float2 x);
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_atan2)
+float3 atan2(float3 y, float3 x);
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_atan2)
+float4 atan2(float4 y, float4 x);
+
 //===----------------------------------------------------------------------===//
 // ceil builtins
 //===----------------------------------------------------------------------===//
@@ -1612,6 +1642,28 @@ uint64_t3 reversebits(uint64_t3);
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse)
 uint64_t4 reversebits(uint64_t4);
 
+//===----------------------------------------------------------------------===//
+// cross builtins
+//===----------------------------------------------------------------------===//
+
+/// \fn T cross(T x, T y)
+/// \brief Returns the cross product of two floating-point, 3D vectors.
+/// \param x [in] The first floating-point, 3D vector.
+/// \param y [in] The second floating-point, 3D vector.
+///
+/// Result is the cross product of x and y, i.e., the resulting
+/// components are, in order :
+/// x[1] * y[2] - y[1] * x[2]
+/// x[2] * y[0] - y[2] * x[0]
+/// x[0] * y[1] - y[0] * x[1]
+
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_cross)
+half3 cross(half3, half3);
+
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_cross)
+float3 cross(float3, float3);
+
 //===----------------------------------------------------------------------===//
 // rcp builtins
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/Headers/lzcntintrin.h b/clang/lib/Headers/lzcntintrin.h
index db00474ffd394..27509021ec258 100644
--- a/clang/lib/Headers/lzcntintrin.h
+++ b/clang/lib/Headers/lzcntintrin.h
@@ -49,8 +49,7 @@
 ///    bits in the operand.
 /// \see _lzcnt_u32
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
-__lzcnt32(unsigned int __X)
-{
+__lzcnt32(unsigned int __X) {
   return __builtin_ia32_lzcnt_u32(__X);
 }
 
@@ -66,8 +65,7 @@ __lzcnt32(unsigned int __X)
 ///    bits in the operand.
 /// \see __lzcnt32
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
-_lzcnt_u32(unsigned int __X)
-{
+_lzcnt_u32(unsigned int __X) {
   return __builtin_ia32_lzcnt_u32(__X);
 }
 
@@ -99,8 +97,7 @@ _lzcnt_u32(unsigned int __X)
 ///    bits in the operand.
 /// \see __lzcnt64
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
-_lzcnt_u64(unsigned long long __X)
-{
+_lzcnt_u64(unsigned long long __X) {
   return __builtin_ia32_lzcnt_u64(__X);
 }
 #endif
diff --git a/clang/lib/Headers/tbmintrin.h b/clang/lib/Headers/tbmintrin.h
index 48a9d07568ff2..cf92d5a7b3b0f 100644
--- a/clang/lib/Headers/tbmintrin.h
+++ b/clang/lib/Headers/tbmintrin.h
@@ -28,56 +28,47 @@
                                            (unsigned int)(b)))
 
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
-__blcfill_u32(unsigned int __a)
-{
+__blcfill_u32(unsigned int __a) {
   return __a & (__a + 1);
 }
 
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
-__blci_u32(unsigned int __a)
-{
+__blci_u32(unsigned int __a) {
   return __a | ~(__a + 1);
 }
 
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
-__blcic_u32(unsigned int __a)
-{
+__blcic_u32(unsigned int __a) {
   return ~__a & (__a + 1);
 }
 
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
-__blcmsk_u32(unsigned int __a)
-{
+__blcmsk_u32(unsigned int __a) {
   return __a ^ (__a + 1);
 }
 
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
-__blcs_u32(unsigned int __a)
-{
+__blcs_u32(unsigned int __a) {
   return __a | (__a + 1);
 }
 
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
-__blsfill_u32(unsigned int __a)
-{
+__blsfill_u32(unsigned int __a) {
   return __a | (__a - 1);
 }
 
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
-__blsic_u32(unsigned int __a)
-{
+__blsic_u32(unsigned int __a) {
   return ~__a | (__a - 1);
 }
 
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
-__t1mskc_u32(unsigned int __a)
-{
+__t1mskc_u32(unsigned int __a) {
   return ~__a | (__a + 1);
 }
 
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
-__tzmsk_u32(unsigned int __a)
-{
+__tzmsk_u32(unsigned int __a) {
   return ~__a & (__a - 1);
 }
 
@@ -87,56 +78,47 @@ __tzmsk_u32(unsigned int __a)
                                                  (unsigned long long)(b)))
 
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
-__blcfill_u64(unsigned long long __a)
-{
+__blcfill_u64(unsigned long long __a) {
   return __a & (__a + 1);
 }
 
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
-__blci_u64(unsigned long long __a)
-{
+__blci_u64(unsigned long long __a) {
   return __a | ~(__a + 1);
 }
 
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
-__blcic_u64(unsigned long long __a)
-{
+__blcic_u64(unsigned long long __a) {
   return ~__a & (__a + 1);
 }
 
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
-__blcmsk_u64(unsigned long long __a)
-{
+__blcmsk_u64(unsigned long long __a) {
   return __a ^ (__a + 1);
 }
 
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
-__blcs_u64(unsigned long long __a)
-{
+__blcs_u64(unsigned long long __a) {
   return __a | (__a + 1);
 }
 
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
-__blsfill_u64(unsigned long long __a)
-{
+__blsfill_u64(unsigned long long __a) {
   return __a | (__a - 1);
 }
 
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
-__blsic_u64(unsigned long long __a)
-{
+__blsic_u64(unsigned long long __a) {
   return ~__a | (__a - 1);
 }
 
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
-__t1mskc_u64(unsigned long long __a)
-{
+__t1mskc_u64(unsigned long long __a) {
   return ~__a | (__a + 1);
 }
 
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
-__tzmsk_u64(unsigned long long __a)
-{
+__tzmsk_u64(unsigned long long __a) {
   return ~__a & (__a - 1);
 }
 #endif
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index a04eed9873c0d..122a05be1c039 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -7950,21 +7950,8 @@ void Parser::ParseParameterDeclarationClause(
     // Parse a C++23 Explicit Object Parameter
     // We do that in all language modes to produce a better diagnostic.
     SourceLocation ThisLoc;
-    if (getLangOpts().CPlusPlus && Tok.is(tok::kw_this)) {
+    if (getLangOpts().CPlusPlus && Tok.is(tok::kw_this))
       ThisLoc = ConsumeToken();
-      // C++23 [dcl.fct]p6:
-      //   An explicit-object-parameter-declaration is a parameter-declaration
-      //   with a this specifier. An explicit-object-parameter-declaration
-      //   shall appear only as the first parameter-declaration of a
-      //   parameter-declaration-list of either:
-      //   - a member-declarator that declares a member function, or
-      //   - a lambda-declarator.
-      //
-      // The parameter-declaration-list of a requires-expression is not such
-      // a context.
-      if (DeclaratorCtx == DeclaratorContext::RequiresExpr)
-        Diag(ThisLoc, diag::err_requires_expr_explicit_object_parameter);
-    }
 
     ParsedTemplateInfo TemplateInfo;
     ParseDeclarationSpecifiers(DS, TemplateInfo, AS_none,
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index e7514500dc53a..2fb4be0035b66 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -3694,7 +3694,7 @@ bool Parser::ParseExpressionList(SmallVectorImpl<Expr *> &Exprs,
       SawError = true;
       if (FailImmediatelyOnInvalidExpr)
         break;
-      SkipUntil(tok::comma, tok::r_paren, StopBeforeMatch);
+      SkipUntil(tok::comma, tok::r_paren, StopAtSemi | StopBeforeMatch);
     } else {
       Exprs.push_back(Expr.get());
     }
diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp
index e66abd6873794..b27e50b147f4a 100644
--- a/clang/lib/Parse/ParseOpenACC.cpp
+++ b/clang/lib/Parse/ParseOpenACC.cpp
@@ -743,38 +743,50 @@ bool Parser::ParseOpenACCDeviceTypeList(
 //    int-expr
 // Note that this is specified under 'gang-arg-list', but also applies to 'tile'
 // via reference.
-bool Parser::ParseOpenACCSizeExpr() {
-  // FIXME: Ensure these are constant expressions.
-
+ExprResult Parser::ParseOpenACCSizeExpr(OpenACCClauseKind CK) {
   // The size-expr ends up being ambiguous when only looking at the current
   // token, as it could be a deref of a variable/expression.
   if (getCurToken().is(tok::star) &&
       NextToken().isOneOf(tok::comma, tok::r_paren,
                           tok::annot_pragma_openacc_end)) {
-    ConsumeToken();
-    return false;
+    SourceLocation AsteriskLoc = ConsumeToken();
+    return getActions().OpenACC().ActOnOpenACCAsteriskSizeExpr(AsteriskLoc);
   }
 
-  return getActions()
-      .CorrectDelayedTyposInExpr(ParseAssignmentExpression())
-      .isInvalid();
+  ExprResult SizeExpr =
+      getActions().CorrectDelayedTyposInExpr(ParseConstantExpression());
+
+  if (!SizeExpr.isUsable())
+    return SizeExpr;
+
+  SizeExpr = getActions().OpenACC().ActOnIntExpr(
+      OpenACCDirectiveKind::Invalid, CK, SizeExpr.get()->getBeginLoc(),
+      SizeExpr.get());
+
+  return SizeExpr;
 }
 
-bool Parser::ParseOpenACCSizeExprList() {
-  if (ParseOpenACCSizeExpr()) {
+bool Parser::ParseOpenACCSizeExprList(
+    OpenACCClauseKind CK, llvm::SmallVectorImpl<Expr *> &SizeExprs) {
+  ExprResult SizeExpr = ParseOpenACCSizeExpr(CK);
+  if (!SizeExpr.isUsable()) {
     SkipUntil(tok::r_paren, tok::annot_pragma_openacc_end,
               Parser::StopBeforeMatch);
-    return false;
+    return true;
   }
 
+  SizeExprs.push_back(SizeExpr.get());
+
   while (!getCurToken().isOneOf(tok::r_paren, tok::annot_pragma_openacc_end)) {
     ExpectAndConsume(tok::comma);
 
-    if (ParseOpenACCSizeExpr()) {
+    SizeExpr = ParseOpenACCSizeExpr(CK);
+    if (!SizeExpr.isUsable()) {
       SkipUntil(tok::r_paren, tok::annot_pragma_openacc_end,
                 Parser::StopBeforeMatch);
-      return false;
+      return true;
     }
+    SizeExprs.push_back(SizeExpr.get());
   }
   return false;
 }
@@ -792,7 +804,7 @@ bool Parser::ParseOpenACCGangArg(SourceLocation GangLoc) {
     // 'static' just takes a size-expr, which is an int-expr or an asterisk.
     ConsumeToken();
     ConsumeToken();
-    return ParseOpenACCSizeExpr();
+    return ParseOpenACCSizeExpr(OpenACCClauseKind::Gang).isInvalid();
   }
 
   if (isOpenACCSpecialToken(OpenACCSpecialTokenKind::Dim, getCurToken()) &&
@@ -1052,12 +1064,16 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
       }
       break;
     }
-    case OpenACCClauseKind::Tile:
-      if (ParseOpenACCSizeExprList()) {
+    case OpenACCClauseKind::Tile: {
+      llvm::SmallVector<Expr *> SizeExprs;
+      if (ParseOpenACCSizeExprList(OpenACCClauseKind::Tile, SizeExprs)) {
         Parens.skipToEnd();
         return OpenACCCanContinue();
       }
+
+      ParsedClause.setIntExprDetails(std::move(SizeExprs));
       break;
+    }
     default:
       llvm_unreachable("Not a required parens type?");
     }
@@ -1461,8 +1477,8 @@ StmtResult Parser::ParseOpenACCDirectiveStmt() {
     return StmtError();
 
   StmtResult AssocStmt;
-  SemaOpenACC::AssociatedStmtRAII AssocStmtRAII(getActions().OpenACC(),
-                                                DirInfo.DirKind);
+  SemaOpenACC::AssociatedStmtRAII AssocStmtRAII(
+      getActions().OpenACC(), DirInfo.DirKind, {}, DirInfo.Clauses);
   if (doesDirectiveHaveAssociatedStmt(DirInfo.DirKind)) {
     ParsingOpenACCDirectiveRAII DirScope(*this, /*Value=*/false);
     ParseScope ACCScope(this, getOpenACCScopeFlags(DirInfo.DirKind));
diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp
index cc6f18b5b319f..12fed448d477c 100644
--- a/clang/lib/Parse/ParsePragma.cpp
+++ b/clang/lib/Parse/ParsePragma.cpp
@@ -2126,6 +2126,7 @@ void PragmaGCCVisibilityHandler::HandlePragma(Preprocessor &PP,
 //   pack '(' [integer] ')'
 //   pack '(' 'show' ')'
 //   pack '(' ('push' | 'pop') [',' identifier] [, integer] ')'
+//   pack '(' 'packed' | 'full' | 'twobyte' | 'reset' ')' with -fzos-extensions
 void PragmaPackHandler::HandlePragma(Preprocessor &PP,
                                      PragmaIntroducer Introducer,
                                      Token &PackTok) {
@@ -2155,10 +2156,35 @@ void PragmaPackHandler::HandlePragma(Preprocessor &PP,
                  ? Sema::PSK_Push_Set
                  : Sema::PSK_Set;
   } else if (Tok.is(tok::identifier)) {
+    // Map pragma pack options to pack (integer).
+    auto MapPack = [&](const char *Literal) {
+      Action = Sema::PSK_Push_Set;
+      Alignment = Tok;
+      Alignment.setKind(tok::numeric_constant);
+      Alignment.setLiteralData(Literal);
+      Alignment.setLength(1);
+    };
+
     const IdentifierInfo *II = Tok.getIdentifierInfo();
     if (II->isStr("show")) {
       Action = Sema::PSK_Show;
       PP.Lex(Tok);
+    } else if (II->isStr("packed") && PP.getLangOpts().ZOSExt) {
+      // #pragma pack(packed) is the same as #pragma pack(1)
+      MapPack("1");
+      PP.Lex(Tok);
+    } else if (II->isStr("full") && PP.getLangOpts().ZOSExt) {
+      // #pragma pack(full) is the same as #pragma pack(4)
+      MapPack("4");
+      PP.Lex(Tok);
+    } else if (II->isStr("twobyte") && PP.getLangOpts().ZOSExt) {
+      // #pragma pack(twobyte) is the same as #pragma pack(2)
+      MapPack("2");
+      PP.Lex(Tok);
+    } else if (II->isStr("reset") && PP.getLangOpts().ZOSExt) {
+      // #pragma pack(reset) is the same as #pragma pack(pop) on XL
+      Action = Sema::PSK_Pop;
+      PP.Lex(Tok);
     } else {
       if (II->isStr("push")) {
         Action = Sema::PSK_Push;
diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp
index 9188799fce13e..6480e88316a7d 100644
--- a/clang/lib/Parse/ParseStmt.cpp
+++ b/clang/lib/Parse/ParseStmt.cpp
@@ -24,6 +24,7 @@
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaCodeCompletion.h"
 #include "clang/Sema/SemaObjC.h"
+#include "clang/Sema/SemaOpenACC.h"
 #include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/TypoCorrection.h"
 #include "llvm/ADT/STLExtras.h"
@@ -1854,6 +1855,11 @@ StmtResult Parser::ParseWhileStatement(SourceLocation *TrailingElseLoc) {
                                 Sema::ConditionKind::Boolean, LParen, RParen))
     return StmtError();
 
+  // OpenACC Restricts a while-loop inside of certain construct/clause
+  // combinations, so diagnose that here in OpenACC mode.
+  SemaOpenACC::LoopInConstructRAII LCR{getActions().OpenACC()};
+  getActions().OpenACC().ActOnWhileStmt(WhileLoc);
+
   // C99 6.8.5p5 - In C99, the body of the while statement is a scope, even if
   // there is no compound stmt.  C90 does not have this clause.  We only do this
   // if the body isn't a compound statement to avoid push/pop in common cases.
@@ -1902,6 +1908,11 @@ StmtResult Parser::ParseDoStatement() {
 
   ParseScope DoScope(this, ScopeFlags);
 
+  // OpenACC Restricts a do-while-loop inside of certain construct/clause
+  // combinations, so diagnose that here in OpenACC mode.
+  SemaOpenACC::LoopInConstructRAII LCR{getActions().OpenACC()};
+  getActions().OpenACC().ActOnDoStmt(DoLoc);
+
   // C99 6.8.5p5 - In C99, the body of the do statement is a scope, even if
   // there is no compound stmt.  C90 does not have this clause. We only do this
   // if the body isn't a compound statement to avoid push/pop in common cases.
@@ -2326,6 +2337,11 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
     }
   }
 
+  // OpenACC Restricts a for-loop inside of certain construct/clause
+  // combinations, so diagnose that here in OpenACC mode.
+  SemaOpenACC::LoopInConstructRAII LCR{getActions().OpenACC()};
+  getActions().OpenACC().ActOnForStmtBegin(ForLoc);
+
   // C99 6.8.5p5 - In C99, the body of the for statement is a scope, even if
   // there is no compound stmt.  C90 does not have this clause.  We only do this
   // if the body isn't a compound statement to avoid push/pop in common cases.
@@ -2358,6 +2374,8 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
   // Pop the body scope if needed.
   InnerScope.Exit();
 
+  getActions().OpenACC().ActOnForStmtEnd(ForLoc, Body);
+
   // Leave the for-scope.
   ForScope.Exit();
 
diff --git a/clang/lib/Sema/CMakeLists.txt b/clang/lib/Sema/CMakeLists.txt
index 2cee4f5ef6e99..719c3a9312ec1 100644
--- a/clang/lib/Sema/CMakeLists.txt
+++ b/clang/lib/Sema/CMakeLists.txt
@@ -55,6 +55,7 @@ add_clang_library(clangSema
   SemaExprMember.cpp
   SemaExprObjC.cpp
   SemaFixItUtils.cpp
+  SemaFunctionEffects.cpp
   SemaHLSL.cpp
   SemaHexagon.cpp
   SemaInit.cpp
diff --git a/clang/lib/Sema/JumpDiagnostics.cpp b/clang/lib/Sema/JumpDiagnostics.cpp
index 8af36d5c24e3d..00fdffca4e9ea 100644
--- a/clang/lib/Sema/JumpDiagnostics.cpp
+++ b/clang/lib/Sema/JumpDiagnostics.cpp
@@ -761,8 +761,7 @@ void JumpScopeChecker::VerifyIndirectJumps() {
       if (CHECK_PERMISSIVE(!LabelAndGotoScopes.count(IG)))
         continue;
       unsigned IGScope = LabelAndGotoScopes[IG];
-      if (!JumpScopesMap.contains(IGScope))
-        JumpScopesMap[IGScope] = IG;
+      JumpScopesMap.try_emplace(IGScope, IG);
     }
     JumpScopes.reserve(JumpScopesMap.size());
     for (auto &Pair : JumpScopesMap)
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index 4be7dfbc29392..f05760428458b 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -643,7 +643,7 @@ void Sema::diagnoseFunctionEffectConversion(QualType DstType, QualType SrcType,
   const auto SrcFX = FunctionEffectsRef::get(SrcType);
   const auto DstFX = FunctionEffectsRef::get(DstType);
   if (SrcFX != DstFX) {
-    for (const auto &Diff : FunctionEffectDifferences(SrcFX, DstFX)) {
+    for (const auto &Diff : FunctionEffectDiffVector(SrcFX, DstFX)) {
       if (Diff.shouldDiagnoseConversion(SrcType, SrcFX, DstType, DstFX))
         Diag(Loc, diag::warn_invalid_add_func_effects) << Diff.effectName();
     }
@@ -1529,6 +1529,9 @@ void Sema::ActOnEndOfTranslationUnit() {
 
   AnalysisWarnings.IssueWarnings(Context.getTranslationUnitDecl());
 
+  if (Context.hasAnyFunctionEffects())
+    performFunctionEffectAnalysis(Context.getTranslationUnitDecl());
+
   // Check we've noticed that we're no longer parsing the initializer for every
   // variable. If we miss cases, then at best we have a performance issue and
   // at worst a rejects-valid bug.
@@ -2774,156 +2777,3 @@ bool Sema::isDeclaratorFunctionLike(Declarator &D) {
   });
   return Result;
 }
-
-FunctionEffectDifferences::FunctionEffectDifferences(
-    const FunctionEffectsRef &Old, const FunctionEffectsRef &New) {
-
-  FunctionEffectsRef::iterator POld = Old.begin();
-  FunctionEffectsRef::iterator OldEnd = Old.end();
-  FunctionEffectsRef::iterator PNew = New.begin();
-  FunctionEffectsRef::iterator NewEnd = New.end();
-
-  while (true) {
-    int cmp = 0;
-    if (POld == OldEnd) {
-      if (PNew == NewEnd)
-        break;
-      cmp = 1;
-    } else if (PNew == NewEnd)
-      cmp = -1;
-    else {
-      FunctionEffectWithCondition Old = *POld;
-      FunctionEffectWithCondition New = *PNew;
-      if (Old.Effect.kind() < New.Effect.kind())
-        cmp = -1;
-      else if (New.Effect.kind() < Old.Effect.kind())
-        cmp = 1;
-      else {
-        cmp = 0;
-        if (Old.Cond.getCondition() != New.Cond.getCondition()) {
-          // FIXME: Cases where the expressions are equivalent but
-          // don't have the same identity.
-          push_back(FunctionEffectDiff{
-              Old.Effect.kind(), FunctionEffectDiff::Kind::ConditionMismatch,
-              Old, New});
-        }
-      }
-    }
-
-    if (cmp < 0) {
-      // removal
-      FunctionEffectWithCondition Old = *POld;
-      push_back(FunctionEffectDiff{
-          Old.Effect.kind(), FunctionEffectDiff::Kind::Removed, Old, {}});
-      ++POld;
-    } else if (cmp > 0) {
-      // addition
-      FunctionEffectWithCondition New = *PNew;
-      push_back(FunctionEffectDiff{
-          New.Effect.kind(), FunctionEffectDiff::Kind::Added, {}, New});
-      ++PNew;
-    } else {
-      ++POld;
-      ++PNew;
-    }
-  }
-}
-
-bool FunctionEffectDiff::shouldDiagnoseConversion(
-    QualType SrcType, const FunctionEffectsRef &SrcFX, QualType DstType,
-    const FunctionEffectsRef &DstFX) const {
-
-  switch (EffectKind) {
-  case FunctionEffect::Kind::NonAllocating:
-    // nonallocating can't be added (spoofed) during a conversion, unless we
-    // have nonblocking.
-    if (DiffKind == Kind::Added) {
-      for (const auto &CFE : SrcFX) {
-        if (CFE.Effect.kind() == FunctionEffect::Kind::NonBlocking)
-          return false;
-      }
-    }
-    [[fallthrough]];
-  case FunctionEffect::Kind::NonBlocking:
-    // nonblocking can't be added (spoofed) during a conversion.
-    switch (DiffKind) {
-    case Kind::Added:
-      return true;
-    case Kind::Removed:
-      return false;
-    case Kind::ConditionMismatch:
-      // FIXME: Condition mismatches are too coarse right now -- expressions
-      // which are equivalent but don't have the same identity are detected as
-      // mismatches. We're going to diagnose those anyhow until expression
-      // matching is better.
-      return true;
-    }
-    llvm_unreachable("Unhandled FunctionEffectDiff::Kind enum");
-  case FunctionEffect::Kind::Blocking:
-  case FunctionEffect::Kind::Allocating:
-    return false;
-  case FunctionEffect::Kind::None:
-    break;
-  }
-  llvm_unreachable("unknown effect kind");
-}
-
-bool FunctionEffectDiff::shouldDiagnoseRedeclaration(
-    const FunctionDecl &OldFunction, const FunctionEffectsRef &OldFX,
-    const FunctionDecl &NewFunction, const FunctionEffectsRef &NewFX) const {
-  switch (EffectKind) {
-  case FunctionEffect::Kind::NonAllocating:
-  case FunctionEffect::Kind::NonBlocking:
-    // nonblocking/nonallocating can't be removed in a redeclaration.
-    switch (DiffKind) {
-    case Kind::Added:
-      return false; // No diagnostic.
-    case Kind::Removed:
-      return true; // Issue diagnostic.
-    case Kind::ConditionMismatch:
-      // All these forms of mismatches are diagnosed.
-      return true;
-    }
-    llvm_unreachable("Unhandled FunctionEffectDiff::Kind enum");
-  case FunctionEffect::Kind::Blocking:
-  case FunctionEffect::Kind::Allocating:
-    return false;
-  case FunctionEffect::Kind::None:
-    break;
-  }
-  llvm_unreachable("unknown effect kind");
-}
-
-FunctionEffectDiff::OverrideResult
-FunctionEffectDiff::shouldDiagnoseMethodOverride(
-    const CXXMethodDecl &OldMethod, const FunctionEffectsRef &OldFX,
-    const CXXMethodDecl &NewMethod, const FunctionEffectsRef &NewFX) const {
-  switch (EffectKind) {
-  case FunctionEffect::Kind::NonAllocating:
-  case FunctionEffect::Kind::NonBlocking:
-    switch (DiffKind) {
-
-    // If added on an override, that's fine and not diagnosed.
-    case Kind::Added:
-      return OverrideResult::NoAction;
-
-    // If missing from an override (removed), propagate from base to derived.
-    case Kind::Removed:
-      return OverrideResult::Merge;
-
-    // If there's a mismatch involving the effect's polarity or condition,
-    // issue a warning.
-    case Kind::ConditionMismatch:
-      return OverrideResult::Warn;
-    }
-    llvm_unreachable("Unhandled FunctionEffectDiff::Kind enum");
-
-  case FunctionEffect::Kind::Blocking:
-  case FunctionEffect::Kind::Allocating:
-    return OverrideResult::NoAction;
-
-  case FunctionEffect::Kind::None:
-    break;
-  }
-  llvm_unreachable("unknown effect kind");
-}
diff --git a/clang/lib/Sema/SemaAPINotes.cpp b/clang/lib/Sema/SemaAPINotes.cpp
index d0236d08c98e6..ec43a0def9c1e 100644
--- a/clang/lib/Sema/SemaAPINotes.cpp
+++ b/clang/lib/Sema/SemaAPINotes.cpp
@@ -913,7 +913,15 @@ void Sema::ProcessAPINotes(Decl *D) {
 
     // Tags
     if (auto Tag = dyn_cast<TagDecl>(D)) {
-      std::string LookupName = Tag->getName().str();
+      // Determine the name of the entity to search for. If this is an
+      // anonymous tag that gets its linked name from a typedef, look for the
+      // typedef name. This allows tag-specific information to be added
+      // to the declaration.
+      std::string LookupName;
+      if (auto typedefName = Tag->getTypedefNameForAnonDecl())
+        LookupName = typedefName->getName().str();
+      else
+        LookupName = Tag->getName().str();
 
       // Use the source location to discern if this Tag is an OPTIONS macro.
       // For now we would like to limit this trick of looking up the APINote tag
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 8634b54b0535d..2bcb930acdcb5 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -2755,16 +2755,12 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
 
   // These builtins restrict the element type to floating point
   // types only, and take in two arguments.
+  case Builtin::BI__builtin_elementwise_minimum:
+  case Builtin::BI__builtin_elementwise_maximum:
+  case Builtin::BI__builtin_elementwise_atan2:
   case Builtin::BI__builtin_elementwise_fmod:
   case Builtin::BI__builtin_elementwise_pow: {
-    if (BuiltinElementwiseMath(TheCall))
-      return ExprError();
-
-    QualType ArgTy = TheCall->getArg(0)->getType();
-    if (checkFPMathBuiltinElementType(*this, TheCall->getArg(0)->getBeginLoc(),
-                                      ArgTy, 1) ||
-        checkFPMathBuiltinElementType(*this, TheCall->getArg(1)->getBeginLoc(),
-                                      ArgTy, 2))
+    if (BuiltinElementwiseMath(TheCall, /*FPOnly=*/true))
       return ExprError();
     break;
   }
@@ -2868,6 +2864,29 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
     TheCall->setType(ElTy);
     break;
   }
+  case Builtin::BI__builtin_reduce_maximum:
+  case Builtin::BI__builtin_reduce_minimum: {
+    if (PrepareBuiltinReduceMathOneArgCall(TheCall))
+      return ExprError();
+
+    const Expr *Arg = TheCall->getArg(0);
+    const auto *TyA = Arg->getType()->getAs<VectorType>();
+
+    QualType ElTy;
+    if (TyA)
+      ElTy = TyA->getElementType();
+    else if (Arg->getType()->isSizelessVectorType())
+      ElTy = Arg->getType()->getSizelessVectorEltType(Context);
+
+    if (ElTy.isNull() || !ElTy->isFloatingType()) {
+      Diag(Arg->getBeginLoc(), diag::err_builtin_invalid_arg_type)
+          << 1 << /* vector of floating points */ 9 << Arg->getType();
+      return ExprError();
+    }
+
+    TheCall->setType(ElTy);
+    break;
+  }
 
   // These builtins support vectors of integers only.
   // TODO: ADD/MUL should support floating-point types.
@@ -14378,9 +14397,9 @@ bool Sema::PrepareBuiltinElementwiseMathOneArgCall(CallExpr *TheCall) {
   return false;
 }
 
-bool Sema::BuiltinElementwiseMath(CallExpr *TheCall) {
+bool Sema::BuiltinElementwiseMath(CallExpr *TheCall, bool FPOnly) {
   QualType Res;
-  if (BuiltinVectorMath(TheCall, Res))
+  if (BuiltinVectorMath(TheCall, Res, FPOnly))
     return true;
   TheCall->setType(Res);
   return false;
@@ -14399,7 +14418,7 @@ bool Sema::BuiltinVectorToScalarMath(CallExpr *TheCall) {
   return false;
 }
 
-bool Sema::BuiltinVectorMath(CallExpr *TheCall, QualType &Res) {
+bool Sema::BuiltinVectorMath(CallExpr *TheCall, QualType &Res, bool FPOnly) {
   if (checkArgCount(TheCall, 2))
     return true;
 
@@ -14419,8 +14438,13 @@ bool Sema::BuiltinVectorMath(CallExpr *TheCall, QualType &Res) {
                 diag::err_typecheck_call_different_arg_types)
            << TyA << TyB;
 
-  if (checkMathBuiltinElementType(*this, A.get()->getBeginLoc(), TyA, 1))
-    return true;
+  if (FPOnly) {
+    if (checkFPMathBuiltinElementType(*this, A.get()->getBeginLoc(), TyA, 1))
+      return true;
+  } else {
+    if (checkMathBuiltinElementType(*this, A.get()->getBeginLoc(), TyA, 1))
+      return true;
+  }
 
   TheCall->setArg(0, A.get());
   TheCall->setArg(1, B.get());
diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp
index 67fc603e9ce1d..998a148a7d24a 100644
--- a/clang/lib/Sema/SemaConcept.cpp
+++ b/clang/lib/Sema/SemaConcept.cpp
@@ -728,6 +728,7 @@ bool Sema::addInstantiatedCapturesToScope(
     ValueDecl *CapturedPattern = CapturePattern.getCapturedVar();
 
     if (!CapturedPattern->isInitCapture()) {
+      Instantiated++;
       continue;
     }
 
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 0e536f71a2f70..2bf610746bc31 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -3803,7 +3803,7 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, NamedDecl *&OldD, Scope *S,
     const auto OldFX = Old->getFunctionEffects();
     const auto NewFX = New->getFunctionEffects();
     if (OldFX != NewFX) {
-      const auto Diffs = FunctionEffectDifferences(OldFX, NewFX);
+      const auto Diffs = FunctionEffectDiffVector(OldFX, NewFX);
       for (const auto &Diff : Diffs) {
         if (Diff.shouldDiagnoseRedeclaration(*Old, OldFX, *New, NewFX)) {
           Diag(New->getLocation(),
@@ -3839,6 +3839,11 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, NamedDecl *&OldD, Scope *S,
           OldQTypeForComparison = Context.getFunctionType(
               OldFPT->getReturnType(), OldFPT->getParamTypes(), EPI);
         }
+        if (OldFX.empty()) {
+          // A redeclaration may add the attribute to a previously seen function
+          // body which needs to be verified.
+          maybeAddDeclWithEffects(Old, MergedFX);
+        }
       }
     }
   }
@@ -4974,6 +4979,9 @@ void Sema::setTagNameForLinkagePurposes(TagDecl *TagFromDeclSpec,
 
   // Otherwise, set this as the anon-decl typedef for the tag.
   TagFromDeclSpec->setTypedefNameForAnonDecl(NewTD);
+
+  // Now that we have a name for the tag, process API notes again.
+  ProcessAPINotes(TagFromDeclSpec);
 }
 
 static unsigned GetDiagnosticTypeSpecifierID(const DeclSpec &DS) {
@@ -9086,9 +9094,14 @@ static NamedDecl *DiagnoseInvalidRedeclaration(
   SemaRef.Diag(NewFD->getLocation(), DiagMsg)
       << Name << NewDC << IsDefinition << NewFD->getLocation();
 
-  bool NewFDisConst = false;
-  if (CXXMethodDecl *NewMD = dyn_cast<CXXMethodDecl>(NewFD))
-    NewFDisConst = NewMD->isConst();
+  CXXMethodDecl *NewMD = dyn_cast<CXXMethodDecl>(NewFD);
+  if (NewMD && DiagMsg == diag::err_member_decl_does_not_match) {
+    CXXRecordDecl *RD = NewMD->getParent();
+    SemaRef.Diag(RD->getLocation(), diag::note_defined_here)
+        << RD->getName() << RD->getLocation();
+  }
+
+  bool NewFDisConst = NewMD && NewMD->isConst();
 
   for (SmallVectorImpl<std::pair<FunctionDecl *, unsigned> >::iterator
        NearMatch = NearMatches.begin(), NearMatchEnd = NearMatches.end();
@@ -15669,6 +15682,8 @@ Decl *Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope, Decl *D,
       getCurLexicalContext()->getDeclKind() != Decl::ObjCImplementation)
     Diag(FD->getLocation(), diag::warn_function_def_in_objc_container);
 
+  maybeAddDeclWithEffects(FD);
+
   return D;
 }
 
@@ -20295,59 +20310,3 @@ bool Sema::shouldIgnoreInHostDeviceCheck(FunctionDecl *Callee) {
   return LangOpts.CUDA && !LangOpts.CUDAIsDevice &&
          CUDA().IdentifyTarget(Callee) == CUDAFunctionTarget::Global;
 }
-
-void Sema::diagnoseFunctionEffectMergeConflicts(
-    const FunctionEffectSet::Conflicts &Errs, SourceLocation NewLoc,
-    SourceLocation OldLoc) {
-  for (const FunctionEffectSet::Conflict &Conflict : Errs) {
-    Diag(NewLoc, diag::warn_conflicting_func_effects)
-        << Conflict.Kept.description() << Conflict.Rejected.description();
-    Diag(OldLoc, diag::note_previous_declaration);
-  }
-}
-
-bool Sema::diagnoseConflictingFunctionEffect(
-    const FunctionEffectsRef &FX, const FunctionEffectWithCondition &NewEC,
-    SourceLocation NewAttrLoc) {
-  // If the new effect has a condition, we can't detect conflicts until the
-  // condition is resolved.
-  if (NewEC.Cond.getCondition() != nullptr)
-    return false;
-
-  // Diagnose the new attribute as incompatible with a previous one.
-  auto Incompatible = [&](const FunctionEffectWithCondition &PrevEC) {
-    Diag(NewAttrLoc, diag::err_attributes_are_not_compatible)
-        << ("'" + NewEC.description() + "'")
-        << ("'" + PrevEC.description() + "'") << false;
-    // We don't necessarily have the location of the previous attribute,
-    // so no note.
-    return true;
-  };
-
-  // Compare against previous attributes.
-  FunctionEffect::Kind NewKind = NewEC.Effect.kind();
-
-  for (const FunctionEffectWithCondition &PrevEC : FX) {
-    // Again, can't check yet when the effect is conditional.
-    if (PrevEC.Cond.getCondition() != nullptr)
-      continue;
-
-    FunctionEffect::Kind PrevKind = PrevEC.Effect.kind();
-    // Note that we allow PrevKind == NewKind; it's redundant and ignored.
-
-    if (PrevEC.Effect.oppositeKind() == NewKind)
-      return Incompatible(PrevEC);
-
-    // A new allocating is incompatible with a previous nonblocking.
-    if (PrevKind == FunctionEffect::Kind::NonBlocking &&
-        NewKind == FunctionEffect::Kind::Allocating)
-      return Incompatible(PrevEC);
-
-    // A new nonblocking is incompatible with a previous allocating.
-    if (PrevKind == FunctionEffect::Kind::Allocating &&
-        NewKind == FunctionEffect::Kind::NonBlocking)
-      return Incompatible(PrevEC);
-  }
-
-  return false;
-}
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index d8cdfcf8c6ec0..9cb2ed02a3f76 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -18181,7 +18181,7 @@ bool Sema::CheckOverridingFunctionAttributes(CXXMethodDecl *New,
 
     if (OldFX != NewFXOrig) {
       FunctionEffectSet NewFX(NewFXOrig);
-      const auto Diffs = FunctionEffectDifferences(OldFX, NewFX);
+      const auto Diffs = FunctionEffectDiffVector(OldFX, NewFX);
       FunctionEffectSet::Conflicts Errs;
       for (const auto &Diff : Diffs) {
         switch (Diff.shouldDiagnoseMethodOverride(*Old, OldFX, *New, NewFX)) {
@@ -18194,7 +18194,7 @@ bool Sema::CheckOverridingFunctionAttributes(CXXMethodDecl *New,
               << Old->getReturnTypeSourceRange();
           break;
         case FunctionEffectDiff::OverrideResult::Merge: {
-          NewFX.insert(Diff.Old, Errs);
+          NewFX.insert(Diff.Old.value(), Errs);
           const auto *NewFT = New->getType()->castAs<FunctionProtoType>();
           FunctionProtoType::ExtProtoInfo EPI = NewFT->getExtProtoInfo();
           EPI.FunctionEffects = FunctionEffectsRef(NewFX);
diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp
index 8aedbfcf878a1..dbddd6c370aa0 100644
--- a/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -1593,6 +1593,8 @@ CanThrowResult Sema::canThrow(const Stmt *S) {
 
   case Stmt::SYCLUniqueStableNameExprClass:
     return CT_Cannot;
+  case Stmt::OpenACCAsteriskSizeExprClass:
+    return CT_Cannot;
   case Stmt::NoStmtClass:
     llvm_unreachable("Invalid class for statement");
   }
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 2db9d1fc69ed1..ae7bcedfb28c7 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -16212,6 +16212,8 @@ ExprResult Sema::ActOnBlockStmtExpr(SourceLocation CaretLoc,
   BlockScopeInfo *BSI = cast<BlockScopeInfo>(FunctionScopes.back());
   BlockDecl *BD = BSI->TheDecl;
 
+  maybeAddDeclWithEffects(BD);
+
   if (BSI->HasImplicitReturnType)
     deduceClosureReturnType(*BSI);
 
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index b30414a8a8277..d490452e91c3b 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -9519,15 +9519,28 @@ Sema::ActOnStartRequiresExpr(SourceLocation RequiresKWLoc,
       } else if (Param->getType().hasQualifiers()) {
         Diag(Param->getBeginLoc(), diag::err_void_param_qualified);
       }
-    }
-
-    if (Param->hasDefaultArg())
+    } else if (Param->hasDefaultArg()) {
       // C++2a [expr.prim.req] p4
       //     [...] A local parameter of a requires-expression shall not have a
       //     default argument. [...]
       Diag(Param->getDefaultArgRange().getBegin(),
            diag::err_requires_expr_local_parameter_default_argument);
-    // Ignore default argument and move on
+      // Ignore default argument and move on
+    } else if (Param->isExplicitObjectParameter()) {
+      // C++23 [dcl.fct]p6:
+      //   An explicit-object-parameter-declaration is a parameter-declaration
+      //   with a this specifier. An explicit-object-parameter-declaration
+      //   shall appear only as the first parameter-declaration of a
+      //   parameter-declaration-list of either:
+      //   - a member-declarator that declares a member function, or
+      //   - a lambda-declarator.
+      //
+      // The parameter-declaration-list of a requires-expression is not such
+      // a context.
+      Diag(Param->getExplicitObjectParamThisLoc(),
+           diag::err_requires_expr_explicit_object_parameter);
+      Param->setExplicitObjectParameterLoc(SourceLocation());
+    }
 
     Param->setDeclContext(Body);
     // If this has an identifier, add it to the scope stack.
diff --git a/clang/lib/Sema/SemaFunctionEffects.cpp b/clang/lib/Sema/SemaFunctionEffects.cpp
new file mode 100644
index 0000000000000..0fb18d207a50b
--- /dev/null
+++ b/clang/lib/Sema/SemaFunctionEffects.cpp
@@ -0,0 +1,1577 @@
+//=== SemaFunctionEffects.cpp - Sema handling of function effects ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements Sema handling of function effects.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclCXX.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/AST/Stmt.h"
+#include "clang/AST/Type.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Sema/SemaInternal.h"
+
+#define DEBUG_TYPE "effectanalysis"
+
+using namespace clang;
+
+namespace {
+
+enum class ViolationID : uint8_t {
+  None = 0, // Sentinel for an empty Violation.
+  // These first 5 map to a %select{} in one of several FunctionEffects
+  // diagnostics, e.g. warn_func_effect_violation.
+  BaseDiagnosticIndex,
+  AllocatesMemory = BaseDiagnosticIndex,
+  ThrowsOrCatchesExceptions,
+  HasStaticLocalVariable,
+  AccessesThreadLocalVariable,
+  AccessesObjCMethodOrProperty,
+
+  // These only apply to callees, where the analysis stops at the Decl.
+  DeclDisallowsInference,
+
+  // These both apply to indirect calls. The difference is that sometimes
+  // we have an actual Decl (generally a variable) which is the function
+  // pointer being called, and sometimes, typically due to a cast, we only
+  // have an expression.
+  CallsDeclWithoutEffect,
+  CallsExprWithoutEffect,
+};
+
+// Information about the AST context in which a violation was found, so
+// that diagnostics can point to the correct source.
+class ViolationSite {
+public:
+  enum class Kind : uint8_t {
+    Default, // Function body.
+    MemberInitializer,
+    DefaultArgExpr
+  };
+
+private:
+  llvm::PointerIntPair<CXXDefaultArgExpr *, 2, Kind> Impl;
+
+public:
+  ViolationSite() = default;
+
+  explicit ViolationSite(CXXDefaultArgExpr *E)
+      : Impl(E, Kind::DefaultArgExpr) {}
+
+  Kind kind() const { return static_cast<Kind>(Impl.getInt()); }
+  CXXDefaultArgExpr *defaultArgExpr() const { return Impl.getPointer(); }
+
+  void setKind(Kind K) { Impl.setPointerAndInt(nullptr, K); }
+};
+
+// Represents a violation of the rules, potentially for the entire duration of
+// the analysis phase, in order to refer to it when explaining why a caller has
+// been made unsafe by a callee. Can be transformed into either a Diagnostic
+// (warning or a note), depending on whether the violation pertains to a
+// function failing to be verifed as holding an effect vs. a function failing to
+// be inferred as holding that effect.
+struct Violation {
+  FunctionEffect Effect;
+  std::optional<FunctionEffect>
+      CalleeEffectPreventingInference; // Only for certain IDs; can be nullopt.
+  ViolationID ID = ViolationID::None;
+  ViolationSite Site;
+  SourceLocation Loc;
+  const Decl *Callee =
+      nullptr; // Only valid for ViolationIDs Calls{Decl,Expr}WithoutEffect.
+
+  Violation(FunctionEffect Effect, ViolationID ID, ViolationSite VS,
+            SourceLocation Loc, const Decl *Callee = nullptr,
+            std::optional<FunctionEffect> CalleeEffect = std::nullopt)
+      : Effect(Effect), CalleeEffectPreventingInference(CalleeEffect), ID(ID),
+        Site(VS), Loc(Loc), Callee(Callee) {}
+
+  unsigned diagnosticSelectIndex() const {
+    return unsigned(ID) - unsigned(ViolationID::BaseDiagnosticIndex);
+  }
+};
+
+enum class SpecialFuncType : uint8_t { None, OperatorNew, OperatorDelete };
+enum class CallableType : uint8_t {
+  // Unknown: probably function pointer.
+  Unknown,
+  Function,
+  Virtual,
+  Block
+};
+
+// Return whether a function's effects CAN be verified.
+// The question of whether it SHOULD be verified is independent.
+static bool functionIsVerifiable(const FunctionDecl *FD) {
+  if (FD->isTrivial()) {
+    // Otherwise `struct x { int a; };` would have an unverifiable default
+    // constructor.
+    return true;
+  }
+  return FD->hasBody();
+}
+
+static bool isNoexcept(const FunctionDecl *FD) {
+  const auto *FPT = FD->getType()->getAs<FunctionProtoType>();
+  return FPT && (FPT->isNothrow() || FD->hasAttr<NoThrowAttr>());
+}
+
+// This list is probably incomplete.
+// FIXME: Investigate:
+// __builtin_eh_return?
+// __builtin_allow_runtime_check?
+// __builtin_unwind_init and other similar things that sound exception-related.
+// va_copy?
+// coroutines?
+static FunctionEffectKindSet getBuiltinFunctionEffects(unsigned BuiltinID) {
+  FunctionEffectKindSet Result;
+
+  switch (BuiltinID) {
+  case 0:  // Not builtin.
+  default: // By default, builtins have no known effects.
+    break;
+
+  // These allocate/deallocate heap memory.
+  case Builtin::ID::BI__builtin_calloc:
+  case Builtin::ID::BI__builtin_malloc:
+  case Builtin::ID::BI__builtin_realloc:
+  case Builtin::ID::BI__builtin_free:
+  case Builtin::ID::BI__builtin_operator_delete:
+  case Builtin::ID::BI__builtin_operator_new:
+  case Builtin::ID::BIaligned_alloc:
+  case Builtin::ID::BIcalloc:
+  case Builtin::ID::BImalloc:
+  case Builtin::ID::BImemalign:
+  case Builtin::ID::BIrealloc:
+  case Builtin::ID::BIfree:
+
+  case Builtin::ID::BIfopen:
+  case Builtin::ID::BIpthread_create:
+  case Builtin::ID::BI_Block_object_dispose:
+    Result.insert(FunctionEffect(FunctionEffect::Kind::Allocating));
+    break;
+
+  // These block in some other way than allocating memory.
+  // longjmp() and friends are presumed unsafe because they are the moral
+  // equivalent of throwing a C++ exception, which is unsafe.
+  case Builtin::ID::BIlongjmp:
+  case Builtin::ID::BI_longjmp:
+  case Builtin::ID::BIsiglongjmp:
+  case Builtin::ID::BI__builtin_longjmp:
+  case Builtin::ID::BIobjc_exception_throw:
+
+  // Objective-C runtime.
+  case Builtin::ID::BIobjc_msgSend:
+  case Builtin::ID::BIobjc_msgSend_fpret:
+  case Builtin::ID::BIobjc_msgSend_fp2ret:
+  case Builtin::ID::BIobjc_msgSend_stret:
+  case Builtin::ID::BIobjc_msgSendSuper:
+  case Builtin::ID::BIobjc_getClass:
+  case Builtin::ID::BIobjc_getMetaClass:
+  case Builtin::ID::BIobjc_enumerationMutation:
+  case Builtin::ID::BIobjc_assign_ivar:
+  case Builtin::ID::BIobjc_assign_global:
+  case Builtin::ID::BIobjc_sync_enter:
+  case Builtin::ID::BIobjc_sync_exit:
+  case Builtin::ID::BINSLog:
+  case Builtin::ID::BINSLogv:
+
+  // stdio.h
+  case Builtin::ID::BIfread:
+  case Builtin::ID::BIfwrite:
+
+  // stdio.h: printf family.
+  case Builtin::ID::BIprintf:
+  case Builtin::ID::BI__builtin_printf:
+  case Builtin::ID::BIfprintf:
+  case Builtin::ID::BIsnprintf:
+  case Builtin::ID::BIsprintf:
+  case Builtin::ID::BIvprintf:
+  case Builtin::ID::BIvfprintf:
+  case Builtin::ID::BIvsnprintf:
+  case Builtin::ID::BIvsprintf:
+
+  // stdio.h: scanf family.
+  case Builtin::ID::BIscanf:
+  case Builtin::ID::BIfscanf:
+  case Builtin::ID::BIsscanf:
+  case Builtin::ID::BIvscanf:
+  case Builtin::ID::BIvfscanf:
+  case Builtin::ID::BIvsscanf:
+    Result.insert(FunctionEffect(FunctionEffect::Kind::Blocking));
+    break;
+  }
+
+  return Result;
+}
+
+// Transitory, more extended information about a callable, which can be a
+// function, block, or function pointer.
+struct CallableInfo {
+  // CDecl holds the function's definition, if any.
+  // FunctionDecl if CallableType::Function or Virtual
+  // BlockDecl if CallableType::Block
+  const Decl *CDecl;
+
+  // Remember whether the callable is a function, block, virtual method,
+  // or (presumed) function pointer.
+  CallableType CType = CallableType::Unknown;
+
+  // Remember whether the callable is an operator new or delete function,
+  // so that calls to them are reported more meaningfully, as memory
+  // allocations.
+  SpecialFuncType FuncType = SpecialFuncType::None;
+
+  // We inevitably want to know the callable's declared effects, so cache them.
+  FunctionEffectKindSet Effects;
+
+  CallableInfo(const Decl &CD, SpecialFuncType FT = SpecialFuncType::None)
+      : CDecl(&CD), FuncType(FT) {
+    FunctionEffectsRef DeclEffects;
+    if (auto *FD = dyn_cast<FunctionDecl>(CDecl)) {
+      // Use the function's definition, if any.
+      if (const FunctionDecl *Def = FD->getDefinition())
+        CDecl = FD = Def;
+      CType = CallableType::Function;
+      if (auto *Method = dyn_cast<CXXMethodDecl>(FD);
+          Method && Method->isVirtual())
+        CType = CallableType::Virtual;
+      DeclEffects = FD->getFunctionEffects();
+    } else if (auto *BD = dyn_cast<BlockDecl>(CDecl)) {
+      CType = CallableType::Block;
+      DeclEffects = BD->getFunctionEffects();
+    } else if (auto *VD = dyn_cast<ValueDecl>(CDecl)) {
+      // ValueDecl is function, enum, or variable, so just look at its type.
+      DeclEffects = FunctionEffectsRef::get(VD->getType());
+    }
+    Effects = FunctionEffectKindSet(DeclEffects);
+  }
+
+  CallableType type() const { return CType; }
+
+  bool isCalledDirectly() const {
+    return CType == CallableType::Function || CType == CallableType::Block;
+  }
+
+  bool isVerifiable() const {
+    switch (CType) {
+    case CallableType::Unknown:
+    case CallableType::Virtual:
+      return false;
+    case CallableType::Block:
+      return true;
+    case CallableType::Function:
+      return functionIsVerifiable(dyn_cast<FunctionDecl>(CDecl));
+    }
+    llvm_unreachable("undefined CallableType");
+  }
+
+  /// Generate a name for logging and diagnostics.
+  std::string getNameForDiagnostic(Sema &S) const {
+    std::string Name;
+    llvm::raw_string_ostream OS(Name);
+
+    if (auto *FD = dyn_cast<FunctionDecl>(CDecl))
+      FD->getNameForDiagnostic(OS, S.getPrintingPolicy(),
+                               /*Qualified=*/true);
+    else if (auto *BD = dyn_cast<BlockDecl>(CDecl))
+      OS << "(block " << BD->getBlockManglingNumber() << ")";
+    else if (auto *VD = dyn_cast<NamedDecl>(CDecl))
+      VD->printQualifiedName(OS);
+    return Name;
+  }
+};
+
+// ----------
+// Map effects to single Violations, to hold the first (of potentially many)
+// violations pertaining to an effect, per function.
+class EffectToViolationMap {
+  // Since we currently only have a tiny number of effects (typically no more
+  // than 1), use a SmallVector with an inline capacity of 1. Since it
+  // is often empty, use a unique_ptr to the SmallVector.
+  // Note that Violation itself contains a FunctionEffect which is the key.
+  // FIXME: Is there a way to simplify this using existing data structures?
+  using ImplVec = llvm::SmallVector<Violation, 1>;
+  std::unique_ptr<ImplVec> Impl;
+
+public:
+  // Insert a new Violation if we do not already have one for its effect.
+  void maybeInsert(const Violation &Viol) {
+    if (Impl == nullptr)
+      Impl = std::make_unique<ImplVec>();
+    else if (lookup(Viol.Effect) != nullptr)
+      return;
+
+    Impl->push_back(Viol);
+  }
+
+  const Violation *lookup(FunctionEffect Key) {
+    if (Impl == nullptr)
+      return nullptr;
+
+    auto *Iter = llvm::find_if(
+        *Impl, [&](const auto &Item) { return Item.Effect == Key; });
+    return Iter != Impl->end() ? &*Iter : nullptr;
+  }
+
+  size_t size() const { return Impl ? Impl->size() : 0; }
+};
+
+// ----------
+// State pertaining to a function whose AST is walked and whose effect analysis
+// is dependent on a subsequent analysis of other functions.
+class PendingFunctionAnalysis {
+  friend class CompleteFunctionAnalysis;
+
+public:
+  struct DirectCall {
+    const Decl *Callee;
+    SourceLocation CallLoc;
+    // Not all recursive calls are detected, just enough
+    // to break cycles.
+    bool Recursed = false;
+    ViolationSite VSite;
+
+    DirectCall(const Decl *D, SourceLocation CallLoc, ViolationSite VSite)
+        : Callee(D), CallLoc(CallLoc), VSite(VSite) {}
+  };
+
+  // We always have two disjoint sets of effects to verify:
+  // 1. Effects declared explicitly by this function.
+  // 2. All other inferrable effects needing verification.
+  FunctionEffectKindSet DeclaredVerifiableEffects;
+  FunctionEffectKindSet EffectsToInfer;
+
+private:
+  // Violations pertaining to the function's explicit effects.
+  SmallVector<Violation, 0> ViolationsForExplicitEffects;
+
+  // Violations pertaining to other, non-explicit, inferrable effects.
+  EffectToViolationMap InferrableEffectToFirstViolation;
+
+  // These unverified direct calls are what keeps the analysis "pending",
+  // until the callees can be verified.
+  SmallVector<DirectCall, 0> UnverifiedDirectCalls;
+
+public:
+  PendingFunctionAnalysis(Sema &S, const CallableInfo &CInfo,
+                          FunctionEffectKindSet AllInferrableEffectsToVerify)
+      : DeclaredVerifiableEffects(CInfo.Effects) {
+    // Check for effects we are not allowed to infer.
+    FunctionEffectKindSet InferrableEffects;
+
+    for (FunctionEffect effect : AllInferrableEffectsToVerify) {
+      std::optional<FunctionEffect> ProblemCalleeEffect =
+          effect.effectProhibitingInference(*CInfo.CDecl, CInfo.Effects);
+      if (!ProblemCalleeEffect)
+        InferrableEffects.insert(effect);
+      else {
+        // Add a Violation for this effect if a caller were to
+        // try to infer it.
+        InferrableEffectToFirstViolation.maybeInsert(Violation(
+            effect, ViolationID::DeclDisallowsInference, ViolationSite{},
+            CInfo.CDecl->getLocation(), nullptr, ProblemCalleeEffect));
+      }
+    }
+    // InferrableEffects is now the set of inferrable effects which are not
+    // prohibited.
+    EffectsToInfer = FunctionEffectKindSet::difference(
+        InferrableEffects, DeclaredVerifiableEffects);
+  }
+
+  // Hide the way that Violations for explicitly required effects vs. inferred
+  // ones are handled differently.
+  void checkAddViolation(bool Inferring, const Violation &NewViol) {
+    if (!Inferring)
+      ViolationsForExplicitEffects.push_back(NewViol);
+    else
+      InferrableEffectToFirstViolation.maybeInsert(NewViol);
+  }
+
+  void addUnverifiedDirectCall(const Decl *D, SourceLocation CallLoc,
+                               ViolationSite VSite) {
+    UnverifiedDirectCalls.emplace_back(D, CallLoc, VSite);
+  }
+
+  // Analysis is complete when there are no unverified direct calls.
+  bool isComplete() const { return UnverifiedDirectCalls.empty(); }
+
+  const Violation *violationForInferrableEffect(FunctionEffect effect) {
+    return InferrableEffectToFirstViolation.lookup(effect);
+  }
+
+  // Mutable because caller may need to set a DirectCall's Recursing flag.
+  MutableArrayRef<DirectCall> unverifiedCalls() {
+    assert(!isComplete());
+    return UnverifiedDirectCalls;
+  }
+
+  ArrayRef<Violation> getSortedViolationsForExplicitEffects(SourceManager &SM) {
+    if (!ViolationsForExplicitEffects.empty())
+      llvm::sort(ViolationsForExplicitEffects,
+                 [&SM](const Violation &LHS, const Violation &RHS) {
+                   return SM.isBeforeInTranslationUnit(LHS.Loc, RHS.Loc);
+                 });
+    return ViolationsForExplicitEffects;
+  }
+
+  void dump(Sema &SemaRef, llvm::raw_ostream &OS) const {
+    OS << "Pending: Declared ";
+    DeclaredVerifiableEffects.dump(OS);
+    OS << ", " << ViolationsForExplicitEffects.size() << " violations; ";
+    OS << " Infer ";
+    EffectsToInfer.dump(OS);
+    OS << ", " << InferrableEffectToFirstViolation.size() << " violations";
+    if (!UnverifiedDirectCalls.empty()) {
+      OS << "; Calls: ";
+      for (const DirectCall &Call : UnverifiedDirectCalls) {
+        CallableInfo CI(*Call.Callee);
+        OS << " " << CI.getNameForDiagnostic(SemaRef);
+      }
+    }
+    OS << "\n";
+  }
+};
+
+// ----------
+class CompleteFunctionAnalysis {
+  // Current size: 2 pointers
+public:
+  // Has effects which are both the declared ones -- not to be inferred -- plus
+  // ones which have been successfully inferred. These are all considered
+  // "verified" for the purposes of callers; any issue with verifying declared
+  // effects has already been reported and is not the problem of any caller.
+  FunctionEffectKindSet VerifiedEffects;
+
+private:
+  // This is used to generate notes about failed inference.
+  EffectToViolationMap InferrableEffectToFirstViolation;
+
+public:
+  // The incoming Pending analysis is consumed (member(s) are moved-from).
+  CompleteFunctionAnalysis(ASTContext &Ctx, PendingFunctionAnalysis &&Pending,
+                           FunctionEffectKindSet DeclaredEffects,
+                           FunctionEffectKindSet AllInferrableEffectsToVerify)
+      : VerifiedEffects(DeclaredEffects) {
+    for (FunctionEffect effect : AllInferrableEffectsToVerify)
+      if (Pending.violationForInferrableEffect(effect) == nullptr)
+        VerifiedEffects.insert(effect);
+
+    InferrableEffectToFirstViolation =
+        std::move(Pending.InferrableEffectToFirstViolation);
+  }
+
+  const Violation *firstViolationForEffect(FunctionEffect Effect) {
+    return InferrableEffectToFirstViolation.lookup(Effect);
+  }
+
+  void dump(llvm::raw_ostream &OS) const {
+    OS << "Complete: Verified ";
+    VerifiedEffects.dump(OS);
+    OS << "; Infer ";
+    OS << InferrableEffectToFirstViolation.size() << " violations\n";
+  }
+};
+
+// ==========
+class Analyzer {
+  Sema &S;
+
+  // Subset of Sema.AllEffectsToVerify
+  FunctionEffectKindSet AllInferrableEffectsToVerify;
+
+  using FuncAnalysisPtr =
+      llvm::PointerUnion<PendingFunctionAnalysis *, CompleteFunctionAnalysis *>;
+
+  // Map all Decls analyzed to FuncAnalysisPtr. Pending state is larger
+  // than complete state, so use different objects to represent them.
+  // The state pointers are owned by the container.
+  class AnalysisMap : llvm::DenseMap<const Decl *, FuncAnalysisPtr> {
+    using Base = llvm::DenseMap<const Decl *, FuncAnalysisPtr>;
+
+  public:
+    ~AnalysisMap();
+
+    // Use non-public inheritance in order to maintain the invariant
+    // that lookups and insertions are via the canonical Decls.
+
+    FuncAnalysisPtr lookup(const Decl *Key) const {
+      return Base::lookup(Key->getCanonicalDecl());
+    }
+
+    FuncAnalysisPtr &operator[](const Decl *Key) {
+      return Base::operator[](Key->getCanonicalDecl());
+    }
+
+    /// Shortcut for the case where we only care about completed analysis.
+    CompleteFunctionAnalysis *completedAnalysisForDecl(const Decl *D) const {
+      if (FuncAnalysisPtr AP = lookup(D);
+          isa_and_nonnull<CompleteFunctionAnalysis *>(AP))
+        return AP.get<CompleteFunctionAnalysis *>();
+      return nullptr;
+    }
+
+    void dump(Sema &SemaRef, llvm::raw_ostream &OS) {
+      OS << "\nAnalysisMap:\n";
+      for (const auto &item : *this) {
+        CallableInfo CI(*item.first);
+        const auto AP = item.second;
+        OS << item.first << " " << CI.getNameForDiagnostic(SemaRef) << " : ";
+        if (AP.isNull()) {
+          OS << "null\n";
+        } else if (isa<CompleteFunctionAnalysis *>(AP)) {
+          auto *CFA = AP.get<CompleteFunctionAnalysis *>();
+          OS << CFA << " ";
+          CFA->dump(OS);
+        } else if (isa<PendingFunctionAnalysis *>(AP)) {
+          auto *PFA = AP.get<PendingFunctionAnalysis *>();
+          OS << PFA << " ";
+          PFA->dump(SemaRef, OS);
+        } else
+          llvm_unreachable("never");
+      }
+      OS << "---\n";
+    }
+  };
+  AnalysisMap DeclAnalysis;
+
+public:
+  Analyzer(Sema &S) : S(S) {}
+
+  void run(const TranslationUnitDecl &TU) {
+    // Gather all of the effects to be verified to see what operations need to
+    // be checked, and to see which ones are inferrable.
+    for (FunctionEffect Effect : S.AllEffectsToVerify) {
+      const FunctionEffect::Flags Flags = Effect.flags();
+      if (Flags & FunctionEffect::FE_InferrableOnCallees)
+        AllInferrableEffectsToVerify.insert(Effect);
+    }
+    LLVM_DEBUG(llvm::dbgs() << "AllInferrableEffectsToVerify: ";
+               AllInferrableEffectsToVerify.dump(llvm::dbgs());
+               llvm::dbgs() << "\n";);
+
+    // We can use DeclsWithEffectsToVerify as a stack for a
+    // depth-first traversal; there's no need for a second container. But first,
+    // reverse it, so when working from the end, Decls are verified in the order
+    // they are declared.
+    SmallVector<const Decl *> &VerificationQueue = S.DeclsWithEffectsToVerify;
+    std::reverse(VerificationQueue.begin(), VerificationQueue.end());
+
+    while (!VerificationQueue.empty()) {
+      const Decl *D = VerificationQueue.back();
+      if (FuncAnalysisPtr AP = DeclAnalysis.lookup(D)) {
+        if (auto *Pending = AP.dyn_cast<PendingFunctionAnalysis *>()) {
+          // All children have been traversed; finish analysis.
+          finishPendingAnalysis(D, Pending);
+        }
+        VerificationQueue.pop_back();
+        continue;
+      }
+
+      // Not previously visited; begin a new analysis for this Decl.
+      PendingFunctionAnalysis *Pending = verifyDecl(D);
+      if (Pending == nullptr) {
+        // Completed now.
+        VerificationQueue.pop_back();
+        continue;
+      }
+
+      // Analysis remains pending because there are direct callees to be
+      // verified first. Push them onto the queue.
+      for (PendingFunctionAnalysis::DirectCall &Call :
+           Pending->unverifiedCalls()) {
+        FuncAnalysisPtr AP = DeclAnalysis.lookup(Call.Callee);
+        if (AP.isNull()) {
+          VerificationQueue.push_back(Call.Callee);
+          continue;
+        }
+
+        // This indicates recursion (not necessarily direct). For the
+        // purposes of effect analysis, we can just ignore it since
+        // no effects forbid recursion.
+        assert(isa<PendingFunctionAnalysis *>(AP));
+        Call.Recursed = true;
+      }
+    }
+  }
+
+private:
+  // Verify a single Decl. Return the pending structure if that was the result,
+  // else null. This method must not recurse.
+  PendingFunctionAnalysis *verifyDecl(const Decl *D) {
+    CallableInfo CInfo(*D);
+    bool isExternC = false;
+
+    if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D))
+      isExternC = FD->getCanonicalDecl()->isExternCContext();
+
+    // For C++, with non-extern "C" linkage only - if any of the Decl's declared
+    // effects forbid throwing (e.g. nonblocking) then the function should also
+    // be declared noexcept.
+    if (S.getLangOpts().CPlusPlus && !isExternC) {
+      for (FunctionEffect Effect : CInfo.Effects) {
+        if (!(Effect.flags() & FunctionEffect::FE_ExcludeThrow))
+          continue;
+
+        bool IsNoexcept = false;
+        if (auto *FD = D->getAsFunction()) {
+          IsNoexcept = isNoexcept(FD);
+        } else if (auto *BD = dyn_cast<BlockDecl>(D)) {
+          if (auto *TSI = BD->getSignatureAsWritten()) {
+            auto *FPT = TSI->getType()->getAs<FunctionProtoType>();
+            IsNoexcept = FPT->isNothrow() || BD->hasAttr<NoThrowAttr>();
+          }
+        }
+        if (!IsNoexcept)
+          S.Diag(D->getBeginLoc(), diag::warn_perf_constraint_implies_noexcept)
+              << GetCallableDeclKind(D, nullptr) << Effect.name();
+        break;
+      }
+    }
+
+    // Build a PendingFunctionAnalysis on the stack. If it turns out to be
+    // complete, we'll have avoided a heap allocation; if it's incomplete, it's
+    // a fairly trivial move to a heap-allocated object.
+    PendingFunctionAnalysis FAnalysis(S, CInfo, AllInferrableEffectsToVerify);
+
+    LLVM_DEBUG(llvm::dbgs()
+                   << "\nVerifying " << CInfo.getNameForDiagnostic(S) << " ";
+               FAnalysis.dump(S, llvm::dbgs()););
+
+    FunctionBodyASTVisitor Visitor(*this, FAnalysis, CInfo);
+
+    Visitor.run();
+    if (FAnalysis.isComplete()) {
+      completeAnalysis(CInfo, std::move(FAnalysis));
+      return nullptr;
+    }
+    // Move the pending analysis to the heap and save it in the map.
+    PendingFunctionAnalysis *PendingPtr =
+        new PendingFunctionAnalysis(std::move(FAnalysis));
+    DeclAnalysis[D] = PendingPtr;
+    LLVM_DEBUG(llvm::dbgs() << "inserted pending " << PendingPtr << "\n";
+               DeclAnalysis.dump(S, llvm::dbgs()););
+    return PendingPtr;
+  }
+
+  // Consume PendingFunctionAnalysis, create with it a CompleteFunctionAnalysis,
+  // inserted in the container.
+  void completeAnalysis(const CallableInfo &CInfo,
+                        PendingFunctionAnalysis &&Pending) {
+    if (ArrayRef<Violation> Viols =
+            Pending.getSortedViolationsForExplicitEffects(S.getSourceManager());
+        !Viols.empty())
+      emitDiagnostics(Viols, CInfo);
+
+    CompleteFunctionAnalysis *CompletePtr = new CompleteFunctionAnalysis(
+        S.getASTContext(), std::move(Pending), CInfo.Effects,
+        AllInferrableEffectsToVerify);
+    DeclAnalysis[CInfo.CDecl] = CompletePtr;
+    LLVM_DEBUG(llvm::dbgs() << "inserted complete " << CompletePtr << "\n";
+               DeclAnalysis.dump(S, llvm::dbgs()););
+  }
+
+  // Called after all direct calls requiring inference have been found -- or
+  // not. Repeats calls to FunctionBodyASTVisitor::followCall() but without
+  // the possibility of inference. Deletes Pending.
+  void finishPendingAnalysis(const Decl *D, PendingFunctionAnalysis *Pending) {
+    CallableInfo Caller(*D);
+    LLVM_DEBUG(llvm::dbgs() << "finishPendingAnalysis for "
+                            << Caller.getNameForDiagnostic(S) << " : ";
+               Pending->dump(S, llvm::dbgs()); llvm::dbgs() << "\n";);
+    for (const PendingFunctionAnalysis::DirectCall &Call :
+         Pending->unverifiedCalls()) {
+      if (Call.Recursed)
+        continue;
+
+      CallableInfo Callee(*Call.Callee);
+      followCall(Caller, *Pending, Callee, Call.CallLoc,
+                 /*AssertNoFurtherInference=*/true, Call.VSite);
+    }
+    completeAnalysis(Caller, std::move(*Pending));
+    delete Pending;
+  }
+
+  // Here we have a call to a Decl, either explicitly via a CallExpr or some
+  // other AST construct. PFA pertains to the caller.
+  void followCall(const CallableInfo &Caller, PendingFunctionAnalysis &PFA,
+                  const CallableInfo &Callee, SourceLocation CallLoc,
+                  bool AssertNoFurtherInference, ViolationSite VSite) {
+    const bool DirectCall = Callee.isCalledDirectly();
+
+    // Initially, the declared effects; inferred effects will be added.
+    FunctionEffectKindSet CalleeEffects = Callee.Effects;
+
+    bool IsInferencePossible = DirectCall;
+
+    if (DirectCall)
+      if (CompleteFunctionAnalysis *CFA =
+              DeclAnalysis.completedAnalysisForDecl(Callee.CDecl)) {
+        // Combine declared effects with those which may have been inferred.
+        CalleeEffects.insert(CFA->VerifiedEffects);
+        IsInferencePossible = false; // We've already traversed it.
+      }
+
+    if (AssertNoFurtherInference) {
+      assert(!IsInferencePossible);
+    }
+
+    if (!Callee.isVerifiable())
+      IsInferencePossible = false;
+
+    LLVM_DEBUG(llvm::dbgs()
+                   << "followCall from " << Caller.getNameForDiagnostic(S)
+                   << " to " << Callee.getNameForDiagnostic(S)
+                   << "; verifiable: " << Callee.isVerifiable() << "; callee ";
+               CalleeEffects.dump(llvm::dbgs()); llvm::dbgs() << "\n";
+               llvm::dbgs() << "  callee " << Callee.CDecl << " canonical "
+                            << Callee.CDecl->getCanonicalDecl() << "\n";);
+
+    auto Check1Effect = [&](FunctionEffect Effect, bool Inferring) {
+      if (!Effect.shouldDiagnoseFunctionCall(DirectCall, CalleeEffects))
+        return;
+
+      // If inference is not allowed, or the target is indirect (virtual
+      // method/function ptr?), generate a Violation now.
+      if (!IsInferencePossible ||
+          !(Effect.flags() & FunctionEffect::FE_InferrableOnCallees)) {
+        if (Callee.FuncType == SpecialFuncType::None)
+          PFA.checkAddViolation(Inferring,
+                                {Effect, ViolationID::CallsDeclWithoutEffect,
+                                 VSite, CallLoc, Callee.CDecl});
+        else
+          PFA.checkAddViolation(
+              Inferring,
+              {Effect, ViolationID::AllocatesMemory, VSite, CallLoc});
+      } else {
+        // Inference is allowed and necessary; defer it.
+        PFA.addUnverifiedDirectCall(Callee.CDecl, CallLoc, VSite);
+      }
+    };
+
+    for (FunctionEffect Effect : PFA.DeclaredVerifiableEffects)
+      Check1Effect(Effect, false);
+
+    for (FunctionEffect Effect : PFA.EffectsToInfer)
+      Check1Effect(Effect, true);
+  }
+
+  // Describe a callable Decl for a diagnostic.
+  // (Not an enum class because the value is always converted to an integer for
+  // use in a diagnostic.)
+  enum CallableDeclKind {
+    CDK_Function,
+    CDK_Constructor,
+    CDK_Destructor,
+    CDK_Lambda,
+    CDK_Block,
+    CDK_MemberInitializer,
+  };
+
+  // Describe a call site or target using an enum mapping to a %select{}
+  // in a diagnostic, e.g. warn_func_effect_violation,
+  // warn_perf_constraint_implies_noexcept, and others.
+  static CallableDeclKind GetCallableDeclKind(const Decl *D,
+                                              const Violation *V) {
+    if (V != nullptr &&
+        V->Site.kind() == ViolationSite::Kind::MemberInitializer)
+      return CDK_MemberInitializer;
+    if (isa<BlockDecl>(D))
+      return CDK_Block;
+    if (auto *Method = dyn_cast<CXXMethodDecl>(D)) {
+      if (isa<CXXConstructorDecl>(D))
+        return CDK_Constructor;
+      if (isa<CXXDestructorDecl>(D))
+        return CDK_Destructor;
+      const CXXRecordDecl *Rec = Method->getParent();
+      if (Rec->isLambda())
+        return CDK_Lambda;
+    }
+    return CDK_Function;
+  };
+
+  // Should only be called when function's analysis is determined to be
+  // complete.
+  void emitDiagnostics(ArrayRef<Violation> Viols, const CallableInfo &CInfo) {
+    if (Viols.empty())
+      return;
+
+    auto MaybeAddTemplateNote = [&](const Decl *D) {
+      if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+        while (FD != nullptr && FD->isTemplateInstantiation()) {
+          S.Diag(FD->getPointOfInstantiation(),
+                 diag::note_func_effect_from_template);
+          FD = FD->getTemplateInstantiationPattern();
+        }
+      }
+    };
+
+    // For note_func_effect_call_indirect.
+    enum { Indirect_VirtualMethod, Indirect_FunctionPtr };
+
+    auto MaybeAddSiteContext = [&](const Decl *D, const Violation &V) {
+      // If a violation site is a member initializer, add a note pointing to
+      // the constructor which invoked it.
+      if (V.Site.kind() == ViolationSite::Kind::MemberInitializer) {
+        unsigned ImplicitCtor = 0;
+        if (auto *Ctor = dyn_cast<CXXConstructorDecl>(D);
+            Ctor && Ctor->isImplicit())
+          ImplicitCtor = 1;
+        S.Diag(D->getLocation(), diag::note_func_effect_in_constructor)
+            << ImplicitCtor;
+      }
+
+      // If a violation site is a default argument expression, add a note
+      // pointing to the call site using the default argument.
+      else if (V.Site.kind() == ViolationSite::Kind::DefaultArgExpr)
+        S.Diag(V.Site.defaultArgExpr()->getUsedLocation(),
+               diag::note_in_evaluating_default_argument);
+    };
+
+    // Top-level violations are warnings.
+    for (const Violation &Viol1 : Viols) {
+      StringRef effectName = Viol1.Effect.name();
+      switch (Viol1.ID) {
+      case ViolationID::None:
+      case ViolationID::DeclDisallowsInference: // Shouldn't happen
+                                                // here.
+        llvm_unreachable("Unexpected violation kind");
+        break;
+      case ViolationID::AllocatesMemory:
+      case ViolationID::ThrowsOrCatchesExceptions:
+      case ViolationID::HasStaticLocalVariable:
+      case ViolationID::AccessesThreadLocalVariable:
+      case ViolationID::AccessesObjCMethodOrProperty:
+        S.Diag(Viol1.Loc, diag::warn_func_effect_violation)
+            << GetCallableDeclKind(CInfo.CDecl, &Viol1) << effectName
+            << Viol1.diagnosticSelectIndex();
+        MaybeAddSiteContext(CInfo.CDecl, Viol1);
+        MaybeAddTemplateNote(CInfo.CDecl);
+        break;
+      case ViolationID::CallsExprWithoutEffect:
+        S.Diag(Viol1.Loc, diag::warn_func_effect_calls_expr_without_effect)
+            << GetCallableDeclKind(CInfo.CDecl, &Viol1) << effectName;
+        MaybeAddSiteContext(CInfo.CDecl, Viol1);
+        MaybeAddTemplateNote(CInfo.CDecl);
+        break;
+
+      case ViolationID::CallsDeclWithoutEffect: {
+        CallableInfo CalleeInfo(*Viol1.Callee);
+        std::string CalleeName = CalleeInfo.getNameForDiagnostic(S);
+
+        S.Diag(Viol1.Loc, diag::warn_func_effect_calls_func_without_effect)
+            << GetCallableDeclKind(CInfo.CDecl, &Viol1) << effectName
+            << GetCallableDeclKind(CalleeInfo.CDecl, nullptr) << CalleeName;
+        MaybeAddSiteContext(CInfo.CDecl, Viol1);
+        MaybeAddTemplateNote(CInfo.CDecl);
+
+        // Emit notes explaining the transitive chain of inferences: Why isn't
+        // the callee safe?
+        for (const Decl *Callee = Viol1.Callee; Callee != nullptr;) {
+          std::optional<CallableInfo> MaybeNextCallee;
+          CompleteFunctionAnalysis *Completed =
+              DeclAnalysis.completedAnalysisForDecl(CalleeInfo.CDecl);
+          if (Completed == nullptr) {
+            // No result - could be
+            // - non-inline and extern
+            // - indirect (virtual or through function pointer)
+            // - effect has been explicitly disclaimed (e.g. "blocking")
+
+            CallableType CType = CalleeInfo.type();
+            if (CType == CallableType::Virtual)
+              S.Diag(Callee->getLocation(),
+                     diag::note_func_effect_call_indirect)
+                  << Indirect_VirtualMethod << effectName;
+            else if (CType == CallableType::Unknown)
+              S.Diag(Callee->getLocation(),
+                     diag::note_func_effect_call_indirect)
+                  << Indirect_FunctionPtr << effectName;
+            else if (CalleeInfo.Effects.contains(Viol1.Effect.oppositeKind()))
+              S.Diag(Callee->getLocation(),
+                     diag::note_func_effect_call_disallows_inference)
+                  << GetCallableDeclKind(CInfo.CDecl, nullptr) << effectName
+                  << FunctionEffect(Viol1.Effect.oppositeKind()).name();
+            else if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(Callee);
+                     FD == nullptr || FD->getBuiltinID() == 0) {
+              // A builtin callee generally doesn't have a useful source
+              // location at which to insert a note.
+              S.Diag(Callee->getLocation(), diag::note_func_effect_call_extern)
+                  << effectName;
+            }
+            break;
+          }
+          const Violation *PtrViol2 =
+              Completed->firstViolationForEffect(Viol1.Effect);
+          if (PtrViol2 == nullptr)
+            break;
+
+          const Violation &Viol2 = *PtrViol2;
+          switch (Viol2.ID) {
+          case ViolationID::None:
+            llvm_unreachable("Unexpected violation kind");
+            break;
+          case ViolationID::DeclDisallowsInference:
+            S.Diag(Viol2.Loc, diag::note_func_effect_call_disallows_inference)
+                << GetCallableDeclKind(CalleeInfo.CDecl, nullptr) << effectName
+                << Viol2.CalleeEffectPreventingInference->name();
+            break;
+          case ViolationID::CallsExprWithoutEffect:
+            S.Diag(Viol2.Loc, diag::note_func_effect_call_indirect)
+                << Indirect_FunctionPtr << effectName;
+            break;
+          case ViolationID::AllocatesMemory:
+          case ViolationID::ThrowsOrCatchesExceptions:
+          case ViolationID::HasStaticLocalVariable:
+          case ViolationID::AccessesThreadLocalVariable:
+          case ViolationID::AccessesObjCMethodOrProperty:
+            S.Diag(Viol2.Loc, diag::note_func_effect_violation)
+                << GetCallableDeclKind(CalleeInfo.CDecl, &Viol2) << effectName
+                << Viol2.diagnosticSelectIndex();
+            MaybeAddSiteContext(CalleeInfo.CDecl, Viol2);
+            break;
+          case ViolationID::CallsDeclWithoutEffect:
+            MaybeNextCallee.emplace(*Viol2.Callee);
+            S.Diag(Viol2.Loc, diag::note_func_effect_calls_func_without_effect)
+                << GetCallableDeclKind(CalleeInfo.CDecl, &Viol2) << effectName
+                << GetCallableDeclKind(Viol2.Callee, nullptr)
+                << MaybeNextCallee->getNameForDiagnostic(S);
+            break;
+          }
+          MaybeAddTemplateNote(Callee);
+          Callee = Viol2.Callee;
+          if (MaybeNextCallee) {
+            CalleeInfo = *MaybeNextCallee;
+            CalleeName = CalleeInfo.getNameForDiagnostic(S);
+          }
+        }
+      } break;
+      }
+    }
+  }
+
+  // ----------
+  // This AST visitor is used to traverse the body of a function during effect
+  // verification. This happens in 2 situations:
+  //  [1] The function has declared effects which need to be validated.
+  //  [2] The function has not explicitly declared an effect in question, and is
+  //      being checked for implicit conformance.
+  //
+  // Violations are always routed to a PendingFunctionAnalysis.
+  struct FunctionBodyASTVisitor : RecursiveASTVisitor<FunctionBodyASTVisitor> {
+    using Base = RecursiveASTVisitor<FunctionBodyASTVisitor>;
+
+    Analyzer &Outer;
+    PendingFunctionAnalysis &CurrentFunction;
+    CallableInfo &CurrentCaller;
+    ViolationSite VSite;
+
+    FunctionBodyASTVisitor(Analyzer &Outer,
+                           PendingFunctionAnalysis &CurrentFunction,
+                           CallableInfo &CurrentCaller)
+        : Outer(Outer), CurrentFunction(CurrentFunction),
+          CurrentCaller(CurrentCaller) {}
+
+    // -- Entry point --
+    void run() {
+      // The target function may have implicit code paths beyond the
+      // body: member and base destructors. Visit these first.
+      if (auto *Dtor = dyn_cast<CXXDestructorDecl>(CurrentCaller.CDecl))
+        followDestructor(dyn_cast<CXXRecordDecl>(Dtor->getParent()), Dtor);
+
+      // Do an AST traversal of the function/block body
+      TraverseDecl(const_cast<Decl *>(CurrentCaller.CDecl));
+    }
+
+    // -- Methods implementing common logic --
+
+    // Handle a language construct forbidden by some effects. Only effects whose
+    // flags include the specified flag receive a violation. \p Flag describes
+    // the construct.
+    void diagnoseLanguageConstruct(FunctionEffect::FlagBit Flag,
+                                   ViolationID VID, SourceLocation Loc,
+                                   const Decl *Callee = nullptr) {
+      // If there are any declared verifiable effects which forbid the construct
+      // represented by the flag, store just one violation.
+      for (FunctionEffect Effect : CurrentFunction.DeclaredVerifiableEffects) {
+        if (Effect.flags() & Flag) {
+          addViolation(/*inferring=*/false, Effect, VID, Loc, Callee);
+          break;
+        }
+      }
+      // For each inferred effect which forbids the construct, store a
+      // violation, if we don't already have a violation for that effect.
+      for (FunctionEffect Effect : CurrentFunction.EffectsToInfer)
+        if (Effect.flags() & Flag)
+          addViolation(/*inferring=*/true, Effect, VID, Loc, Callee);
+    }
+
+    void addViolation(bool Inferring, FunctionEffect Effect, ViolationID VID,
+                      SourceLocation Loc, const Decl *Callee = nullptr) {
+      CurrentFunction.checkAddViolation(
+          Inferring, Violation(Effect, VID, VSite, Loc, Callee));
+    }
+
+    // Here we have a call to a Decl, either explicitly via a CallExpr or some
+    // other AST construct. CallableInfo pertains to the callee.
+    void followCall(CallableInfo &CI, SourceLocation CallLoc) {
+      // Check for a call to a builtin function, whose effects are
+      // handled specially.
+      if (const auto *FD = dyn_cast<FunctionDecl>(CI.CDecl)) {
+        if (unsigned BuiltinID = FD->getBuiltinID()) {
+          CI.Effects = getBuiltinFunctionEffects(BuiltinID);
+          if (CI.Effects.empty()) {
+            // A builtin with no known effects is assumed safe.
+            return;
+          }
+          // A builtin WITH effects doesn't get any special treatment for
+          // being noreturn/noexcept, e.g. longjmp(), so we skip the check
+          // below.
+        } else {
+          // If the callee is both `noreturn` and `noexcept`, it presumably
+          // terminates. Ignore it for the purposes of effect analysis.
+          // If not C++, `noreturn` alone is sufficient.
+          if (FD->isNoReturn() &&
+              (!Outer.S.getLangOpts().CPlusPlus || isNoexcept(FD)))
+            return;
+        }
+      }
+
+      Outer.followCall(CurrentCaller, CurrentFunction, CI, CallLoc,
+                       /*AssertNoFurtherInference=*/false, VSite);
+    }
+
+    void checkIndirectCall(CallExpr *Call, QualType CalleeType) {
+      auto *FPT =
+          CalleeType->getAs<FunctionProtoType>(); // Null if FunctionType.
+      FunctionEffectKindSet CalleeEffects;
+      if (FPT)
+        CalleeEffects.insert(FPT->getFunctionEffects());
+
+      auto Check1Effect = [&](FunctionEffect Effect, bool Inferring) {
+        if (FPT == nullptr || Effect.shouldDiagnoseFunctionCall(
+                                  /*direct=*/false, CalleeEffects))
+          addViolation(Inferring, Effect, ViolationID::CallsExprWithoutEffect,
+                       Call->getBeginLoc());
+      };
+
+      for (FunctionEffect Effect : CurrentFunction.DeclaredVerifiableEffects)
+        Check1Effect(Effect, false);
+
+      for (FunctionEffect Effect : CurrentFunction.EffectsToInfer)
+        Check1Effect(Effect, true);
+    }
+
+    // This destructor's body should be followed by the caller, but here we
+    // follow the field and base destructors.
+    void followDestructor(const CXXRecordDecl *Rec,
+                          const CXXDestructorDecl *Dtor) {
+      SourceLocation DtorLoc = Dtor->getLocation();
+      for (const FieldDecl *Field : Rec->fields())
+        followTypeDtor(Field->getType(), DtorLoc);
+
+      if (const auto *Class = dyn_cast<CXXRecordDecl>(Rec))
+        for (const CXXBaseSpecifier &Base : Class->bases())
+          followTypeDtor(Base.getType(), DtorLoc);
+    }
+
+    void followTypeDtor(QualType QT, SourceLocation CallSite) {
+      const Type *Ty = QT.getTypePtr();
+      while (Ty->isArrayType()) {
+        const ArrayType *Arr = Ty->getAsArrayTypeUnsafe();
+        QT = Arr->getElementType();
+        Ty = QT.getTypePtr();
+      }
+
+      if (Ty->isRecordType()) {
+        if (const CXXRecordDecl *Class = Ty->getAsCXXRecordDecl()) {
+          if (CXXDestructorDecl *Dtor = Class->getDestructor();
+              Dtor && !Dtor->isDeleted()) {
+            CallableInfo CI(*Dtor);
+            followCall(CI, CallSite);
+          }
+        }
+      }
+    }
+
+    // -- Methods for use of RecursiveASTVisitor --
+
+    bool shouldVisitImplicitCode() const { return true; }
+
+    bool shouldWalkTypesOfTypeLocs() const { return false; }
+
+    bool VisitCXXThrowExpr(CXXThrowExpr *Throw) {
+      diagnoseLanguageConstruct(FunctionEffect::FE_ExcludeThrow,
+                                ViolationID::ThrowsOrCatchesExceptions,
+                                Throw->getThrowLoc());
+      return true;
+    }
+
+    bool VisitCXXCatchStmt(CXXCatchStmt *Catch) {
+      diagnoseLanguageConstruct(FunctionEffect::FE_ExcludeCatch,
+                                ViolationID::ThrowsOrCatchesExceptions,
+                                Catch->getCatchLoc());
+      return true;
+    }
+
+    bool VisitObjCAtThrowStmt(ObjCAtThrowStmt *Throw) {
+      diagnoseLanguageConstruct(FunctionEffect::FE_ExcludeThrow,
+                                ViolationID::ThrowsOrCatchesExceptions,
+                                Throw->getThrowLoc());
+      return true;
+    }
+
+    bool VisitObjCAtCatchStmt(ObjCAtCatchStmt *Catch) {
+      diagnoseLanguageConstruct(FunctionEffect::FE_ExcludeCatch,
+                                ViolationID::ThrowsOrCatchesExceptions,
+                                Catch->getAtCatchLoc());
+      return true;
+    }
+
+    bool VisitObjCMessageExpr(ObjCMessageExpr *Msg) {
+      diagnoseLanguageConstruct(FunctionEffect::FE_ExcludeObjCMessageSend,
+                                ViolationID::AccessesObjCMethodOrProperty,
+                                Msg->getBeginLoc());
+      return true;
+    }
+
+    bool VisitSEHExceptStmt(SEHExceptStmt *Exc) {
+      diagnoseLanguageConstruct(FunctionEffect::FE_ExcludeCatch,
+                                ViolationID::ThrowsOrCatchesExceptions,
+                                Exc->getExceptLoc());
+      return true;
+    }
+
+    bool VisitCallExpr(CallExpr *Call) {
+      LLVM_DEBUG(llvm::dbgs()
+                     << "VisitCallExpr : "
+                     << Call->getBeginLoc().printToString(Outer.S.SourceMgr)
+                     << "\n";);
+
+      Expr *CalleeExpr = Call->getCallee();
+      if (const Decl *Callee = CalleeExpr->getReferencedDeclOfCallee()) {
+        CallableInfo CI(*Callee);
+        followCall(CI, Call->getBeginLoc());
+        return true;
+      }
+
+      if (isa<CXXPseudoDestructorExpr>(CalleeExpr)) {
+        // Just destroying a scalar, fine.
+        return true;
+      }
+
+      // No Decl, just an Expr. Just check based on its type.
+      checkIndirectCall(Call, CalleeExpr->getType());
+
+      return true;
+    }
+
+    bool VisitVarDecl(VarDecl *Var) {
+      LLVM_DEBUG(llvm::dbgs()
+                     << "VisitVarDecl : "
+                     << Var->getBeginLoc().printToString(Outer.S.SourceMgr)
+                     << "\n";);
+
+      if (Var->isStaticLocal())
+        diagnoseLanguageConstruct(FunctionEffect::FE_ExcludeStaticLocalVars,
+                                  ViolationID::HasStaticLocalVariable,
+                                  Var->getLocation());
+
+      const QualType::DestructionKind DK =
+          Var->needsDestruction(Outer.S.getASTContext());
+      if (DK == QualType::DK_cxx_destructor)
+        followTypeDtor(Var->getType(), Var->getLocation());
+      return true;
+    }
+
+    bool VisitCXXNewExpr(CXXNewExpr *New) {
+      // RecursiveASTVisitor does not visit the implicit call to operator new.
+      if (FunctionDecl *FD = New->getOperatorNew()) {
+        CallableInfo CI(*FD, SpecialFuncType::OperatorNew);
+        followCall(CI, New->getBeginLoc());
+      }
+
+      // It's a bit excessive to check operator delete here, since it's
+      // just a fallback for operator new followed by a failed constructor.
+      // We could check it via New->getOperatorDelete().
+
+      // It DOES however visit the called constructor
+      return true;
+    }
+
+    bool VisitCXXDeleteExpr(CXXDeleteExpr *Delete) {
+      // RecursiveASTVisitor does not visit the implicit call to operator
+      // delete.
+      if (FunctionDecl *FD = Delete->getOperatorDelete()) {
+        CallableInfo CI(*FD, SpecialFuncType::OperatorDelete);
+        followCall(CI, Delete->getBeginLoc());
+      }
+
+      // It DOES however visit the called destructor
+
+      return true;
+    }
+
+    bool VisitCXXConstructExpr(CXXConstructExpr *Construct) {
+      LLVM_DEBUG(llvm::dbgs() << "VisitCXXConstructExpr : "
+                              << Construct->getBeginLoc().printToString(
+                                     Outer.S.SourceMgr)
+                              << "\n";);
+
+      // RecursiveASTVisitor does not visit the implicit call to the
+      // constructor.
+      const CXXConstructorDecl *Ctor = Construct->getConstructor();
+      CallableInfo CI(*Ctor);
+      followCall(CI, Construct->getLocation());
+
+      return true;
+    }
+
+    bool TraverseConstructorInitializer(CXXCtorInitializer *Init) {
+      ViolationSite PrevVS = VSite;
+      if (Init->isAnyMemberInitializer())
+        VSite.setKind(ViolationSite::Kind::MemberInitializer);
+      bool Result = Base::TraverseConstructorInitializer(Init);
+      VSite = PrevVS;
+      return Result;
+    }
+
+    bool TraverseCXXDefaultArgExpr(CXXDefaultArgExpr *E) {
+      LLVM_DEBUG(llvm::dbgs()
+                     << "TraverseCXXDefaultArgExpr : "
+                     << E->getUsedLocation().printToString(Outer.S.SourceMgr)
+                     << "\n";);
+
+      ViolationSite PrevVS = VSite;
+      if (VSite.kind() == ViolationSite::Kind::Default)
+        VSite = ViolationSite{E};
+
+      bool Result = Base::TraverseCXXDefaultArgExpr(E);
+      VSite = PrevVS;
+      return Result;
+    }
+
+    bool TraverseLambdaExpr(LambdaExpr *Lambda) {
+      // We override this so as to be able to skip traversal of the lambda's
+      // body. We have to explicitly traverse the captures. Why not return
+      // false from shouldVisitLambdaBody()? Because we need to visit a lambda's
+      // body when we are verifying the lambda itself; we only want to skip it
+      // in the context of the outer function.
+      for (unsigned I = 0, N = Lambda->capture_size(); I < N; ++I)
+        TraverseLambdaCapture(Lambda, Lambda->capture_begin() + I,
+                              Lambda->capture_init_begin()[I]);
+
+      return true;
+    }
+
+    bool TraverseBlockExpr(BlockExpr * /*unused*/) {
+      // TODO: are the capture expressions (ctor call?) safe?
+      return true;
+    }
+
+    bool VisitDeclRefExpr(const DeclRefExpr *E) {
+      const ValueDecl *Val = E->getDecl();
+      if (const auto *Var = dyn_cast<VarDecl>(Val)) {
+        if (Var->getTLSKind() != VarDecl::TLS_None) {
+          // At least on macOS, thread-local variables are initialized on
+          // first access, including a heap allocation.
+          diagnoseLanguageConstruct(FunctionEffect::FE_ExcludeThreadLocalVars,
+                                    ViolationID::AccessesThreadLocalVariable,
+                                    E->getLocation());
+        }
+      }
+      return true;
+    }
+
+    bool TraverseGenericSelectionExpr(GenericSelectionExpr *Node) {
+      return TraverseStmt(Node->getResultExpr());
+    }
+    bool TraverseUnaryExprOrTypeTraitExpr(UnaryExprOrTypeTraitExpr *Node) {
+      return true;
+    }
+
+    bool TraverseTypeOfExprTypeLoc(TypeOfExprTypeLoc Node) { return true; }
+
+    bool TraverseDecltypeTypeLoc(DecltypeTypeLoc Node) { return true; }
+
+    bool TraverseCXXNoexceptExpr(CXXNoexceptExpr *Node) { return true; }
+
+    bool TraverseCXXTypeidExpr(CXXTypeidExpr *Node) { return true; }
+
+    // Skip concept requirements since they don't generate code.
+    bool TraverseConceptRequirement(concepts::Requirement *R) { return true; }
+  };
+};
+
+Analyzer::AnalysisMap::~AnalysisMap() {
+  for (const auto &Item : *this) {
+    FuncAnalysisPtr AP = Item.second;
+    if (isa<PendingFunctionAnalysis *>(AP))
+      delete AP.get<PendingFunctionAnalysis *>();
+    else
+      delete AP.get<CompleteFunctionAnalysis *>();
+  }
+}
+
+} // anonymous namespace
+
+namespace clang {
+
+bool Sema::diagnoseConflictingFunctionEffect(
+    const FunctionEffectsRef &FX, const FunctionEffectWithCondition &NewEC,
+    SourceLocation NewAttrLoc) {
+  // If the new effect has a condition, we can't detect conflicts until the
+  // condition is resolved.
+  if (NewEC.Cond.getCondition() != nullptr)
+    return false;
+
+  // Diagnose the new attribute as incompatible with a previous one.
+  auto Incompatible = [&](const FunctionEffectWithCondition &PrevEC) {
+    Diag(NewAttrLoc, diag::err_attributes_are_not_compatible)
+        << ("'" + NewEC.description() + "'")
+        << ("'" + PrevEC.description() + "'") << false;
+    // We don't necessarily have the location of the previous attribute,
+    // so no note.
+    return true;
+  };
+
+  // Compare against previous attributes.
+  FunctionEffect::Kind NewKind = NewEC.Effect.kind();
+
+  for (const FunctionEffectWithCondition &PrevEC : FX) {
+    // Again, can't check yet when the effect is conditional.
+    if (PrevEC.Cond.getCondition() != nullptr)
+      continue;
+
+    FunctionEffect::Kind PrevKind = PrevEC.Effect.kind();
+    // Note that we allow PrevKind == NewKind; it's redundant and ignored.
+
+    if (PrevEC.Effect.oppositeKind() == NewKind)
+      return Incompatible(PrevEC);
+
+    // A new allocating is incompatible with a previous nonblocking.
+    if (PrevKind == FunctionEffect::Kind::NonBlocking &&
+        NewKind == FunctionEffect::Kind::Allocating)
+      return Incompatible(PrevEC);
+
+    // A new nonblocking is incompatible with a previous allocating.
+    if (PrevKind == FunctionEffect::Kind::Allocating &&
+        NewKind == FunctionEffect::Kind::NonBlocking)
+      return Incompatible(PrevEC);
+  }
+
+  return false;
+}
+
+void Sema::diagnoseFunctionEffectMergeConflicts(
+    const FunctionEffectSet::Conflicts &Errs, SourceLocation NewLoc,
+    SourceLocation OldLoc) {
+  for (const FunctionEffectSet::Conflict &Conflict : Errs) {
+    Diag(NewLoc, diag::warn_conflicting_func_effects)
+        << Conflict.Kept.description() << Conflict.Rejected.description();
+    Diag(OldLoc, diag::note_previous_declaration);
+  }
+}
+
+// Decl should be a FunctionDecl or BlockDecl.
+void Sema::maybeAddDeclWithEffects(const Decl *D,
+                                   const FunctionEffectsRef &FX) {
+  if (!D->hasBody()) {
+    if (const auto *FD = D->getAsFunction(); FD && !FD->willHaveBody())
+      return;
+  }
+
+  if (Diags.getIgnoreAllWarnings() ||
+      (Diags.getSuppressSystemWarnings() &&
+       SourceMgr.isInSystemHeader(D->getLocation())))
+    return;
+
+  if (hasUncompilableErrorOccurred())
+    return;
+
+  // For code in dependent contexts, we'll do this at instantiation time.
+  // Without this check, we would analyze the function based on placeholder
+  // template parameters, and potentially generate spurious diagnostics.
+  if (cast<DeclContext>(D)->isDependentContext())
+    return;
+
+  addDeclWithEffects(D, FX);
+}
+
+void Sema::addDeclWithEffects(const Decl *D, const FunctionEffectsRef &FX) {
+  // To avoid the possibility of conflict, don't add effects which are
+  // not FE_InferrableOnCallees and therefore not verified; this removes
+  // blocking/allocating but keeps nonblocking/nonallocating.
+  // Also, ignore any conditions when building the list of effects.
+  bool AnyVerifiable = false;
+  for (const FunctionEffectWithCondition &EC : FX)
+    if (EC.Effect.flags() & FunctionEffect::FE_InferrableOnCallees) {
+      AllEffectsToVerify.insert(EC.Effect);
+      AnyVerifiable = true;
+    }
+
+  // Record the declaration for later analysis.
+  if (AnyVerifiable)
+    DeclsWithEffectsToVerify.push_back(D);
+}
+
+void Sema::performFunctionEffectAnalysis(TranslationUnitDecl *TU) {
+  if (hasUncompilableErrorOccurred() || Diags.getIgnoreAllWarnings())
+    return;
+  if (TU == nullptr)
+    return;
+  Analyzer{*this}.run(*TU);
+}
+
+Sema::FunctionEffectDiffVector::FunctionEffectDiffVector(
+    const FunctionEffectsRef &Old, const FunctionEffectsRef &New) {
+
+  FunctionEffectsRef::iterator POld = Old.begin();
+  FunctionEffectsRef::iterator OldEnd = Old.end();
+  FunctionEffectsRef::iterator PNew = New.begin();
+  FunctionEffectsRef::iterator NewEnd = New.end();
+
+  while (true) {
+    int cmp = 0;
+    if (POld == OldEnd) {
+      if (PNew == NewEnd)
+        break;
+      cmp = 1;
+    } else if (PNew == NewEnd)
+      cmp = -1;
+    else {
+      FunctionEffectWithCondition Old = *POld;
+      FunctionEffectWithCondition New = *PNew;
+      if (Old.Effect.kind() < New.Effect.kind())
+        cmp = -1;
+      else if (New.Effect.kind() < Old.Effect.kind())
+        cmp = 1;
+      else {
+        cmp = 0;
+        if (Old.Cond.getCondition() != New.Cond.getCondition()) {
+          // FIXME: Cases where the expressions are equivalent but
+          // don't have the same identity.
+          push_back(FunctionEffectDiff{
+              Old.Effect.kind(), FunctionEffectDiff::Kind::ConditionMismatch,
+              Old, New});
+        }
+      }
+    }
+
+    if (cmp < 0) {
+      // removal
+      FunctionEffectWithCondition Old = *POld;
+      push_back(FunctionEffectDiff{Old.Effect.kind(),
+                                   FunctionEffectDiff::Kind::Removed, Old,
+                                   std::nullopt});
+      ++POld;
+    } else if (cmp > 0) {
+      // addition
+      FunctionEffectWithCondition New = *PNew;
+      push_back(FunctionEffectDiff{New.Effect.kind(),
+                                   FunctionEffectDiff::Kind::Added,
+                                   std::nullopt, New});
+      ++PNew;
+    } else {
+      ++POld;
+      ++PNew;
+    }
+  }
+}
+
+bool Sema::FunctionEffectDiff::shouldDiagnoseConversion(
+    QualType SrcType, const FunctionEffectsRef &SrcFX, QualType DstType,
+    const FunctionEffectsRef &DstFX) const {
+
+  switch (EffectKind) {
+  case FunctionEffect::Kind::NonAllocating:
+    // nonallocating can't be added (spoofed) during a conversion, unless we
+    // have nonblocking.
+    if (DiffKind == Kind::Added) {
+      for (const auto &CFE : SrcFX) {
+        if (CFE.Effect.kind() == FunctionEffect::Kind::NonBlocking)
+          return false;
+      }
+    }
+    [[fallthrough]];
+  case FunctionEffect::Kind::NonBlocking:
+    // nonblocking can't be added (spoofed) during a conversion.
+    switch (DiffKind) {
+    case Kind::Added:
+      return true;
+    case Kind::Removed:
+      return false;
+    case Kind::ConditionMismatch:
+      // FIXME: Condition mismatches are too coarse right now -- expressions
+      // which are equivalent but don't have the same identity are detected as
+      // mismatches. We're going to diagnose those anyhow until expression
+      // matching is better.
+      return true;
+    }
+  case FunctionEffect::Kind::Blocking:
+  case FunctionEffect::Kind::Allocating:
+    return false;
+  }
+  llvm_unreachable("unknown effect kind");
+}
+
+bool Sema::FunctionEffectDiff::shouldDiagnoseRedeclaration(
+    const FunctionDecl &OldFunction, const FunctionEffectsRef &OldFX,
+    const FunctionDecl &NewFunction, const FunctionEffectsRef &NewFX) const {
+  switch (EffectKind) {
+  case FunctionEffect::Kind::NonAllocating:
+  case FunctionEffect::Kind::NonBlocking:
+    // nonblocking/nonallocating can't be removed in a redeclaration.
+    switch (DiffKind) {
+    case Kind::Added:
+      return false; // No diagnostic.
+    case Kind::Removed:
+      return true; // Issue diagnostic.
+    case Kind::ConditionMismatch:
+      // All these forms of mismatches are diagnosed.
+      return true;
+    }
+  case FunctionEffect::Kind::Blocking:
+  case FunctionEffect::Kind::Allocating:
+    return false;
+  }
+  llvm_unreachable("unknown effect kind");
+}
+
+Sema::FunctionEffectDiff::OverrideResult
+Sema::FunctionEffectDiff::shouldDiagnoseMethodOverride(
+    const CXXMethodDecl &OldMethod, const FunctionEffectsRef &OldFX,
+    const CXXMethodDecl &NewMethod, const FunctionEffectsRef &NewFX) const {
+  switch (EffectKind) {
+  case FunctionEffect::Kind::NonAllocating:
+  case FunctionEffect::Kind::NonBlocking:
+    switch (DiffKind) {
+
+    // If added on an override, that's fine and not diagnosed.
+    case Kind::Added:
+      return OverrideResult::NoAction;
+
+    // If missing from an override (removed), propagate from base to derived.
+    case Kind::Removed:
+      return OverrideResult::Merge;
+
+    // If there's a mismatch involving the effect's polarity or condition,
+    // issue a warning.
+    case Kind::ConditionMismatch:
+      return OverrideResult::Warn;
+    }
+
+  case FunctionEffect::Kind::Blocking:
+  case FunctionEffect::Kind::Allocating:
+    return OverrideResult::NoAction;
+  }
+  llvm_unreachable("unknown effect kind");
+}
+
+} // namespace clang
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 43cc6c81ae5cb..fbcba201a351a 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -1828,6 +1828,41 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
       return true;
     break;
   }
+  case Builtin::BI__builtin_hlsl_cross: {
+    if (SemaRef.checkArgCount(TheCall, 2))
+      return true;
+    if (CheckVectorElementCallArgs(&SemaRef, TheCall))
+      return true;
+    if (CheckFloatOrHalfRepresentations(&SemaRef, TheCall))
+      return true;
+    // ensure both args have 3 elements
+    int NumElementsArg1 =
+        TheCall->getArg(0)->getType()->getAs<VectorType>()->getNumElements();
+    int NumElementsArg2 =
+        TheCall->getArg(1)->getType()->getAs<VectorType>()->getNumElements();
+
+    if (NumElementsArg1 != 3) {
+      int LessOrMore = NumElementsArg1 > 3 ? 1 : 0;
+      SemaRef.Diag(TheCall->getBeginLoc(),
+                   diag::err_vector_incorrect_num_elements)
+          << LessOrMore << 3 << NumElementsArg1 << /*operand*/ 1;
+      return true;
+    }
+    if (NumElementsArg2 != 3) {
+      int LessOrMore = NumElementsArg2 > 3 ? 1 : 0;
+
+      SemaRef.Diag(TheCall->getBeginLoc(),
+                   diag::err_vector_incorrect_num_elements)
+          << LessOrMore << 3 << NumElementsArg2 << /*operand*/ 1;
+      return true;
+    }
+
+    ExprResult A = TheCall->getArg(0);
+    QualType ArgTyA = A.get()->getType();
+    // return type is the same as the input type
+    TheCall->setType(ArgTyA);
+    break;
+  }
   case Builtin::BI__builtin_hlsl_dot: {
     if (SemaRef.checkArgCount(TheCall, 2))
       return true;
@@ -1959,6 +1994,7 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
   case Builtin::BI__builtin_elementwise_acos:
   case Builtin::BI__builtin_elementwise_asin:
   case Builtin::BI__builtin_elementwise_atan:
+  case Builtin::BI__builtin_elementwise_atan2:
   case Builtin::BI__builtin_elementwise_ceil:
   case Builtin::BI__builtin_elementwise_cos:
   case Builtin::BI__builtin_elementwise_cosh:
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 4d11f2a43fcc6..edd1fe40fdf27 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -1976,8 +1976,9 @@ void InitListChecker::CheckVectorType(const InitializedEntity &Entity,
   if (numEltsInit != maxElements) {
     if (!VerifyOnly)
       SemaRef.Diag(IList->getBeginLoc(),
-                   diag::err_vector_incorrect_num_initializers)
-          << (numEltsInit < maxElements) << maxElements << numEltsInit;
+                   diag::err_vector_incorrect_num_elements)
+          << (numEltsInit < maxElements) << maxElements << numEltsInit
+          << /*initialization*/ 0;
     hadError = true;
   }
 }
diff --git a/clang/lib/Sema/SemaLambda.cpp b/clang/lib/Sema/SemaLambda.cpp
index 15ed8572e6084..aeb20299b714a 100644
--- a/clang/lib/Sema/SemaLambda.cpp
+++ b/clang/lib/Sema/SemaLambda.cpp
@@ -423,11 +423,11 @@ bool Sema::DiagnoseInvalidExplicitObjectParameterInLambda(
   // is an empty cast path for the method stored in the context (signalling that
   // we've already diagnosed it) and then just not building the call, but that
   // doesn't really seem any simpler than diagnosing it at the call site...
-  if (auto It = Context.LambdaCastPaths.find(Method);
-      It != Context.LambdaCastPaths.end())
+  auto [It, Inserted] = Context.LambdaCastPaths.try_emplace(Method);
+  if (!Inserted)
     return It->second.empty();
 
-  CXXCastPath &Path = Context.LambdaCastPaths[Method];
+  CXXCastPath &Path = It->second;
   CXXBasePaths Paths(/*FindAmbiguities=*/true, /*RecordPaths=*/true,
                      /*DetectVirtual=*/false);
   if (!IsDerivedFrom(RD->getLocation(), ExplicitObjectParameterType, LambdaType,
@@ -1951,6 +1951,9 @@ ExprResult Sema::BuildCaptureInit(const Capture &Cap,
 ExprResult Sema::ActOnLambdaExpr(SourceLocation StartLoc, Stmt *Body) {
   LambdaScopeInfo LSI = *cast<LambdaScopeInfo>(FunctionScopes.back());
   ActOnFinishFunctionBody(LSI.CallOperator, Body);
+
+  maybeAddDeclWithEffects(LSI.CallOperator);
+
   return BuildLambdaExpr(StartLoc, Body->getEndLoc(), &LSI);
 }
 
diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp
index 89142b837e60a..66f8029a2754b 100644
--- a/clang/lib/Sema/SemaOpenACC.cpp
+++ b/clang/lib/Sema/SemaOpenACC.cpp
@@ -354,6 +354,17 @@ bool doesClauseApplyToDirective(OpenACCDirectiveKind DirectiveKind,
       return false;
     }
   }
+  case OpenACCClauseKind::Tile: {
+    switch (DirectiveKind) {
+    case OpenACCDirectiveKind::Loop:
+    case OpenACCDirectiveKind::ParallelLoop:
+    case OpenACCDirectiveKind::SerialLoop:
+    case OpenACCDirectiveKind::KernelsLoop:
+      return true;
+    default:
+      return false;
+    }
+  }
 
   default:
     // Do nothing so we can go to the 'unimplemented' diagnostic instead.
@@ -526,6 +537,36 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitDefaultClause(
       Clause.getLParenLoc(), Clause.getEndLoc());
 }
 
+OpenACCClause *SemaOpenACCClauseVisitor::VisitTileClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Loop)
+    return isNotImplemented();
+
+  // Duplicates here are not really sensible.  We could possible permit
+  // multiples if they all had the same value, but there isn't really a good
+  // reason to do so. Also, this simplifies the suppression of duplicates, in
+  // that we know if we 'find' one after instantiation, that it is the same
+  // clause, which simplifies instantiation/checking/etc.
+  if (checkAlreadyHasClauseOfKind(SemaRef, ExistingClauses, Clause))
+    return nullptr;
+
+  llvm::SmallVector<Expr *> NewSizeExprs;
+
+  // Make sure these are all positive constant expressions or *.
+  for (Expr *E : Clause.getIntExprs()) {
+    ExprResult Res = SemaRef.CheckTileSizeExpr(E);
+
+    if (!Res.isUsable())
+      return nullptr;
+
+    NewSizeExprs.push_back(Res.get());
+  }
+
+  return OpenACCTileClause::Create(Ctx, Clause.getBeginLoc(),
+                                   Clause.getLParenLoc(), NewSizeExprs,
+                                   Clause.getEndLoc());
+}
+
 OpenACCClause *SemaOpenACCClauseVisitor::VisitIfClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
   // Restrictions only properly implemented on 'compute' constructs, and
@@ -1073,19 +1114,90 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitCollapseClause(
 
 SemaOpenACC::SemaOpenACC(Sema &S) : SemaBase(S) {}
 
-SemaOpenACC::AssociatedStmtRAII::AssociatedStmtRAII(SemaOpenACC &S,
-                                                    OpenACCDirectiveKind DK)
+SemaOpenACC::AssociatedStmtRAII::AssociatedStmtRAII(
+    SemaOpenACC &S, OpenACCDirectiveKind DK,
+    ArrayRef<const OpenACCClause *> UnInstClauses,
+    ArrayRef<OpenACCClause *> Clauses)
     : SemaRef(S), WasInsideComputeConstruct(S.InsideComputeConstruct),
-      DirKind(DK) {
+      DirKind(DK), LoopRAII(SemaRef, /*PreserveDepth=*/false) {
   // Compute constructs end up taking their 'loop'.
   if (DirKind == OpenACCDirectiveKind::Parallel ||
       DirKind == OpenACCDirectiveKind::Serial ||
       DirKind == OpenACCDirectiveKind::Kernels) {
     SemaRef.InsideComputeConstruct = true;
     SemaRef.ParentlessLoopConstructs.swap(ParentlessLoopConstructs);
+  } else if (DirKind == OpenACCDirectiveKind::Loop) {
+    SetCollapseInfoBeforeAssociatedStmt(UnInstClauses, Clauses);
+    SetTileInfoBeforeAssociatedStmt(UnInstClauses, Clauses);
   }
 }
 
+void SemaOpenACC::AssociatedStmtRAII::SetCollapseInfoBeforeAssociatedStmt(
+    ArrayRef<const OpenACCClause *> UnInstClauses,
+    ArrayRef<OpenACCClause *> Clauses) {
+
+  // Reset this checking for loops that aren't covered in a RAII object.
+  SemaRef.LoopInfo.CurLevelHasLoopAlready = false;
+  SemaRef.CollapseInfo.CollapseDepthSatisfied = true;
+  SemaRef.TileInfo.TileDepthSatisfied = true;
+
+  // We make sure to take an optional list of uninstantiated clauses, so that
+  // we can check to make sure we don't 'double diagnose' in the event that
+  // the value of 'N' was not dependent in a template. We also ensure during
+  // Sema that there is only 1 collapse on each construct, so we can count on
+  // the fact that if both find a 'collapse', that they are the same one.
+  auto *CollapseClauseItr =
+      llvm::find_if(Clauses, llvm::IsaPred<OpenACCCollapseClause>);
+  auto *UnInstCollapseClauseItr =
+      llvm::find_if(UnInstClauses, llvm::IsaPred<OpenACCCollapseClause>);
+
+  if (Clauses.end() == CollapseClauseItr)
+    return;
+
+  OpenACCCollapseClause *CollapseClause =
+      cast<OpenACCCollapseClause>(*CollapseClauseItr);
+
+  SemaRef.CollapseInfo.ActiveCollapse = CollapseClause;
+  Expr *LoopCount = CollapseClause->getLoopCount();
+
+  // If the loop count is still instantiation dependent, setting the depth
+  // counter isn't necessary, so return here.
+  if (!LoopCount || LoopCount->isInstantiationDependent())
+    return;
+
+  // Suppress diagnostics if we've done a 'transform' where the previous version
+  // wasn't dependent, meaning we already diagnosed it.
+  if (UnInstCollapseClauseItr != UnInstClauses.end() &&
+      !cast<OpenACCCollapseClause>(*UnInstCollapseClauseItr)
+           ->getLoopCount()
+           ->isInstantiationDependent())
+    return;
+
+  SemaRef.CollapseInfo.CollapseDepthSatisfied = false;
+  SemaRef.CollapseInfo.CurCollapseCount =
+      cast<ConstantExpr>(LoopCount)->getResultAsAPSInt();
+}
+
+void SemaOpenACC::AssociatedStmtRAII::SetTileInfoBeforeAssociatedStmt(
+    ArrayRef<const OpenACCClause *> UnInstClauses,
+    ArrayRef<OpenACCClause *> Clauses) {
+  // We don't diagnose if this is during instantiation, since the only thing we
+  // care about is the number of arguments, which we can figure out without
+  // instantiation, so we don't want to double-diagnose.
+  if (UnInstClauses.size() > 0)
+    return;
+  auto *TileClauseItr =
+      llvm::find_if(Clauses, llvm::IsaPred<OpenACCTileClause>);
+
+  if (Clauses.end() == TileClauseItr)
+    return;
+
+  OpenACCTileClause *TileClause = cast<OpenACCTileClause>(*TileClauseItr);
+  SemaRef.TileInfo.ActiveTile = TileClause;
+  SemaRef.TileInfo.TileDepthSatisfied = false;
+  SemaRef.TileInfo.CurTileCount = TileClause->getSizeExprs().size();
+}
+
 SemaOpenACC::AssociatedStmtRAII::~AssociatedStmtRAII() {
   SemaRef.InsideComputeConstruct = WasInsideComputeConstruct;
   if (DirKind == OpenACCDirectiveKind::Parallel ||
@@ -1094,6 +1206,9 @@ SemaOpenACC::AssociatedStmtRAII::~AssociatedStmtRAII() {
     assert(SemaRef.ParentlessLoopConstructs.empty() &&
            "Didn't consume loop construct list?");
     SemaRef.ParentlessLoopConstructs.swap(ParentlessLoopConstructs);
+  } else if (DirKind == OpenACCDirectiveKind::Loop) {
+    // Nothing really to do here, the LoopInConstruct should handle restorations
+    // correctly.
   }
 }
 
@@ -1646,10 +1761,242 @@ ExprResult SemaOpenACC::CheckCollapseLoopCount(Expr *LoopCount) {
       ConstantExpr::Create(getASTContext(), LoopCount, APValue{*ICE})};
 }
 
+ExprResult SemaOpenACC::CheckTileSizeExpr(Expr *SizeExpr) {
+  if (!SizeExpr)
+    return ExprError();
+
+  assert((SizeExpr->isInstantiationDependent() ||
+          SizeExpr->getType()->isIntegerType()) &&
+         "size argument non integer?");
+
+  // If dependent, or an asterisk, the expression is fine.
+  if (SizeExpr->isInstantiationDependent() ||
+      isa<OpenACCAsteriskSizeExpr>(SizeExpr))
+    return ExprResult{SizeExpr};
+
+  std::optional<llvm::APSInt> ICE =
+      SizeExpr->getIntegerConstantExpr(getASTContext());
+
+  // OpenACC 3.3 2.9.8
+  // where each tile size is a constant positive integer expression or asterisk.
+  if (!ICE || *ICE <= 0) {
+    Diag(SizeExpr->getBeginLoc(), diag::err_acc_size_expr_value)
+        << ICE.has_value() << ICE.value_or(llvm::APSInt{}).getExtValue();
+    return ExprError();
+  }
+
+  return ExprResult{
+      ConstantExpr::Create(getASTContext(), SizeExpr, APValue{*ICE})};
+}
+
+void SemaOpenACC::ActOnWhileStmt(SourceLocation WhileLoc) {
+  if (!getLangOpts().OpenACC)
+    return;
+
+  if (!LoopInfo.TopLevelLoopSeen)
+    return;
+
+  if (CollapseInfo.CurCollapseCount && *CollapseInfo.CurCollapseCount > 0) {
+    Diag(WhileLoc, diag::err_acc_invalid_in_loop)
+        << /*while loop*/ 1 << OpenACCClauseKind::Collapse;
+    assert(CollapseInfo.ActiveCollapse && "Collapse count without object?");
+    Diag(CollapseInfo.ActiveCollapse->getBeginLoc(),
+         diag::note_acc_active_clause_here)
+        << OpenACCClauseKind::Collapse;
+
+    // Remove the value so that we don't get cascading errors in the body. The
+    // caller RAII object will restore this.
+    CollapseInfo.CurCollapseCount = std::nullopt;
+  }
+
+  if (TileInfo.CurTileCount && *TileInfo.CurTileCount > 0) {
+    Diag(WhileLoc, diag::err_acc_invalid_in_loop)
+        << /*while loop*/ 1 << OpenACCClauseKind::Tile;
+    assert(TileInfo.ActiveTile && "tile count without object?");
+    Diag(TileInfo.ActiveTile->getBeginLoc(), diag::note_acc_active_clause_here)
+        << OpenACCClauseKind::Tile;
+
+    // Remove the value so that we don't get cascading errors in the body. The
+    // caller RAII object will restore this.
+    TileInfo.CurTileCount = std::nullopt;
+  }
+}
+
+void SemaOpenACC::ActOnDoStmt(SourceLocation DoLoc) {
+  if (!getLangOpts().OpenACC)
+    return;
+
+  if (!LoopInfo.TopLevelLoopSeen)
+    return;
+
+  if (CollapseInfo.CurCollapseCount && *CollapseInfo.CurCollapseCount > 0) {
+    Diag(DoLoc, diag::err_acc_invalid_in_loop)
+        << /*do loop*/ 2 << OpenACCClauseKind::Collapse;
+    assert(CollapseInfo.ActiveCollapse && "Collapse count without object?");
+    Diag(CollapseInfo.ActiveCollapse->getBeginLoc(),
+         diag::note_acc_active_clause_here)
+        << OpenACCClauseKind::Collapse;
+
+    // Remove the value so that we don't get cascading errors in the body. The
+    // caller RAII object will restore this.
+    CollapseInfo.CurCollapseCount = std::nullopt;
+  }
+
+  if (TileInfo.CurTileCount && *TileInfo.CurTileCount > 0) {
+    Diag(DoLoc, diag::err_acc_invalid_in_loop)
+        << /*do loop*/ 2 << OpenACCClauseKind::Tile;
+    assert(TileInfo.ActiveTile && "tile count without object?");
+    Diag(TileInfo.ActiveTile->getBeginLoc(), diag::note_acc_active_clause_here)
+        << OpenACCClauseKind::Tile;
+
+    // Remove the value so that we don't get cascading errors in the body. The
+    // caller RAII object will restore this.
+    TileInfo.CurTileCount = std::nullopt;
+  }
+}
+
+void SemaOpenACC::ActOnForStmtBegin(SourceLocation ForLoc) {
+  if (!getLangOpts().OpenACC)
+    return;
+
+  // Enable the while/do-while checking.
+  LoopInfo.TopLevelLoopSeen = true;
+
+  if (CollapseInfo.CurCollapseCount && *CollapseInfo.CurCollapseCount > 0) {
+
+    // OpenACC 3.3 2.9.1:
+    // Each associated loop, except the innermost, must contain exactly one loop
+    // or loop nest.
+    // This checks for more than 1 loop at the current level, the
+    // 'depth'-satisifed checking manages the 'not zero' case.
+    if (LoopInfo.CurLevelHasLoopAlready) {
+      Diag(ForLoc, diag::err_acc_clause_multiple_loops) << /*Collapse*/ 0;
+      assert(CollapseInfo.ActiveCollapse && "No collapse object?");
+      Diag(CollapseInfo.ActiveCollapse->getBeginLoc(),
+           diag::note_acc_active_clause_here)
+          << OpenACCClauseKind::Collapse;
+    } else {
+      --(*CollapseInfo.CurCollapseCount);
+
+      // Once we've hit zero here, we know we have deep enough 'for' loops to
+      // get to the bottom.
+      if (*CollapseInfo.CurCollapseCount == 0)
+        CollapseInfo.CollapseDepthSatisfied = true;
+    }
+  }
+
+  if (TileInfo.CurTileCount && *TileInfo.CurTileCount > 0) {
+    if (LoopInfo.CurLevelHasLoopAlready) {
+      Diag(ForLoc, diag::err_acc_clause_multiple_loops) << /*Tile*/ 1;
+      assert(TileInfo.ActiveTile && "No tile object?");
+      Diag(TileInfo.ActiveTile->getBeginLoc(),
+           diag::note_acc_active_clause_here)
+          << OpenACCClauseKind::Tile;
+    } else {
+      --(*TileInfo.CurTileCount);
+      // Once we've hit zero here, we know we have deep enough 'for' loops to
+      // get to the bottom.
+      if (*TileInfo.CurTileCount == 0)
+        TileInfo.TileDepthSatisfied = true;
+    }
+  }
+
+  // Set this to 'false' for the body of this loop, so that the next level
+  // checks independently.
+  LoopInfo.CurLevelHasLoopAlready = false;
+}
+
+namespace {
+SourceLocation FindInterveningCodeInLoop(const Stmt *CurStmt) {
+  // We should diagnose on anything except `CompoundStmt`, `NullStmt`,
+  // `ForStmt`, `CXXForRangeStmt`, since those are legal, and `WhileStmt` and
+  // `DoStmt`, as those are caught as a violation elsewhere.
+  // For `CompoundStmt` we need to search inside of it.
+  if (!CurStmt ||
+      isa<ForStmt, NullStmt, ForStmt, CXXForRangeStmt, WhileStmt, DoStmt>(
+          CurStmt))
+    return SourceLocation{};
+
+  // Any other construct is an error anyway, so it has already been diagnosed.
+  if (isa<OpenACCConstructStmt>(CurStmt))
+    return SourceLocation{};
+
+  // Search inside the compound statement, this allows for arbitrary nesting
+  // of compound statements, as long as there isn't any code inside.
+  if (const auto *CS = dyn_cast<CompoundStmt>(CurStmt)) {
+    for (const auto *ChildStmt : CS->children()) {
+      SourceLocation ChildStmtLoc = FindInterveningCodeInLoop(ChildStmt);
+      if (ChildStmtLoc.isValid())
+        return ChildStmtLoc;
+    }
+    // Empty/not invalid compound statements are legal.
+    return SourceLocation{};
+  }
+  return CurStmt->getBeginLoc();
+}
+} // namespace
+
+void SemaOpenACC::ActOnForStmtEnd(SourceLocation ForLoc, StmtResult Body) {
+  if (!getLangOpts().OpenACC)
+    return;
+  // Set this to 'true' so if we find another one at this level we can diagnose.
+  LoopInfo.CurLevelHasLoopAlready = true;
+
+  if (!Body.isUsable())
+    return;
+
+  bool IsActiveCollapse = CollapseInfo.CurCollapseCount &&
+                          *CollapseInfo.CurCollapseCount > 0 &&
+                          !CollapseInfo.ActiveCollapse->hasForce();
+  bool IsActiveTile = TileInfo.CurTileCount && *TileInfo.CurTileCount > 0;
+
+  if (IsActiveCollapse || IsActiveTile) {
+    SourceLocation OtherStmtLoc = FindInterveningCodeInLoop(Body.get());
+
+    if (OtherStmtLoc.isValid() && IsActiveCollapse) {
+      Diag(OtherStmtLoc, diag::err_acc_intervening_code)
+          << OpenACCClauseKind::Collapse;
+      Diag(CollapseInfo.ActiveCollapse->getBeginLoc(),
+           diag::note_acc_active_clause_here)
+          << OpenACCClauseKind::Collapse;
+    }
+
+    if (OtherStmtLoc.isValid() && IsActiveTile) {
+      Diag(OtherStmtLoc, diag::err_acc_intervening_code)
+          << OpenACCClauseKind::Tile;
+      Diag(TileInfo.ActiveTile->getBeginLoc(),
+           diag::note_acc_active_clause_here)
+          << OpenACCClauseKind::Tile;
+    }
+  }
+}
+
 bool SemaOpenACC::ActOnStartStmtDirective(OpenACCDirectiveKind K,
                                           SourceLocation StartLoc) {
   SemaRef.DiscardCleanupsInEvaluationContext();
   SemaRef.PopExpressionEvaluationContext();
+
+  // OpenACC 3.3 2.9.1:
+  // Intervening code must not contain other OpenACC directives or calls to API
+  // routines.
+  //
+  // ALL constructs are ill-formed if there is an active 'collapse'
+  if (CollapseInfo.CurCollapseCount && *CollapseInfo.CurCollapseCount > 0) {
+    Diag(StartLoc, diag::err_acc_invalid_in_loop)
+        << /*OpenACC Construct*/ 0 << OpenACCClauseKind::Collapse << K;
+    assert(CollapseInfo.ActiveCollapse && "Collapse count without object?");
+    Diag(CollapseInfo.ActiveCollapse->getBeginLoc(),
+         diag::note_acc_active_clause_here)
+        << OpenACCClauseKind::Collapse;
+  }
+  if (TileInfo.CurTileCount && *TileInfo.CurTileCount > 0) {
+    Diag(StartLoc, diag::err_acc_invalid_in_loop)
+        << /*OpenACC Construct*/ 0 << OpenACCClauseKind::Tile << K;
+    assert(TileInfo.ActiveTile && "Tile count without object?");
+    Diag(TileInfo.ActiveTile->getBeginLoc(), diag::note_acc_active_clause_here)
+        << OpenACCClauseKind::Tile;
+  }
+
   return diagnoseConstructAppertainment(*this, K, StartLoc, /*IsStmt=*/true);
 }
 
@@ -1713,12 +2060,36 @@ StmtResult SemaOpenACC::ActOnAssociatedStmt(SourceLocation DirectiveLoc,
     // the 'structured block'.
     return AssocStmt;
   case OpenACCDirectiveKind::Loop:
-    if (AssocStmt.isUsable() &&
-        !isa<CXXForRangeStmt, ForStmt>(AssocStmt.get())) {
+    if (!AssocStmt.isUsable())
+      return StmtError();
+
+    if (!isa<CXXForRangeStmt, ForStmt>(AssocStmt.get())) {
       Diag(AssocStmt.get()->getBeginLoc(), diag::err_acc_loop_not_for_loop);
       Diag(DirectiveLoc, diag::note_acc_construct_here) << K;
       return StmtError();
     }
+
+    if (!CollapseInfo.CollapseDepthSatisfied || !TileInfo.TileDepthSatisfied) {
+      if (!CollapseInfo.CollapseDepthSatisfied) {
+        Diag(DirectiveLoc, diag::err_acc_insufficient_loops)
+            << OpenACCClauseKind::Collapse;
+        assert(CollapseInfo.ActiveCollapse && "Collapse count without object?");
+        Diag(CollapseInfo.ActiveCollapse->getBeginLoc(),
+             diag::note_acc_active_clause_here)
+            << OpenACCClauseKind::Collapse;
+      }
+
+      if (!TileInfo.TileDepthSatisfied) {
+        Diag(DirectiveLoc, diag::err_acc_insufficient_loops)
+            << OpenACCClauseKind::Tile;
+        assert(TileInfo.ActiveTile && "Collapse count without object?");
+        Diag(TileInfo.ActiveTile->getBeginLoc(),
+             diag::note_acc_active_clause_here)
+            << OpenACCClauseKind::Tile;
+      }
+      return StmtError();
+    }
+
     // TODO OpenACC: 2.9 ~ line 2010 specifies that the associated loop has some
     // restrictions when there is a 'seq' clause in place. We probably need to
     // implement that, including piping in the clauses here.
@@ -1738,3 +2109,13 @@ bool SemaOpenACC::ActOnStartDeclDirective(OpenACCDirectiveKind K,
 }
 
 DeclGroupRef SemaOpenACC::ActOnEndDeclDirective() { return DeclGroupRef{}; }
+
+ExprResult
+SemaOpenACC::BuildOpenACCAsteriskSizeExpr(SourceLocation AsteriskLoc) {
+  return OpenACCAsteriskSizeExpr::Create(getASTContext(), AsteriskLoc);
+}
+
+ExprResult
+SemaOpenACC::ActOnOpenACCAsteriskSizeExpr(SourceLocation AsteriskLoc) {
+  return BuildOpenACCAsteriskSizeExpr(AsteriskLoc);
+}
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 0c1e054f7c30a..2cde8131108fb 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -2232,16 +2232,21 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
     // just strip the qualifiers because they don't matter.
     FromType = FromType.getUnqualifiedType();
   } else if (S.getLangOpts().HLSL && FromType->isConstantArrayType() &&
-             ToType->isArrayParameterType()) {
+             ToType->isConstantArrayType()) {
     // HLSL constant array parameters do not decay, so if the argument is a
     // constant array and the parameter is an ArrayParameterType we have special
     // handling here.
-    FromType = S.Context.getArrayParameterType(FromType);
+    if (ToType->isArrayParameterType()) {
+      FromType = S.Context.getArrayParameterType(FromType);
+      SCS.First = ICK_HLSL_Array_RValue;
+    } else {
+      SCS.First = ICK_Identity;
+    }
+
     if (S.Context.getCanonicalType(FromType) !=
         S.Context.getCanonicalType(ToType))
       return false;
 
-    SCS.First = ICK_HLSL_Array_RValue;
     SCS.setAllToTypes(ToType);
     return true;
   } else if (FromType->isArrayType()) {
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index c7d48b81bc034..eeaa1ebd7ba83 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -5502,8 +5502,7 @@ bool Sema::CheckTemplateArgumentList(
         DefaultArgs && ParamIdx >= DefaultArgs.StartPos) {
       // All written arguments should have been consumed by this point.
       assert(ArgIdx == NumArgs && "bad default argument deduction");
-      // FIXME: Don't ignore parameter packs.
-      if (ParamIdx == DefaultArgs.StartPos && !(*Param)->isParameterPack()) {
+      if (ParamIdx == DefaultArgs.StartPos) {
         assert(Param + DefaultArgs.Args.size() <= ParamEnd);
         // Default arguments from a DeducedTemplateName are already converted.
         for (const TemplateArgument &DefArg : DefaultArgs.Args) {
@@ -5580,9 +5579,6 @@ bool Sema::CheckTemplateArgumentList(
         return true;
       }
 
-      // We're now done with this argument.
-      ++ArgIdx;
-
       if ((*Param)->isTemplateParameterPack()) {
         // The template parameter was a template parameter pack, so take the
         // deduced argument and place it on the argument pack. Note that we
@@ -5593,8 +5589,19 @@ bool Sema::CheckTemplateArgumentList(
       } else {
         // Move to the next template parameter.
         ++Param;
+        if (PartialOrderingTTP && PackExpansionIntoNonPack) {
+          // Keep converting the pattern in the argument against
+          // subsequent parameters. The argument is converted
+          // in place and will be added back when we are done.
+          SugaredConverted.pop_back();
+          CanonicalConverted.pop_back();
+          continue;
+        }
       }
 
+      // We're now done with this argument.
+      ++ArgIdx;
+
       // If we just saw a pack expansion into a non-pack, then directly convert
       // the remaining arguments, because we don't know what parameters they'll
       // match up with.
@@ -5728,14 +5735,10 @@ bool Sema::CheckTemplateArgumentList(
   // pack expansions; they might be empty. This can happen even if
   // PartialTemplateArgs is false (the list of arguments is complete but
   // still dependent).
-  if (ArgIdx < NumArgs && CurrentInstantiationScope &&
-      CurrentInstantiationScope->getPartiallySubstitutedPack()) {
-    while (ArgIdx < NumArgs &&
-           NewArgs[ArgIdx].getArgument().isPackExpansion()) {
-      const TemplateArgument &Arg = NewArgs[ArgIdx++].getArgument();
-      SugaredConverted.push_back(Arg);
-      CanonicalConverted.push_back(Context.getCanonicalTemplateArgument(Arg));
-    }
+  while (ArgIdx < NumArgs && NewArgs[ArgIdx].getArgument().isPackExpansion()) {
+    const TemplateArgument &Arg = NewArgs[ArgIdx++].getArgument();
+    SugaredConverted.push_back(Arg);
+    CanonicalConverted.push_back(Context.getCanonicalTemplateArgument(Arg));
   }
 
   // If we have any leftover arguments, then there were too many arguments.
@@ -7327,64 +7330,46 @@ bool Sema::CheckTemplateTemplateArgument(TemplateTemplateParmDecl *Param,
       << Template;
   }
 
+  if (!getLangOpts().RelaxedTemplateTemplateArgs)
+    return !TemplateParameterListsAreEqual(
+        Template->getTemplateParameters(), Params, /*Complain=*/true,
+        TPL_TemplateTemplateArgumentMatch, Arg.getLocation());
+
   // C++1z [temp.arg.template]p3: (DR 150)
   //   A template-argument matches a template template-parameter P when P
   //   is at least as specialized as the template-argument A.
-  if (getLangOpts().RelaxedTemplateTemplateArgs) {
-    // Quick check for the common case:
-    //   If P contains a parameter pack, then A [...] matches P if each of A's
-    //   template parameters matches the corresponding template parameter in
-    //   the template-parameter-list of P.
-    if (TemplateParameterListsAreEqual(
-            Template->getTemplateParameters(), Params, false,
-            TPL_TemplateTemplateArgumentMatch, Arg.getLocation()) &&
-        // If the argument has no associated constraints, then the parameter is
-        // definitely at least as specialized as the argument.
-        // Otherwise - we need a more thorough check.
-        !Template->hasAssociatedConstraints())
-      return false;
-
-    if (isTemplateTemplateParameterAtLeastAsSpecializedAs(
-            Params, Template, DefaultArgs, Arg.getLocation(), IsDeduced)) {
-      // P2113
-      // C++20[temp.func.order]p2
-      //   [...] If both deductions succeed, the partial ordering selects the
-      // more constrained template (if one exists) as determined below.
-      SmallVector<const Expr *, 3> ParamsAC, TemplateAC;
-      Params->getAssociatedConstraints(ParamsAC);
-      // C++2a[temp.arg.template]p3
-      //   [...] In this comparison, if P is unconstrained, the constraints on A
-      //   are not considered.
-      if (ParamsAC.empty())
-        return false;
+  if (!isTemplateTemplateParameterAtLeastAsSpecializedAs(
+          Params, Param, Template, DefaultArgs, Arg.getLocation(), IsDeduced))
+    return true;
+  // P2113
+  // C++20[temp.func.order]p2
+  //   [...] If both deductions succeed, the partial ordering selects the
+  // more constrained template (if one exists) as determined below.
+  SmallVector<const Expr *, 3> ParamsAC, TemplateAC;
+  Params->getAssociatedConstraints(ParamsAC);
+  // C++20[temp.arg.template]p3
+  //   [...] In this comparison, if P is unconstrained, the constraints on A
+  //   are not considered.
+  if (ParamsAC.empty())
+    return false;
 
-      Template->getAssociatedConstraints(TemplateAC);
+  Template->getAssociatedConstraints(TemplateAC);
 
-      bool IsParamAtLeastAsConstrained;
-      if (IsAtLeastAsConstrained(Param, ParamsAC, Template, TemplateAC,
-                                 IsParamAtLeastAsConstrained))
-        return true;
-      if (!IsParamAtLeastAsConstrained) {
-        Diag(Arg.getLocation(),
-             diag::err_template_template_parameter_not_at_least_as_constrained)
-            << Template << Param << Arg.getSourceRange();
-        Diag(Param->getLocation(), diag::note_entity_declared_at) << Param;
-        Diag(Template->getLocation(), diag::note_entity_declared_at)
-            << Template;
-        MaybeEmitAmbiguousAtomicConstraintsDiagnostic(Param, ParamsAC, Template,
-                                                      TemplateAC);
-        return true;
-      }
-      return false;
-    }
-    // FIXME: Produce better diagnostics for deduction failures.
+  bool IsParamAtLeastAsConstrained;
+  if (IsAtLeastAsConstrained(Param, ParamsAC, Template, TemplateAC,
+                             IsParamAtLeastAsConstrained))
+    return true;
+  if (!IsParamAtLeastAsConstrained) {
+    Diag(Arg.getLocation(),
+         diag::err_template_template_parameter_not_at_least_as_constrained)
+        << Template << Param << Arg.getSourceRange();
+    Diag(Param->getLocation(), diag::note_entity_declared_at) << Param;
+    Diag(Template->getLocation(), diag::note_entity_declared_at) << Template;
+    MaybeEmitAmbiguousAtomicConstraintsDiagnostic(Param, ParamsAC, Template,
+                                                  TemplateAC);
+    return true;
   }
-
-  return !TemplateParameterListsAreEqual(Template->getTemplateParameters(),
-                                         Params,
-                                         true,
-                                         TPL_TemplateTemplateArgumentMatch,
-                                         Arg.getLocation());
+  return false;
 }
 
 static Sema::SemaDiagnosticBuilder noteLocation(Sema &S, const NamedDecl &Decl,
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index cc095ae67ac40..ce3317db5a820 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -145,7 +145,9 @@ static TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch(
     PartialOrderingKind POK, bool DeducedFromArrayBound,
     bool *HasDeducedAnyParam);
 
-enum class PackFold { ParameterToArgument, ArgumentToParameter };
+/// What directions packs are allowed to match non-packs.
+enum class PackFold { ParameterToArgument, ArgumentToParameter, Both };
+
 static TemplateDeductionResult
 DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
                         ArrayRef<TemplateArgument> Ps,
@@ -440,18 +442,18 @@ DeduceNonTypeTemplateArgument(Sema &S, TemplateParameterList *TemplateParams,
 
   // FIXME: It's not clear how deduction of a parameter of reference
   // type from an argument (of non-reference type) should be performed.
-  // For now, we just remove reference types from both sides and let
-  // the final check for matching types sort out the mess.
-  ValueType = ValueType.getNonReferenceType();
-  if (ParamType->isReferenceType())
-    ParamType = ParamType.getNonReferenceType();
-  else
-    // Top-level cv-qualifiers are irrelevant for a non-reference type.
-    ValueType = ValueType.getUnqualifiedType();
+  // For now, we just make the argument have same reference type as the
+  // parameter.
+  if (ParamType->isReferenceType() && !ValueType->isReferenceType()) {
+    if (ParamType->isRValueReferenceType())
+      ValueType = S.Context.getRValueReferenceType(ValueType);
+    else
+      ValueType = S.Context.getLValueReferenceType(ValueType);
+  }
 
   return DeduceTemplateArgumentsByTypeMatch(
       S, TemplateParams, ParamType, ValueType, Info, Deduced,
-      TDF_SkipNonDependent,
+      TDF_SkipNonDependent | TDF_IgnoreQualifiers,
       PartialOrdering ? PartialOrderingKind::NonCall
                       : PartialOrderingKind::None,
       /*ArrayBound=*/NewDeduced.wasDeducedFromArrayBound(), HasDeducedAnyParam);
@@ -1711,7 +1713,21 @@ static TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch(
     DeducedTemplateArgument Result =
         checkDeducedTemplateArguments(S.Context, Deduced[Index], NewDeduced);
     if (Result.isNull()) {
-      Info.Param = cast<TemplateTypeParmDecl>(TemplateParams->getParam(Index));
+      // We can also get inconsistencies when matching NTTP type.
+      switch (NamedDecl *Param = TemplateParams->getParam(Index);
+              Param->getKind()) {
+      case Decl::TemplateTypeParm:
+        Info.Param = cast<TemplateTypeParmDecl>(Param);
+        break;
+      case Decl::NonTypeTemplateParm:
+        Info.Param = cast<NonTypeTemplateParmDecl>(Param);
+        break;
+      case Decl::TemplateTemplateParm:
+        Info.Param = cast<TemplateTemplateParmDecl>(Param);
+        break;
+      default:
+        llvm_unreachable("unexpected kind");
+      }
       Info.FirstArg = Deduced[Index];
       Info.SecondArg = NewDeduced;
       return TemplateDeductionResult::Inconsistent;
@@ -2549,8 +2565,31 @@ DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
     if (const NonTypeTemplateParmDecl *NTTP =
             getDeducedParameterFromExpr(Info, P.getAsExpr())) {
       switch (A.getKind()) {
+      case TemplateArgument::Expression: {
+        const Expr *E = A.getAsExpr();
+        // When checking NTTP, if either the parameter or the argument is
+        // dependent, as there would be otherwise nothing to deduce, we force
+        // the argument to the parameter type using this dependent implicit
+        // cast, in order to maintain invariants. Now we can deduce the
+        // resulting type from the original type, and deduce the original type
+        // against the parameter we are checking.
+        if (const auto *ICE = dyn_cast<ImplicitCastExpr>(E);
+            ICE && ICE->getCastKind() == clang::CK_Dependent) {
+          E = ICE->getSubExpr();
+          if (auto Result = DeduceTemplateArgumentsByTypeMatch(
+                  S, TemplateParams, ICE->getType(), E->getType(), Info,
+                  Deduced, TDF_SkipNonDependent,
+                  PartialOrdering ? PartialOrderingKind::NonCall
+                                  : PartialOrderingKind::None,
+                  /*DeducedFromArrayBound=*/false, HasDeducedAnyParam);
+              Result != TemplateDeductionResult::Success)
+            return Result;
+        }
+        return DeduceNonTypeTemplateArgument(
+            S, TemplateParams, NTTP, DeducedTemplateArgument(A), E->getType(),
+            Info, PartialOrdering, Deduced, HasDeducedAnyParam);
+      }
       case TemplateArgument::Integral:
-      case TemplateArgument::Expression:
       case TemplateArgument::StructuralValue:
         return DeduceNonTypeTemplateArgument(
             S, TemplateParams, NTTP, DeducedTemplateArgument(A),
@@ -2639,50 +2678,75 @@ DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
                         SmallVectorImpl<DeducedTemplateArgument> &Deduced,
                         bool NumberOfArgumentsMustMatch, bool PartialOrdering,
                         PackFold PackFold, bool *HasDeducedAnyParam) {
-  if (PackFold == PackFold::ArgumentToParameter)
-    std::swap(Ps, As);
+  bool FoldPackParameter = PackFold == PackFold::ParameterToArgument ||
+                           PackFold == PackFold::Both,
+       FoldPackArgument = PackFold == PackFold::ArgumentToParameter ||
+                          PackFold == PackFold::Both;
+
   // C++0x [temp.deduct.type]p9:
   //   If the template argument list of P contains a pack expansion that is not
   //   the last template argument, the entire template argument list is a
   //   non-deduced context.
-  if (hasPackExpansionBeforeEnd(Ps))
+  if (FoldPackParameter && hasPackExpansionBeforeEnd(Ps))
+    return TemplateDeductionResult::Success;
+
+  if (FoldPackArgument && hasPackExpansionBeforeEnd(As))
     return TemplateDeductionResult::Success;
 
   // C++0x [temp.deduct.type]p9:
   //   If P has a form that contains <T> or <i>, then each argument Pi of the
   //   respective template argument list P is compared with the corresponding
   //   argument Ai of the corresponding template argument list of A.
-  unsigned ArgIdx = 0, ParamIdx = 0;
-  for (; hasTemplateArgumentForDeduction(Ps, ParamIdx); ++ParamIdx) {
-    const TemplateArgument &P = Ps[ParamIdx];
-    if (!P.isPackExpansion()) {
+  for (unsigned ArgIdx = 0, ParamIdx = 0; /**/; /**/) {
+    if (!hasTemplateArgumentForDeduction(Ps, ParamIdx))
+      return !FoldPackParameter && hasTemplateArgumentForDeduction(As, ArgIdx)
+                 ? TemplateDeductionResult::MiscellaneousDeductionFailure
+                 : TemplateDeductionResult::Success;
+
+    if (!Ps[ParamIdx].isPackExpansion()) {
       // The simple case: deduce template arguments by matching Pi and Ai.
 
       // Check whether we have enough arguments.
       if (!hasTemplateArgumentForDeduction(As, ArgIdx))
-        return NumberOfArgumentsMustMatch
+        return !FoldPackArgument && NumberOfArgumentsMustMatch
                    ? TemplateDeductionResult::MiscellaneousDeductionFailure
                    : TemplateDeductionResult::Success;
 
-      // C++1z [temp.deduct.type]p9:
-      //   During partial ordering, if Ai was originally a pack expansion [and]
-      //   Pi is not a pack expansion, template argument deduction fails.
-      if (As[ArgIdx].isPackExpansion())
-        return TemplateDeductionResult::MiscellaneousDeductionFailure;
+      if (As[ArgIdx].isPackExpansion()) {
+        // C++1z [temp.deduct.type]p9:
+        //   During partial ordering, if Ai was originally a pack expansion
+        //   [and] Pi is not a pack expansion, template argument deduction
+        //   fails.
+        if (!FoldPackArgument)
+          return TemplateDeductionResult::MiscellaneousDeductionFailure;
+
+        TemplateArgument Pattern = As[ArgIdx].getPackExpansionPattern();
+        for (;;) {
+          // Deduce template parameters from the pattern.
+          if (auto Result = DeduceTemplateArguments(
+                  S, TemplateParams, Ps[ParamIdx], Pattern, Info,
+                  PartialOrdering, Deduced, HasDeducedAnyParam);
+              Result != TemplateDeductionResult::Success)
+            return Result;
 
-      // Perform deduction for this Pi/Ai pair.
-      TemplateArgument Pi = P, Ai = As[ArgIdx];
-      if (PackFold == PackFold::ArgumentToParameter)
-        std::swap(Pi, Ai);
-      if (auto Result = DeduceTemplateArguments(S, TemplateParams, Pi, Ai, Info,
-                                                PartialOrdering, Deduced,
-                                                HasDeducedAnyParam);
-          Result != TemplateDeductionResult::Success)
-        return Result;
+          ++ParamIdx;
+          if (!hasTemplateArgumentForDeduction(Ps, ParamIdx))
+            return TemplateDeductionResult::Success;
+          if (Ps[ParamIdx].isPackExpansion())
+            break;
+        }
+      } else {
+        // Perform deduction for this Pi/Ai pair.
+        if (auto Result = DeduceTemplateArguments(
+                S, TemplateParams, Ps[ParamIdx], As[ArgIdx], Info,
+                PartialOrdering, Deduced, HasDeducedAnyParam);
+            Result != TemplateDeductionResult::Success)
+          return Result;
 
-      // Move to the next argument.
-      ++ArgIdx;
-      continue;
+        ++ArgIdx;
+        ++ParamIdx;
+        continue;
+      }
     }
 
     // The parameter is a pack expansion.
@@ -2692,7 +2756,7 @@ DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
     //   each remaining argument in the template argument list of A. Each
     //   comparison deduces template arguments for subsequent positions in the
     //   template parameter packs expanded by Pi.
-    TemplateArgument Pattern = P.getPackExpansionPattern();
+    TemplateArgument Pattern = Ps[ParamIdx].getPackExpansionPattern();
 
     // Prepare to deduce the packs within the pattern.
     PackDeductionScope PackScope(S, TemplateParams, Deduced, Info, Pattern);
@@ -2703,13 +2767,12 @@ DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
     for (; hasTemplateArgumentForDeduction(As, ArgIdx) &&
            PackScope.hasNextElement();
          ++ArgIdx) {
-      TemplateArgument Pi = Pattern, Ai = As[ArgIdx];
-      if (PackFold == PackFold::ArgumentToParameter)
-        std::swap(Pi, Ai);
+      if (!FoldPackParameter && !As[ArgIdx].isPackExpansion())
+        return TemplateDeductionResult::MiscellaneousDeductionFailure;
       // Deduce template arguments from the pattern.
-      if (auto Result = DeduceTemplateArguments(S, TemplateParams, Pi, Ai, Info,
-                                                PartialOrdering, Deduced,
-                                                HasDeducedAnyParam);
+      if (auto Result = DeduceTemplateArguments(
+              S, TemplateParams, Pattern, As[ArgIdx], Info, PartialOrdering,
+              Deduced, HasDeducedAnyParam);
           Result != TemplateDeductionResult::Success)
         return Result;
 
@@ -2718,12 +2781,8 @@ DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
 
     // Build argument packs for each of the parameter packs expanded by this
     // pack expansion.
-    if (auto Result = PackScope.finish();
-        Result != TemplateDeductionResult::Success)
-      return Result;
+    return PackScope.finish();
   }
-
-  return TemplateDeductionResult::Success;
 }
 
 TemplateDeductionResult Sema::DeduceTemplateArguments(
@@ -3302,7 +3361,6 @@ static TemplateDeductionResult FinishTemplateArgumentDeduction(
   // Unevaluated SFINAE context.
   EnterExpressionEvaluationContext Unevaluated(
       S, Sema::ExpressionEvaluationContext::Unevaluated);
-  Sema::SFINAETrap Trap(S);
 
   Sema::ContextRAII SavedContext(S, getAsDeclContextOrEnclosing(Template));
 
@@ -3319,20 +3377,41 @@ static TemplateDeductionResult FinishTemplateArgumentDeduction(
     return Result;
 
   // Check that we produced the correct argument list.
-  TemplateParameterList *TemplateParams = Template->getTemplateParameters();
-  for (unsigned I = 0, E = TemplateParams->size(); I != E; ++I) {
-    TemplateArgument InstArg = CanonicalBuilder[I];
-    if (!isSameTemplateArg(S.Context, TemplateArgs[I], InstArg, PartialOrdering,
-                           /*PackExpansionMatchesPack=*/true)) {
-      Info.Param = makeTemplateParameter(TemplateParams->getParam(I));
-      Info.FirstArg = TemplateArgs[I];
-      Info.SecondArg = InstArg;
-      return TemplateDeductionResult::NonDeducedMismatch;
+  for (ArrayRef<TemplateArgument> Ps = TemplateArgs, As = CanonicalBuilder;
+       !Ps.empty() && !As.empty();
+       /**/) {
+    TemplateArgument P = Ps.front(), A = As.front();
+    if (P.getKind() == TemplateArgument::Pack) {
+      assert(Ps.size() == 1 && "Pack not last element?");
+      Ps = P.getPackAsArray();
+      continue;
+    }
+    if (A.getKind() == TemplateArgument::Pack) {
+      assert(As.size() == 1 && "Pack not last element?");
+      As = A.getPackAsArray();
+      continue;
     }
-  }
 
-  if (Trap.hasErrorOccurred())
-    return TemplateDeductionResult::SubstitutionFailure;
+    if (P.isPackExpansion())
+      P = P.getPackExpansionPattern();
+    else
+      Ps = Ps.drop_front();
+    if (A.isPackExpansion())
+      A = A.getPackExpansionPattern();
+    else
+      As = As.drop_front();
+
+    if (isSameTemplateArg(S.Context, P, A, PartialOrdering))
+      continue;
+    unsigned I = As.end() == CanonicalBuilder.end()
+                     ? As.begin() - CanonicalBuilder.begin()
+                     : CanonicalBuilder.size() - 1;
+    Info.Param =
+        makeTemplateParameter(Template->getTemplateParameters()->getParam(I));
+    Info.FirstArg = P;
+    Info.SecondArg = A;
+    return TemplateDeductionResult::NonDeducedMismatch;
+  }
 
   if (!PartialOrdering) {
     if (auto Result = CheckDeducedArgumentConstraints(
@@ -3354,7 +3433,6 @@ static TemplateDeductionResult FinishTemplateArgumentDeduction(
   // Unevaluated SFINAE context.
   EnterExpressionEvaluationContext Unevaluated(
       S, Sema::ExpressionEvaluationContext::Unevaluated);
-  Sema::SFINAETrap Trap(S);
 
   Sema::ContextRAII SavedContext(S, getAsDeclContextOrEnclosing(TD));
 
@@ -3363,20 +3441,13 @@ static TemplateDeductionResult FinishTemplateArgumentDeduction(
   //   explicitly specified, template argument deduction fails.
   SmallVector<TemplateArgument, 4> SugaredBuilder, CanonicalBuilder;
   if (auto Result = ConvertDeducedTemplateArguments(
-          S, TD, /*IsPartialOrdering=*/false, Deduced, Info, SugaredBuilder,
+          S, TD, /*IsDeduced=*/false, Deduced, Info, SugaredBuilder,
           CanonicalBuilder);
       Result != TemplateDeductionResult::Success)
     return Result;
 
-  if (Trap.hasErrorOccurred())
-    return TemplateDeductionResult::SubstitutionFailure;
-
-  if (auto Result = CheckDeducedArgumentConstraints(S, TD, SugaredBuilder,
-                                                    CanonicalBuilder, Info);
-      Result != TemplateDeductionResult::Success)
-    return Result;
-
-  return TemplateDeductionResult::Success;
+  return ::CheckDeducedArgumentConstraints(S, TD, SugaredBuilder,
+                                           CanonicalBuilder, Info);
 }
 
 /// Perform template argument deduction to determine whether the given template
@@ -3423,16 +3494,20 @@ DeduceTemplateArguments(Sema &S, T *Partial,
   if (Inst.isInvalid())
     return TemplateDeductionResult::InstantiationDepth;
 
-  if (Trap.hasErrorOccurred())
-    return TemplateDeductionResult::SubstitutionFailure;
-
   TemplateDeductionResult Result;
   S.runWithSufficientStackSpace(Info.getLocation(), [&] {
     Result = ::FinishTemplateArgumentDeduction(S, Partial,
                                                /*IsPartialOrdering=*/false,
                                                TemplateArgs, Deduced, Info);
   });
-  return Result;
+
+  if (Result != TemplateDeductionResult::Success)
+    return Result;
+
+  if (Trap.hasErrorOccurred())
+    return TemplateDeductionResult::SubstitutionFailure;
+
+  return TemplateDeductionResult::Success;
 }
 
 TemplateDeductionResult
@@ -3488,14 +3563,18 @@ Sema::DeduceTemplateArgumentsFromType(TemplateDecl *TD, QualType FromType,
   if (Inst.isInvalid())
     return TemplateDeductionResult::InstantiationDepth;
 
-  if (Trap.hasErrorOccurred())
-    return TemplateDeductionResult::SubstitutionFailure;
-
   TemplateDeductionResult Result;
   runWithSufficientStackSpace(Info.getLocation(), [&] {
     Result = ::FinishTemplateArgumentDeduction(*this, TD, Deduced, Info);
   });
-  return Result;
+
+  if (Result != TemplateDeductionResult::Success)
+    return Result;
+
+  if (Trap.hasErrorOccurred())
+    return TemplateDeductionResult::SubstitutionFailure;
+
+  return TemplateDeductionResult::Success;
 }
 
 /// Determine whether the given type T is a simple-template-id type.
@@ -6136,14 +6215,23 @@ static bool isAtLeastAsSpecializedAs(Sema &S, QualType T1, QualType T2,
     return false;
 
   const auto *TST1 = cast<TemplateSpecializationType>(T1);
-  bool AtLeastAsSpecialized;
+
+  Sema::SFINAETrap Trap(S);
+
+  TemplateDeductionResult Result;
   S.runWithSufficientStackSpace(Info.getLocation(), [&] {
-    AtLeastAsSpecialized =
-        FinishTemplateArgumentDeduction(
-            S, P2, /*IsPartialOrdering=*/true, TST1->template_arguments(),
-            Deduced, Info) == TemplateDeductionResult::Success;
+    Result = ::FinishTemplateArgumentDeduction(
+        S, P2, /*IsPartialOrdering=*/true, TST1->template_arguments(), Deduced,
+        Info);
   });
-  return AtLeastAsSpecialized;
+
+  if (Result != TemplateDeductionResult::Success)
+    return false;
+
+  if (Trap.hasErrorOccurred())
+    return false;
+
+  return true;
 }
 
 namespace {
@@ -6381,8 +6469,9 @@ bool Sema::isMoreSpecializedThanPrimary(
 }
 
 bool Sema::isTemplateTemplateParameterAtLeastAsSpecializedAs(
-    TemplateParameterList *P, TemplateDecl *AArg,
-    const DefaultArguments &DefaultArgs, SourceLocation Loc, bool IsDeduced) {
+    TemplateParameterList *P, TemplateDecl *PArg, TemplateDecl *AArg,
+    const DefaultArguments &DefaultArgs, SourceLocation ArgLoc,
+    bool IsDeduced) {
   // C++1z [temp.arg.template]p4: (DR 150)
   //   A template template-parameter P is at least as specialized as a
   //   template template-argument A if, given the following rewrite to two
@@ -6394,6 +6483,12 @@ bool Sema::isTemplateTemplateParameterAtLeastAsSpecializedAs(
   //
   TemplateParameterList *A = AArg->getTemplateParameters();
 
+  Sema::InstantiatingTemplate Inst(
+      *this, ArgLoc, Sema::InstantiatingTemplate::PartialOrderingTTP(), PArg,
+      SourceRange(P->getTemplateLoc(), P->getRAngleLoc()));
+  if (Inst.isInvalid())
+    return false;
+
   //   Given an invented class template X with the template parameter list of
   //   A (including default arguments):
   //    - Each function template has a single function parameter whose type is
@@ -6408,8 +6503,6 @@ bool Sema::isTemplateTemplateParameterAtLeastAsSpecializedAs(
   // templates.
   SmallVector<TemplateArgument, 4> PArgs;
   {
-    SFINAETrap Trap(*this);
-
     Context.getInjectedTemplateArgs(P, PArgs);
     TemplateArgumentListInfo PArgList(P->getLAngleLoc(),
                                       P->getRAngleLoc());
@@ -6429,18 +6522,17 @@ bool Sema::isTemplateTemplateParameterAtLeastAsSpecializedAs(
     // C++1z [temp.arg.template]p3:
     //   If the rewrite produces an invalid type, then P is not at least as
     //   specialized as A.
-    SmallVector<TemplateArgument, 4> SugaredPArgs;
-    if (CheckTemplateArgumentList(AArg, Loc, PArgList, DefaultArgs, false,
-                                  SugaredPArgs, PArgs,
+    SmallVector<TemplateArgument, 4> CanonicalPArgs;
+    if (CheckTemplateArgumentList(AArg, ArgLoc, PArgList, DefaultArgs, false,
+                                  PArgs, CanonicalPArgs,
                                   /*UpdateArgsWithConversions=*/true,
                                   /*ConstraintsNotSatisfied=*/nullptr,
-                                  /*PartialOrderTTP=*/true) ||
-        Trap.hasErrorOccurred())
+                                  /*PartialOrderingTTP=*/true))
       return false;
   }
 
   // Determine whether P1 is at least as specialized as P2.
-  TemplateDeductionInfo Info(Loc, A->getDepth());
+  TemplateDeductionInfo Info(ArgLoc, A->getDepth());
   SmallVector<DeducedTemplateArgument, 4> Deduced;
   Deduced.resize(A->size());
 
@@ -6455,29 +6547,89 @@ bool Sema::isTemplateTemplateParameterAtLeastAsSpecializedAs(
   //   be inverted between Ps and As. On non-deduced context, matching needs to
   //   happen both ways, according to [temp.arg.template]p3, but this is
   //   currently implemented as a special case elsewhere.
-  if (::DeduceTemplateArguments(*this, A, AArgs, PArgs, Info, Deduced,
-                                /*NumberOfArgumentsMustMatch=*/false,
-                                /*PartialOrdering=*/true,
-                                IsDeduced ? PackFold::ArgumentToParameter
-                                          : PackFold::ParameterToArgument,
-                                /*HasDeducedAnyParam=*/nullptr) !=
-      TemplateDeductionResult::Success)
+  switch (::DeduceTemplateArguments(
+      *this, A, AArgs, PArgs, Info, Deduced,
+      /*NumberOfArgumentsMustMatch=*/false, /*PartialOrdering=*/true,
+      IsDeduced ? PackFold::ArgumentToParameter : PackFold::Both,
+      /*HasDeducedAnyParam=*/nullptr)) {
+  case clang::TemplateDeductionResult::Success:
+    break;
+
+  case TemplateDeductionResult::MiscellaneousDeductionFailure:
+    Diag(AArg->getLocation(), diag::err_template_param_list_different_arity)
+        << (A->size() > P->size()) << /*isTemplateTemplateParameter=*/true
+        << SourceRange(A->getTemplateLoc(), P->getRAngleLoc());
     return false;
+  case TemplateDeductionResult::NonDeducedMismatch:
+    Diag(AArg->getLocation(), diag::err_non_deduced_mismatch)
+        << Info.FirstArg << Info.SecondArg;
+    return false;
+  case TemplateDeductionResult::Inconsistent:
+    Diag(getAsNamedDecl(Info.Param)->getLocation(),
+         diag::err_inconsistent_deduction)
+        << Info.FirstArg << Info.SecondArg;
+    return false;
+  case TemplateDeductionResult::AlreadyDiagnosed:
+    return false;
+
+  // None of these should happen for a plain deduction.
+  case TemplateDeductionResult::Invalid:
+  case TemplateDeductionResult::InstantiationDepth:
+  case TemplateDeductionResult::Incomplete:
+  case TemplateDeductionResult::IncompletePack:
+  case TemplateDeductionResult::Underqualified:
+  case TemplateDeductionResult::SubstitutionFailure:
+  case TemplateDeductionResult::DeducedMismatch:
+  case TemplateDeductionResult::DeducedMismatchNested:
+  case TemplateDeductionResult::TooManyArguments:
+  case TemplateDeductionResult::TooFewArguments:
+  case TemplateDeductionResult::InvalidExplicitArguments:
+  case TemplateDeductionResult::NonDependentConversionFailure:
+  case TemplateDeductionResult::ConstraintsNotSatisfied:
+  case TemplateDeductionResult::CUDATargetMismatch:
+    llvm_unreachable("Unexpected Result");
+  }
 
   SmallVector<TemplateArgument, 4> DeducedArgs(Deduced.begin(), Deduced.end());
-  Sema::InstantiatingTemplate Inst(*this, Info.getLocation(), AArg, DeducedArgs,
-                                   Info);
-  if (Inst.isInvalid())
-    return false;
 
-  bool AtLeastAsSpecialized;
+  TemplateDeductionResult TDK;
   runWithSufficientStackSpace(Info.getLocation(), [&] {
-    AtLeastAsSpecialized =
-        ::FinishTemplateArgumentDeduction(
-            *this, AArg, /*IsPartialOrdering=*/true, PArgs, Deduced, Info) ==
-        TemplateDeductionResult::Success;
+    TDK = ::FinishTemplateArgumentDeduction(
+        *this, AArg, /*IsPartialOrdering=*/true, PArgs, Deduced, Info);
   });
-  return AtLeastAsSpecialized;
+  switch (TDK) {
+  case TemplateDeductionResult::Success:
+    return true;
+
+  // It doesn't seem possible to get a non-deduced mismatch when partial
+  // ordering TTPs.
+  case TemplateDeductionResult::NonDeducedMismatch:
+    llvm_unreachable("Unexpected NonDeducedMismatch");
+
+  // Substitution failures should have already been diagnosed.
+  case TemplateDeductionResult::AlreadyDiagnosed:
+  case TemplateDeductionResult::SubstitutionFailure:
+  case TemplateDeductionResult::InstantiationDepth:
+    return false;
+
+  // None of these should happen when just converting deduced arguments.
+  case TemplateDeductionResult::Invalid:
+  case TemplateDeductionResult::Incomplete:
+  case TemplateDeductionResult::IncompletePack:
+  case TemplateDeductionResult::Inconsistent:
+  case TemplateDeductionResult::Underqualified:
+  case TemplateDeductionResult::DeducedMismatch:
+  case TemplateDeductionResult::DeducedMismatchNested:
+  case TemplateDeductionResult::TooManyArguments:
+  case TemplateDeductionResult::TooFewArguments:
+  case TemplateDeductionResult::InvalidExplicitArguments:
+  case TemplateDeductionResult::NonDependentConversionFailure:
+  case TemplateDeductionResult::ConstraintsNotSatisfied:
+  case TemplateDeductionResult::MiscellaneousDeductionFailure:
+  case TemplateDeductionResult::CUDATargetMismatch:
+    llvm_unreachable("Unexpected Result");
+  }
+  llvm_unreachable("Unexpected TDK");
 }
 
 namespace {
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index b36381422851f..898255ff7c6a3 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -572,6 +572,7 @@ bool Sema::CodeSynthesisContext::isInstantiationRecord() const {
   case LambdaExpressionSubstitution:
   case BuildingDeductionGuides:
   case TypeAliasTemplateInstantiation:
+  case PartialOrderingTTP:
     return false;
 
   // This function should never be called when Kind's value is Memoization.
@@ -804,6 +805,11 @@ Sema::InstantiatingTemplate::InstantiatingTemplate(
           SemaRef, CodeSynthesisContext::BuildingDeductionGuides,
           PointOfInstantiation, InstantiationRange, Entity) {}
 
+Sema::InstantiatingTemplate::InstantiatingTemplate(
+    Sema &SemaRef, SourceLocation ArgLoc, PartialOrderingTTP,
+    TemplateDecl *PArg, SourceRange InstantiationRange)
+    : InstantiatingTemplate(SemaRef, CodeSynthesisContext::PartialOrderingTTP,
+                            ArgLoc, InstantiationRange, PArg) {}
 
 void Sema::pushCodeSynthesisContext(CodeSynthesisContext Ctx) {
   Ctx.SavedInNonInstantiationSFINAEContext = InNonInstantiationSFINAEContext;
@@ -1243,6 +1249,14 @@ void Sema::PrintInstantiationStack() {
           << cast<TypeAliasTemplateDecl>(Active->Entity)
           << Active->InstantiationRange;
       break;
+    case CodeSynthesisContext::PartialOrderingTTP:
+      Diags.Report(Active->PointOfInstantiation,
+                   diag::note_template_arg_template_params_mismatch);
+      if (SourceLocation ParamLoc = Active->Entity->getLocation();
+          ParamLoc.isValid())
+        Diags.Report(ParamLoc, diag::note_template_prev_declaration)
+            << /*isTemplateTemplateParam=*/true << Active->InstantiationRange;
+      break;
     }
   }
 }
@@ -1285,6 +1299,7 @@ std::optional<TemplateDeductionInfo *> Sema::isSFINAEContext() const {
     case CodeSynthesisContext::PriorTemplateArgumentSubstitution:
     case CodeSynthesisContext::DefaultTemplateArgumentChecking:
     case CodeSynthesisContext::RewritingOperatorAsSpaceship:
+    case CodeSynthesisContext::PartialOrderingTTP:
       // A default template argument instantiation and substitution into
       // template parameters with arguments for prior parameters may or may
       // not be a SFINAE context; look further up the stack.
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 6fdb18d51acef..76efc9fe83185 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -4090,6 +4090,10 @@ class TreeTransform {
         OpenACCDirectiveKind::Loop, BeginLoc, DirLoc, EndLoc, Clauses, Loop);
   }
 
+  ExprResult RebuildOpenACCAsteriskSizeExpr(SourceLocation AsteriskLoc) {
+    return getSema().OpenACC().ActOnOpenACCAsteriskSizeExpr(AsteriskLoc);
+  }
+
 private:
   TypeLoc TransformTypeInObjectScope(TypeLoc TL,
                                      QualType ObjectType,
@@ -8200,6 +8204,11 @@ TreeTransform<Derived>::TransformWhileStmt(WhileStmt *S) {
   if (Cond.isInvalid())
     return StmtError();
 
+  // OpenACC Restricts a while-loop inside of certain construct/clause
+  // combinations, so diagnose that here in OpenACC mode.
+  SemaOpenACC::LoopInConstructRAII LCR{SemaRef.OpenACC()};
+  SemaRef.OpenACC().ActOnWhileStmt(S->getBeginLoc());
+
   // Transform the body
   StmtResult Body = getDerived().TransformStmt(S->getBody());
   if (Body.isInvalid())
@@ -8217,6 +8226,11 @@ TreeTransform<Derived>::TransformWhileStmt(WhileStmt *S) {
 template<typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformDoStmt(DoStmt *S) {
+  // OpenACC Restricts a do-loop inside of certain construct/clause
+  // combinations, so diagnose that here in OpenACC mode.
+  SemaOpenACC::LoopInConstructRAII LCR{SemaRef.OpenACC()};
+  SemaRef.OpenACC().ActOnDoStmt(S->getBeginLoc());
+
   // Transform the body
   StmtResult Body = getDerived().TransformStmt(S->getBody());
   if (Body.isInvalid())
@@ -8270,11 +8284,18 @@ TreeTransform<Derived>::TransformForStmt(ForStmt *S) {
   if (S->getInc() && !FullInc.get())
     return StmtError();
 
+  // OpenACC Restricts a for-loop inside of certain construct/clause
+  // combinations, so diagnose that here in OpenACC mode.
+  SemaOpenACC::LoopInConstructRAII LCR{SemaRef.OpenACC()};
+  SemaRef.OpenACC().ActOnForStmtBegin(S->getBeginLoc());
+
   // Transform the body
   StmtResult Body = getDerived().TransformStmt(S->getBody());
   if (Body.isInvalid())
     return StmtError();
 
+  SemaRef.OpenACC().ActOnForStmtEnd(S->getBeginLoc(), Body);
+
   if (!getDerived().AlwaysRebuild() &&
       Init.get() == S->getInit() &&
       Cond.get() == std::make_pair(S->getConditionVariable(), S->getCond()) &&
@@ -9008,10 +9029,17 @@ TreeTransform<Derived>::TransformCXXForRangeStmt(CXXForRangeStmt *S) {
     }
   }
 
+  // OpenACC Restricts a while-loop inside of certain construct/clause
+  // combinations, so diagnose that here in OpenACC mode.
+  SemaOpenACC::LoopInConstructRAII LCR{SemaRef.OpenACC()};
+  SemaRef.OpenACC().ActOnForStmtBegin(S->getBeginLoc());
+
   StmtResult Body = getDerived().TransformStmt(S->getBody());
   if (Body.isInvalid())
     return StmtError();
 
+  SemaRef.OpenACC().ActOnForStmtEnd(S->getBeginLoc(), Body);
+
   // Body has changed but we didn't rebuild the for-range statement. Rebuild
   // it now so we have a new statement to attach the body to.
   if (Body.get() != S->getBody() && NewStmt.get() == S) {
@@ -11847,6 +11875,36 @@ void OpenACCClauseTransform<Derived>::VisitCollapseClause(
       ParsedClause.getLParenLoc(), ParsedClause.isForce(),
       ParsedClause.getLoopCount(), ParsedClause.getEndLoc());
 }
+
+template <typename Derived>
+void OpenACCClauseTransform<Derived>::VisitTileClause(
+    const OpenACCTileClause &C) {
+
+  llvm::SmallVector<Expr *> TransformedExprs;
+
+  for (Expr *E : C.getSizeExprs()) {
+    ExprResult NewSizeExpr = Self.TransformExpr(E);
+
+    if (!NewSizeExpr.isUsable())
+      return;
+
+    NewSizeExpr = Self.getSema().OpenACC().ActOnIntExpr(
+        OpenACCDirectiveKind::Invalid, ParsedClause.getClauseKind(),
+        NewSizeExpr.get()->getBeginLoc(), NewSizeExpr.get());
+
+    NewSizeExpr = Self.getSema().OpenACC().CheckTileSizeExpr(NewSizeExpr.get());
+
+    if (!NewSizeExpr.isUsable())
+      return;
+    TransformedExprs.push_back(NewSizeExpr.get());
+  }
+
+  ParsedClause.setIntExprDetails(TransformedExprs);
+  NewClause = OpenACCTileClause::Create(
+      Self.getSema().getASTContext(), ParsedClause.getBeginLoc(),
+      ParsedClause.getLParenLoc(), ParsedClause.getIntExprs(),
+      ParsedClause.getEndLoc());
+}
 } // namespace
 template <typename Derived>
 OpenACCClause *TreeTransform<Derived>::TransformOpenACCClause(
@@ -11894,8 +11952,9 @@ StmtResult TreeTransform<Derived>::TransformOpenACCComputeConstruct(
     return StmtError();
 
   // Transform Structured Block.
-  SemaOpenACC::AssociatedStmtRAII AssocStmtRAII(getSema().OpenACC(),
-                                                C->getDirectiveKind());
+  SemaOpenACC::AssociatedStmtRAII AssocStmtRAII(
+      getSema().OpenACC(), C->getDirectiveKind(), C->clauses(),
+      TransformedClauses);
   StmtResult StrBlock = getDerived().TransformStmt(C->getStructuredBlock());
   StrBlock = getSema().OpenACC().ActOnAssociatedStmt(
       C->getBeginLoc(), C->getDirectiveKind(), StrBlock);
@@ -11920,8 +11979,9 @@ TreeTransform<Derived>::TransformOpenACCLoopConstruct(OpenACCLoopConstruct *C) {
     return StmtError();
 
   // Transform Loop.
-  SemaOpenACC::AssociatedStmtRAII AssocStmtRAII(getSema().OpenACC(),
-                                                C->getDirectiveKind());
+  SemaOpenACC::AssociatedStmtRAII AssocStmtRAII(
+      getSema().OpenACC(), C->getDirectiveKind(), C->clauses(),
+      TransformedClauses);
   StmtResult Loop = getDerived().TransformStmt(C->getLoop());
   Loop = getSema().OpenACC().ActOnAssociatedStmt(C->getBeginLoc(),
                                                  C->getDirectiveKind(), Loop);
@@ -11931,6 +11991,15 @@ TreeTransform<Derived>::TransformOpenACCLoopConstruct(OpenACCLoopConstruct *C) {
       TransformedClauses, Loop);
 }
 
+template <typename Derived>
+ExprResult TreeTransform<Derived>::TransformOpenACCAsteriskSizeExpr(
+    OpenACCAsteriskSizeExpr *E) {
+  if (getDerived().AlwaysRebuild())
+    return getDerived().RebuildOpenACCAsteriskSizeExpr(E->getLocation());
+  // Nothing can ever change, so there is never anything to transform.
+  return E;
+}
+
 //===----------------------------------------------------------------------===//
 // Expression transformation
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 0a4251c0e5240..97b79bd1381c0 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -3912,6 +3912,11 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
       FPPragmaOptions.swap(Record);
       break;
 
+    case DECLS_WITH_EFFECTS_TO_VERIFY:
+      for (unsigned I = 0, N = Record.size(); I != N; /*in loop*/)
+        DeclsWithEffectsToVerify.push_back(ReadDeclID(F, Record, I));
+      break;
+
     case OPENCL_EXTENSIONS:
       for (unsigned I = 0, E = Record.size(); I != E; ) {
         auto Name = ReadString(Record, I);
@@ -8413,6 +8418,17 @@ void ASTReader::InitializeSema(Sema &S) {
         NewOverrides.applyOverrides(SemaObj->getLangOpts());
   }
 
+  for (GlobalDeclID ID : DeclsWithEffectsToVerify) {
+    Decl *D = GetDecl(ID);
+    if (auto *FD = dyn_cast<FunctionDecl>(D))
+      SemaObj->addDeclWithEffects(FD, FD->getFunctionEffects());
+    else if (auto *BD = dyn_cast<BlockDecl>(D))
+      SemaObj->addDeclWithEffects(BD, BD->getFunctionEffects());
+    else
+      llvm_unreachable("unexpected Decl type in DeclsWithEffectsToVerify");
+  }
+  DeclsWithEffectsToVerify.clear();
+
   SemaObj->OpenCLFeatures = OpenCLExtensions;
 
   UpdateSema();
@@ -12290,6 +12306,15 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
     return OpenACCCollapseClause::Create(getContext(), BeginLoc, LParenLoc,
                                          HasForce, LoopCount, EndLoc);
   }
+  case OpenACCClauseKind::Tile: {
+    SourceLocation LParenLoc = readSourceLocation();
+    unsigned NumClauses = readInt();
+    llvm::SmallVector<Expr *> SizeExprs;
+    for (unsigned I = 0; I < NumClauses; ++I)
+      SizeExprs.push_back(readSubExpr());
+    return OpenACCTileClause::Create(getContext(), BeginLoc, LParenLoc,
+                                     SizeExprs, EndLoc);
+  }
 
   case OpenACCClauseKind::Finalize:
   case OpenACCClauseKind::IfPresent:
@@ -12306,7 +12331,6 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
   case OpenACCClauseKind::Bind:
   case OpenACCClauseKind::DeviceNum:
   case OpenACCClauseKind::DefaultAsync:
-  case OpenACCClauseKind::Tile:
   case OpenACCClauseKind::Gang:
   case OpenACCClauseKind::Invalid:
     llvm_unreachable("Clause serialization not yet implemented");
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 84743a52d4c8b..2038fe7829af9 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -575,6 +575,11 @@ void ASTStmtReader::VisitConstantExpr(ConstantExpr *E) {
   E->setSubExpr(Record.readSubExpr());
 }
 
+void ASTStmtReader::VisitOpenACCAsteriskSizeExpr(OpenACCAsteriskSizeExpr *E) {
+  VisitExpr(E);
+  E->setAsteriskLocation(readSourceLocation());
+}
+
 void ASTStmtReader::VisitSYCLUniqueStableNameExpr(SYCLUniqueStableNameExpr *E) {
   VisitExpr(E);
 
@@ -3054,6 +3059,10 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
       S = SYCLUniqueStableNameExpr::CreateEmpty(Context);
       break;
 
+    case EXPR_OPENACC_ASTERISK_SIZE:
+      S = OpenACCAsteriskSizeExpr::CreateEmpty(Context);
+      break;
+
     case EXPR_PREDEFINED:
       S = PredefinedExpr::CreateEmpty(
           Context,
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index aa9764e25c323..836532ca402ff 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -4642,6 +4642,17 @@ void ASTWriter::WriteFloatControlPragmaOptions(Sema &SemaRef) {
   Stream.EmitRecord(FLOAT_CONTROL_PRAGMA_OPTIONS, Record);
 }
 
+/// Write Sema's collected list of declarations with unverified effects.
+void ASTWriter::WriteDeclsWithEffectsToVerify(Sema &SemaRef) {
+  if (SemaRef.DeclsWithEffectsToVerify.empty())
+    return;
+  RecordData Record;
+  for (const auto *D : SemaRef.DeclsWithEffectsToVerify) {
+    AddDeclRef(D, Record);
+  }
+  Stream.EmitRecord(DECLS_WITH_EFFECTS_TO_VERIFY, Record);
+}
+
 void ASTWriter::WriteModuleFileExtension(Sema &SemaRef,
                                          ModuleFileExtensionWriter &Writer) {
   // Enter the extension block.
@@ -5599,6 +5610,7 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot,
   }
   WritePackPragmaOptions(SemaRef);
   WriteFloatControlPragmaOptions(SemaRef);
+  WriteDeclsWithEffectsToVerify(SemaRef);
 
   // Some simple statistics
   RecordData::value_type Record[] = {
@@ -8155,6 +8167,14 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
     AddStmt(const_cast<Expr *>(CC->getLoopCount()));
     return;
   }
+  case OpenACCClauseKind::Tile: {
+    const auto *TC = cast<OpenACCTileClause>(C);
+    writeSourceLocation(TC->getLParenLoc());
+    writeUInt32(TC->getSizeExprs().size());
+    for (Expr *E : TC->getSizeExprs())
+      AddStmt(E);
+    return;
+  }
 
   case OpenACCClauseKind::Finalize:
   case OpenACCClauseKind::IfPresent:
@@ -8171,7 +8191,6 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
   case OpenACCClauseKind::Bind:
   case OpenACCClauseKind::DeviceNum:
   case OpenACCClauseKind::DefaultAsync:
-  case OpenACCClauseKind::Tile:
   case OpenACCClauseKind::Gang:
   case OpenACCClauseKind::Invalid:
     llvm_unreachable("Clause serialization not yet implemented");
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index 837136600181c..64e4894dc29fb 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -643,6 +643,12 @@ void ASTStmtWriter::VisitConstantExpr(ConstantExpr *E) {
   Code = serialization::EXPR_CONSTANT;
 }
 
+void ASTStmtWriter::VisitOpenACCAsteriskSizeExpr(OpenACCAsteriskSizeExpr *E) {
+  VisitExpr(E);
+  Record.AddSourceLocation(E->getLocation());
+  Code = serialization::EXPR_OPENACC_ASTERISK_SIZE;
+}
+
 void ASTStmtWriter::VisitSYCLUniqueStableNameExpr(SYCLUniqueStableNameExpr *E) {
   VisitExpr(E);
 
diff --git a/clang/lib/Serialization/GlobalModuleIndex.cpp b/clang/lib/Serialization/GlobalModuleIndex.cpp
index 1163943c5dffa..9c48712a0b3fb 100644
--- a/clang/lib/Serialization/GlobalModuleIndex.cpp
+++ b/clang/lib/Serialization/GlobalModuleIndex.cpp
@@ -430,14 +430,13 @@ namespace {
 
     /// Retrieve the module file information for the given file.
     ModuleFileInfo &getModuleFileInfo(FileEntryRef File) {
-      auto Known = ModuleFiles.find(File);
-      if (Known != ModuleFiles.end())
-        return Known->second;
-
-      unsigned NewID = ModuleFiles.size();
-      ModuleFileInfo &Info = ModuleFiles[File];
-      Info.ID = NewID;
-      return Info;
+      auto [It, Inserted] = ModuleFiles.try_emplace(File);
+      if (Inserted) {
+        unsigned NewID = ModuleFiles.size();
+        ModuleFileInfo &Info = It->second;
+        Info.ID = NewID;
+      }
+      return It->second;
     }
 
   public:
diff --git a/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
index b198b1c2ff4d1..69d8e968283b3 100644
--- a/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
@@ -16,21 +16,93 @@
 
 #include "clang/Basic/Builtins.h"
 #include "clang/StaticAnalyzer/Checkers/BuiltinCheckerRegistration.h"
+#include "clang/StaticAnalyzer/Checkers/Taint.h"
 #include "clang/StaticAnalyzer/Core/Checker.h"
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/DynamicExtent.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/SVals.h"
 
 using namespace clang;
 using namespace ento;
+using namespace taint;
 
 namespace {
 
+QualType getSufficientTypeForOverflowOp(CheckerContext &C, const QualType &T) {
+  // Calling a builtin with a non-integer type result produces compiler error.
+  assert(T->isIntegerType());
+
+  ASTContext &ACtx = C.getASTContext();
+
+  unsigned BitWidth = ACtx.getIntWidth(T);
+  return ACtx.getIntTypeForBitwidth(BitWidth * 2, T->isSignedIntegerType());
+}
+
+QualType getOverflowBuiltinResultType(const CallEvent &Call) {
+  // Calling a builtin with an incorrect argument count produces compiler error.
+  assert(Call.getNumArgs() == 3);
+
+  return Call.getArgExpr(2)->getType()->getPointeeType();
+}
+
+QualType getOverflowBuiltinResultType(const CallEvent &Call, CheckerContext &C,
+                                      unsigned BI) {
+  // Calling a builtin with an incorrect argument count produces compiler error.
+  assert(Call.getNumArgs() == 3);
+
+  ASTContext &ACtx = C.getASTContext();
+
+  switch (BI) {
+  case Builtin::BI__builtin_smul_overflow:
+  case Builtin::BI__builtin_ssub_overflow:
+  case Builtin::BI__builtin_sadd_overflow:
+    return ACtx.IntTy;
+  case Builtin::BI__builtin_smull_overflow:
+  case Builtin::BI__builtin_ssubl_overflow:
+  case Builtin::BI__builtin_saddl_overflow:
+    return ACtx.LongTy;
+  case Builtin::BI__builtin_smulll_overflow:
+  case Builtin::BI__builtin_ssubll_overflow:
+  case Builtin::BI__builtin_saddll_overflow:
+    return ACtx.LongLongTy;
+  case Builtin::BI__builtin_umul_overflow:
+  case Builtin::BI__builtin_usub_overflow:
+  case Builtin::BI__builtin_uadd_overflow:
+    return ACtx.UnsignedIntTy;
+  case Builtin::BI__builtin_umull_overflow:
+  case Builtin::BI__builtin_usubl_overflow:
+  case Builtin::BI__builtin_uaddl_overflow:
+    return ACtx.UnsignedLongTy;
+  case Builtin::BI__builtin_umulll_overflow:
+  case Builtin::BI__builtin_usubll_overflow:
+  case Builtin::BI__builtin_uaddll_overflow:
+    return ACtx.UnsignedLongLongTy;
+  case Builtin::BI__builtin_mul_overflow:
+  case Builtin::BI__builtin_sub_overflow:
+  case Builtin::BI__builtin_add_overflow:
+    return getOverflowBuiltinResultType(Call);
+  default:
+    assert(false && "Unknown overflow builtin");
+    return ACtx.IntTy;
+  }
+}
+
 class BuiltinFunctionChecker : public Checker<eval::Call> {
 public:
   bool evalCall(const CallEvent &Call, CheckerContext &C) const;
+  void handleOverflowBuiltin(const CallEvent &Call, CheckerContext &C,
+                             BinaryOperator::Opcode Op,
+                             QualType ResultType) const;
+  const NoteTag *createBuiltinNoOverflowNoteTag(CheckerContext &C,
+                                                bool BothFeasible, SVal Arg1,
+                                                SVal Arg2, SVal Result) const;
+  const NoteTag *createBuiltinOverflowNoteTag(CheckerContext &C) const;
+  std::pair<bool, bool> checkOverflow(CheckerContext &C, SVal RetVal,
+                                      QualType Res) const;
 
 private:
   // From: clang/include/clang/Basic/Builtins.def
@@ -50,6 +122,102 @@ class BuiltinFunctionChecker : public Checker<eval::Call> {
 
 } // namespace
 
+const NoteTag *BuiltinFunctionChecker::createBuiltinNoOverflowNoteTag(
+    CheckerContext &C, bool BothFeasible, SVal Arg1, SVal Arg2,
+    SVal Result) const {
+  return C.getNoteTag([Result, Arg1, Arg2, BothFeasible](
+                          PathSensitiveBugReport &BR, llvm::raw_ostream &OS) {
+    if (!BR.isInteresting(Result))
+      return;
+
+    // Propagate interestingness to input argumets if result is interesting.
+    BR.markInteresting(Arg1);
+    BR.markInteresting(Arg2);
+
+    if (BothFeasible)
+      OS << "Assuming no overflow";
+  });
+}
+
+const NoteTag *
+BuiltinFunctionChecker::createBuiltinOverflowNoteTag(CheckerContext &C) const {
+  return C.getNoteTag([](PathSensitiveBugReport &BR,
+                         llvm::raw_ostream &OS) { OS << "Assuming overflow"; },
+                      /*isPrunable=*/true);
+}
+
+std::pair<bool, bool>
+BuiltinFunctionChecker::checkOverflow(CheckerContext &C, SVal RetVal,
+                                      QualType Res) const {
+  // Calling a builtin with a non-integer type result produces compiler error.
+  assert(Res->isIntegerType());
+
+  unsigned BitWidth = C.getASTContext().getIntWidth(Res);
+  bool IsUnsigned = Res->isUnsignedIntegerType();
+
+  auto MinValType = llvm::APSInt::getMinValue(BitWidth, IsUnsigned);
+  auto MaxValType = llvm::APSInt::getMaxValue(BitWidth, IsUnsigned);
+  nonloc::ConcreteInt MinVal{MinValType};
+  nonloc::ConcreteInt MaxVal{MaxValType};
+
+  SValBuilder &SVB = C.getSValBuilder();
+  ProgramStateRef State = C.getState();
+  SVal IsLeMax = SVB.evalBinOp(State, BO_LE, RetVal, MaxVal, Res);
+  SVal IsGeMin = SVB.evalBinOp(State, BO_GE, RetVal, MinVal, Res);
+
+  auto [MayNotOverflow, MayOverflow] =
+      State->assume(IsLeMax.castAs<DefinedOrUnknownSVal>());
+  auto [MayNotUnderflow, MayUnderflow] =
+      State->assume(IsGeMin.castAs<DefinedOrUnknownSVal>());
+
+  return {MayOverflow || MayUnderflow, MayNotOverflow && MayNotUnderflow};
+}
+
+void BuiltinFunctionChecker::handleOverflowBuiltin(const CallEvent &Call,
+                                                   CheckerContext &C,
+                                                   BinaryOperator::Opcode Op,
+                                                   QualType ResultType) const {
+  // Calling a builtin with an incorrect argument count produces compiler error.
+  assert(Call.getNumArgs() == 3);
+
+  ProgramStateRef State = C.getState();
+  SValBuilder &SVB = C.getSValBuilder();
+  const Expr *CE = Call.getOriginExpr();
+
+  SVal Arg1 = Call.getArgSVal(0);
+  SVal Arg2 = Call.getArgSVal(1);
+
+  SVal RetValMax = SVB.evalBinOp(State, Op, Arg1, Arg2,
+                                 getSufficientTypeForOverflowOp(C, ResultType));
+  SVal RetVal = SVB.evalBinOp(State, Op, Arg1, Arg2, ResultType);
+
+  auto [Overflow, NotOverflow] = checkOverflow(C, RetValMax, ResultType);
+  if (NotOverflow) {
+    ProgramStateRef StateNoOverflow =
+        State->BindExpr(CE, C.getLocationContext(), SVB.makeTruthVal(false));
+
+    if (auto L = Call.getArgSVal(2).getAs<Loc>()) {
+      StateNoOverflow =
+          StateNoOverflow->bindLoc(*L, RetVal, C.getLocationContext());
+
+      // Propagate taint if any of the argumets were tainted
+      if (isTainted(State, Arg1) || isTainted(State, Arg2))
+        StateNoOverflow = addTaint(StateNoOverflow, *L);
+    }
+
+    C.addTransition(
+        StateNoOverflow,
+        createBuiltinNoOverflowNoteTag(
+            C, /*BothFeasible=*/NotOverflow && Overflow, Arg1, Arg2, RetVal));
+  }
+
+  if (Overflow) {
+    C.addTransition(
+        State->BindExpr(CE, C.getLocationContext(), SVB.makeTruthVal(true)),
+        createBuiltinOverflowNoteTag(C));
+  }
+}
+
 bool BuiltinFunctionChecker::isBuiltinLikeFunction(
     const CallEvent &Call) const {
   const auto *FD = llvm::dyn_cast_or_null<FunctionDecl>(Call.getDecl());
@@ -82,10 +250,41 @@ bool BuiltinFunctionChecker::evalCall(const CallEvent &Call,
     return true;
   }
 
-  switch (FD->getBuiltinID()) {
+  unsigned BI = FD->getBuiltinID();
+
+  switch (BI) {
   default:
     return false;
-
+  case Builtin::BI__builtin_mul_overflow:
+  case Builtin::BI__builtin_smul_overflow:
+  case Builtin::BI__builtin_smull_overflow:
+  case Builtin::BI__builtin_smulll_overflow:
+  case Builtin::BI__builtin_umul_overflow:
+  case Builtin::BI__builtin_umull_overflow:
+  case Builtin::BI__builtin_umulll_overflow:
+    handleOverflowBuiltin(Call, C, BO_Mul,
+                          getOverflowBuiltinResultType(Call, C, BI));
+    return true;
+  case Builtin::BI__builtin_sub_overflow:
+  case Builtin::BI__builtin_ssub_overflow:
+  case Builtin::BI__builtin_ssubl_overflow:
+  case Builtin::BI__builtin_ssubll_overflow:
+  case Builtin::BI__builtin_usub_overflow:
+  case Builtin::BI__builtin_usubl_overflow:
+  case Builtin::BI__builtin_usubll_overflow:
+    handleOverflowBuiltin(Call, C, BO_Sub,
+                          getOverflowBuiltinResultType(Call, C, BI));
+    return true;
+  case Builtin::BI__builtin_add_overflow:
+  case Builtin::BI__builtin_sadd_overflow:
+  case Builtin::BI__builtin_saddl_overflow:
+  case Builtin::BI__builtin_saddll_overflow:
+  case Builtin::BI__builtin_uadd_overflow:
+  case Builtin::BI__builtin_uaddl_overflow:
+  case Builtin::BI__builtin_uaddll_overflow:
+    handleOverflowBuiltin(Call, C, BO_Add,
+                          getOverflowBuiltinResultType(Call, C, BI));
+    return true;
   case Builtin::BI__builtin_assume:
   case Builtin::BI__assume: {
     assert (Call.getNumArgs() > 0);
diff --git a/clang/lib/StaticAnalyzer/Checkers/ErrnoChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ErrnoChecker.cpp
index 72fd6781a7561..beb3c8574b5fd 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ErrnoChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ErrnoChecker.cpp
@@ -42,7 +42,6 @@ class ErrnoChecker
                      ArrayRef<const MemRegion *> ExplicitRegions,
                      ArrayRef<const MemRegion *> Regions,
                      const LocationContext *LCtx, const CallEvent *Call) const;
-  void checkBranchCondition(const Stmt *Condition, CheckerContext &Ctx) const;
 
   /// Indicates if a read (load) of \c errno is allowed in a non-condition part
   /// of \c if, \c switch, loop and conditional statements when the errno
diff --git a/clang/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp
index e7fde3edc7f9e..f7fd92db7757e 100644
--- a/clang/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp
@@ -43,6 +43,12 @@ void FixedAddressChecker::checkPreStmt(const BinaryOperator *B,
   if (!T->isPointerType())
     return;
 
+  // Omit warning if the RHS has already pointer type. Without this passing
+  // around one fixed value in several pointer variables would produce several
+  // redundant warnings.
+  if (B->getRHS()->IgnoreParenCasts()->getType()->isPointerType())
+    return;
+
   SVal RV = C.getSVal(B->getRHS());
 
   if (!RV.isConstant() || RV.isZeroConstant())
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index fdabba46992b0..43ab646d398b3 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1966,6 +1966,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::OMPArrayShapingExprClass:
     case Stmt::OMPIteratorExprClass:
     case Stmt::SYCLUniqueStableNameExprClass:
+    case Stmt::OpenACCAsteriskSizeExprClass:
     case Stmt::TypeTraitExprClass: {
       Bldr.takeNodes(Pred);
       ExplodedNodeSet preVisit;
@@ -3810,7 +3811,9 @@ void ExprEngine::VisitGCCAsmStmt(const GCCAsmStmt *A, ExplodedNode *Pred,
     assert(!isa<NonLoc>(X)); // Should be an Lval, or unknown, undef.
 
     if (std::optional<Loc> LV = X.getAs<Loc>())
-      state = state->bindLoc(*LV, UnknownVal(), Pred->getLocationContext());
+      state = state->invalidateRegions(*LV, A, currBldrCtx->blockCount(),
+                                       Pred->getLocationContext(),
+                                       /*CausedByPointerEscape=*/true);
   }
 
   // Do not reason about locations passed inside inline assembly.
@@ -3818,7 +3821,9 @@ void ExprEngine::VisitGCCAsmStmt(const GCCAsmStmt *A, ExplodedNode *Pred,
     SVal X = state->getSVal(I, Pred->getLocationContext());
 
     if (std::optional<Loc> LV = X.getAs<Loc>())
-      state = state->bindLoc(*LV, UnknownVal(), Pred->getLocationContext());
+      state = state->invalidateRegions(*LV, A, currBldrCtx->blockCount(),
+                                       Pred->getLocationContext(),
+                                       /*CausedByPointerEscape=*/true);
   }
 
   Bldr.generateNode(A, Pred, state);
diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
index 677f426590ab9..77f9d07175c2c 100644
--- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
+++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
@@ -549,7 +549,7 @@ void ModuleDepCollectorPP::EndOfMainFile() {
     auto It = MDC.ModularDeps.find(M);
     // Only report direct dependencies that were successfully handled.
     if (It != MDC.ModularDeps.end())
-      MDC.Consumer.handleDirectModuleDependency(MDC.ModularDeps[M]->ID);
+      MDC.Consumer.handleDirectModuleDependency(It->second->ID);
   }
 
   for (auto &&I : MDC.FileDeps)
diff --git a/clang/test/APINotes/Inputs/Frameworks/SimpleKit.framework/Headers/SimpleKit.apinotes b/clang/test/APINotes/Inputs/Frameworks/SimpleKit.framework/Headers/SimpleKit.apinotes
index ef6e44c51c21c..f51811354eb00 100644
--- a/clang/test/APINotes/Inputs/Frameworks/SimpleKit.framework/Headers/SimpleKit.apinotes
+++ b/clang/test/APINotes/Inputs/Frameworks/SimpleKit.framework/Headers/SimpleKit.apinotes
@@ -46,3 +46,5 @@ Tags:
     SwiftName:       SuccessfullyRenamedA
   - Name:            RenamedAgainInAPINotesB
     SwiftName:       SuccessfullyRenamedB
+  - Name:            AnonEnumWithTypedefName
+    SwiftName:       SuccessfullyRenamedC
diff --git a/clang/test/APINotes/Inputs/Frameworks/SimpleKit.framework/Headers/SimpleKit.h b/clang/test/APINotes/Inputs/Frameworks/SimpleKit.framework/Headers/SimpleKit.h
index bd73926e9d6af..7342c3f83141b 100644
--- a/clang/test/APINotes/Inputs/Frameworks/SimpleKit.framework/Headers/SimpleKit.h
+++ b/clang/test/APINotes/Inputs/Frameworks/SimpleKit.framework/Headers/SimpleKit.h
@@ -27,3 +27,7 @@ void *getCFAuditedToNone_DUMP(void);
 - (id)getOwnedToUnowned __attribute__((__ns_returns_retained__));
 - (id)getUnownedToOwned __attribute__((__ns_returns_not_retained__));
 @end
+
+typedef enum {
+  kConstantInAnonEnum
+} AnonEnumWithTypedefName;
diff --git a/clang/test/APINotes/types.m b/clang/test/APINotes/types.m
index 133d504713d76..752f102643284 100644
--- a/clang/test/APINotes/types.m
+++ b/clang/test/APINotes/types.m
@@ -7,6 +7,9 @@
 
 // CHECK: struct __attribute__((swift_name("SuccessfullyRenamedA"))) RenamedAgainInAPINotesA {
 // CHECK: struct __attribute__((swift_name("SuccessfullyRenamedB"))) RenamedAgainInAPINotesB {
+// CHECK: typedef enum __attribute__((swift_name("SuccessfullyRenamedC"))) {
+// CHECK-NEXT: kConstantInAnonEnum
+// CHECK-NEXT: } AnonEnumWithTypedefName
 
 void test(OverriddenTypes *overridden) {
   int *ip1 = global_int_ptr; // expected-warning{{incompatible pointer types initializing 'int *' with an expression of type 'double (*)(int, int)'}}
diff --git a/clang/test/APINotes/yaml-roundtrip-2.test b/clang/test/APINotes/yaml-roundtrip-2.test
index b0b777b595060..63717bda7c099 100644
--- a/clang/test/APINotes/yaml-roundtrip-2.test
+++ b/clang/test/APINotes/yaml-roundtrip-2.test
@@ -7,5 +7,5 @@ REQUIRES: shell
 
 We expect only the document markers to be emitted
 
-CHECK: 50d
+CHECK: 52d
 CHECK: 1d
diff --git a/clang/test/AST/HLSL/ArrayAssignable.hlsl b/clang/test/AST/HLSL/ArrayAssignable.hlsl
new file mode 100644
index 0000000000000..b32d478983edf
--- /dev/null
+++ b/clang/test/AST/HLSL/ArrayAssignable.hlsl
@@ -0,0 +1,106 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -x hlsl -ast-dump %s | FileCheck %s
+
+// CHECK-LABEL: arr_assign1
+// CHECK: BinaryOperator 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue '='
+// CHECK: DeclRefExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue Var 0x{{[0-9a-f]+}} 'Arr' 'int[2]'
+// CHECK: DeclRefExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue Var 0x{{[0-9a-f]+}} 'Arr2' 'int[2]'
+void arr_assign1() {
+  int Arr[2] = {0, 1};
+  int Arr2[2] = {0, 0};
+  Arr = Arr2;
+}
+
+// CHECK-LABEL: arr_assign2
+// CHECK: BinaryOperator 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue '='
+// CHECK: DeclRefExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue Var 0x{{[0-9a-f]+}} 'Arr' 'int[2]'
+// CHECK: BinaryOperator 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue '='
+// CHECK: DeclRefExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue Var 0x{{[0-9a-f]+}} 'Arr2' 'int[2]'
+// CHECK: DeclRefExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue Var 0x{{[0-9a-f]+}} 'Arr3' 'int[2]'
+void arr_assign2() {
+  int Arr[2] = {0, 1};
+  int Arr2[2] = {0, 0};
+  int Arr3[2] = {2, 2};
+  Arr = Arr2 = Arr3;
+}
+
+// CHECK-LABEL: arr_assign3
+// CHECK: BinaryOperator 0x{{[0-9a-f]+}} {{.*}} 'int[2][2]' lvalue '='
+// CHECK: DeclRefExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2][2]' lvalue Var 0x{{[0-9a-f]+}} 'Arr' 'int[2][2]'
+// CHECK: DeclRefExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2][2]' lvalue Var 0x{{[0-9a-f]+}} 'Arr2' 'int[2][2]'
+void arr_assign3() {
+  int Arr[2][2] = {{0, 1}, {2, 3}};
+  int Arr2[2][2] = {{0, 0}, {1, 1}};
+  Arr = Arr2;
+}
+
+// CHECK-LABEL: arr_assign4
+// CHECK: BinaryOperator 0x{{[0-9a-f]+}} {{.*}} 'int' lvalue '='
+// CHECK: ArraySubscriptExpr 0x{{[0-9a-f]+}} {{.*}} 'int' lvalue
+// CHECK: ImplicitCastExpr 0x{{[0-9a-f]+}} {{.*}} 'int *' <ArrayToPointerDecay>
+// CHECK: ParenExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue
+// CHECK: BinaryOperator 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue '='
+// CHECK: DeclRefExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue Var 0x{{[0-9a-f]+}} 'Arr' 'int[2]'
+// CHECK: DeclRefExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue Var 0x{{[0-9a-f]+}} 'Arr2' 'int[2]'
+// CHECK: IntegerLiteral 0x{{[0-9a-f]+}} {{.*}} 'int' 0
+// CHECK: IntegerLiteral 0x{{[0-9a-f]+}} {{.*}} 'int' 6
+void arr_assign4() {
+  int Arr[2] = {0, 1};
+  int Arr2[2] = {0, 0};
+  (Arr = Arr2)[0] = 6;
+}
+
+// CHECK-LABEL: arr_assign5
+// CHECK: BinaryOperator 0x{{[0-9a-f]+}} {{.*}} 'int' lvalue '='
+// CHECK: ArraySubscriptExpr 0x{{[0-9a-f]+}} {{.*}} 'int' lvalue
+// CHECK: ImplicitCastExpr 0x{{[0-9a-f]+}} {{.*}} 'int *' <ArrayToPointerDecay>
+// CHECK: ParenExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue
+// CHECK: BinaryOperator 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue '='
+// CHECK: DeclRefExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue Var 0x{{[0-9a-f]+}} 'Arr' 'int[2]'
+// CHECK: BinaryOperator 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue '='
+// CHECK: DeclRefExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue Var 0x{{[0-9a-f]+}} 'Arr2' 'int[2]'
+// CHECK: DeclRefExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue Var 0x{{[0-9a-f]+}} 'Arr3' 'int[2]'
+// CHECK: IntegerLiteral 0x{{[0-9a-f]+}} {{.*}} 'int' 0
+// CHECK: IntegerLiteral 0x{{[0-9a-f]+}} {{.*}} 'int' 6
+void arr_assign5() {
+  int Arr[2] = {0, 1};
+  int Arr2[2] = {0, 0};
+  int Arr3[2] = {3, 4};
+  (Arr = Arr2 = Arr3)[0] = 6;
+}
+
+// CHECK-LABEL: arr_assign6
+// CHECK: BinaryOperator 0x{{[0-9a-f]+}} {{.*}} 'int' lvalue '='
+// CHECK: ArraySubscriptExpr 0x{{[0-9a-f]+}} {{.*}} 'int' lvalue
+// CHECK: ImplicitCastExpr 0x{{[0-9a-f]+}} {{.*}} 'int *' <ArrayToPointerDecay>
+// CHECK: ArraySubscriptExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue
+// CHECK: ImplicitCastExpr 0x{{[0-9a-f]+}} {{.*}} 'int (*)[2]' <ArrayToPointerDecay>
+// CHECK: ParenExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2][2]' lvalue
+// CHECK: BinaryOperator 0x{{[0-9a-f]+}} {{.*}} 'int[2][2]' lvalue '='
+// CHECK: DeclRefExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2][2]' lvalue Var 0x{{[0-9a-f]+}} 'Arr' 'int[2][2]'
+// CHECK: DeclRefExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2][2]' lvalue Var 0x{{[0-9a-f]+}} 'Arr2' 'int[2][2]'
+// CHECK: IntegerLiteral 0x{{[0-9a-f]+}} {{.*}} 'int' 0
+// CHECK: IntegerLiteral 0x{{[0-9a-f]+}} {{.*}} 'int' 0
+// CHECK: IntegerLiteral 0x{{[0-9a-f]+}} {{.*}} 'int' 6
+void arr_assign6() {
+  int Arr[2][2] = {{0, 1}, {2, 3}};
+  int Arr2[2][2] = {{0, 0}, {1, 1}};
+  (Arr = Arr2)[0][0] = 6;
+}
+
+// CHECK-LABEL: arr_assign7
+// CHECK: BinaryOperator 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue '='
+// CHECK: ArraySubscriptExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2]' lvalue
+// CHECK: ImplicitCastExpr 0x{{[0-9a-f]+}} {{.*}} 'int (*)[2]' <ArrayToPointerDecay>
+// CHECK: ParenExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2][2]' lvalue
+// CHECK: BinaryOperator 0x{{[0-9a-f]+}} {{.*}} 'int[2][2]' lvalue '='
+// CHECK: DeclRefExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2][2]' lvalue Var 0x{{[0-9a-f]+}} 'Arr' 'int[2][2]'
+// CHECK: DeclRefExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2][2]' lvalue Var 0x{{[0-9a-f]+}} 'Arr2' 'int[2][2]'
+// CHECK: IntegerLiteral 0x{{[0-9a-f]+}} {{.*}} 'int' 0
+// CHECK: InitListExpr 0x{{[0-9a-f]+}} {{.*}} 'int[2]'
+// CHECK: IntegerLiteral 0x{{[0-9a-f]+}} {{.*}} 'int' 6
+// CHECK: IntegerLiteral 0x{{[0-9a-f]+}} {{.*}} 'int' 6
+void arr_assign7() {
+  int Arr[2][2] = {{0, 1}, {2, 3}};
+  int Arr2[2][2] = {{0, 0}, {1, 1}};
+  (Arr = Arr2)[0] = {6, 6};
+}
diff --git a/clang/test/AST/ast-print-openacc-loop-construct.cpp b/clang/test/AST/ast-print-openacc-loop-construct.cpp
index ae1f7964f019e..aee4591cab428 100644
--- a/clang/test/AST/ast-print-openacc-loop-construct.cpp
+++ b/clang/test/AST/ast-print-openacc-loop-construct.cpp
@@ -2,6 +2,7 @@
 
 struct SomeStruct{};
 
+constexpr int get_value() { return 1; }
 void foo() {
 // CHECK: #pragma acc loop
 // CHECK-NEXT: for (;;)
@@ -82,4 +83,16 @@ void foo() {
 #pragma acc loop collapse(force:2)
   for(;;)
     for(;;);
+
+// CHECK: #pragma acc loop tile(1, 3, *, get_value())
+// CHECK-NEXT: for (;;)
+// CHECK-NEXT: for (;;)
+// CHECK-NEXT: for (;;)
+// CHECK-NEXT: for (;;)
+// CHECK-NEXT: ;
+#pragma acc loop tile(1, 3, *, get_value())
+  for(;;)
+    for(;;)
+      for(;;)
+        for(;;);
 }
diff --git a/clang/test/AST/new-unknown-type.cpp b/clang/test/AST/new-unknown-type.cpp
new file mode 100644
index 0000000000000..e7dd3c90145d2
--- /dev/null
+++ b/clang/test/AST/new-unknown-type.cpp
@@ -0,0 +1,25 @@
+// RUN: %clang_cc1 -verify -ast-dump %s | FileCheck %s
+
+extern void foo(Unknown*); // expected-error {{unknown type name 'Unknown'}}
+
+namespace a {
+  void computeSomething() {
+    foo(new Unknown()); // expected-error {{unknown type name 'Unknown'}}
+    foo(new Unknown{}); // expected-error {{unknown type name 'Unknown'}}
+    foo(new Unknown);   // expected-error {{unknown type name 'Unknown'}}
+  }
+} // namespace a
+
+namespace b {
+  struct Bar{};
+} // namespace b
+
+// CHECK:      |-NamespaceDecl 0x{{[^ ]*}} <line:5:1, line:11:1> line:5:11 a
+// CHECK-NEXT: | `-FunctionDecl 0x{{[^ ]*}} <line:6:3, line:10:3> line:6:8 computeSomething 'void ()'
+// CHECK-NEXT: |   `-CompoundStmt 0x{{[^ ]*}} <col:27, line:10:3>
+// CHECK-NEXT: |-NamespaceDecl 0x{{[^ ]*}} <line:13:1, line:15:1> line:13:11 b
+// CHECK-NEXT: | `-CXXRecordDecl 0x{{[^ ]*}} <line:14:3, col:14> col:10 referenced struct Bar definition
+
+static b::Bar bar;
+// CHECK:      `-VarDecl 0x{{[^ ]*}} <line:23:1, col:15> col:15 bar 'b::Bar' static callinit
+// CHECK-NEXT:   `-CXXConstructExpr 0x{{[^ ]*}} <col:15> 'b::Bar' 'void () noexcept'
diff --git a/clang/test/Analysis/asm.cpp b/clang/test/Analysis/asm.cpp
index e0691dc4d794f..a795400ebbcaf 100644
--- a/clang/test/Analysis/asm.cpp
+++ b/clang/test/Analysis/asm.cpp
@@ -7,11 +7,10 @@ void clang_analyzer_dump_ptr(void *);
 
 int global;
 void testRValueOutput() {
-  int &ref = global;
-  ref = 1;
+  int origVal = global;
   __asm__("" : "=r"(((int)(global))));  // don't crash on rvalue output operand
-  clang_analyzer_eval(global == 1); // expected-warning{{UNKNOWN}}
-  clang_analyzer_eval(ref == 1);    // expected-warning{{UNKNOWN}}
+  int newVal = global; // Value "after" the invalidation.
+  clang_analyzer_eval(origVal == newVal); // expected-warning{{TRUE}} expected-warning{{FALSE}}
 }
 
 void *MyMemcpy(void *d, const void *s, const int n) {
@@ -40,7 +39,20 @@ void testInlineAsmMemcpyUninit(void)
 {
     int a[10], b[10] = {}, c;
     MyMemcpy(&a[1], &b[1], sizeof(b) - sizeof(b[1]));
-    c = a[0]; // expected-warning{{Assigned value is garbage or undefined}}
+    c = a[0]; // FIXME: should be warning about uninitialized value, but invalidateRegions() also
+              // invalidates super region.
+}
+
+void testInlineAsmMemcpyUninitLoop(const void *src, unsigned long len)
+{
+    int a[10], c;
+    unsigned long toCopy = sizeof(a) < len ? sizeof(a) : len;
+
+    MyMemcpy(a, src, toCopy);
+
+    // Use index 1, since before use of invalidateRegions in VisitGCCAsmStmt, engine bound unknown SVal only to
+    // first element.
+    c = a[1]; // no-warning
 }
 
 void testAsmWithVoidPtrArgument()
@@ -49,6 +61,6 @@ void testAsmWithVoidPtrArgument()
   clang_analyzer_dump(*(int *)globalVoidPtr); // expected-warning-re {{reg_${{[0-9]+}}<int Element{SymRegion{reg_${{[0-9]+}}<void * globalVoidPtr>},0 S64b,int}>}}
   clang_analyzer_dump_ptr(globalVoidPtr); // expected-warning-re {{&SymRegion{reg_${{[0-9]+}}<void * globalVoidPtr>}}}
   asm ("" : : "a"(globalVoidPtr)); // no crash
-  clang_analyzer_dump(*(int *)globalVoidPtr); // expected-warning {{Unknown}}
+  clang_analyzer_dump(*(int *)globalVoidPtr); // expected-warning {{derived_}}
   clang_analyzer_dump_ptr(globalVoidPtr); // expected-warning-re {{&SymRegion{reg_${{[0-9]+}}<void * globalVoidPtr>}}}
 }
diff --git a/clang/test/Analysis/builtin_overflow.c b/clang/test/Analysis/builtin_overflow.c
new file mode 100644
index 0000000000000..5c61795661d09
--- /dev/null
+++ b/clang/test/Analysis/builtin_overflow.c
@@ -0,0 +1,157 @@
+// RUN: %clang_analyze_cc1 -triple x86_64-unknown-unknown -verify %s \
+// RUN:   -analyzer-checker=core,debug.ExprInspection
+
+#define __UINT_MAX__ (__INT_MAX__ * 2U + 1U)
+#define __INT_MIN__  (-__INT_MAX__ - 1)
+
+void clang_analyzer_dump_int(int);
+void clang_analyzer_dump_long(long);
+void clang_analyzer_eval(int);
+void clang_analyzer_warnIfReached(void);
+
+void test_add_nooverflow(void)
+{
+   int res;
+
+   if (__builtin_add_overflow(10, 20, &res)) {
+     clang_analyzer_warnIfReached();
+     return;
+   }
+
+   clang_analyzer_dump_int(res); //expected-warning{{30 S32b}}
+}
+
+void test_add_overflow(void)
+{
+   int res;
+
+   if (__builtin_add_overflow(__INT_MAX__, 1, &res)) {
+     clang_analyzer_dump_int(res); //expected-warning{{1st function call argument is an uninitialized value}}
+     return;
+   }
+
+   clang_analyzer_warnIfReached();
+}
+
+void test_add_underoverflow(void)
+{
+   int res;
+
+   if (__builtin_add_overflow(__INT_MIN__, -1, &res)) {
+     clang_analyzer_dump_int(res); //expected-warning{{1st function call argument is an uninitialized value}}
+     return;
+   }
+
+   clang_analyzer_warnIfReached();
+}
+
+void test_sub_underflow(void)
+{
+   int res;
+
+   if (__builtin_sub_overflow(__INT_MIN__, 10, &res)) {
+     return;
+   }
+
+   clang_analyzer_warnIfReached();
+}
+
+void test_sub_overflow(void)
+{
+   int res;
+
+   if (__builtin_sub_overflow(__INT_MAX__, -1, &res)) {
+     return;
+   }
+
+   clang_analyzer_warnIfReached();
+}
+
+void test_sub_nooverflow(void)
+{
+   int res;
+
+   if (__builtin_sub_overflow(__INT_MAX__, 1, &res)) {
+     clang_analyzer_warnIfReached();
+     return;
+   }
+
+   clang_analyzer_dump_int(res); //expected-warning{{2147483646 S32b}}
+}
+
+void test_mul_overflow(void)
+{
+   int res;
+
+   if (__builtin_mul_overflow(__INT_MAX__, 2, &res)) {
+     return;
+   }
+
+   clang_analyzer_warnIfReached();
+}
+
+void test_mul_underflow(void)
+{
+   int res;
+
+   if (__builtin_mul_overflow(__INT_MIN__, -2, &res)) {
+     return;
+   }
+
+   clang_analyzer_warnIfReached();
+}
+
+void test_mul_nooverflow(void)
+{
+   int res;
+
+   if (__builtin_mul_overflow(10, -2, &res)) {
+     clang_analyzer_warnIfReached();
+     return;
+   }
+
+   clang_analyzer_dump_int(res); //expected-warning{{-20 S32b}}
+}
+
+void test_nooverflow_diff_types(void)
+{
+   long res;
+
+   // This is not an overflow, since result type is long.
+   if (__builtin_add_overflow(__INT_MAX__, 1, &res)) {
+     clang_analyzer_warnIfReached();
+     return;
+   }
+
+   clang_analyzer_dump_long(res); //expected-warning{{2147483648 S64b}}
+}
+
+void test_uaddll_overflow_contraints(unsigned long a, unsigned long b)
+{
+   unsigned long long res;
+
+   if (a != 10)
+     return;
+   if (b != 10)
+     return;
+
+   if (__builtin_uaddll_overflow(a, b, &res)) {
+     clang_analyzer_warnIfReached();
+     return;
+   }
+}
+
+void test_uadd_overflow_contraints(unsigned a, unsigned b)
+{
+   unsigned res;
+
+   if (a > 5)
+     return;
+   if (b != 10)
+     return;
+
+   if (__builtin_uadd_overflow(a, b, &res)) {
+     clang_analyzer_warnIfReached();
+     return;
+   }
+}
diff --git a/clang/test/Analysis/builtin_overflow_notes.c b/clang/test/Analysis/builtin_overflow_notes.c
new file mode 100644
index 0000000000000..20f333a4a6cca
--- /dev/null
+++ b/clang/test/Analysis/builtin_overflow_notes.c
@@ -0,0 +1,30 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core -analyzer-output text \
+// RUN:   -verify %s
+
+void test_no_overflow_note(int a, int b)
+{
+   int res;
+
+   if (__builtin_add_overflow(a, b, &res)) // expected-note {{Assuming no overflow}}
+                                           // expected-note@-1 {{Taking false branch}}
+     return;
+
+   if (res) { // expected-note {{Assuming 'res' is not equal to 0}}
+              // expected-note@-1 {{Taking true branch}}
+     int *ptr = 0; // expected-note {{'ptr' initialized to a null pointer value}}
+     int var = *(int *) ptr; //expected-warning {{Dereference of null pointer}}
+                             //expected-note@-1 {{Dereference of null pointer}}
+   }
+}
+
+void test_overflow_note(int a, int b)
+{
+   int res; // expected-note{{'res' declared without an initial value}}
+
+   if (__builtin_add_overflow(a, b, &res)) { // expected-note {{Assuming overflow}}
+                                             // expected-note@-1 {{Taking true branch}}
+     int var = res; // expected-warning{{Assigned value is garbage or undefined}}
+                    // expected-note@-1 {{Assigned value is garbage or undefined}}
+     return;
+   }
+}
diff --git a/clang/test/Analysis/out-of-bounds-diagnostics.c b/clang/test/Analysis/out-of-bounds-diagnostics.c
index 8ecad7036c331..65bc28f58276f 100644
--- a/clang/test/Analysis/out-of-bounds-diagnostics.c
+++ b/clang/test/Analysis/out-of-bounds-diagnostics.c
@@ -278,6 +278,21 @@ int *mallocRegion(void) {
   return mem;
 }
 
+int *custom_calloc(size_t a, size_t b) {
+  size_t res;
+
+  return __builtin_mul_overflow(a, b, &res) ? 0 : malloc(res);
+}
+
+int *mallocRegionOverflow(void) {
+  int *mem = (int*)custom_calloc(10, sizeof(int));
+
+  mem[20] = 10;
+  // expected-warning@-1 {{Out of bound access to memory after the end of the heap area}}
+  // expected-note@-2 {{Access of the heap area at index 20, while it holds only 10 'int' elements}}
+  return mem;
+}
+
 int *mallocRegionDeref(void) {
   int *mem = (int*)malloc(2*sizeof(int));
 
diff --git a/clang/test/Analysis/ptr-arith.c b/clang/test/Analysis/ptr-arith.c
index 020a500629230..b5ccd2c207ead 100644
--- a/clang/test/Analysis/ptr-arith.c
+++ b/clang/test/Analysis/ptr-arith.c
@@ -42,6 +42,10 @@ domain_port (const char *domain_b, const char *domain_e,
 void f4(void) {
   int *p;
   p = (int*) 0x10000; // expected-warning{{Using a fixed address is not portable because that address will probably not be valid in all environments or platforms}}
+
+  int *p1;
+  p1 = p; // no warning
+
   long x = 0x10100;
   x += 10;
   p = (int*) x; // expected-warning{{Using a fixed address is not portable because that address will probably not be valid in all environments or platforms}}
diff --git a/clang/test/Analysis/taint-tester.c b/clang/test/Analysis/taint-tester.c
index 479a96c92ecec..66d54eab19f39 100644
--- a/clang/test/Analysis/taint-tester.c
+++ b/clang/test/Analysis/taint-tester.c
@@ -196,3 +196,19 @@ void noCrashTest(void) {
     __builtin___memcpy_chk(pointer2, pointer1, 0, 0); // no-crash
   }
 }
+
+void builtin_overflow_test(void) {
+  int input, input2, res;
+
+  scanf("%d", &input);
+  scanf("%d", &input2);
+
+  if (__builtin_add_overflow(input, 10, &res)) // expected-warning + {{tainted}}
+    return;
+
+  if (__builtin_add_overflow(10, input, &res)) // expected-warning + {{tainted}}
+    return;
+
+  if (__builtin_add_overflow(input2, input, &res)) // expected-warning + {{tainted}}
+    return;
+}
diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p1.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p1.cpp
index 9e890204c78bd..752451f3d9749 100644
--- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p1.cpp
+++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p1.cpp
@@ -164,11 +164,12 @@ namespace {
 
 #if __cplusplus < 201402L
 namespace ImplicitConstexprDef {
-  struct A {
+  struct A { // #defined-here
     void f(); // expected-note {{member declaration does not match because it is not const qualified}}
   };
 
   constexpr void A::f() { } // expected-warning {{'constexpr' non-static member function will not be implicitly 'const' in C++14; add 'const' to avoid a change in behavior}}
                             // expected-error@-1 {{out-of-line definition of 'f' does not match any declaration in 'ImplicitConstexprDef::A'}}
+                            // expected-note@#defined-here {{defined here}}
 }
 #endif
diff --git a/clang/test/CXX/dcl.decl/dcl.meaning/dcl.fct/p8.cpp b/clang/test/CXX/dcl.decl/dcl.meaning/dcl.fct/p8.cpp
index 0454412229fad..32ad25e2b7054 100644
--- a/clang/test/CXX/dcl.decl/dcl.meaning/dcl.fct/p8.cpp
+++ b/clang/test/CXX/dcl.decl/dcl.meaning/dcl.fct/p8.cpp
@@ -1,8 +1,9 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s
 
-struct A { };
+struct A { }; // #defined-here
 A::A (enum { e1 }) {} // expected-error{{cannot be defined in a parameter}}
 void A::f(enum { e2 }) {} // expected-error{{cannot be defined in a parameter}}
 
 enum { e3 } A::g() { } // expected-error{{cannot be defined in the result type}} \
 // expected-error{{out-of-line definition}}
+// expected-note@#defined-here{{defined here}}
diff --git a/clang/test/CXX/dcl/dcl.fct/p17.cpp b/clang/test/CXX/dcl/dcl.fct/p17.cpp
index d7487233f5d5c..4a81875cb4f89 100644
--- a/clang/test/CXX/dcl/dcl.fct/p17.cpp
+++ b/clang/test/CXX/dcl/dcl.fct/p17.cpp
@@ -96,7 +96,7 @@ namespace unconstrained {
   // expected-error@-1{{no matching}}
 
   template<typename T>
-  struct S {
+  struct S { // #defined-here
     constexpr auto f1(auto x, T t) -> decltype(x + t);
 
     template<typename U>
@@ -110,6 +110,7 @@ namespace unconstrained {
   template<typename U>
   constexpr auto S<T>::f2(auto x, U u, T t) -> decltype(x + u + t) { return x + u + t; }
   // expected-error@-1 {{out-of-line definition of 'f2' does not match any declaration in 'S<T>'}}
+  // expected-note@#defined-here {{S defined here}}
 
   template<typename T>
   template<typename U>
diff --git a/clang/test/CXX/drs/cwg22xx.cpp b/clang/test/CXX/drs/cwg22xx.cpp
index 797c3ed8546ef..0614d4f34ad5e 100644
--- a/clang/test/CXX/drs/cwg22xx.cpp
+++ b/clang/test/CXX/drs/cwg22xx.cpp
@@ -102,12 +102,13 @@ namespace MultilevelSpecialization {
   // default argument -- how far back do we look when determining whether a
   // parameter was expanded from a pack?
   //   -- zygoloid 2020-06-02
-  template<typename ...T> struct B {
+  template<typename ...T> struct B { // #cwg2233-B
     template <T... V> void f(int i = 0, int (&... arr)[V]);
   };
   template<> template<int a, int b>
     void B<int, int>::f(int i, int (&arr1)[a], int (&arr2)[b]) {}
     // since-cxx11-error@-1 {{out-of-line definition of 'f' does not match any declaration in 'cwg2233::MultilevelSpecialization::B<int, int>'}}
+    //   since-cxx11-note@#cwg2233-B {{defined here}}
   template<> template<>
     void B<int, int>::f<1, 1>(int i, int (&arr1a)[1], int (&arr2a)[1]) {}
 }
diff --git a/clang/test/CXX/drs/cwg27xx.cpp b/clang/test/CXX/drs/cwg27xx.cpp
index 581e2af822d55..fb5c8b1d1fbf8 100644
--- a/clang/test/CXX/drs/cwg27xx.cpp
+++ b/clang/test/CXX/drs/cwg27xx.cpp
@@ -2,9 +2,9 @@
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -std=c++11 -pedantic-errors -verify=expected %s
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -std=c++14 -pedantic-errors -verify=expected %s
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -std=c++17 -pedantic-errors -verify=expected %s
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -std=c++20 -pedantic-errors -verify=expected %s
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -std=c++23 -pedantic-errors -verify=expected,since-cxx23 %s
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -std=c++2c -pedantic-errors -verify=expected,since-cxx23,since-cxx26 %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -std=c++20 -pedantic-errors -verify=expected,since-cxx20 %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -std=c++23 -pedantic-errors -verify=expected,since-cxx20,since-cxx23 %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -std=c++2c -pedantic-errors -verify=expected,since-cxx20,since-cxx23,since-cxx26 %s
 
 #if __cplusplus == 199711L
 #define static_assert(...) __extension__ _Static_assert(__VA_ARGS__)
@@ -29,6 +29,35 @@ namespace std {
 #endif
 } // namespace std
 
+namespace cwg2707 { // cwg2707: 20
+
+#if __cplusplus >= 202002L
+
+template <class T, unsigned N> struct A { // #cwg2707-A
+  T value[N];
+};
+
+template <typename... T>
+A(T...) -> A<int, sizeof...(T)> requires (sizeof...(T) == 2); // #cwg2707-guide-A
+
+// Brace elision is not allowed for synthesized CTAD guides if the array size
+// is value-dependent.
+// So this should pick up our explicit deduction guide.
+A a = {1, 2};
+
+A b = {3, 4, 5};
+// since-cxx20-error@-1 {{no viable constructor or deduction guide}}
+//   since-cxx20-note@#cwg2707-A {{candidate function template not viable}}
+//   since-cxx20-note@#cwg2707-A {{implicit deduction guide}}
+//   since-cxx20-note@#cwg2707-guide-A {{constraints not satisfied}}
+//   since-cxx20-note@#cwg2707-guide-A {{because 'sizeof...(T) == 2' (3 == 2) evaluated to false}}
+//   since-cxx20-note@#cwg2707-A {{candidate function template not viable}}
+//   since-cxx20-note@#cwg2707-A {{implicit deduction guide}}
+
+#endif
+
+} // namespace cwg2707
+
 namespace cwg2718 { // cwg2718: 2.7
 struct B {};
 struct D;
@@ -200,32 +229,3 @@ static_assert(false, f().s);
 // since-cxx26-error@-1 {{static assertion failed: Hello}}
 #endif
 } // namespace cwg2798
-
-namespace cwg2707 { // cwg2707: 20
-
-#if __cplusplus >= 202002L
-
-template <class T, unsigned N> struct A {
-  T value[N];
-};
-
-template <typename... T>
-A(T...) -> A<int, sizeof...(T)> requires (sizeof...(T) == 2);
-
-// Brace elision is not allowed for synthesized CTAD guides if the array size
-// is value-dependent.
-// So this should pick up our explicit deduction guide.
-A a = {1, 2};
-
-A b = {3, 4, 5};
-// expected-error@-1 {{no viable constructor or deduction guide}} \
-// expected-note@-13 {{candidate function template not viable}} \
-// expected-note@-13 {{implicit deduction guide}} \
-// expected-note@-8 {{constraints not satisfied}} \
-// expected-note@-8 {{because 'sizeof...(T) == 2' (3 == 2) evaluated to false}} \
-// expected-note@-13 {{candidate function template not viable}} \
-// expected-note@-13 {{implicit deduction guide}}
-
-#endif
-
-} // namespace cwg2707
diff --git a/clang/test/CXX/drs/cwg3xx.cpp b/clang/test/CXX/drs/cwg3xx.cpp
index 7157feed3f762..f20054c3701b1 100644
--- a/clang/test/CXX/drs/cwg3xx.cpp
+++ b/clang/test/CXX/drs/cwg3xx.cpp
@@ -597,9 +597,10 @@ namespace cwg336 { // cwg336: yes
         void mf2();
       };
     };
-    template<> template<class X> class A<int>::B {};
+    template<> template<class X> class A<int>::B {}; // #cwg336-B
     template<> template<> template<class T> void A<int>::B<double>::mf1(T t) {}
     // expected-error@-1 {{out-of-line definition of 'mf1' does not match any declaration in 'cwg336::Pre::A<int>::B<double>'}}
+    //   expected-note@#cwg336-B {{defined here}}
     template<class Y> template<> void A<Y>::B<double>::mf2() {}
     // expected-error@-1 {{nested name specifier 'A<Y>::B<double>::' for declaration does not refer into a class, class template or class template partial specialization}}
   }
@@ -758,7 +759,7 @@ namespace cwg347 { // cwg347: yes
     void g();
   };
 
-  struct derived : base {};
+  struct derived : base {}; // #cwg347-derived
 
   struct derived::nested {};
   // expected-error@-1 {{no struct named 'nested' in 'cwg347::derived'}}
@@ -766,8 +767,10 @@ namespace cwg347 { // cwg347: yes
   // expected-error@-1 {{no member named 'n' in 'cwg347::derived'}}
   void derived::f() {}
   // expected-error@-1 {{out-of-line definition of 'f' does not match any declaration in 'cwg347::derived'}}
+  //   expected-note@#cwg347-derived {{defined here}}
   void derived::g() {}
   // expected-error@-1 {{out-of-line definition of 'g' does not match any declaration in 'cwg347::derived'}}
+  //   expected-note@#cwg347-derived {{defined here}}
 }
 
 // cwg348: na
@@ -1009,18 +1012,20 @@ namespace cwg355 { struct ::cwg355_S s; }
 // cwg356: na
 
 namespace cwg357 { // cwg357: yes
-  template<typename T> struct A {
+  template<typename T> struct A { // #cwg357-A
     void f() const; // #cwg357-f
   };
   template<typename T> void A<T>::f() {}
   // expected-error@-1 {{out-of-line definition of 'f' does not match any declaration in 'A<T>'}}
+  //   expected-note@#cwg357-A {{defined here}}
   //   expected-note@#cwg357-f {{member declaration does not match because it is const qualified}}
 
-  struct B {
+  struct B { // #cwg357-B
     template<typename T> void f();
   };
   template<typename T> void B::f() const {}
   // expected-error@-1 {{out-of-line definition of 'f' does not match any declaration in 'cwg357::B'}}
+  //   expected-note@#cwg357-B {{defined here}}
 }
 
 namespace cwg358 { // cwg358: yes
diff --git a/clang/test/CXX/special/class.inhctor/p8.cpp b/clang/test/CXX/special/class.inhctor/p8.cpp
index 58c01d2b912d4..dbc357c1a8d4f 100644
--- a/clang/test/CXX/special/class.inhctor/p8.cpp
+++ b/clang/test/CXX/special/class.inhctor/p8.cpp
@@ -24,9 +24,10 @@ struct C {
   template<typename T> constexpr C(T t) : v(t) {}
   int v;
 };
-struct D : C {
+struct D : C { // #defined-here
   using C::C;
 };
 static_assert(D(123).v == 123, "");
 
 template<typename T> constexpr D::D(T t) : C(t) {} // expected-error {{does not match any declaration in 'D'}}
+                                                   // expected-note@#defined-here {{defined here}}
diff --git a/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-0x.cpp b/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-0x.cpp
index 19793fe826372..ce27e6aa83c3b 100644
--- a/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-0x.cpp
+++ b/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-0x.cpp
@@ -2,13 +2,13 @@
 
 template <class T> struct eval; // expected-note 3{{template is declared here}}
 
-template <template <class, class...> class TT, class T1, class... Rest> 
+template <template <class, class...> class TT, class T1, class... Rest>
 struct eval<TT<T1, Rest...>> { };
 
-template <class T1> struct A; 
-template <class T1, class T2> struct B; 
-template <int N> struct C; 
-template <class T1, int N> struct D; 
+template <class T1> struct A;
+template <class T1, class T2> struct B;
+template <int N> struct C;
+template <class T1, int N> struct D;
 template <class T1, class T2, int N = 17> struct E;
 
 eval<A<int>> eA;
@@ -17,27 +17,32 @@ eval<C<17>> eC; // expected-error{{implicit instantiation of undefined template
 eval<D<int, 17>> eD; // expected-error{{implicit instantiation of undefined template 'eval<D<int, 17>>'}}
 eval<E<int, float>> eE; // expected-error{{implicit instantiation of undefined template 'eval<E<int, float>>}}
 
-template<template <int ...N> class TT> struct X0 { }; // expected-note{{previous non-type template parameter with type 'int' is here}}
+template<
+  template <int ...N> // expected-error{{deduced non-type template argument does not have the same type as the corresponding template parameter ('long' vs 'int')}}
+  class TT // expected-note {{previous template template parameter is here}}
+> struct X0 { };
+
 template<int I, int J, int ...Rest> struct X0a;
 template<int ...Rest> struct X0b;
-template<int I, long J> struct X0c; // expected-note{{template non-type parameter has a different type 'long' in template argument}}
+template<int I, long J> struct X0c; // expected-note{{template parameter is declared here}}
 
 X0<X0a> inst_x0a;
 X0<X0b> inst_x0b;
-X0<X0c> inst_x0c; // expected-error{{template template argument has different template parameters than its corresponding template template parameter}}
+X0<X0c> inst_x0c; // expected-note{{template template argument has different template parameters than its corresponding template template parameter}}
 
-template<typename T, 
-         template <T ...N> class TT>  // expected-note{{previous non-type template parameter with type 'short' is here}}
-struct X1 { };
+template<typename T,
+         template <T ...N> // expected-error{{deduced non-type template argument does not have the same type as the corresponding template parameter ('long' vs 'short')}}
+         class TT // expected-note {{previous template template parameter is here}}
+> struct X1 { };
 template<int I, int J, int ...Rest> struct X1a;
 template<long I, long ...Rest> struct X1b;
 template<short I, short J> struct X1c;
-template<short I, long J> struct X1d; // expected-note{{template non-type parameter has a different type 'long' in template argument}}
+template<short I, long J> struct X1d; // expected-note{{template parameter is declared here}}
 
 X1<int, X1a> inst_x1a;
 X1<long, X1b> inst_x1b;
 X1<short, X1c> inst_x1c;
-X1<short, X1d> inst_x1d; // expected-error{{template template argument has different template parameters than its corresponding template template paramete}}
+X1<short, X1d> inst_x1d; // expected-note{{template template argument has different template parameters than its corresponding template template parameter}}
 
 template <int> class X2; // expected-note{{template is declared here}} \
                          // expected-note{{template is declared here}}
diff --git a/clang/test/CXX/temp/temp.constr/temp.constr.decl/func-template-decl.cpp b/clang/test/CXX/temp/temp.constr/temp.constr.decl/func-template-decl.cpp
index 30fbec64eea78..9a8148bdf625f 100644
--- a/clang/test/CXX/temp/temp.constr/temp.constr.decl/func-template-decl.cpp
+++ b/clang/test/CXX/temp/temp.constr/temp.constr.decl/func-template-decl.cpp
@@ -47,7 +47,7 @@ int AA::A() { return sizeof(T); }
 namespace diag {
 
 template <unsigned N>
-struct TA {
+struct TA { // #defined-here
   template <template <unsigned> class TT> requires TT<N>::happy
   int A();
 };
@@ -55,5 +55,6 @@ struct TA {
 template <unsigned N>
 template <template <unsigned> class TT> int TA<N>::A() { return sizeof(TT<N>); }
 // expected-error@-1{{out-of-line definition of 'A' does not match any declaration in 'TA<N>'}}
+// expected-note@#defined-here{{defined here}}
 
 } // end namespace diag
diff --git a/clang/test/CXX/temp/temp.fct.spec/temp.deduct/temp.deduct.type/p9-0x.cpp b/clang/test/CXX/temp/temp.fct.spec/temp.deduct/temp.deduct.type/p9-0x.cpp
index 51df1e0b14541..fbda7cbe696bb 100644
--- a/clang/test/CXX/temp/temp.fct.spec/temp.deduct/temp.deduct.type/p9-0x.cpp
+++ b/clang/test/CXX/temp/temp.fct.spec/temp.deduct/temp.deduct.type/p9-0x.cpp
@@ -29,10 +29,14 @@ namespace PackExpansionNotAtEnd {
                                                 >::value? 1 : -1];
 
   template<typename ... Types> struct UselessPartialSpec;
+  // expected-note@-1 {{template is declared here}}
 
+  // FIXME: We should simply disallow a pack expansion which is not at
+  // the end of the partial spec template argument list.
   template<typename ... Types, // expected-note{{non-deducible template parameter 'Types'}}
            typename Tail> // expected-note{{non-deducible template parameter 'Tail'}}
   struct UselessPartialSpec<Types..., Tail>; // expected-error{{class template partial specialization contains template parameters that cannot be deduced; this partial specialization will never be used}}
+  // expected-error@-1 {{is not more specialized than the primary template}}
 }
 
 // When a pack expansion occurs within a template argument list, the entire
diff --git a/clang/test/CXX/temp/temp.param/p12.cpp b/clang/test/CXX/temp/temp.param/p12.cpp
index 7be38790905fa..8317e7f24152c 100644
--- a/clang/test/CXX/temp/temp.param/p12.cpp
+++ b/clang/test/CXX/temp/temp.param/p12.cpp
@@ -1,39 +1,40 @@
-// RUN: %clang_cc1 -fsyntax-only -verify %s 
-template<typename> struct Y1; // expected-note{{too few template parameters in template template argument}}
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+template<typename> struct Y1; // expected-note{{template is declared here}}
 template<typename, int> struct Y2;
 
 // C++ [temp.param]p12:
-template<class T1, 
+template<class T1,
          class T2 = int> // expected-note{{previous default template argument defined here}}
   class B3;
 template<class T1, typename T2> class B3;
-template<class T1, 
+template<class T1,
          typename T2 = float> // expected-error{{template parameter redefines default argument}}
   class B3;
 
-template<template<class, int> class, 
+template<template<class, int> class,
          template<class> class = Y1> // expected-note{{previous default template argument defined here}}
   class B3t;
 
 template<template<class, int> class, template<class> class> class B3t;
 
-template<template<class, int> class, 
+template<template<class, int> class,
          template<class> class = Y1> // expected-error{{template parameter redefines default argument}}
   class B3t;
 
-template<int N, 
+template<int N,
          int M = 5> // expected-note{{previous default template argument defined here}}
   class B3n;
 
 template<int N, int M> class B3n;
 
-template<int N, 
+template<int N,
          int M = 7>  // expected-error{{template parameter redefines default argument}}
   class B3n;
 
 // Check validity of default arguments
-template<template<class, int> class // expected-note{{previous template template parameter is here}}
-           = Y1> // expected-error{{template template argument has different template parameters than its corresponding template template parameter}}
+template<template<class, int> class =// expected-note {{previous template template parameter is here}}
+           Y1> // expected-error{{too many template arguments for class template 'Y1'}}
+               // expected-note@-1 {{template template argument has different template parameters than its corresponding template template parameter}}
   class C1 {};
 
 C1<> c1; // expected-note{{while checking a default template argument}}
diff --git a/clang/test/CXX/temp/temp.res/temp.local/p8.cpp b/clang/test/CXX/temp/temp.res/temp.local/p8.cpp
index 56599985da06e..f503b4f15dd3f 100644
--- a/clang/test/CXX/temp/temp.res/temp.local/p8.cpp
+++ b/clang/test/CXX/temp/temp.res/temp.local/p8.cpp
@@ -69,7 +69,7 @@ namespace SearchClassBetweenTemplateParameterLists {
 
   template<typename T> struct A {
     using AA = void;
-    template<typename U> struct B {
+    template<typename U> struct B { // #defined-here
       using BB = void;
       void f(U);
       void g(U);
@@ -127,19 +127,19 @@ namespace SearchClassBetweenTemplateParameterLists {
   template<typename T> template<typename BB>
   void A<T>::B<BB>::g(BB) { // expected-error {{does not match}}
     BB bb; // expected-error {{incomplete type}}
-  }
+  } // expected-note@#defined-here {{defined here}}
 
   // error, 'AA' found in (4)
   template<typename AA> template<typename U>
   void A<AA>::B<U>::h(AA) { // expected-error {{does not match}}
     AA aa; // expected-error {{incomplete type}}
-  }
+  } // expected-note@#defined-here {{defined here}}
 
   // error, 'BB' found in (2)
   template<typename BB> template<typename U>
   void A<BB>::B<U>::i(BB) { // expected-error {{does not match}}
     BB bb; // expected-error {{incomplete type}}
-  }
+  } // expected-note@#defined-here {{defined here}}
 
   // OK, 'BB' found in (1)
   template<typename T> template<typename U> template<typename BB>
@@ -151,7 +151,7 @@ namespace SearchClassBetweenTemplateParameterLists {
   template<typename T> template<typename BB> template<typename V>
   void A<T>::B<BB>::k(V) { // expected-error {{does not match}}
     BB bb; // expected-error {{incomplete type}}
-  }
+  } // expected-note@#defined-here {{defined here}}
 
   int CC;
   template <typename> struct C;
diff --git a/clang/test/CXX/temp/temp.spec/temp.expl.spec/p12.cpp b/clang/test/CXX/temp/temp.spec/temp.expl.spec/p12.cpp
index 2a57489083695..9717fbf419b0a 100644
--- a/clang/test/CXX/temp/temp.spec/temp.expl.spec/p12.cpp
+++ b/clang/test/CXX/temp/temp.spec/temp.expl.spec/p12.cpp
@@ -31,7 +31,7 @@ constexpr void A::f1<long>(); // since-cxx14-error {{no function template matche
 // members of a class template explicitly specialized for an implicitly
 // instantiated specialization of that template.
 template<typename T>
-struct B {
+struct B { // #defined-here
   void g0(); // since-cxx14-note {{previous declaration is here}}
              // cxx11-note@-1 {{member declaration does not match because it is not const qualified}}
 
@@ -49,11 +49,13 @@ template<>
 constexpr void B<short>::g0(); // since-cxx14-error {{constexpr declaration of 'g0' follows non-constexpr declaration}}
                                // cxx11-error@-1 {{out-of-line declaration of 'g0' does not match any declaration in 'B<short>'}}
                                // cxx11-warning@-2 {{'constexpr' non-static member function will not be implicitly 'const' in C++14; add 'const'}}
+                               // expected-note@#defined-here {{defined here}}
 
 template<>
 constexpr void B<short>::g1(); // since-cxx14-error {{out-of-line declaration of 'g1' does not match any declaration in 'B<short>'}}
                                // cxx11-error@-1 {{constexpr declaration of 'g1' follows non-constexpr declaration}}
                                // cxx11-warning@-2 {{'constexpr' non-static member function will not be implicitly 'const' in C++14; add 'const'}}
+                               // expected-note@#defined-here {{defined here}}
 
 template<>
 template<typename U>
@@ -66,5 +68,3 @@ template<typename U>
 constexpr void B<long>::h1(); // since-cxx14-error {{out-of-line declaration of 'h1' does not match any declaration in 'B<long>'}}
                               // cxx11-error@-1 {{constexpr declaration of 'h1' follows non-constexpr declaration}}
                               // cxx11-warning@-2 {{'constexpr' non-static member function will not be implicitly 'const' in C++14; add 'const'}}
-
-
diff --git a/clang/test/CXX/temp/temp.spec/temp.expl.spec/p14-23.cpp b/clang/test/CXX/temp/temp.spec/temp.expl.spec/p14-23.cpp
index a023bf84137d7..062c2db7db0e2 100644
--- a/clang/test/CXX/temp/temp.spec/temp.expl.spec/p14-23.cpp
+++ b/clang/test/CXX/temp/temp.spec/temp.expl.spec/p14-23.cpp
@@ -8,7 +8,7 @@ namespace N0 {
   concept D = I < 8;
 
   template<int I>
-  struct A {
+  struct A { // #defined-here
     constexpr static int f() { return 0; }
     constexpr static int f() requires C<I> && D<I> { return 1; }
     constexpr static int f() requires C<I> { return 2; }
@@ -56,6 +56,7 @@ namespace N0 {
 
   template<>
   constexpr int A<0>::h() { return 2; } // expected-error {{out-of-line definition of 'h' does not match any declaration in 'N0::A<0>'}}
+                                        // expected-note@#defined-here {{defined here}}
 
   static_assert(A<5>::h() == 0);
   static_assert(A<4>::h() == 1);
diff --git a/clang/test/CXX/temp/temp.spec/temp.expl.spec/p18.cpp b/clang/test/CXX/temp/temp.spec/temp.expl.spec/p18.cpp
index 4d175a8860870..5351773fafe89 100644
--- a/clang/test/CXX/temp/temp.spec/temp.expl.spec/p18.cpp
+++ b/clang/test/CXX/temp/temp.spec/temp.expl.spec/p18.cpp
@@ -7,13 +7,14 @@ template<class T1> class A {
 }; 
 
 template<> template<class X>
-class A<long>::B { }; 
+class A<long>::B { }; // #defined-here
 
 template<> template<> template<class T>
   void A<int>::B<double>::mf1(T t) { } 
 
 template<> template<> template<class T>
 void A<long>::B<double>::mf1(T t) { } // expected-error{{does not match}}
+                                      // expected-note@#defined-here{{defined here}}
 
 // FIXME: This diagnostic could probably be better.
 template<class Y> template<>
diff --git a/clang/test/CodeGen/2008-08-07-AlignPadding1.c b/clang/test/CodeGen/2008-08-07-AlignPadding1.c
index 17e88ce02659f..b51bcbc0244da 100644
--- a/clang/test/CodeGen/2008-08-07-AlignPadding1.c
+++ b/clang/test/CodeGen/2008-08-07-AlignPadding1.c
@@ -1,4 +1,5 @@
 /* RUN: %clang_cc1 %s -emit-llvm -triple x86_64-apple-darwin -o - | FileCheck %s
+/* RUN: %clang_cc1 %s -emit-llvm -triple x86_64-apple-darwin -o - -fexperimental-new-constant-interpreter | FileCheck %s
 
 The FE must generate padding here both at the end of each PyG_Head and
 between array elements.  Reduced from Python. */
diff --git a/clang/test/CodeGen/PowerPC/ppc-xmmintrin.c b/clang/test/CodeGen/PowerPC/ppc-xmmintrin.c
index 4a15fa9f76cee..8681bb3f9b4f8 100644
--- a/clang/test/CodeGen/PowerPC/ppc-xmmintrin.c
+++ b/clang/test/CodeGen/PowerPC/ppc-xmmintrin.c
@@ -23,7 +23,7 @@
 
 // RUN: %clang -S -emit-llvm -target powerpc64-ibm-aix -mcpu=pwr8 -ffreestanding -DNO_WARN_X86_INTRINSICS %s \
 // RUN:   -ffp-contract=off -fno-discard-value-names -mllvm -disable-llvm-optzns -o - | llvm-cxxfilt -n | FileCheck %s --check-prefixes=CHECK,CHECK-BE
-// RUN: %clang -x c++ -fsyntax-only -target powerpc64-ibm-aix -mcpu=pwr8 -ffreestanding -DNO_WARN_X86_INTRINSICS %s \
+// RUN: %clang -x c++ -fsyntax-only -target powerpc64-ibm-aix -mcpu=pwr8 -ffreestanding -nostdlibinc -DNO_WARN_X86_INTRINSICS %s \
 // RUN:   -fno-discard-value-names -mllvm -disable-llvm-optzns
 // RUN: %clang -S -emit-llvm -target powerpc64-ibm-aix -mcpu=pwr10 -ffreestanding -DNO_WARN_X86_INTRINSICS %s \
 // RUN:   -ffp-contract=off -fno-discard-value-names -mllvm -disable-llvm-optzns -o - | llvm-cxxfilt -n | FileCheck %s --check-prefixes=CHECK,CHECK-P10-BE
diff --git a/clang/test/CodeGen/X86/adc-builtins.c b/clang/test/CodeGen/X86/adc-builtins.c
index 162a78c474f90..87e3b9a662bce 100644
--- a/clang/test/CodeGen/X86/adc-builtins.c
+++ b/clang/test/CodeGen/X86/adc-builtins.c
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -ffreestanding -triple x86_64-unknown-unknown -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding -triple x86_64-unknown-unknown -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding -triple x86_64-unknown-unknown -emit-llvm -o - %s | FileCheck %s
 
 #include <x86intrin.h>
 
@@ -43,3 +44,81 @@ unsigned char test_subborrow_u64(unsigned char __cf, unsigned long long __x,
 // CHECK: [[CF:%.*]] = extractvalue { i8, i64 } [[SBB]], 0
   return _subborrow_u64(__cf, __x, __y, __p);
 }
+
+// Test constexpr handling.
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+
+template<typename X>
+struct Result {
+  unsigned char A;
+  X B;
+  constexpr bool operator==(const Result<X> &Other) {
+    return A == Other.A && B == Other.B;
+  }
+};
+
+constexpr Result<unsigned int>
+const_test_addcarry_u32(unsigned char __cf, unsigned int __x, unsigned int __y)
+{
+  unsigned int __r{};
+  return { _addcarry_u32(__cf, __x, __y, &__r), __r };
+}
+
+void constexpr adcu32() {
+  static_assert(const_test_addcarry_u32(0, 0x00000000, 0x00000000) == Result<unsigned int>{0, 0x00000000});
+  static_assert(const_test_addcarry_u32(1, 0xFFFFFFFE, 0x00000000) == Result<unsigned int>{0, 0xFFFFFFFF});
+  static_assert(const_test_addcarry_u32(1, 0xFFFFFFFE, 0x00000001) == Result<unsigned int>{1, 0x00000000});
+  static_assert(const_test_addcarry_u32(0, 0xFFFFFFFF, 0xFFFFFFFF) == Result<unsigned int>{1, 0xFFFFFFFE});
+  static_assert(const_test_addcarry_u32(1, 0xFFFFFFFF, 0xFFFFFFFF) == Result<unsigned int>{1, 0xFFFFFFFF});
+}
+
+constexpr Result<unsigned int>
+const_test_subborrow_u32(unsigned char __cf, unsigned int __x, unsigned int __y)
+{
+  unsigned int __r{};
+  return { _subborrow_u32(__cf, __x, __y, &__r), __r };
+}
+
+void constexpr sbbu32() {
+  static_assert(const_test_subborrow_u32(0, 0x00000000, 0x00000000) == Result<unsigned int>{0, 0x00000000});
+  static_assert(const_test_subborrow_u32(0, 0x00000000, 0x00000001) == Result<unsigned int>{1, 0xFFFFFFFF});
+  static_assert(const_test_subborrow_u32(1, 0x00000000, 0x00000001) == Result<unsigned int>{1, 0xFFFFFFFE});
+  static_assert(const_test_subborrow_u32(1, 0xFFFFFFFE, 0x00000000) == Result<unsigned int>{0, 0xFFFFFFFD});
+  static_assert(const_test_subborrow_u32(1, 0xFFFFFFFE, 0x00000001) == Result<unsigned int>{0, 0xFFFFFFFC});
+  static_assert(const_test_subborrow_u32(0, 0xFFFFFFFF, 0xFFFFFFFF) == Result<unsigned int>{0, 0x00000000});
+  static_assert(const_test_subborrow_u32(1, 0xFFFFFFFF, 0xFFFFFFFF) == Result<unsigned int>{1, 0xFFFFFFFF});
+}
+
+constexpr Result<unsigned long long>
+const_test_addcarry_u64(unsigned char __cf, unsigned long long __x, unsigned long long __y)
+{
+  unsigned long long __r{};
+  return { _addcarry_u64(__cf, __x, __y, &__r), __r };
+}
+
+void constexpr adcu64() {
+  static_assert(const_test_addcarry_u64(0, 0x0000000000000000ULL, 0x0000000000000000ULL) == Result<unsigned long long>{0, 0x0000000000000000ULL});
+  static_assert(const_test_addcarry_u64(1, 0xFFFFFFFFFFFFFFFEULL, 0x0000000000000000ULL) == Result<unsigned long long>{0, 0xFFFFFFFFFFFFFFFFULL});
+  static_assert(const_test_addcarry_u64(1, 0xFFFFFFFFFFFFFFFEULL, 0x0000000000000001ULL) == Result<unsigned long long>{1, 0x0000000000000000ULL});
+  static_assert(const_test_addcarry_u64(0, 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL) == Result<unsigned long long>{1, 0xFFFFFFFFFFFFFFFEULL});
+  static_assert(const_test_addcarry_u64(1, 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL) == Result<unsigned long long>{1, 0xFFFFFFFFFFFFFFFFULL});
+}
+
+constexpr Result<unsigned long long>
+const_test_subborrow_u64(unsigned char __cf, unsigned long long __x, unsigned long long __y)
+{
+  unsigned long long __r{};
+  return { _subborrow_u64(__cf, __x, __y, &__r), __r };
+}
+
+void constexpr sbbu64() {
+  static_assert(const_test_subborrow_u64(0, 0x0000000000000000ULL, 0x0000000000000000ULL) == Result<unsigned long long>{0, 0x0000000000000000ULL});
+  static_assert(const_test_subborrow_u64(0, 0x0000000000000000ULL, 0x0000000000000001ULL) == Result<unsigned long long>{1, 0xFFFFFFFFFFFFFFFFULL});
+  static_assert(const_test_subborrow_u64(1, 0x0000000000000000ULL, 0x0000000000000001ULL) == Result<unsigned long long>{1, 0xFFFFFFFFFFFFFFFEULL});
+  static_assert(const_test_subborrow_u64(1, 0xFFFFFFFFFFFFFFFEULL, 0x0000000000000000ULL) == Result<unsigned long long>{0, 0xFFFFFFFFFFFFFFFDULL});
+  static_assert(const_test_subborrow_u64(1, 0xFFFFFFFFFFFFFFFEULL, 0x0000000000000001ULL) == Result<unsigned long long>{0, 0xFFFFFFFFFFFFFFFCULL});
+  static_assert(const_test_subborrow_u64(0, 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL) == Result<unsigned long long>{0, 0x0000000000000000ULL});
+  static_assert(const_test_subborrow_u64(1, 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL) == Result<unsigned long long>{1, 0xFFFFFFFFFFFFFFFFULL});
+}
+
+#endif
\ No newline at end of file
diff --git a/clang/test/CodeGen/X86/adx-builtins.c b/clang/test/CodeGen/X86/adx-builtins.c
index 563a50b9bc84e..53861bc3ceb72 100644
--- a/clang/test/CodeGen/X86/adx-builtins.c
+++ b/clang/test/CodeGen/X86/adx-builtins.c
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple x86_64-unknown-unknown -ffreestanding -target-feature +adx -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -x c -triple x86_64-unknown-unknown -ffreestanding -target-feature +adx -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -x c++ -triple x86_64-unknown-unknown -ffreestanding -target-feature +adx -emit-llvm -o - %s | FileCheck %s
 
 #include <immintrin.h>
 
@@ -22,3 +23,47 @@ unsigned char test_addcarryx_u64(unsigned char __cf, unsigned long long __x,
 // CHECK: [[CF:%.*]] = extractvalue { i8, i64 } [[ADC]], 0
   return _addcarryx_u64(__cf, __x, __y, __p);
 }
+
+// Test constexpr handling.
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+
+template<typename X>
+struct Result {
+  unsigned char A;
+  X B;
+  constexpr bool operator==(const Result<X> &Other) {
+    return A == Other.A && B == Other.B;
+  }
+};
+
+constexpr Result<unsigned int>
+const_test_addcarryx_u32(unsigned char __cf, unsigned int __x, unsigned int __y)
+{
+  unsigned int __r{};
+  return { _addcarryx_u32(__cf, __x, __y, &__r), __r };
+}
+
+void constexpr addxu32() {
+  static_assert(const_test_addcarryx_u32(0, 0x00000000, 0x00000000) == Result<unsigned int>{0, 0x00000000});
+  static_assert(const_test_addcarryx_u32(1, 0xFFFFFFFE, 0x00000000) == Result<unsigned int>{0, 0xFFFFFFFF});
+  static_assert(const_test_addcarryx_u32(1, 0xFFFFFFFE, 0x00000001) == Result<unsigned int>{1, 0x00000000});
+  static_assert(const_test_addcarryx_u32(0, 0xFFFFFFFF, 0xFFFFFFFF) == Result<unsigned int>{1, 0xFFFFFFFE});
+  static_assert(const_test_addcarryx_u32(1, 0xFFFFFFFF, 0xFFFFFFFF) == Result<unsigned int>{1, 0xFFFFFFFF});
+}
+
+constexpr Result<unsigned long long>
+const_test_addcarryx_u64(unsigned char __cf, unsigned long long __x, unsigned long long __y)
+{
+  unsigned long long __r{};
+  return { _addcarryx_u64(__cf, __x, __y, &__r), __r };
+}
+
+void constexpr addxu64() {
+  static_assert(const_test_addcarryx_u64(0, 0x0000000000000000ULL, 0x0000000000000000ULL) == Result<unsigned long long>{0, 0x0000000000000000ULL});
+  static_assert(const_test_addcarryx_u64(1, 0xFFFFFFFFFFFFFFFEULL, 0x0000000000000000ULL) == Result<unsigned long long>{0, 0xFFFFFFFFFFFFFFFFULL});
+  static_assert(const_test_addcarryx_u64(1, 0xFFFFFFFFFFFFFFFEULL, 0x0000000000000001ULL) == Result<unsigned long long>{1, 0x0000000000000000ULL});
+  static_assert(const_test_addcarryx_u64(0, 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL) == Result<unsigned long long>{1, 0xFFFFFFFFFFFFFFFEULL});
+  static_assert(const_test_addcarryx_u64(1, 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL) == Result<unsigned long long>{1, 0xFFFFFFFFFFFFFFFFULL});
+}
+
+#endif
\ No newline at end of file
diff --git a/clang/test/CodeGen/X86/cx-complex-range.c b/clang/test/CodeGen/X86/cx-complex-range.c
index 14887637d516e..a0e6dc219b36f 100644
--- a/clang/test/CodeGen/X86/cx-complex-range.c
+++ b/clang/test/CodeGen/X86/cx-complex-range.c
@@ -1220,7 +1220,7 @@ _Complex _Float16 mulf16(_Complex _Float16 a, _Complex _Float16 b) {
 // FULL_FAST-NEXT:    [[C_IMAG:%.*]] = load half, ptr [[C_IMAGP]], align 2
 // FULL_FAST-NEXT:    [[CONV:%.*]] = fpext half [[C_REAL]] to x86_fp80
 // FULL_FAST-NEXT:    [[CONV1:%.*]] = fpext half [[C_IMAG]] to x86_fp80
-// FULL_FAST-NEXT:    [[CALL:%.*]] = call { x86_fp80, x86_fp80 } @__divxc3(x86_fp80 noundef nofpclass(nan inf) [[B_REAL]], x86_fp80 noundef nofpclass(nan inf) [[B_IMAG]], x86_fp80 noundef nofpclass(nan inf) [[CONV]], x86_fp80 noundef nofpclass(nan inf) [[CONV1]]) #[[ATTR1]]
+// FULL_FAST-NEXT:    [[CALL:%.*]] = call reassoc nnan ninf nsz arcp afn nofpclass(nan inf) { x86_fp80, x86_fp80 } @__divxc3(x86_fp80 noundef nofpclass(nan inf) [[B_REAL]], x86_fp80 noundef nofpclass(nan inf) [[B_IMAG]], x86_fp80 noundef nofpclass(nan inf) [[CONV]], x86_fp80 noundef nofpclass(nan inf) [[CONV1]]) #[[ATTR1]]
 // FULL_FAST-NEXT:    [[TMP0:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 0
 // FULL_FAST-NEXT:    [[TMP1:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 1
 // FULL_FAST-NEXT:    [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP0]] to half
diff --git a/clang/test/CodeGen/X86/pr51324.c b/clang/test/CodeGen/X86/pr51324.c
new file mode 100644
index 0000000000000..10d1ba3c84b85
--- /dev/null
+++ b/clang/test/CodeGen/X86/pr51324.c
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse4.1 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse4.1 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s
+
+
+#include <immintrin.h>
+
+// Make sure brackets work after macro intrinsics.
+float pr51324(__m128 a) {
+  // CHECK-LABEL: pr51324
+  // CHECK: call <4 x float> @llvm.x86.sse41.round.ps(<4 x float> %{{.*}}, i32 0)
+  // CHECK: extractelement <4 x float> %{{.*}}, i32 0
+  return _mm_round_ps(a, 0)[0];
+}
diff --git a/clang/test/CodeGen/X86/pr60006.c b/clang/test/CodeGen/X86/pr60006.c
new file mode 100644
index 0000000000000..4ef8504a52f07
--- /dev/null
+++ b/clang/test/CodeGen/X86/pr60006.c
@@ -0,0 +1,17 @@
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse4.1 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse4.1 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s
+
+
+#include <immintrin.h>
+
+// Ensure _mm_test_all_ones macro doesn't reuse argument
+__m128i expensive_call();
+int pr60006() {
+  // CHECK-LABEL: pr60006
+  // CHECK: call {{.*}} @expensive_call
+  // CHECK-NOT: call {{.*}} @expensive_call
+  // CHECK: call i32 @llvm.x86.sse41.ptestc(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  return _mm_test_all_ones(expensive_call());
+}
diff --git a/clang/test/CodeGen/X86/sse-builtins-constrained-cmp.c b/clang/test/CodeGen/X86/sse-builtins-constrained-cmp.c
index eb1488885e5dd..563fe3d86d821 100644
--- a/clang/test/CodeGen/X86/sse-builtins-constrained-cmp.c
+++ b/clang/test/CodeGen/X86/sse-builtins-constrained-cmp.c
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse -emit-llvm -ffp-exception-behavior=strict -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse -emit-llvm -ffp-exception-behavior=strict -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse -emit-llvm -ffp-exception-behavior=strict -o - -Wall -Werror | FileCheck %s
 
 
 #include <immintrin.h>
diff --git a/clang/test/CodeGen/X86/sse-builtins-constrained.c b/clang/test/CodeGen/X86/sse-builtins-constrained.c
index c67e230c2ff0f..92240bbc5bb31 100644
--- a/clang/test/CodeGen/X86/sse-builtins-constrained.c
+++ b/clang/test/CodeGen/X86/sse-builtins-constrained.c
@@ -1,8 +1,12 @@
 // REQUIRES: x86-registered-target
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-unknown-linux-gnu -target-feature +sse -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefix=UNCONSTRAINED --check-prefix=COMMON --check-prefix=COMMONIR
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-unknown-linux-gnu -target-feature +sse -ffp-exception-behavior=maytrap -DSTRICT=1 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefix=CONSTRAINED --check-prefix=COMMON --check-prefix=COMMONIR
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-unknown-linux-gnu -target-feature +sse -S -o - -Wall -Werror | FileCheck %s --check-prefix=CHECK-ASM --check-prefix=COMMON
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-unknown-linux-gnu -target-feature +sse -ffp-exception-behavior=maytrap -DSTRICT=1 -S -o - -Wall -Werror | FileCheck %s --check-prefix=CHECK-ASM --check-prefix=COMMON
+// RUN: %clang_cc1 -x c -ffreestanding %s -triple=x86_64-unknown-linux-gnu -target-feature +sse -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefix=UNCONSTRAINED --check-prefix=COMMON --check-prefix=COMMONIR
+// RUN: %clang_cc1 -x c -ffreestanding %s -triple=x86_64-unknown-linux-gnu -target-feature +sse -ffp-exception-behavior=maytrap -DSTRICT=1 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefix=CONSTRAINED --check-prefix=COMMON --check-prefix=COMMONIR
+// RUN: %clang_cc1 -x c -ffreestanding %s -triple=x86_64-unknown-linux-gnu -target-feature +sse -S -o - -Wall -Werror | FileCheck %s --check-prefix=CHECK-ASM --check-prefix=COMMON
+// RUN: %clang_cc1 -x c -ffreestanding %s -triple=x86_64-unknown-linux-gnu -target-feature +sse -ffp-exception-behavior=maytrap -DSTRICT=1 -S -o - -Wall -Werror | FileCheck %s --check-prefix=CHECK-ASM --check-prefix=COMMON
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -triple=x86_64-unknown-linux-gnu -target-feature +sse -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefix=UNCONSTRAINED --check-prefix=COMMON --check-prefix=COMMONIR
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -triple=x86_64-unknown-linux-gnu -target-feature +sse -ffp-exception-behavior=maytrap -DSTRICT=1 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefix=CONSTRAINED --check-prefix=COMMON --check-prefix=COMMONIR
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -triple=x86_64-unknown-linux-gnu -target-feature +sse -S -o - -Wall -Werror | FileCheck %s --check-prefix=CHECK-ASM --check-prefix=COMMON
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -triple=x86_64-unknown-linux-gnu -target-feature +sse -ffp-exception-behavior=maytrap -DSTRICT=1 -S -o - -Wall -Werror | FileCheck %s --check-prefix=CHECK-ASM --check-prefix=COMMON
 
 #ifdef STRICT
 // Test that the constrained intrinsics are picking up the exception
@@ -16,8 +20,8 @@
 
 __m128 test_mm_sqrt_ps(__m128 x) {
   // COMMON-LABEL: test_mm_sqrt_ps
-  // UNCONSTRAINED: call <4 x float> @llvm.sqrt.v4f32(<4 x float> {{.*}})
-  // CONSTRAINED: call <4 x float> @llvm.experimental.constrained.sqrt.v4f32(<4 x float> {{.*}}, metadata !{{.*}})
+  // UNCONSTRAINED: call {{.*}}<4 x float> @llvm.sqrt.v4f32(<4 x float> {{.*}})
+  // CONSTRAINED: call {{.*}}<4 x float> @llvm.experimental.constrained.sqrt.v4f32(<4 x float> {{.*}}, metadata !{{.*}})
   // CHECK-ASM: sqrtps
   return _mm_sqrt_ps(x);
 }
diff --git a/clang/test/CodeGen/X86/sse-builtins-dbg.c b/clang/test/CodeGen/X86/sse-builtins-dbg.c
index 25678942a89f0..e8d70d3fabd05 100644
--- a/clang/test/CodeGen/X86/sse-builtins-dbg.c
+++ b/clang/test/CodeGen/X86/sse-builtins-dbg.c
@@ -1,12 +1,13 @@
-// RUN: %clang_cc1 -ffreestanding -triple x86_64-apple-macosx10.8.0 -target-feature +sse4.1 -debug-info-kind=limited -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding -triple x86_64-apple-macosx10.8.0 -target-feature +sse4.1 -debug-info-kind=limited -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding -triple x86_64-apple-macosx10.8.0 -target-feature +sse4.1 -debug-info-kind=limited -emit-llvm %s -o - | FileCheck %s
 
 // Test that intrinsic calls inlined from _mm_* wrappers have debug metadata.
 
 #include <xmmintrin.h>
 
 __m128 test_rsqrt_ss(__m128 x) {
-  // CHECK: define {{.*}} @test_rsqrt_ss
-  // CHECK: call <4 x float> @llvm.x86.sse.rsqrt.ss({{.*}}, !dbg !{{.*}}
+  // CHECK-LABEL: test_rsqrt_ss
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.sse.rsqrt.ss({{.*}}, !dbg !{{.*}}
   // CHECK: ret <4 x float>
   return _mm_rsqrt_ss(x);
 }
diff --git a/clang/test/CodeGen/X86/sse-builtins.c b/clang/test/CodeGen/X86/sse-builtins.c
index aae5cfb8bb1d9..74f62e93c800b 100644
--- a/clang/test/CodeGen/X86/sse-builtins.c
+++ b/clang/test/CodeGen/X86/sse-builtins.c
@@ -1,5 +1,7 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 -flax-vector-conversions=none -fms-extensions -fms-compatibility -ffreestanding %s -triple=x86_64-windows-msvc -target-feature +sse -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -fms-extensions -fms-compatibility -ffreestanding %s -triple=x86_64-windows-msvc -target-feature +sse -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -fms-extensions -fms-compatibility -ffreestanding %s -triple=x86_64-windows-msvc -target-feature +sse -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 
 #include <immintrin.h>
@@ -84,7 +86,7 @@ __m128 test_mm_cmp_ps_ord_q(__m128 a, __m128 b) {
 
 __m128 test_mm_cmp_ss(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_cmp_ss
-  // CHECK: call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}}, i8 7)
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}}, i8 7)
   return _mm_cmp_ss(A, B, _CMP_ORD_Q);
 }
 
@@ -274,43 +276,43 @@ __m128 test_mm_cmpunord_ss(__m128 __a, __m128 __b) {
 
 int test_mm_comieq_ss(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_comieq_ss
-  // CHECK: call i32 @llvm.x86.sse.comieq.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse.comieq.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_comieq_ss(A, B);
 }
 
 int test_mm_comige_ss(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_comige_ss
-  // CHECK: call i32 @llvm.x86.sse.comige.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse.comige.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_comige_ss(A, B);
 }
 
 int test_mm_comigt_ss(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_comigt_ss
-  // CHECK: call i32 @llvm.x86.sse.comigt.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse.comigt.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_comigt_ss(A, B);
 }
 
 int test_mm_comile_ss(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_comile_ss
-  // CHECK: call i32 @llvm.x86.sse.comile.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse.comile.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_comile_ss(A, B);
 }
 
 int test_mm_comilt_ss(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_comilt_ss
-  // CHECK: call i32 @llvm.x86.sse.comilt.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse.comilt.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_comilt_ss(A, B);
 }
 
 int test_mm_comineq_ss(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_comineq_ss
-  // CHECK: call i32 @llvm.x86.sse.comineq.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse.comineq.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_comineq_ss(A, B);
 }
 
 int test_mm_cvt_ss2si(__m128 A) {
   // CHECK-LABEL: test_mm_cvt_ss2si
-  // CHECK: call i32 @llvm.x86.sse.cvtss2si(<4 x float> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse.cvtss2si(<4 x float> %{{.*}})
   return _mm_cvt_ss2si(A);
 }
 
@@ -338,34 +340,34 @@ float test_mm_cvtss_f32(__m128 A) {
 
 int test_mm_cvtss_si32(__m128 A) {
   // CHECK-LABEL: test_mm_cvtss_si32
-  // CHECK: call i32 @llvm.x86.sse.cvtss2si(<4 x float> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse.cvtss2si(<4 x float> %{{.*}})
   return _mm_cvtss_si32(A);
 }
 
 #ifdef __x86_64__
 long long test_mm_cvtss_si64(__m128 A) {
   // CHECK-LABEL: test_mm_cvtss_si64
-  // CHECK: call i64 @llvm.x86.sse.cvtss2si64(<4 x float> %{{.*}})
+  // CHECK: call {{.*}}i64 @llvm.x86.sse.cvtss2si64(<4 x float> %{{.*}})
   return _mm_cvtss_si64(A);
 }
 #endif
 
 int test_mm_cvtt_ss2si(__m128 A) {
   // CHECK-LABEL: test_mm_cvtt_ss2si
-  // CHECK: call i32 @llvm.x86.sse.cvttss2si(<4 x float> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse.cvttss2si(<4 x float> %{{.*}})
   return _mm_cvtt_ss2si(A);
 }
 
 int test_mm_cvttss_si32(__m128 A) {
   // CHECK-LABEL: test_mm_cvttss_si32
-  // CHECK: call i32 @llvm.x86.sse.cvttss2si(<4 x float> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse.cvttss2si(<4 x float> %{{.*}})
   return _mm_cvttss_si32(A);
 }
 
 #ifdef __x86_64__
 long long test_mm_cvttss_si64(__m128 A) {
   // CHECK-LABEL: test_mm_cvttss_si64
-  // CHECK: call i64 @llvm.x86.sse.cvttss2si64(<4 x float> %{{.*}})
+  // CHECK: call {{.*}}i64 @llvm.x86.sse.cvttss2si64(<4 x float> %{{.*}})
   return _mm_cvttss_si64(A);
 }
 #endif
@@ -530,7 +532,7 @@ __m128 test_mm_movelh_ps(__m128 A, __m128 B) {
 
 int test_mm_movemask_ps(__m128 A) {
   // CHECK-LABEL: test_mm_movemask_ps
-  // CHECK: call i32 @llvm.x86.sse.movmsk.ps(<4 x float> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse.movmsk.ps(<4 x float> %{{.*}})
   return _mm_movemask_ps(A);
 }
 
@@ -563,25 +565,25 @@ void test_mm_prefetch(char const* p) {
 
 __m128 test_mm_rcp_ps(__m128 x) {
   // CHECK-LABEL: test_mm_rcp_ps
-  // CHECK: call <4 x float> @llvm.x86.sse.rcp.ps(<4 x float> {{.*}})
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.sse.rcp.ps(<4 x float> {{.*}})
   return _mm_rcp_ps(x);
 }
 
 __m128 test_mm_rcp_ss(__m128 x) {
   // CHECK-LABEL: test_mm_rcp_ss
-  // CHECK: call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> {{.*}})
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.sse.rcp.ss(<4 x float> {{.*}})
   return _mm_rcp_ss(x);
 }
 
 __m128 test_mm_rsqrt_ps(__m128 x) {
   // CHECK-LABEL: test_mm_rsqrt_ps
-  // CHECK: call <4 x float> @llvm.x86.sse.rsqrt.ps(<4 x float> {{.*}})
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.sse.rsqrt.ps(<4 x float> {{.*}})
   return _mm_rsqrt_ps(x);
 }
 
 __m128 test_mm_rsqrt_ss(__m128 x) {
   // CHECK-LABEL: test_mm_rsqrt_ss
-  // CHECK: call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> {{.*}})
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> {{.*}})
   return _mm_rsqrt_ss(x);
 }
 
@@ -701,7 +703,7 @@ __m128 test_mm_shuffle_ps(__m128 A, __m128 B) {
 
 __m128 test_mm_sqrt_ps(__m128 x) {
   // CHECK-LABEL: test_mm_sqrt_ps
-  // CHECK: call <4 x float> @llvm.sqrt.v4f32(<4 x float> {{.*}})
+  // CHECK: call {{.*}}<4 x float> @llvm.sqrt.v4f32(<4 x float> {{.*}})
   return _mm_sqrt_ps(x);
 }
 
@@ -810,37 +812,37 @@ void test_MM_TRANSPOSE4_PS(__m128 *A, __m128 *B, __m128 *C, __m128 *D) {
 
 int test_mm_ucomieq_ss(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_ucomieq_ss
-  // CHECK: call i32 @llvm.x86.sse.ucomieq.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse.ucomieq.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_ucomieq_ss(A, B);
 }
 
 int test_mm_ucomige_ss(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_ucomige_ss
-  // CHECK: call i32 @llvm.x86.sse.ucomige.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse.ucomige.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_ucomige_ss(A, B);
 }
 
 int test_mm_ucomigt_ss(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_ucomigt_ss
-  // CHECK: call i32 @llvm.x86.sse.ucomigt.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse.ucomigt.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_ucomigt_ss(A, B);
 }
 
 int test_mm_ucomile_ss(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_ucomile_ss
-  // CHECK: call i32 @llvm.x86.sse.ucomile.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse.ucomile.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_ucomile_ss(A, B);
 }
 
 int test_mm_ucomilt_ss(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_ucomilt_ss
-  // CHECK: call i32 @llvm.x86.sse.ucomilt.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse.ucomilt.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_ucomilt_ss(A, B);
 }
 
 int test_mm_ucomineq_ss(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_ucomineq_ss
-  // CHECK: call i32 @llvm.x86.sse.ucomineq.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse.ucomineq.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_ucomineq_ss(A, B);
 }
 
diff --git a/clang/test/CodeGen/X86/sse2-builtins-constrained-cmp.c b/clang/test/CodeGen/X86/sse2-builtins-constrained-cmp.c
index f430bba9d06e8..732d4d53e4cf8 100644
--- a/clang/test/CodeGen/X86/sse2-builtins-constrained-cmp.c
+++ b/clang/test/CodeGen/X86/sse2-builtins-constrained-cmp.c
@@ -1,5 +1,7 @@
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse2 -emit-llvm -ffp-exception-behavior=strict -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse2 -emit-llvm -ffp-exception-behavior=strict -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse2 -emit-llvm -ffp-exception-behavior=strict -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse2 -emit-llvm -ffp-exception-behavior=strict -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse2 -emit-llvm -ffp-exception-behavior=strict -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse2 -emit-llvm -ffp-exception-behavior=strict -o - -Wall -Werror | FileCheck %s
 
 
 #include <immintrin.h>
diff --git a/clang/test/CodeGen/X86/sse2-builtins.c b/clang/test/CodeGen/X86/sse2-builtins.c
index 3ca7777e76335..a5348761ec84e 100644
--- a/clang/test/CodeGen/X86/sse2-builtins.c
+++ b/clang/test/CodeGen/X86/sse2-builtins.c
@@ -1,8 +1,13 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse2 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse2 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse2 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X86
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse2 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X86
-// RUN: %clang_cc1 -flax-vector-conversions=none -fms-extensions -fms-compatibility -ffreestanding %s -triple=x86_64-windows-msvc -target-feature +sse2 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse2 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse2 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse2 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X86
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse2 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X86
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -fms-extensions -fms-compatibility -ffreestanding %s -triple=x86_64-windows-msvc -target-feature +sse2 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse2 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse2 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse2 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X86
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse2 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X86
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -fms-extensions -fms-compatibility -ffreestanding %s -triple=x86_64-windows-msvc -target-feature +sse2 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
 
 
 #include <immintrin.h>
@@ -210,7 +215,7 @@ __m128d test_mm_cmp_pd_ord_q(__m128d a, __m128d b) {
 
 __m128d test_mm_cmp_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_cmp_sd
-  // CHECK: call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 7)
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 7)
   return _mm_cmp_sd(A, B, _CMP_ORD_Q);
 }
 
@@ -242,7 +247,7 @@ __m128d test_mm_cmpeq_pd(__m128d A, __m128d B) {
 
 __m128d test_mm_cmpeq_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_cmpeq_sd
-  // CHECK: call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 0)
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 0)
   return _mm_cmpeq_sd(A, B);
 }
 
@@ -256,7 +261,7 @@ __m128d test_mm_cmpge_pd(__m128d A, __m128d B) {
 
 __m128d test_mm_cmpge_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_cmpge_sd
-  // CHECK: call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 2)
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 2)
   // CHECK: extractelement <2 x double> %{{.*}}, i32 0
   // CHECK: insertelement <2 x double> poison, double %{{.*}}, i32 0
   // CHECK: extractelement <2 x double> %{{.*}}, i32 1
@@ -292,7 +297,7 @@ __m128d test_mm_cmpgt_pd(__m128d A, __m128d B) {
 
 __m128d test_mm_cmpgt_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_cmpgt_sd
-  // CHECK: call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 1)
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 1)
   // CHECK: extractelement <2 x double> %{{.*}}, i32 0
   // CHECK: insertelement <2 x double> poison, double %{{.*}}, i32 0
   // CHECK: extractelement <2 x double> %{{.*}}, i32 1
@@ -310,7 +315,7 @@ __m128d test_mm_cmple_pd(__m128d A, __m128d B) {
 
 __m128d test_mm_cmple_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_cmple_sd
-  // CHECK: call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 2)
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 2)
   return _mm_cmple_sd(A, B);
 }
 
@@ -342,7 +347,7 @@ __m128d test_mm_cmplt_pd(__m128d A, __m128d B) {
 
 __m128d test_mm_cmplt_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_cmplt_sd
-  // CHECK: call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 1)
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 1)
   return _mm_cmplt_sd(A, B);
 }
 
@@ -356,7 +361,7 @@ __m128d test_mm_cmpneq_pd(__m128d A, __m128d B) {
 
 __m128d test_mm_cmpneq_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_cmpneq_sd
-  // CHECK: call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 4)
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 4)
   return _mm_cmpneq_sd(A, B);
 }
 
@@ -370,7 +375,7 @@ __m128d test_mm_cmpnge_pd(__m128d A, __m128d B) {
 
 __m128d test_mm_cmpnge_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_cmpnge_sd
-  // CHECK: call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 6)
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 6)
   // CHECK: extractelement <2 x double> %{{.*}}, i32 0
   // CHECK: insertelement <2 x double> poison, double %{{.*}}, i32 0
   // CHECK: extractelement <2 x double> %{{.*}}, i32 1
@@ -388,7 +393,7 @@ __m128d test_mm_cmpngt_pd(__m128d A, __m128d B) {
 
 __m128d test_mm_cmpngt_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_cmpngt_sd
-  // CHECK: call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 5)
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 5)
   // CHECK: extractelement <2 x double> %{{.*}}, i32 0
   // CHECK: insertelement <2 x double> poison, double %{{.*}}, i32 0
   // CHECK: extractelement <2 x double> %{{.*}}, i32 1
@@ -406,7 +411,7 @@ __m128d test_mm_cmpnle_pd(__m128d A, __m128d B) {
 
 __m128d test_mm_cmpnle_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_cmpnle_sd
-  // CHECK: call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 6)
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 6)
   return _mm_cmpnle_sd(A, B);
 }
 
@@ -420,7 +425,7 @@ __m128d test_mm_cmpnlt_pd(__m128d A, __m128d B) {
 
 __m128d test_mm_cmpnlt_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_cmpnlt_sd
-  // CHECK: call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 5)
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 5)
   return _mm_cmpnlt_sd(A, B);
 }
 
@@ -434,7 +439,7 @@ __m128d test_mm_cmpord_pd(__m128d A, __m128d B) {
 
 __m128d test_mm_cmpord_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_cmpord_sd
-  // CHECK: call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 7)
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 7)
   return _mm_cmpord_sd(A, B);
 }
 
@@ -448,43 +453,43 @@ __m128d test_mm_cmpunord_pd(__m128d A, __m128d B) {
 
 __m128d test_mm_cmpunord_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_cmpunord_sd
-  // CHECK: call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 3)
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i8 3)
   return _mm_cmpunord_sd(A, B);
 }
 
 int test_mm_comieq_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_comieq_sd
-  // CHECK: call i32 @llvm.x86.sse2.comieq.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse2.comieq.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_comieq_sd(A, B);
 }
 
 int test_mm_comige_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_comige_sd
-  // CHECK: call i32 @llvm.x86.sse2.comige.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse2.comige.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_comige_sd(A, B);
 }
 
 int test_mm_comigt_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_comigt_sd
-  // CHECK: call i32 @llvm.x86.sse2.comigt.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse2.comigt.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_comigt_sd(A, B);
 }
 
 int test_mm_comile_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_comile_sd
-  // CHECK: call i32 @llvm.x86.sse2.comile.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse2.comile.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_comile_sd(A, B);
 }
 
 int test_mm_comilt_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_comilt_sd
-  // CHECK: call i32 @llvm.x86.sse2.comilt.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse2.comilt.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_comilt_sd(A, B);
 }
 
 int test_mm_comineq_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_comineq_sd
-  // CHECK: call i32 @llvm.x86.sse2.comineq.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse2.comineq.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_comineq_sd(A, B);
 }
 
@@ -509,7 +514,7 @@ __m128i test_mm_cvtpd_epi32(__m128d A) {
 
 __m128 test_mm_cvtpd_ps(__m128d A) {
   // CHECK-LABEL: test_mm_cvtpd_ps
-  // CHECK: call <4 x float> @llvm.x86.sse2.cvtpd2ps(<2 x double> %{{.*}})
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.sse2.cvtpd2ps(<2 x double> %{{.*}})
   return _mm_cvtpd_ps(A);
 }
 
@@ -534,21 +539,21 @@ double test_mm_cvtsd_f64(__m128d A) {
 
 int test_mm_cvtsd_si32(__m128d A) {
   // CHECK-LABEL: test_mm_cvtsd_si32
-  // CHECK: call i32 @llvm.x86.sse2.cvtsd2si(<2 x double> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse2.cvtsd2si(<2 x double> %{{.*}})
   return _mm_cvtsd_si32(A);
 }
 
 #ifdef __x86_64__
 long long test_mm_cvtsd_si64(__m128d A) {
   // X64-LABEL: test_mm_cvtsd_si64
-  // X64: call i64 @llvm.x86.sse2.cvtsd2si64(<2 x double> %{{.*}})
+  // X64: call {{.*}}i64 @llvm.x86.sse2.cvtsd2si64(<2 x double> %{{.*}})
   return _mm_cvtsd_si64(A);
 }
 #endif
 
 __m128 test_mm_cvtsd_ss(__m128 A, __m128d B) {
   // CHECK-LABEL: test_mm_cvtsd_ss
-  // CHECK: call <4 x float> @llvm.x86.sse2.cvtsd2ss(<4 x float> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.sse2.cvtsd2ss(<4 x float> %{{.*}}, <2 x double> %{{.*}})
   return _mm_cvtsd_ss(A, B);
 }
 
@@ -618,14 +623,14 @@ __m128i test_mm_cvttps_epi32(__m128 A) {
 
 int test_mm_cvttsd_si32(__m128d A) {
   // CHECK-LABEL: test_mm_cvttsd_si32
-  // CHECK: call i32 @llvm.x86.sse2.cvttsd2si(<2 x double> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse2.cvttsd2si(<2 x double> %{{.*}})
   return _mm_cvttsd_si32(A);
 }
 
 #ifdef __x86_64__
 long long test_mm_cvttsd_si64(__m128d A) {
   // X64-LABEL: test_mm_cvttsd_si64
-  // X64: call i64 @llvm.x86.sse2.cvttsd2si64(<2 x double> %{{.*}})
+  // X64: call {{.*}}i64 @llvm.x86.sse2.cvttsd2si64(<2 x double> %{{.*}})
   return _mm_cvttsd_si64(A);
 }
 #endif
@@ -699,7 +704,7 @@ __m128d test_mm_load1_pd(double const* A) {
   return _mm_load1_pd(A);
 }
 
-__m128d test_mm_loadh_pd(__m128d x, void* y) {
+__m128d test_mm_loadh_pd(__m128d x, double const* y) {
   // CHECK-LABEL: test_mm_loadh_pd
   // CHECK: load double, ptr %{{.*}}, align 1{{$}}
   // CHECK: insertelement <2 x double> %{{.*}}, double %{{.*}}, i32 1
@@ -714,7 +719,7 @@ __m128i test_mm_loadl_epi64(__m128i* y) {
   return _mm_loadl_epi64(y);
 }
 
-__m128d test_mm_loadl_pd(__m128d x, void* y) {
+__m128d test_mm_loadl_pd(__m128d x, double const* y) {
   // CHECK-LABEL: test_mm_loadl_pd
   // CHECK: load double, ptr %{{.*}}, align 1{{$}}
   // CHECK: insertelement <2 x double> poison, double %{{.*}}, i32 0
@@ -800,13 +805,13 @@ __m128i test_mm_max_epu8(__m128i A, __m128i B) {
 
 __m128d test_mm_max_pd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_max_pd
-  // CHECK: call <2 x double> @llvm.x86.sse2.max.pd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse2.max.pd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_max_pd(A, B);
 }
 
 __m128d test_mm_max_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_max_sd
-  // CHECK: call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse2.max.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_max_sd(A, B);
 }
 
@@ -830,13 +835,13 @@ __m128i test_mm_min_epu8(__m128i A, __m128i B) {
 
 __m128d test_mm_min_pd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_min_pd
-  // CHECK: call <2 x double> @llvm.x86.sse2.min.pd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse2.min.pd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_min_pd(A, B);
 }
 
 __m128d test_mm_min_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_min_sd
-  // CHECK: call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse2.min.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_min_sd(A, B);
 }
 
@@ -871,13 +876,13 @@ __m128d test_mm_move_sd(__m128d A, __m128d B) {
 
 int test_mm_movemask_epi8(__m128i A) {
   // CHECK-LABEL: test_mm_movemask_epi8
-  // CHECK: call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %{{.*}})
   return _mm_movemask_epi8(A);
 }
 
 int test_mm_movemask_pd(__m128d A) {
   // CHECK-LABEL: test_mm_movemask_pd
-  // CHECK: call i32 @llvm.x86.sse2.movmsk.pd(<2 x double> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse2.movmsk.pd(<2 x double> %{{.*}})
   return _mm_movemask_pd(A);
 }
 
@@ -960,7 +965,7 @@ void test_mm_pause(void) {
 
 __m128i test_mm_sad_epu8(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_sad_epu8
-  // CHECK: call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   return _mm_sad_epu8(A, B);
 }
 
@@ -1221,7 +1226,7 @@ __m128i test_mm_sll_epi32(__m128i A, __m128i B) {
 
 __m128i test_mm_sll_epi64(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_sll_epi64
-  // CHECK: call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_sll_epi64(A, B);
 }
 
@@ -1263,19 +1268,19 @@ __m128i test_mm_slli_epi32_2(__m128i A, int B) {
 
 __m128i test_mm_slli_epi64(__m128i A) {
   // CHECK-LABEL: test_mm_slli_epi64
-  // CHECK: call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %{{.*}}, i32 %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %{{.*}}, i32 %{{.*}})
   return _mm_slli_epi64(A, 1);
 }
 
 __m128i test_mm_slli_epi64_1(__m128i A) {
   // CHECK-LABEL: test_mm_slli_epi64_1
-  // CHECK: call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %{{.*}}, i32 %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %{{.*}}, i32 %{{.*}})
   return _mm_slli_epi64(A, -1);
 }
 
 __m128i test_mm_slli_epi64_2(__m128i A, int B) {
   // CHECK-LABEL: test_mm_slli_epi64_2
-  // CHECK: call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %{{.*}}, i32 %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %{{.*}}, i32 %{{.*}})
   return _mm_slli_epi64(A, B);
 }
 
@@ -1293,7 +1298,7 @@ __m128i test_mm_slli_si128_2(__m128i A) {
 
 __m128d test_mm_sqrt_pd(__m128d A) {
   // CHECK-LABEL: test_mm_sqrt_pd
-  // CHECK: call <2 x double> @llvm.sqrt.v2f64(<2 x double> %{{.*}})
+  // CHECK: call {{.*}}<2 x double> @llvm.sqrt.v2f64(<2 x double> %{{.*}})
   return _mm_sqrt_pd(A);
 }
 
@@ -1367,7 +1372,7 @@ __m128i test_mm_srl_epi32(__m128i A, __m128i B) {
 
 __m128i test_mm_srl_epi64(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_srl_epi64
-  // CHECK: call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_srl_epi64(A, B);
 }
 
@@ -1409,19 +1414,19 @@ __m128i test_mm_srli_epi32_2(__m128i A, int B) {
 
 __m128i test_mm_srli_epi64(__m128i A) {
   // CHECK-LABEL: test_mm_srli_epi64
-  // CHECK: call <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %{{.*}}, i32 %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %{{.*}}, i32 %{{.*}})
   return _mm_srli_epi64(A, 1);
 }
 
 __m128i test_mm_srli_epi64_1(__m128i A) {
   // CHECK-LABEL: test_mm_srli_epi64_1
-  // CHECK: call <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %{{.*}}, i32 %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %{{.*}}, i32 %{{.*}})
   return _mm_srli_epi64(A, -1);
 }
 
 __m128i test_mm_srli_epi64_2(__m128i A, int B) {
   // CHECK-LABEL: test_mm_srli_epi64_2
-  // CHECK: call <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %{{.*}}, i32 %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %{{.*}}, i32 %{{.*}})
   return _mm_srli_epi64(A, B);
 }
 
@@ -1481,7 +1486,7 @@ void test_mm_storel_epi64(__m128i x, void* y) {
   // CHECK-LABEL: test_mm_storel_epi64
   // CHECK: extractelement <2 x i64> %{{.*}}, i32 0
   // CHECK: store {{.*}} ptr {{.*}}, align 1{{$}}
-  _mm_storel_epi64(y, x);
+  _mm_storel_epi64((__m128i_u*)y, x);
 }
 
 void test_mm_storel_pd(double* A, __m128d B) {
@@ -1653,37 +1658,37 @@ __m128i test_mm_subs_epu16(__m128i A, __m128i B) {
 
 int test_mm_ucomieq_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_ucomieq_sd
-  // CHECK: call i32 @llvm.x86.sse2.ucomieq.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse2.ucomieq.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_ucomieq_sd(A, B);
 }
 
 int test_mm_ucomige_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_ucomige_sd
-  // CHECK: call i32 @llvm.x86.sse2.ucomige.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse2.ucomige.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_ucomige_sd(A, B);
 }
 
 int test_mm_ucomigt_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_ucomigt_sd
-  // CHECK: call i32 @llvm.x86.sse2.ucomigt.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse2.ucomigt.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_ucomigt_sd(A, B);
 }
 
 int test_mm_ucomile_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_ucomile_sd
-  // CHECK: call i32 @llvm.x86.sse2.ucomile.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse2.ucomile.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_ucomile_sd(A, B);
 }
 
 int test_mm_ucomilt_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_ucomilt_sd
-  // CHECK: call i32 @llvm.x86.sse2.ucomilt.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse2.ucomilt.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_ucomilt_sd(A, B);
 }
 
 int test_mm_ucomineq_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_ucomineq_sd
-  // CHECK: call i32 @llvm.x86.sse2.ucomineq.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse2.ucomineq.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_ucomineq_sd(A, B);
 }
 
diff --git a/clang/test/CodeGen/X86/sse3-builtins.c b/clang/test/CodeGen/X86/sse3-builtins.c
index efc9c31d356d9..57a18dadefaa8 100644
--- a/clang/test/CodeGen/X86/sse3-builtins.c
+++ b/clang/test/CodeGen/X86/sse3-builtins.c
@@ -1,5 +1,7 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse3 -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse3 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse3 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse3 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse3 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse3 -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 
 #include <immintrin.h>
@@ -8,37 +10,37 @@
 
 __m128d test_mm_addsub_pd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_addsub_pd
-  // CHECK: call <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse3.addsub.pd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_addsub_pd(A, B);
 }
 
 __m128 test_mm_addsub_ps(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_addsub_ps
-  // CHECK: call <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> %{{.*}}, <4 x float> %{{.*}})
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_addsub_ps(A, B);
 }
 
 __m128d test_mm_hadd_pd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_hadd_pd
-  // CHECK: call <2 x double> @llvm.x86.sse3.hadd.pd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse3.hadd.pd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_hadd_pd(A, B);
 }
 
 __m128 test_mm_hadd_ps(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_hadd_ps
-  // CHECK: call <4 x float> @llvm.x86.sse3.hadd.ps(<4 x float> %{{.*}}, <4 x float> %{{.*}})
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.sse3.hadd.ps(<4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_hadd_ps(A, B);
 }
 
 __m128d test_mm_hsub_pd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_hsub_pd
-  // CHECK: call <2 x double> @llvm.x86.sse3.hsub.pd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse3.hsub.pd(<2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_hsub_pd(A, B);
 }
 
 __m128 test_mm_hsub_ps(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_hsub_ps
-  // CHECK: call <4 x float> @llvm.x86.sse3.hsub.ps(<4 x float> %{{.*}}, <4 x float> %{{.*}})
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.sse3.hsub.ps(<4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_hsub_ps(A, B);
 }
 
diff --git a/clang/test/CodeGen/X86/sse41-builtins.c b/clang/test/CodeGen/X86/sse41-builtins.c
index bfe7a917a8855..340874a4fdd3c 100644
--- a/clang/test/CodeGen/X86/sse41-builtins.c
+++ b/clang/test/CodeGen/X86/sse41-builtins.c
@@ -1,7 +1,11 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse4.1 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse4.1 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse4.1 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse4.1 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse4.1 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse4.1 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK
 
 
 #include <immintrin.h>
@@ -34,37 +38,37 @@ __m128i test_mm_blendv_epi8(__m128i V1, __m128i V2, __m128i V3) {
 
 __m128d test_mm_blendv_pd(__m128d V1, __m128d V2, __m128d V3) {
   // CHECK-LABEL: test_mm_blendv_pd
-  // CHECK: call <2 x double> @llvm.x86.sse41.blendvpd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse41.blendvpd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_blendv_pd(V1, V2, V3);
 }
 
 __m128 test_mm_blendv_ps(__m128 V1, __m128 V2, __m128 V3) {
   // CHECK-LABEL: test_mm_blendv_ps
-  // CHECK: call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.sse41.blendvps(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_blendv_ps(V1, V2, V3);
 }
 
 __m128d test_mm_ceil_pd(__m128d x) {
   // CHECK-LABEL: test_mm_ceil_pd
-  // CHECK: call <2 x double> @llvm.x86.sse41.round.pd(<2 x double> %{{.*}}, i32 2)
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse41.round.pd(<2 x double> %{{.*}}, i32 2)
   return _mm_ceil_pd(x);
 }
 
 __m128 test_mm_ceil_ps(__m128 x) {
   // CHECK-LABEL: test_mm_ceil_ps
-  // CHECK: call <4 x float> @llvm.x86.sse41.round.ps(<4 x float> %{{.*}}, i32 2)
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.sse41.round.ps(<4 x float> %{{.*}}, i32 2)
   return _mm_ceil_ps(x);
 }
 
 __m128d test_mm_ceil_sd(__m128d x, __m128d y) {
   // CHECK-LABEL: test_mm_ceil_sd
-  // CHECK: call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i32 2)
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse41.round.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i32 2)
   return _mm_ceil_sd(x, y);
 }
 
 __m128 test_mm_ceil_ss(__m128 x, __m128 y) {
   // CHECK-LABEL: test_mm_ceil_ss
-  // CHECK: call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}}, i32 2)
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.sse41.round.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}}, i32 2)
   return _mm_ceil_ss(x, y);
 }
 
@@ -161,13 +165,13 @@ __m128i test_mm_cvtepu32_epi64(__m128i a) {
 
 __m128d test_mm_dp_pd(__m128d x, __m128d y) {
   // CHECK-LABEL: test_mm_dp_pd
-  // CHECK: call <2 x double> @llvm.x86.sse41.dppd(<2 x double> {{.*}}, <2 x double> {{.*}}, i8 7)
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse41.dppd(<2 x double> {{.*}}, <2 x double> {{.*}}, i8 7)
   return _mm_dp_pd(x, y, 7);
 }
 
 __m128 test_mm_dp_ps(__m128 x, __m128 y) {
   // CHECK-LABEL: test_mm_dp_ps
-  // CHECK: call <4 x float> @llvm.x86.sse41.dpps(<4 x float> {{.*}}, <4 x float> {{.*}}, i8 7)
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.sse41.dpps(<4 x float> {{.*}}, <4 x float> {{.*}}, i8 7)
   return _mm_dp_ps(x, y, 7);
 }
 
@@ -198,25 +202,25 @@ int test_mm_extract_ps(__m128 x) {
 
 __m128d test_mm_floor_pd(__m128d x) {
   // CHECK-LABEL: test_mm_floor_pd
-  // CHECK: call <2 x double> @llvm.x86.sse41.round.pd(<2 x double> %{{.*}}, i32 1)
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse41.round.pd(<2 x double> %{{.*}}, i32 1)
   return _mm_floor_pd(x);
 }
 
 __m128 test_mm_floor_ps(__m128 x) {
   // CHECK-LABEL: test_mm_floor_ps
-  // CHECK: call <4 x float> @llvm.x86.sse41.round.ps(<4 x float> %{{.*}}, i32 1)
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.sse41.round.ps(<4 x float> %{{.*}}, i32 1)
   return _mm_floor_ps(x);
 }
 
 __m128d test_mm_floor_sd(__m128d x, __m128d y) {
   // CHECK-LABEL: test_mm_floor_sd
-  // CHECK: call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i32 1)
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse41.round.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i32 1)
   return _mm_floor_sd(x, y);
 }
 
 __m128 test_mm_floor_ss(__m128 x, __m128 y) {
   // CHECK-LABEL: test_mm_floor_ss
-  // CHECK: call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}}, i32 1)
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.sse41.round.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}}, i32 1)
   return _mm_floor_ss(x, y);
 }
 
@@ -242,7 +246,7 @@ __m128i test_mm_insert_epi64(__m128i x, long long b) {
 
 __m128 test_mm_insert_ps(__m128 x, __m128 y) {
   // CHECK-LABEL: test_mm_insert_ps
-  // CHECK: call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %{{.*}}, <4 x float> %{{.*}}, i8 4)
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.sse41.insertps(<4 x float> %{{.*}}, <4 x float> %{{.*}}, i8 4)
   return _mm_insert_ps(x, y, 4);
 }
 
@@ -330,25 +334,25 @@ __m128i test_mm_packus_epi32(__m128i x, __m128i y) {
 
 __m128d test_mm_round_pd(__m128d x) {
   // CHECK-LABEL: test_mm_round_pd
-  // CHECK: call <2 x double> @llvm.x86.sse41.round.pd(<2 x double> %{{.*}}, i32 4)
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse41.round.pd(<2 x double> %{{.*}}, i32 4)
   return _mm_round_pd(x, 4);
 }
 
 __m128 test_mm_round_ps(__m128 x) {
   // CHECK-LABEL: test_mm_round_ps
-  // CHECK: call <4 x float> @llvm.x86.sse41.round.ps(<4 x float> %{{.*}}, i32 4)
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.sse41.round.ps(<4 x float> %{{.*}}, i32 4)
   return _mm_round_ps(x, 4);
 }
 
 __m128d test_mm_round_sd(__m128d x, __m128d y) {
   // CHECK-LABEL: test_mm_round_sd
-  // CHECK: call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i32 4)
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.sse41.round.sd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, i32 4)
   return _mm_round_sd(x, y, 4);
 }
 
 __m128 test_mm_round_ss(__m128 x, __m128 y) {
   // CHECK-LABEL: test_mm_round_ss
-  // CHECK: call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}}, i32 4)
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.sse41.round.ss(<4 x float> %{{.*}}, <4 x float> %{{.*}}, i32 4)
   return _mm_round_ss(x, y, 4);
 }
 
@@ -366,54 +370,37 @@ __m128i test_mm_stream_load_si128_void(const void *a) {
 
 int test_mm_test_all_ones(__m128i x) {
   // CHECK-LABEL: test_mm_test_all_ones
-  // CHECK: call i32 @llvm.x86.sse41.ptestc(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse41.ptestc(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_test_all_ones(x);
 }
 
 int test_mm_test_all_zeros(__m128i x, __m128i y) {
   // CHECK-LABEL: test_mm_test_all_zeros
-  // CHECK: call i32 @llvm.x86.sse41.ptestz(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse41.ptestz(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_test_all_zeros(x, y);
 }
 
 int test_mm_test_mix_ones_zeros(__m128i x, __m128i y) {
   // CHECK-LABEL: test_mm_test_mix_ones_zeros
-  // CHECK: call i32 @llvm.x86.sse41.ptestnzc(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse41.ptestnzc(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_test_mix_ones_zeros(x, y);
 }
 
 int test_mm_testc_si128(__m128i x, __m128i y) {
   // CHECK-LABEL: test_mm_testc_si128
-  // CHECK: call i32 @llvm.x86.sse41.ptestc(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse41.ptestc(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_testc_si128(x, y);
 }
 
 int test_mm_testnzc_si128(__m128i x, __m128i y) {
   // CHECK-LABEL: test_mm_testnzc_si128
-  // CHECK: call i32 @llvm.x86.sse41.ptestnzc(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse41.ptestnzc(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_testnzc_si128(x, y);
 }
 
 int test_mm_testz_si128(__m128i x, __m128i y) {
   // CHECK-LABEL: test_mm_testz_si128
-  // CHECK: call i32 @llvm.x86.sse41.ptestz(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse41.ptestz(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_testz_si128(x, y);
 }
 
-// Make sure brackets work after macro intrinsics.
-float pr51324(__m128 a) {
-  // CHECK-LABEL: pr51324
-  // CHECK: call <4 x float> @llvm.x86.sse41.round.ps(<4 x float> %{{.*}}, i32 0)
-  // CHECK: extractelement <4 x float> %{{.*}}, i32 0
-  return _mm_round_ps(a, 0)[0];
-}
-
-// Ensure _mm_test_all_ones macro doesn't reuse argument
-__m128i expensive_call();
-int pr60006() {
-  // CHECK-LABEL: pr60006
-  // CHECK: call {{.*}} @expensive_call
-  // CHECK-NOT: call {{.*}} @expensive_call
-  // CHECK: call i32 @llvm.x86.sse41.ptestc(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
-  return _mm_test_all_ones(expensive_call());
-}
diff --git a/clang/test/CodeGen/X86/sse42-builtins.c b/clang/test/CodeGen/X86/sse42-builtins.c
index 92fc7b301a0e2..d0c0cce33e1d0 100644
--- a/clang/test/CodeGen/X86/sse42-builtins.c
+++ b/clang/test/CodeGen/X86/sse42-builtins.c
@@ -1,7 +1,11 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.2 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.2 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse4.2 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse4.2 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.2 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.2 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse4.2 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse4.2 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.2 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.2 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse4.2 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse4.2 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK
 
 
 #include <immintrin.h>
@@ -10,19 +14,19 @@
 
 int test_mm_cmpestra(__m128i A, int LA, __m128i B, int LB) {
   // CHECK-LABEL: test_mm_cmpestra
-  // CHECK: call i32 @llvm.x86.sse42.pcmpestria128(<16 x i8> %{{.*}}, i32 %{{.*}}, <16 x i8> %{{.*}}, i32 %{{.*}}, i8 7)
+  // CHECK: call {{.*}}i32 @llvm.x86.sse42.pcmpestria128(<16 x i8> %{{.*}}, i32 %{{.*}}, <16 x i8> %{{.*}}, i32 %{{.*}}, i8 7)
   return _mm_cmpestra(A, LA, B, LB, 7);
 }
 
 int test_mm_cmpestrc(__m128i A, int LA, __m128i B, int LB) {
   // CHECK-LABEL: test_mm_cmpestrc
-  // CHECK: call i32 @llvm.x86.sse42.pcmpestric128(<16 x i8> %{{.*}}, i32 %{{.*}}, <16 x i8> %{{.*}}, i32 %{{.*}}, i8 7)
+  // CHECK: call {{.*}}i32 @llvm.x86.sse42.pcmpestric128(<16 x i8> %{{.*}}, i32 %{{.*}}, <16 x i8> %{{.*}}, i32 %{{.*}}, i8 7)
   return _mm_cmpestrc(A, LA, B, LB, 7);
 }
 
 int test_mm_cmpestri(__m128i A, int LA, __m128i B, int LB) {
   // CHECK-LABEL: test_mm_cmpestri
-  // CHECK: call i32 @llvm.x86.sse42.pcmpestri128(<16 x i8> %{{.*}}, i32 %{{.*}}, <16 x i8> %{{.*}}, i32 %{{.*}}, i8 7)
+  // CHECK: call {{.*}}i32 @llvm.x86.sse42.pcmpestri128(<16 x i8> %{{.*}}, i32 %{{.*}}, <16 x i8> %{{.*}}, i32 %{{.*}}, i8 7)
   return _mm_cmpestri(A, LA, B, LB, 7);
 }
 
@@ -34,19 +38,19 @@ __m128i test_mm_cmpestrm(__m128i A, int LA, __m128i B, int LB) {
 
 int test_mm_cmpestro(__m128i A, int LA, __m128i B, int LB) {
   // CHECK-LABEL: test_mm_cmpestro
-  // CHECK: call i32 @llvm.x86.sse42.pcmpestrio128(<16 x i8> %{{.*}}, i32 %{{.*}}, <16 x i8> %{{.*}}, i32 %{{.*}}, i8 7)
+  // CHECK: call {{.*}}i32 @llvm.x86.sse42.pcmpestrio128(<16 x i8> %{{.*}}, i32 %{{.*}}, <16 x i8> %{{.*}}, i32 %{{.*}}, i8 7)
   return _mm_cmpestro(A, LA, B, LB, 7);
 }
 
 int test_mm_cmpestrs(__m128i A, int LA, __m128i B, int LB) {
   // CHECK-LABEL: test_mm_cmpestrs
-  // CHECK: call i32 @llvm.x86.sse42.pcmpestris128(<16 x i8> %{{.*}}, i32 %{{.*}}, <16 x i8> %{{.*}}, i32 %{{.*}}, i8 7)
+  // CHECK: call {{.*}}i32 @llvm.x86.sse42.pcmpestris128(<16 x i8> %{{.*}}, i32 %{{.*}}, <16 x i8> %{{.*}}, i32 %{{.*}}, i8 7)
   return _mm_cmpestrs(A, LA, B, LB, 7);
 }
 
 int test_mm_cmpestrz(__m128i A, int LA, __m128i B, int LB) {
   // CHECK-LABEL: test_mm_cmpestrz
-  // CHECK: call i32 @llvm.x86.sse42.pcmpestriz128(<16 x i8> %{{.*}}, i32 %{{.*}}, <16 x i8> %{{.*}}, i32 %{{.*}}, i8 7)
+  // CHECK: call {{.*}}i32 @llvm.x86.sse42.pcmpestriz128(<16 x i8> %{{.*}}, i32 %{{.*}}, <16 x i8> %{{.*}}, i32 %{{.*}}, i8 7)
   return _mm_cmpestrz(A, LA, B, LB, 7);
 }
 
@@ -58,19 +62,19 @@ __m128i test_mm_cmpgt_epi64(__m128i A, __m128i B) {
 
 int test_mm_cmpistra(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_cmpistra
-  // CHECK: call i32 @llvm.x86.sse42.pcmpistria128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}}, i8 7)
+  // CHECK: call {{.*}}i32 @llvm.x86.sse42.pcmpistria128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}}, i8 7)
   return _mm_cmpistra(A, B, 7);
 }
 
 int test_mm_cmpistrc(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_cmpistrc
-  // CHECK: call i32 @llvm.x86.sse42.pcmpistric128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}}, i8 7)
+  // CHECK: call {{.*}}i32 @llvm.x86.sse42.pcmpistric128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}}, i8 7)
   return _mm_cmpistrc(A, B, 7);
 }
 
 int test_mm_cmpistri(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_cmpistri
-  // CHECK: call i32 @llvm.x86.sse42.pcmpistri128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}}, i8 7)
+  // CHECK: call {{.*}}i32 @llvm.x86.sse42.pcmpistri128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}}, i8 7)
   return _mm_cmpistri(A, B, 7);
 }
 
@@ -82,44 +86,44 @@ __m128i test_mm_cmpistrm(__m128i A, __m128i B) {
 
 int test_mm_cmpistro(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_cmpistro
-  // CHECK: call i32 @llvm.x86.sse42.pcmpistrio128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}}, i8 7)
+  // CHECK: call {{.*}}i32 @llvm.x86.sse42.pcmpistrio128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}}, i8 7)
   return _mm_cmpistro(A, B, 7);
 }
 
 int test_mm_cmpistrs(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_cmpistrs
-  // CHECK: call i32 @llvm.x86.sse42.pcmpistris128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}}, i8 7)
+  // CHECK: call {{.*}}i32 @llvm.x86.sse42.pcmpistris128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}}, i8 7)
   return _mm_cmpistrs(A, B, 7);
 }
 
 int test_mm_cmpistrz(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_cmpistrz
-  // CHECK: call i32 @llvm.x86.sse42.pcmpistriz128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}}, i8 7)
+  // CHECK: call {{.*}}i32 @llvm.x86.sse42.pcmpistriz128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}}, i8 7)
   return _mm_cmpistrz(A, B, 7);
 }
 
 unsigned int test_mm_crc32_u8(unsigned int CRC, unsigned char V) {
   // CHECK-LABEL: test_mm_crc32_u8
-  // CHECK: call i32 @llvm.x86.sse42.crc32.32.8(i32 %{{.*}}, i8 %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse42.crc32.32.8(i32 %{{.*}}, i8 %{{.*}})
   return _mm_crc32_u8(CRC, V);
 }
 
 unsigned int test_mm_crc32_u16(unsigned int CRC, unsigned short V) {
   // CHECK-LABEL: test_mm_crc32_u16
-  // CHECK: call i32 @llvm.x86.sse42.crc32.32.16(i32 %{{.*}}, i16 %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse42.crc32.32.16(i32 %{{.*}}, i16 %{{.*}})
   return _mm_crc32_u16(CRC, V);
 }
 
 unsigned int test_mm_crc32_u32(unsigned int CRC, unsigned int V) {
   // CHECK-LABEL: test_mm_crc32_u32
-  // CHECK: call i32 @llvm.x86.sse42.crc32.32.32(i32 %{{.*}}, i32 %{{.*}})
+  // CHECK: call {{.*}}i32 @llvm.x86.sse42.crc32.32.32(i32 %{{.*}}, i32 %{{.*}})
   return _mm_crc32_u32(CRC, V);
 }
 
 #ifdef __x86_64__
 unsigned long long test_mm_crc32_u64(unsigned long long CRC, unsigned long long V) {
   // X64-LABEL: test_mm_crc32_u64
-  // X64: call i64 @llvm.x86.sse42.crc32.64.64(i64 %{{.*}}, i64 %{{.*}})
+  // X64: call {{.*}}i64 @llvm.x86.sse42.crc32.64.64(i64 %{{.*}}, i64 %{{.*}})
   return _mm_crc32_u64(CRC, V);
 }
 #endif
diff --git a/clang/test/CodeGen/X86/sse4a-builtins.c b/clang/test/CodeGen/X86/sse4a-builtins.c
index 5d38a9cde329d..68fd03eb06140 100644
--- a/clang/test/CodeGen/X86/sse4a-builtins.c
+++ b/clang/test/CodeGen/X86/sse4a-builtins.c
@@ -1,5 +1,7 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4a -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse4a -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4a -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse4a -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4a -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +sse4a -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 
 #include <x86intrin.h>
@@ -8,25 +10,25 @@
 
 __m128i test_mm_extracti_si64(__m128i x) {
   // CHECK-LABEL: test_mm_extracti_si64
-  // CHECK: call <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64> %{{[^,]+}}, i8 3, i8 2)
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64> %{{[^,]+}}, i8 3, i8 2)
   return _mm_extracti_si64(x, 3, 2);
 }
 
 __m128i test_mm_extract_si64(__m128i x, __m128i y) {
   // CHECK-LABEL: test_mm_extract_si64
-  // CHECK: call <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> %{{[^,]+}}, <16 x i8> %{{[^,]+}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> %{{[^,]+}}, <16 x i8> %{{[^,]+}})
   return _mm_extract_si64(x, y);
 }
 
 __m128i test_mm_inserti_si64(__m128i x, __m128i y) {
   // CHECK-LABEL: test_mm_inserti_si64
-  // CHECK: call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %{{[^,]+}}, <2 x i64> %{{[^,]+}}, i8 5, i8 6)
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %{{[^,]+}}, <2 x i64> %{{[^,]+}}, i8 5, i8 6)
   return _mm_inserti_si64(x, y, 5, 6);
 }
 
 __m128i test_mm_insert_si64(__m128i x, __m128i y) {
   // CHECK-LABEL: test_mm_insert_si64
-  // CHECK: call <2 x i64> @llvm.x86.sse4a.insertq(<2 x i64> %{{[^,]+}}, <2 x i64> %{{[^,]+}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.sse4a.insertq(<2 x i64> %{{[^,]+}}, <2 x i64> %{{[^,]+}})
   return _mm_insert_si64(x, y);
 }
 
diff --git a/clang/test/CodeGen/X86/ssse3-builtins.c b/clang/test/CodeGen/X86/ssse3-builtins.c
index ea65af0447e91..982c74c5fc3b4 100644
--- a/clang/test/CodeGen/X86/ssse3-builtins.c
+++ b/clang/test/CodeGen/X86/ssse3-builtins.c
@@ -1,5 +1,7 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +ssse3 -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +ssse3 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +ssse3 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +ssse3 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +ssse3 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +ssse3 -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 
 #include <immintrin.h>
diff --git a/clang/test/CodeGen/X86/x86-intrinsics-imm.c b/clang/test/CodeGen/X86/x86-intrinsics-imm.c
index 58bcf187b6f44..76c0d2f01dc8b 100644
--- a/clang/test/CodeGen/X86/x86-intrinsics-imm.c
+++ b/clang/test/CodeGen/X86/x86-intrinsics-imm.c
@@ -23,7 +23,6 @@ __m128i check__mm256_cvtps_ph(__m256 val, const int I) {
   return _mm256_cvtps_ph(val, I); // expected-error  {{argument to '__builtin_ia32_vcvtps2ph256' must be a constant integer}}
 }
 
-// SCE_ARCH begin: bug 16381
 void check__mm_slli_si128(__m128i a, const int count) {
   _mm_slli_si128(a, count); // expected-error {{argument to '__builtin_ia32_pslldqi128_byteshift' must be a constant integer}}
 }
@@ -31,7 +30,6 @@ void check__mm_slli_si128(__m128i a, const int count) {
 void check__mm_srli_si128(__m128i a, const int count) {
   _mm_srli_si128(a, count); // expected-error {{argument to '__builtin_ia32_psrldqi128_byteshift' must be a constant integer}}
 }
-// SCE_ARCH end
 
 void check__mm_shuffle_epi32(__m128i  a, const int imm) {
   _mm_shuffle_epi32(a, imm); // expected-error {{argument to '__builtin_ia32_pshufd' must be a constant integer}}
diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_write_lane_zt.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_write_lane_zt.c
index e73697a569473..f295bfb2d18b4 100644
--- a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_write_lane_zt.c
+++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_write_lane_zt.c
@@ -1,10 +1,10 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
 
-// RUN: %clang_cc1 -triple aarch64 -target-feature +bf16 -target-feature +sme -target-feature +sme2 -target-feature +sme-lutv2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
-// RUN: %clang_cc1 -triple aarch64 -target-feature +bf16 -target-feature +sme -target-feature +sme2 -target-feature +sme-lutv2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
-// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +bf16 -target-feature +sme -target-feature +sme2 -target-feature +sme-lutv2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
-// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +bf16 -target-feature +sme -target-feature +sme2 -target-feature +sme-lutv2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
-// RUN: %clang_cc1 -triple aarch64 -target-feature +bf16 -target-feature +sme -target-feature +sme2 -target-feature +sme-lutv2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+// RUN: %clang_cc1 -triple aarch64 -target-feature +bf16 -target-feature +sme2  -target-feature +sme  -target-feature +sme-lutv2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64 -target-feature +bf16 -target-feature +sme2  -target-feature +sme  -target-feature +sme-lutv2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +bf16 -target-feature +sme  -target-feature +sme2  -target-feature +sme-lutv2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +bf16 -target-feature +sme  -target-feature +sme2 -target-feature +sme-lutv2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64 -target-feature +bf16 -target-feature +sme  -target-feature +sme2 -target-feature +sme-lutv2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
 
 // REQUIRES: aarch64-registered-target
 
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvt-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvt-bfloat.c
index 145d60db6eda3..cbeac2f384f9a 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvt-bfloat.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvt-bfloat.c
@@ -24,14 +24,14 @@
 
 // CHECK-LABEL: @test_svcvt_bf16_f32_x(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32.v2(<vscale x 8 x bfloat> undef, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
 // CPP-CHECK-LABEL: @_Z21test_svcvt_bf16_f32_xu10__SVBool_tu13__SVFloat32_t(
 // CPP-CHECK-NEXT:  entry:
-// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32.v2(<vscale x 8 x bfloat> undef, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
 svbfloat16_t test_svcvt_bf16_f32_x(svbool_t pg, svfloat32_t op) MODE_ATTR {
@@ -40,14 +40,14 @@ svbfloat16_t test_svcvt_bf16_f32_x(svbool_t pg, svfloat32_t op) MODE_ATTR {
 
 // CHECK-LABEL: @test_svcvt_bf16_f32_z(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32.v2(<vscale x 8 x bfloat> zeroinitializer, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
 // CPP-CHECK-LABEL: @_Z21test_svcvt_bf16_f32_zu10__SVBool_tu13__SVFloat32_t(
 // CPP-CHECK-NEXT:  entry:
-// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32.v2(<vscale x 8 x bfloat> zeroinitializer, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
 svbfloat16_t test_svcvt_bf16_f32_z(svbool_t pg, svfloat32_t op) MODE_ATTR {
@@ -56,14 +56,14 @@ svbfloat16_t test_svcvt_bf16_f32_z(svbool_t pg, svfloat32_t op) MODE_ATTR {
 
 // CHECK-LABEL: @test_svcvt_bf16_f32_m(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32(<vscale x 8 x bfloat> [[INACTIVE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32.v2(<vscale x 8 x bfloat> [[INACTIVE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
 // CPP-CHECK-LABEL: @_Z21test_svcvt_bf16_f32_mu14__SVBfloat16_tu10__SVBool_tu13__SVFloat32_t(
 // CPP-CHECK-NEXT:  entry:
-// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32(<vscale x 8 x bfloat> [[INACTIVE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32.v2(<vscale x 8 x bfloat> [[INACTIVE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
 svbfloat16_t test_svcvt_bf16_f32_m(svbfloat16_t inactive, svbool_t pg, svfloat32_t op) MODE_ATTR {
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvtnt.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvtnt.c
index ce719f92674c0..9a92ddc448f19 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvtnt.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cvtnt.c
@@ -24,14 +24,14 @@
 
 // CHECK-LABEL: @test_svcvtnt_bf16_f32_x(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32(<vscale x 8 x bfloat> [[EVEN:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32.v2(<vscale x 8 x bfloat> [[EVEN:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
 // CPP-CHECK-LABEL: @_Z23test_svcvtnt_bf16_f32_xu14__SVBfloat16_tu10__SVBool_tu13__SVFloat32_t(
 // CPP-CHECK-NEXT:  entry:
-// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32(<vscale x 8 x bfloat> [[EVEN:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32.v2(<vscale x 8 x bfloat> [[EVEN:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
 svbfloat16_t test_svcvtnt_bf16_f32_x(svbfloat16_t even, svbool_t pg, svfloat32_t op) MODE_ATTR {
@@ -40,14 +40,14 @@ svbfloat16_t test_svcvtnt_bf16_f32_x(svbfloat16_t even, svbool_t pg, svfloat32_t
 
 // CHECK-LABEL: @test_svcvtnt_bf16_f32_m(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32(<vscale x 8 x bfloat> [[EVEN:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32.v2(<vscale x 8 x bfloat> [[EVEN:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
 // CPP-CHECK-LABEL: @_Z23test_svcvtnt_bf16_f32_mu14__SVBfloat16_tu10__SVBool_tu13__SVFloat32_t(
 // CPP-CHECK-NEXT:  entry:
-// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32(<vscale x 8 x bfloat> [[EVEN:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32.v2(<vscale x 8 x bfloat> [[EVEN:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
 svbfloat16_t test_svcvtnt_bf16_f32_m(svbfloat16_t even, svbool_t pg, svfloat32_t op) MODE_ATTR {
diff --git a/clang/test/CodeGen/align-loops.c b/clang/test/CodeGen/align-loops.c
index 7744b4ab0f24e..25f8cded59f32 100644
--- a/clang/test/CodeGen/align-loops.c
+++ b/clang/test/CodeGen/align-loops.c
@@ -5,8 +5,8 @@
 // RUN: %clang_cc1 -triple=x86_64 -S %s -falign-loops=32 -O -o - | FileCheck %s --check-prefixes=CHECK,CHECK_32
 
 // CHECK-LABEL: foo:
-// CHECK_8:       .p2align 3, 0x90
-// CHECK_32:      .p2align 5, 0x90
+// CHECK_8:       .p2align 3
+// CHECK_32:      .p2align 5
 
 void bar(void);
 void foo(void) {
diff --git a/clang/test/CodeGen/arm-acle-coproc.c b/clang/test/CodeGen/arm-acle-coproc.c
index 0354d1297ece1..93b713ba97318 100644
--- a/clang/test/CodeGen/arm-acle-coproc.c
+++ b/clang/test/CodeGen/arm-acle-coproc.c
@@ -25,6 +25,7 @@
 // RUN: %clang_cc1 -triple armv9.3a %s -E -dD -o - | FileCheck --check-prefix=CHECK-V8 %s
 // RUN: %clang_cc1 -triple armv9.4a %s -E -dD -o - | FileCheck --check-prefix=CHECK-V8 %s
 // RUN: %clang_cc1 -triple armv9.5a %s -E -dD -o - | FileCheck --check-prefix=CHECK-V8 %s
+// RUN: %clang_cc1 -triple armv9.6a %s -E -dD -o - | FileCheck --check-prefix=CHECK-V8 %s
 // RUN: %clang_cc1 -triple thumbv4 %s -E -dD -o - | FileCheck --check-prefix=CHECK-V4-THUMB %s
 // RUN: %clang_cc1 -triple thumbv4t %s -E -dD -o - | FileCheck --check-prefix=CHECK-V4-THUMB %s
 // RUN: %clang_cc1 -triple thumbv5 %s -E -dD -o - | FileCheck --check-prefix=CHECK-V5-THUMB %s
@@ -54,6 +55,7 @@
 // RUN: %clang_cc1 -triple thumbv9.3a %s -E -dD -o - | FileCheck --check-prefix=CHECK-V8 %s
 // RUN: %clang_cc1 -triple thumbv9.4a %s -E -dD -o - | FileCheck --check-prefix=CHECK-V8 %s
 // RUN: %clang_cc1 -triple thumbv9.5a %s -E -dD -o - | FileCheck --check-prefix=CHECK-V8 %s
+// RUN: %clang_cc1 -triple thumbv9.6a %s -E -dD -o - | FileCheck --check-prefix=CHECK-V8 %s
 // RUN: %clang_cc1 -triple thumbv8m.base %s -E -dD -o - | FileCheck --check-prefix=CHECK-V8-BASE %s
 // RUN: %clang_cc1 -triple thumbv8m.main %s -E -dD -o - | FileCheck --check-prefix=CHECK-V8-MAIN %s
 // RUN: %clang_cc1 -triple thumbv8.1m.main %s -E -dD -o - | FileCheck --check-prefix=CHECK-V8-MAIN %s
diff --git a/clang/test/CodeGen/attr-counted-by-pr110385.c b/clang/test/CodeGen/attr-counted-by-pr110385.c
new file mode 100644
index 0000000000000..e120dcc583578
--- /dev/null
+++ b/clang/test/CodeGen/attr-counted-by-pr110385.c
@@ -0,0 +1,62 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -O2 -Wno-missing-declarations -emit-llvm -o - %s | FileCheck %s
+
+// See #110385
+// Based on reproducer from Kees Cook:
+// https://lore.kernel.org/all/202409170436.C3C6E7F7A@keescook/
+
+struct variable {
+        int a;
+        int b;
+        int length;
+        short array[] __attribute__((counted_by(length)));
+};
+
+struct bucket {
+        int a;
+        struct variable *growable;
+        int b;
+};
+
+struct bucket2 {
+        int a;
+        struct variable growable;
+};
+
+void init(void * __attribute__((pass_dynamic_object_size(0))));
+
+// CHECK-LABEL: define dso_local void @test1(
+// CHECK-SAME: ptr nocapture noundef readonly [[FOO:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[GROWABLE:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO]], i64 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[GROWABLE]], align 8, !tbaa [[TBAA2:![0-9]+]]
+// CHECK-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 12
+// CHECK-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
+// CHECK-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[DOT_COUNTED_BY_LOAD]] to i64
+// CHECK-NEXT:    [[TMP2:%.*]] = shl nsw i64 [[TMP1]], 1
+// CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[DOT_COUNTED_BY_LOAD]], -1
+// CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i64 [[TMP2]], i64 0
+// CHECK-NEXT:    tail call void @init(ptr noundef nonnull [[ARRAY]], i64 noundef [[TMP4]]) #[[ATTR2:[0-9]+]]
+// CHECK-NEXT:    ret void
+//
+void test1(struct bucket *foo) {
+        init(foo->growable->array);
+}
+
+// CHECK-LABEL: define dso_local void @test2(
+// CHECK-SAME: ptr noundef [[FOO:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw i8, ptr [[FOO]], i64 16
+// CHECK-NEXT:    [[DOT_COUNTED_BY_GEP:%.*]] = getelementptr inbounds i8, ptr [[FOO]], i64 12
+// CHECK-NEXT:    [[DOT_COUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[DOT_COUNTED_BY_LOAD]] to i64
+// CHECK-NEXT:    [[TMP1:%.*]] = shl nsw i64 [[TMP0]], 1
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[DOT_COUNTED_BY_LOAD]], -1
+// CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i64 [[TMP1]], i64 0
+// CHECK-NEXT:    tail call void @init(ptr noundef nonnull [[ARRAY]], i64 noundef [[TMP3]]) #[[ATTR2]]
+// CHECK-NEXT:    ret void
+//
+void test2(struct bucket2 *foo) {
+        init(foo->growable.array);
+}
diff --git a/clang/test/CodeGen/attr-counted-by.c b/clang/test/CodeGen/attr-counted-by.c
index 2e2d9cdd4da20..4a130c5e3d401 100644
--- a/clang/test/CodeGen/attr-counted-by.c
+++ b/clang/test/CodeGen/attr-counted-by.c
@@ -318,36 +318,36 @@ size_t test3_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[CONV1:%.*]] = select i1 [[TMP2]], i32 [[TMP5]], i32 0
 // SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM]]
 // SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV1]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA4]]
-// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD7:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
+// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD6:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
 // SANITIZE-WITH-ATTR-NEXT:    [[ADD:%.*]] = add nsw i32 [[INDEX]], 1
-// SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM13:%.*]] = sext i32 [[ADD]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP6:%.*]] = zext i32 [[DOT_COUNTED_BY_LOAD7]] to i64, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[IDXPROM13]], [[TMP6]], !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP7]], label [[CONT20:%.*]], label [[HANDLER_OUT_OF_BOUNDS16:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR:       handler.out_of_bounds16:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB6:[0-9]+]], i64 [[IDXPROM13]]) #[[ATTR10]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM12:%.*]] = sext i32 [[ADD]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP6:%.*]] = zext i32 [[DOT_COUNTED_BY_LOAD6]] to i64, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[IDXPROM12]], [[TMP6]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP7]], label [[CONT19:%.*]], label [[HANDLER_OUT_OF_BOUNDS15:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       handler.out_of_bounds15:
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB6:[0-9]+]], i64 [[IDXPROM12]]) #[[ATTR10]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR:       cont20:
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP8:%.*]] = icmp sgt i32 [[DOT_COUNTED_BY_LOAD7]], 3
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP9:%.*]] = shl i32 [[DOT_COUNTED_BY_LOAD7]], 2
+// SANITIZE-WITH-ATTR:       cont19:
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP8:%.*]] = icmp sgt i32 [[DOT_COUNTED_BY_LOAD6]], 3
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP9:%.*]] = shl i32 [[DOT_COUNTED_BY_LOAD6]], 2
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP10:%.*]] = add i32 [[TMP9]], 240
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP11:%.*]] = and i32 [[TMP10]], 252
-// SANITIZE-WITH-ATTR-NEXT:    [[CONV9:%.*]] = select i1 [[TMP8]], i32 [[TMP11]], i32 0
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM13]]
-// SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV9]], ptr [[ARRAYIDX18]], align 4, !tbaa [[TBAA4]]
-// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD23:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
-// SANITIZE-WITH-ATTR-NEXT:    [[ADD29:%.*]] = add nsw i32 [[INDEX]], 2
-// SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM30:%.*]] = sext i32 [[ADD29]] to i64
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP12:%.*]] = zext i32 [[DOT_COUNTED_BY_LOAD23]] to i64, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP13:%.*]] = icmp ult i64 [[IDXPROM30]], [[TMP12]], !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP13]], label [[CONT37:%.*]], label [[HANDLER_OUT_OF_BOUNDS33:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR:       handler.out_of_bounds33:
-// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB7:[0-9]+]], i64 [[IDXPROM30]]) #[[ATTR10]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[CONV8:%.*]] = select i1 [[TMP8]], i32 [[TMP11]], i32 0
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM12]]
+// SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV8]], ptr [[ARRAYIDX17]], align 4, !tbaa [[TBAA4]]
+// SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD21:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
+// SANITIZE-WITH-ATTR-NEXT:    [[ADD27:%.*]] = add nsw i32 [[INDEX]], 2
+// SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM28:%.*]] = sext i32 [[ADD27]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP12:%.*]] = zext i32 [[DOT_COUNTED_BY_LOAD21]] to i64, !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP13:%.*]] = icmp ult i64 [[IDXPROM28]], [[TMP12]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP13]], label [[CONT35:%.*]], label [[HANDLER_OUT_OF_BOUNDS31:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
+// SANITIZE-WITH-ATTR:       handler.out_of_bounds31:
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB7:[0-9]+]], i64 [[IDXPROM28]]) #[[ATTR10]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize [[META2]]
-// SANITIZE-WITH-ATTR:       cont37:
-// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM30]]
+// SANITIZE-WITH-ATTR:       cont35:
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM28]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP14:%.*]] = icmp sgt i32 [[FAM_IDX]], -1
-// SANITIZE-WITH-ATTR-NEXT:    [[TMP15:%.*]] = sext i32 [[DOT_COUNTED_BY_LOAD23]] to i64
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP15:%.*]] = sext i32 [[DOT_COUNTED_BY_LOAD21]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP16:%.*]] = sext i32 [[FAM_IDX]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP17:%.*]] = sub nsw i64 [[TMP15]], [[TMP16]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP18:%.*]] = icmp sgt i64 [[TMP17]], -1
@@ -355,8 +355,8 @@ size_t test3_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTTR:%.*]] = trunc i64 [[TMP17]] to i32
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP20:%.*]] = shl i32 [[DOTTR]], 2
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP21:%.*]] = and i32 [[TMP20]], 252
-// SANITIZE-WITH-ATTR-NEXT:    [[CONV25:%.*]] = select i1 [[TMP19]], i32 [[TMP21]], i32 0
-// SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV25]], ptr [[ARRAYIDX35]], align 4, !tbaa [[TBAA4]]
+// SANITIZE-WITH-ATTR-NEXT:    [[CONV23:%.*]] = select i1 [[TMP19]], i32 [[TMP21]], i32 0
+// SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV23]], ptr [[ARRAYIDX33]], align 4, !tbaa [[TBAA4]]
 // SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test4(
@@ -373,18 +373,18 @@ size_t test3_bdos(struct annotated *p) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV1]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD4:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = shl i32 [[DOT_COUNTED_BY_LOAD4]], 2
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD3:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = shl i32 [[DOT_COUNTED_BY_LOAD3]], 2
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP5:%.*]] = add i32 [[TMP4]], 240
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP6:%.*]] = icmp sgt i32 [[DOT_COUNTED_BY_LOAD4]], 3
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP6:%.*]] = icmp sgt i32 [[DOT_COUNTED_BY_LOAD3]], 3
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP7:%.*]] = and i32 [[TMP5]], 252
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV6:%.*]] = select i1 [[TMP6]], i32 [[TMP7]], i32 0
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV5:%.*]] = select i1 [[TMP6]], i32 [[TMP7]], i32 0
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ADD:%.*]] = add nsw i32 [[INDEX]], 1
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM8:%.*]] = sext i32 [[ADD]] to i64
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM8]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV6]], ptr [[ARRAYIDX9]], align 4, !tbaa [[TBAA2]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD12:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP8:%.*]] = sext i32 [[DOT_COUNTED_BY_LOAD12]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[ADD]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM7]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV5]], ptr [[ARRAYIDX8]], align 4, !tbaa [[TBAA2]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[DOT_COUNTED_BY_LOAD10:%.*]] = load i32, ptr [[DOT_COUNTED_BY_GEP]], align 4
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP8:%.*]] = sext i32 [[DOT_COUNTED_BY_LOAD10]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP9:%.*]] = sext i32 [[FAM_IDX]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP10:%.*]] = sub nsw i64 [[TMP8]], [[TMP9]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP11:%.*]] = icmp sgt i64 [[TMP10]], -1
@@ -393,11 +393,11 @@ size_t test3_bdos(struct annotated *p) {
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[DOTTR:%.*]] = trunc i64 [[TMP10]] to i32
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP14:%.*]] = shl i32 [[DOTTR]], 2
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP15:%.*]] = and i32 [[TMP14]], 252
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV14:%.*]] = select i1 [[TMP13]], i32 [[TMP15]], i32 0
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[ADD16:%.*]] = add nsw i32 [[INDEX]], 2
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM17:%.*]] = sext i32 [[ADD16]] to i64
-// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM17]]
-// NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV14]], ptr [[ARRAYIDX18]], align 4, !tbaa [[TBAA2]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV12:%.*]] = select i1 [[TMP13]], i32 [[TMP15]], i32 0
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ADD14:%.*]] = add nsw i32 [[INDEX]], 2
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM15:%.*]] = sext i32 [[ADD14]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM15]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV12]], ptr [[ARRAYIDX16]], align 4, !tbaa [[TBAA2]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    ret void
 //
 // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test4(
diff --git a/clang/test/CodeGen/builtins-elementwise-math.c b/clang/test/CodeGen/builtins-elementwise-math.c
index 0e53d3e141b01..2850125a54c44 100644
--- a/clang/test/CodeGen/builtins-elementwise-math.c
+++ b/clang/test/CodeGen/builtins-elementwise-math.c
@@ -169,6 +169,82 @@ void test_builtin_elementwise_sub_sat(float f1, float f2, double d1, double d2,
   i1 = __builtin_elementwise_sub_sat(1, 'a');
 }
 
+void test_builtin_elementwise_maximum(float f1, float f2, double d1, double d2,
+                                      float4 vf1, float4 vf2, long long int i1,
+                                      long long int i2, si8 vi1, si8 vi2,
+                                      unsigned u1, unsigned u2, u4 vu1, u4 vu2,
+                                      _BitInt(31) bi1, _BitInt(31) bi2,
+                                      unsigned _BitInt(55) bu1, unsigned _BitInt(55) bu2) {
+  // CHECK-LABEL: define void @test_builtin_elementwise_maximum(
+  // CHECK:      [[F1:%.+]] = load float, ptr %f1.addr, align 4
+  // CHECK-NEXT: [[F2:%.+]] = load float, ptr %f2.addr, align 4
+  // CHECK-NEXT:  call float @llvm.maximum.f32(float [[F1]], float [[F2]])
+  f1 = __builtin_elementwise_maximum(f1, f2);
+
+  // CHECK:      [[D1:%.+]] = load double, ptr %d1.addr, align 8
+  // CHECK-NEXT: [[D2:%.+]] = load double, ptr %d2.addr, align 8
+  // CHECK-NEXT: call double @llvm.maximum.f64(double [[D1]], double [[D2]])
+  d1 = __builtin_elementwise_maximum(d1, d2);
+
+  // CHECK:      [[D2:%.+]] = load double, ptr %d2.addr, align 8
+  // CHECK-NEXT: call double @llvm.maximum.f64(double 2.000000e+01, double [[D2]])
+  d1 = __builtin_elementwise_maximum(20.0, d2);
+
+  // CHECK:      [[VF1:%.+]] = load <4 x float>, ptr %vf1.addr, align 16
+  // CHECK-NEXT: [[VF2:%.+]] = load <4 x float>, ptr %vf2.addr, align 16
+  // CHECK-NEXT: call <4 x float> @llvm.maximum.v4f32(<4 x float> [[VF1]], <4 x float> [[VF2]])
+  vf1 = __builtin_elementwise_maximum(vf1, vf2);
+
+  // CHECK:      [[CVF1:%.+]] = load <4 x float>, ptr %cvf1, align 16
+  // CHECK-NEXT: [[VF2:%.+]] = load <4 x float>, ptr %vf2.addr, align 16
+  // CHECK-NEXT: call <4 x float> @llvm.maximum.v4f32(<4 x float> [[CVF1]], <4 x float> [[VF2]])
+  const float4 cvf1 = vf1;
+  vf1 = __builtin_elementwise_maximum(cvf1, vf2);
+
+  // CHECK:      [[VF2:%.+]] = load <4 x float>, ptr %vf2.addr, align 16
+  // CHECK-NEXT: [[CVF1:%.+]] = load <4 x float>, ptr %cvf1, align 16
+  // CHECK-NEXT: call <4 x float> @llvm.maximum.v4f32(<4 x float> [[VF2]], <4 x float> [[CVF1]])
+  vf1 = __builtin_elementwise_maximum(vf2, cvf1);
+}
+
+void test_builtin_elementwise_minimum(float f1, float f2, double d1, double d2,
+                                      float4 vf1, float4 vf2, long long int i1,
+                                      long long int i2, si8 vi1, si8 vi2,
+                                      unsigned u1, unsigned u2, u4 vu1, u4 vu2,
+                                      _BitInt(31) bi1, _BitInt(31) bi2,
+                                      unsigned _BitInt(55) bu1, unsigned _BitInt(55) bu2) {
+  // CHECK-LABEL: define void @test_builtin_elementwise_minimum(
+  // CHECK:      [[F1:%.+]] = load float, ptr %f1.addr, align 4
+  // CHECK-NEXT: [[F2:%.+]] = load float, ptr %f2.addr, align 4
+  // CHECK-NEXT:  call float @llvm.minimum.f32(float [[F1]], float [[F2]])
+  f1 = __builtin_elementwise_minimum(f1, f2);
+
+  // CHECK:      [[D1:%.+]] = load double, ptr %d1.addr, align 8
+  // CHECK-NEXT: [[D2:%.+]] = load double, ptr %d2.addr, align 8
+  // CHECK-NEXT: call double @llvm.minimum.f64(double [[D1]], double [[D2]])
+  d1 = __builtin_elementwise_minimum(d1, d2);
+
+  // CHECK:      [[D1:%.+]] = load double, ptr %d1.addr, align 8
+  // CHECK-NEXT: call double @llvm.minimum.f64(double [[D1]], double 2.000000e+00)
+  d1 = __builtin_elementwise_minimum(d1, 2.0);
+
+  // CHECK:      [[VF1:%.+]] = load <4 x float>, ptr %vf1.addr, align 16
+  // CHECK-NEXT: [[VF2:%.+]] = load <4 x float>, ptr %vf2.addr, align 16
+  // CHECK-NEXT: call <4 x float> @llvm.minimum.v4f32(<4 x float> [[VF1]], <4 x float> [[VF2]])
+  vf1 = __builtin_elementwise_minimum(vf1, vf2);
+
+  // CHECK:      [[CVF1:%.+]] = load <4 x float>, ptr %cvf1, align 16
+  // CHECK-NEXT: [[VF2:%.+]] = load <4 x float>, ptr %vf2.addr, align 16
+  // CHECK-NEXT: call <4 x float> @llvm.minimum.v4f32(<4 x float> [[CVF1]], <4 x float> [[VF2]])
+  const float4 cvf1 = vf1;
+  vf1 = __builtin_elementwise_minimum(cvf1, vf2);
+
+  // CHECK:      [[VF2:%.+]] = load <4 x float>, ptr %vf2.addr, align 16
+  // CHECK-NEXT: [[CVF1:%.+]] = load <4 x float>, ptr %cvf1, align 16
+  // CHECK-NEXT: call <4 x float> @llvm.minimum.v4f32(<4 x float> [[VF2]], <4 x float> [[CVF1]])
+  vf1 = __builtin_elementwise_minimum(vf2, cvf1);
+}
+
 void test_builtin_elementwise_max(float f1, float f2, double d1, double d2,
                                   float4 vf1, float4 vf2, long long int i1,
                                   long long int i2, si8 vi1, si8 vi2,
@@ -441,6 +517,26 @@ void test_builtin_elementwise_atan(float f1, float f2, double d1, double d2,
   vf2 = __builtin_elementwise_atan(vf1);
 }
 
+void test_builtin_elementwise_atan2(float f1, float f2, float f3, double d1,
+                                    double d2, double d3, float4 vf1,
+                                    float4 vf2, float4 vf3) {
+  // CHECK-LABEL: define void @test_builtin_elementwise_atan2(
+  // CHECK:      [[F1:%.+]] = load float, ptr %f1.addr, align 4
+  // CHECK-NEXT: [[F2:%.+]] = load float, ptr %f2.addr, align 4
+  // CHECK-NEXT: call float @llvm.atan2.f32(float [[F1]], float [[F2]])
+  f3 = __builtin_elementwise_atan2(f1, f2);
+
+  // CHECK:      [[D1:%.+]] = load double, ptr %d1.addr, align 8
+  // CHECK-NEXT: [[D2:%.+]] = load double, ptr %d2.addr, align 8
+  // CHECK-NEXT: call double @llvm.atan2.f64(double [[D1]], double [[D2]])
+  d3 = __builtin_elementwise_atan2(d1, d2);
+
+  // CHECK:      [[VF1:%.+]] = load <4 x float>, ptr %vf1.addr, align 16
+  // CHECK-NEXT: [[VF2:%.+]] = load <4 x float>, ptr %vf2.addr, align 16
+  // CHECK-NEXT: call <4 x float> @llvm.atan2.v4f32(<4 x float> [[VF1]], <4 x float> [[VF2]])
+  vf3 = __builtin_elementwise_atan2(vf1, vf2);
+}
+
 void test_builtin_elementwise_cos(float f1, float f2, double d1, double d2,
                                   float4 vf1, float4 vf2) {
   // CHECK-LABEL: define void @test_builtin_elementwise_cos(
diff --git a/clang/test/CodeGen/builtins-reduction-math.c b/clang/test/CodeGen/builtins-reduction-math.c
index acafe9222d59f..e12fd729c84c0 100644
--- a/clang/test/CodeGen/builtins-reduction-math.c
+++ b/clang/test/CodeGen/builtins-reduction-math.c
@@ -138,6 +138,30 @@ void test_builtin_reduce_and(si8 vi1, u4 vu1) {
   unsigned r3 = __builtin_reduce_and(vu1);
 }
 
+void test_builtin_reduce_maximum(float4 vf1) {
+  // CHECK-LABEL: define void @test_builtin_reduce_maximum(
+  // CHECK:      [[VF1:%.+]] = load <4 x float>, ptr %vf1.addr, align 16
+  // CHECK-NEXT: call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> [[VF1]])
+  float r1 = __builtin_reduce_maximum(vf1);
+
+  // CHECK:      [[VF1_AS1:%.+]] = load <4 x float>, ptr addrspace(1) @vf1_as_one, align 16
+  // CHECK-NEXT: [[RDX1:%.+]] = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> [[VF1_AS1]])
+  // CHECK-NEXT: fpext float [[RDX1]] to double
+  const double r4 = __builtin_reduce_maximum(vf1_as_one);
+}
+
+void test_builtin_reduce_minimum(float4 vf1) {
+  // CHECK-LABEL: define void @test_builtin_reduce_minimum(
+  // CHECK:      [[VF1:%.+]] = load <4 x float>, ptr %vf1.addr, align 16
+  // CHECK-NEXT: call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> [[VF1]])
+  float r1 = __builtin_reduce_minimum(vf1);
+
+  // CHECK:      [[VF1_AS1:%.+]] = load <4 x float>, ptr addrspace(1) @vf1_as_one, align 16
+  // CHECK-NEXT: [[RDX1:%.+]] = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> [[VF1_AS1]])
+  // CHECK-NEXT: fpext float [[RDX1]] to double
+  const double r4 = __builtin_reduce_minimum(vf1_as_one);
+}
+
 #if defined(__ARM_FEATURE_SVE)
 #include <arm_sve.h>
 
diff --git a/clang/test/CodeGen/cx-complex-range.c b/clang/test/CodeGen/cx-complex-range.c
index d4fb62a7dfec3..d83d4d02ac199 100644
--- a/clang/test/CodeGen/cx-complex-range.c
+++ b/clang/test/CodeGen/cx-complex-range.c
@@ -1382,7 +1382,7 @@ _Complex float mulf(_Complex float a, _Complex float b) {
 // AVRFP64-NEXT:    store double [[AGG_RESULT_IMAG]], ptr [[AGG_RESULT_IMAGP4]], align 1
 // AVRFP64-NEXT:    ret void
 //
-// BASIC_FAST-LABEL: define dso_local { double, double } @divd(
+// BASIC_FAST-LABEL: define dso_local nofpclass(nan inf) { double, double } @divd(
 // BASIC_FAST-SAME: double noundef nofpclass(nan inf) [[A_COERCE0:%.*]], double noundef nofpclass(nan inf) [[A_COERCE1:%.*]], double noundef nofpclass(nan inf) [[B_COERCE0:%.*]], double noundef nofpclass(nan inf) [[B_COERCE1:%.*]]) #[[ATTR1:[0-9]+]] {
 // BASIC_FAST-NEXT:  entry:
 // BASIC_FAST-NEXT:    [[RETVAL:%.*]] = alloca { double, double }, align 8
@@ -1422,7 +1422,7 @@ _Complex float mulf(_Complex float a, _Complex float b) {
 // BASIC_FAST-NEXT:    [[TMP15:%.*]] = load { double, double }, ptr [[RETVAL]], align 8
 // BASIC_FAST-NEXT:    ret { double, double } [[TMP15]]
 //
-// FULL_FAST-LABEL: define dso_local { double, double } @divd(
+// FULL_FAST-LABEL: define dso_local nofpclass(nan inf) { double, double } @divd(
 // FULL_FAST-SAME: double noundef nofpclass(nan inf) [[A_COERCE0:%.*]], double noundef nofpclass(nan inf) [[A_COERCE1:%.*]], double noundef nofpclass(nan inf) [[B_COERCE0:%.*]], double noundef nofpclass(nan inf) [[B_COERCE1:%.*]]) #[[ATTR1:[0-9]+]] {
 // FULL_FAST-NEXT:  entry:
 // FULL_FAST-NEXT:    [[RETVAL:%.*]] = alloca { double, double }, align 8
@@ -1444,7 +1444,7 @@ _Complex float mulf(_Complex float a, _Complex float b) {
 // FULL_FAST-NEXT:    [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
 // FULL_FAST-NEXT:    [[B_IMAGP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 1
 // FULL_FAST-NEXT:    [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8
-// FULL_FAST-NEXT:    [[CALL:%.*]] = call { double, double } @__divdc3(double noundef nofpclass(nan inf) [[A_REAL]], double noundef nofpclass(nan inf) [[A_IMAG]], double noundef nofpclass(nan inf) [[B_REAL]], double noundef nofpclass(nan inf) [[B_IMAG]]) #[[ATTR2]]
+// FULL_FAST-NEXT:    [[CALL:%.*]] = call reassoc nnan ninf nsz arcp afn nofpclass(nan inf) { double, double } @__divdc3(double noundef nofpclass(nan inf) [[A_REAL]], double noundef nofpclass(nan inf) [[A_IMAG]], double noundef nofpclass(nan inf) [[B_REAL]], double noundef nofpclass(nan inf) [[B_IMAG]]) #[[ATTR2]]
 // FULL_FAST-NEXT:    [[TMP4:%.*]] = extractvalue { double, double } [[CALL]], 0
 // FULL_FAST-NEXT:    [[TMP5:%.*]] = extractvalue { double, double } [[CALL]], 1
 // FULL_FAST-NEXT:    [[RETVAL_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RETVAL]], i32 0, i32 0
@@ -1454,7 +1454,7 @@ _Complex float mulf(_Complex float a, _Complex float b) {
 // FULL_FAST-NEXT:    [[TMP6:%.*]] = load { double, double }, ptr [[RETVAL]], align 8
 // FULL_FAST-NEXT:    ret { double, double } [[TMP6]]
 //
-// IMPRVD_FAST-LABEL: define dso_local { double, double } @divd(
+// IMPRVD_FAST-LABEL: define dso_local nofpclass(nan inf) { double, double } @divd(
 // IMPRVD_FAST-SAME: double noundef nofpclass(nan inf) [[A_COERCE0:%.*]], double noundef nofpclass(nan inf) [[A_COERCE1:%.*]], double noundef nofpclass(nan inf) [[B_COERCE0:%.*]], double noundef nofpclass(nan inf) [[B_COERCE1:%.*]]) #[[ATTR2:[0-9]+]] {
 // IMPRVD_FAST-NEXT:  entry:
 // IMPRVD_FAST-NEXT:    [[RETVAL:%.*]] = alloca { double, double }, align 8
@@ -1512,7 +1512,7 @@ _Complex float mulf(_Complex float a, _Complex float b) {
 // IMPRVD_FAST-NEXT:    [[TMP26:%.*]] = load { double, double }, ptr [[RETVAL]], align 8
 // IMPRVD_FAST-NEXT:    ret { double, double } [[TMP26]]
 //
-// PRMTD_FAST-LABEL: define dso_local { double, double } @divd(
+// PRMTD_FAST-LABEL: define dso_local nofpclass(nan inf) { double, double } @divd(
 // PRMTD_FAST-SAME: double noundef nofpclass(nan inf) [[A_COERCE0:%.*]], double noundef nofpclass(nan inf) [[A_COERCE1:%.*]], double noundef nofpclass(nan inf) [[B_COERCE0:%.*]], double noundef nofpclass(nan inf) [[B_COERCE1:%.*]]) #[[ATTR1:[0-9]+]] {
 // PRMTD_FAST-NEXT:  entry:
 // PRMTD_FAST-NEXT:    [[RETVAL:%.*]] = alloca { double, double }, align 8
@@ -1934,7 +1934,7 @@ _Complex double divd(_Complex double a, _Complex double b) {
 // AVRFP64-NEXT:    store double [[AGG_RESULT_IMAG]], ptr [[AGG_RESULT_IMAGP4]], align 1
 // AVRFP64-NEXT:    ret void
 //
-// BASIC_FAST-LABEL: define dso_local { double, double } @muld(
+// BASIC_FAST-LABEL: define dso_local nofpclass(nan inf) { double, double } @muld(
 // BASIC_FAST-SAME: double noundef nofpclass(nan inf) [[A_COERCE0:%.*]], double noundef nofpclass(nan inf) [[A_COERCE1:%.*]], double noundef nofpclass(nan inf) [[B_COERCE0:%.*]], double noundef nofpclass(nan inf) [[B_COERCE1:%.*]]) #[[ATTR1]] {
 // BASIC_FAST-NEXT:  entry:
 // BASIC_FAST-NEXT:    [[RETVAL:%.*]] = alloca { double, double }, align 8
@@ -1969,7 +1969,7 @@ _Complex double divd(_Complex double a, _Complex double b) {
 // BASIC_FAST-NEXT:    [[TMP4:%.*]] = load { double, double }, ptr [[RETVAL]], align 8
 // BASIC_FAST-NEXT:    ret { double, double } [[TMP4]]
 //
-// FULL_FAST-LABEL: define dso_local { double, double } @muld(
+// FULL_FAST-LABEL: define dso_local nofpclass(nan inf) { double, double } @muld(
 // FULL_FAST-SAME: double noundef nofpclass(nan inf) [[A_COERCE0:%.*]], double noundef nofpclass(nan inf) [[A_COERCE1:%.*]], double noundef nofpclass(nan inf) [[B_COERCE0:%.*]], double noundef nofpclass(nan inf) [[B_COERCE1:%.*]]) #[[ATTR1]] {
 // FULL_FAST-NEXT:  entry:
 // FULL_FAST-NEXT:    [[RETVAL:%.*]] = alloca { double, double }, align 8
@@ -2003,7 +2003,7 @@ _Complex double divd(_Complex double a, _Complex double b) {
 // FULL_FAST-NEXT:    [[ISNAN_CMP1:%.*]] = fcmp reassoc nnan ninf nsz arcp afn uno double [[MUL_I]], [[MUL_I]]
 // FULL_FAST-NEXT:    br i1 [[ISNAN_CMP1]], label [[COMPLEX_MUL_LIBCALL:%.*]], label [[COMPLEX_MUL_CONT]], !prof [[PROF2]]
 // FULL_FAST:       complex_mul_libcall:
-// FULL_FAST-NEXT:    [[CALL:%.*]] = call { double, double } @__muldc3(double noundef nofpclass(nan inf) [[A_REAL]], double noundef nofpclass(nan inf) [[A_IMAG]], double noundef nofpclass(nan inf) [[B_REAL]], double noundef nofpclass(nan inf) [[B_IMAG]]) #[[ATTR2]]
+// FULL_FAST-NEXT:    [[CALL:%.*]] = call reassoc nnan ninf nsz arcp afn nofpclass(nan inf) { double, double } @__muldc3(double noundef nofpclass(nan inf) [[A_REAL]], double noundef nofpclass(nan inf) [[A_IMAG]], double noundef nofpclass(nan inf) [[B_REAL]], double noundef nofpclass(nan inf) [[B_IMAG]]) #[[ATTR2]]
 // FULL_FAST-NEXT:    [[TMP4:%.*]] = extractvalue { double, double } [[CALL]], 0
 // FULL_FAST-NEXT:    [[TMP5:%.*]] = extractvalue { double, double } [[CALL]], 1
 // FULL_FAST-NEXT:    br label [[COMPLEX_MUL_CONT]]
@@ -2017,7 +2017,7 @@ _Complex double divd(_Complex double a, _Complex double b) {
 // FULL_FAST-NEXT:    [[TMP6:%.*]] = load { double, double }, ptr [[RETVAL]], align 8
 // FULL_FAST-NEXT:    ret { double, double } [[TMP6]]
 //
-// IMPRVD_FAST-LABEL: define dso_local { double, double } @muld(
+// IMPRVD_FAST-LABEL: define dso_local nofpclass(nan inf) { double, double } @muld(
 // IMPRVD_FAST-SAME: double noundef nofpclass(nan inf) [[A_COERCE0:%.*]], double noundef nofpclass(nan inf) [[A_COERCE1:%.*]], double noundef nofpclass(nan inf) [[B_COERCE0:%.*]], double noundef nofpclass(nan inf) [[B_COERCE1:%.*]]) #[[ATTR2]] {
 // IMPRVD_FAST-NEXT:  entry:
 // IMPRVD_FAST-NEXT:    [[RETVAL:%.*]] = alloca { double, double }, align 8
@@ -2052,7 +2052,7 @@ _Complex double divd(_Complex double a, _Complex double b) {
 // IMPRVD_FAST-NEXT:    [[TMP4:%.*]] = load { double, double }, ptr [[RETVAL]], align 8
 // IMPRVD_FAST-NEXT:    ret { double, double } [[TMP4]]
 //
-// PRMTD_FAST-LABEL: define dso_local { double, double } @muld(
+// PRMTD_FAST-LABEL: define dso_local nofpclass(nan inf) { double, double } @muld(
 // PRMTD_FAST-SAME: double noundef nofpclass(nan inf) [[A_COERCE0:%.*]], double noundef nofpclass(nan inf) [[A_COERCE1:%.*]], double noundef nofpclass(nan inf) [[B_COERCE0:%.*]], double noundef nofpclass(nan inf) [[B_COERCE1:%.*]]) #[[ATTR1]] {
 // PRMTD_FAST-NEXT:  entry:
 // PRMTD_FAST-NEXT:    [[RETVAL:%.*]] = alloca { double, double }, align 8
@@ -2493,7 +2493,7 @@ _Complex double muld(_Complex double a, _Complex double b) {
 // AVRFP64-NEXT:    store double [[AGG_RESULT_IMAG]], ptr [[AGG_RESULT_IMAGP4]], align 1
 // AVRFP64-NEXT:    ret void
 //
-// BASIC_FAST-LABEL: define dso_local { x86_fp80, x86_fp80 } @divld(
+// BASIC_FAST-LABEL: define dso_local nofpclass(nan inf) { x86_fp80, x86_fp80 } @divld(
 // BASIC_FAST-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR1]] {
 // BASIC_FAST-NEXT:  entry:
 // BASIC_FAST-NEXT:    [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16
@@ -2523,7 +2523,7 @@ _Complex double muld(_Complex double a, _Complex double b) {
 // BASIC_FAST-NEXT:    [[TMP11:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16
 // BASIC_FAST-NEXT:    ret { x86_fp80, x86_fp80 } [[TMP11]]
 //
-// FULL_FAST-LABEL: define dso_local { x86_fp80, x86_fp80 } @divld(
+// FULL_FAST-LABEL: define dso_local nofpclass(nan inf) { x86_fp80, x86_fp80 } @divld(
 // FULL_FAST-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR1]] {
 // FULL_FAST-NEXT:  entry:
 // FULL_FAST-NEXT:    [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16
@@ -2535,7 +2535,7 @@ _Complex double muld(_Complex double a, _Complex double b) {
 // FULL_FAST-NEXT:    [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16
 // FULL_FAST-NEXT:    [[B_IMAGP:%.*]] = getelementptr inbounds nuw { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1
 // FULL_FAST-NEXT:    [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16
-// FULL_FAST-NEXT:    [[CALL:%.*]] = call { x86_fp80, x86_fp80 } @__divxc3(x86_fp80 noundef nofpclass(nan inf) [[A_REAL]], x86_fp80 noundef nofpclass(nan inf) [[A_IMAG]], x86_fp80 noundef nofpclass(nan inf) [[B_REAL]], x86_fp80 noundef nofpclass(nan inf) [[B_IMAG]]) #[[ATTR2]]
+// FULL_FAST-NEXT:    [[CALL:%.*]] = call reassoc nnan ninf nsz arcp afn nofpclass(nan inf) { x86_fp80, x86_fp80 } @__divxc3(x86_fp80 noundef nofpclass(nan inf) [[A_REAL]], x86_fp80 noundef nofpclass(nan inf) [[A_IMAG]], x86_fp80 noundef nofpclass(nan inf) [[B_REAL]], x86_fp80 noundef nofpclass(nan inf) [[B_IMAG]]) #[[ATTR2]]
 // FULL_FAST-NEXT:    [[TMP0:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 0
 // FULL_FAST-NEXT:    [[TMP1:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 1
 // FULL_FAST-NEXT:    [[RETVAL_REALP:%.*]] = getelementptr inbounds nuw { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 0
@@ -2545,7 +2545,7 @@ _Complex double muld(_Complex double a, _Complex double b) {
 // FULL_FAST-NEXT:    [[TMP2:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16
 // FULL_FAST-NEXT:    ret { x86_fp80, x86_fp80 } [[TMP2]]
 //
-// IMPRVD_FAST-LABEL: define dso_local { x86_fp80, x86_fp80 } @divld(
+// IMPRVD_FAST-LABEL: define dso_local nofpclass(nan inf) { x86_fp80, x86_fp80 } @divld(
 // IMPRVD_FAST-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR2]] {
 // IMPRVD_FAST-NEXT:  entry:
 // IMPRVD_FAST-NEXT:    [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16
@@ -2593,7 +2593,7 @@ _Complex double muld(_Complex double a, _Complex double b) {
 // IMPRVD_FAST-NEXT:    [[TMP22:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16
 // IMPRVD_FAST-NEXT:    ret { x86_fp80, x86_fp80 } [[TMP22]]
 //
-// PRMTD_FAST-LABEL: define dso_local { x86_fp80, x86_fp80 } @divld(
+// PRMTD_FAST-LABEL: define dso_local nofpclass(nan inf) { x86_fp80, x86_fp80 } @divld(
 // PRMTD_FAST-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR1]] {
 // PRMTD_FAST-NEXT:  entry:
 // PRMTD_FAST-NEXT:    [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16
@@ -2979,7 +2979,7 @@ _Complex long double divld(_Complex long double a, _Complex long double b) {
 // AVRFP64-NEXT:    store double [[AGG_RESULT_IMAG]], ptr [[AGG_RESULT_IMAGP4]], align 1
 // AVRFP64-NEXT:    ret void
 //
-// BASIC_FAST-LABEL: define dso_local { x86_fp80, x86_fp80 } @mulld(
+// BASIC_FAST-LABEL: define dso_local nofpclass(nan inf) { x86_fp80, x86_fp80 } @mulld(
 // BASIC_FAST-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR1]] {
 // BASIC_FAST-NEXT:  entry:
 // BASIC_FAST-NEXT:    [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16
@@ -3004,7 +3004,7 @@ _Complex long double divld(_Complex long double a, _Complex long double b) {
 // BASIC_FAST-NEXT:    [[TMP0:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16
 // BASIC_FAST-NEXT:    ret { x86_fp80, x86_fp80 } [[TMP0]]
 //
-// FULL_FAST-LABEL: define dso_local { x86_fp80, x86_fp80 } @mulld(
+// FULL_FAST-LABEL: define dso_local nofpclass(nan inf) { x86_fp80, x86_fp80 } @mulld(
 // FULL_FAST-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR1]] {
 // FULL_FAST-NEXT:  entry:
 // FULL_FAST-NEXT:    [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16
@@ -3028,7 +3028,7 @@ _Complex long double divld(_Complex long double a, _Complex long double b) {
 // FULL_FAST-NEXT:    [[ISNAN_CMP1:%.*]] = fcmp reassoc nnan ninf nsz arcp afn uno x86_fp80 [[MUL_I]], [[MUL_I]]
 // FULL_FAST-NEXT:    br i1 [[ISNAN_CMP1]], label [[COMPLEX_MUL_LIBCALL:%.*]], label [[COMPLEX_MUL_CONT]], !prof [[PROF2]]
 // FULL_FAST:       complex_mul_libcall:
-// FULL_FAST-NEXT:    [[CALL:%.*]] = call { x86_fp80, x86_fp80 } @__mulxc3(x86_fp80 noundef nofpclass(nan inf) [[A_REAL]], x86_fp80 noundef nofpclass(nan inf) [[A_IMAG]], x86_fp80 noundef nofpclass(nan inf) [[B_REAL]], x86_fp80 noundef nofpclass(nan inf) [[B_IMAG]]) #[[ATTR2]]
+// FULL_FAST-NEXT:    [[CALL:%.*]] = call reassoc nnan ninf nsz arcp afn nofpclass(nan inf) { x86_fp80, x86_fp80 } @__mulxc3(x86_fp80 noundef nofpclass(nan inf) [[A_REAL]], x86_fp80 noundef nofpclass(nan inf) [[A_IMAG]], x86_fp80 noundef nofpclass(nan inf) [[B_REAL]], x86_fp80 noundef nofpclass(nan inf) [[B_IMAG]]) #[[ATTR2]]
 // FULL_FAST-NEXT:    [[TMP0:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 0
 // FULL_FAST-NEXT:    [[TMP1:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 1
 // FULL_FAST-NEXT:    br label [[COMPLEX_MUL_CONT]]
@@ -3042,7 +3042,7 @@ _Complex long double divld(_Complex long double a, _Complex long double b) {
 // FULL_FAST-NEXT:    [[TMP2:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16
 // FULL_FAST-NEXT:    ret { x86_fp80, x86_fp80 } [[TMP2]]
 //
-// IMPRVD_FAST-LABEL: define dso_local { x86_fp80, x86_fp80 } @mulld(
+// IMPRVD_FAST-LABEL: define dso_local nofpclass(nan inf) { x86_fp80, x86_fp80 } @mulld(
 // IMPRVD_FAST-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR2]] {
 // IMPRVD_FAST-NEXT:  entry:
 // IMPRVD_FAST-NEXT:    [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16
@@ -3067,7 +3067,7 @@ _Complex long double divld(_Complex long double a, _Complex long double b) {
 // IMPRVD_FAST-NEXT:    [[TMP0:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16
 // IMPRVD_FAST-NEXT:    ret { x86_fp80, x86_fp80 } [[TMP0]]
 //
-// PRMTD_FAST-LABEL: define dso_local { x86_fp80, x86_fp80 } @mulld(
+// PRMTD_FAST-LABEL: define dso_local nofpclass(nan inf) { x86_fp80, x86_fp80 } @mulld(
 // PRMTD_FAST-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR1]] {
 // PRMTD_FAST-NEXT:  entry:
 // PRMTD_FAST-NEXT:    [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16
@@ -3753,7 +3753,7 @@ _Complex long double mulld(_Complex long double a, _Complex long double b) {
 // FULL_FAST-NEXT:    [[C_IMAG:%.*]] = load float, ptr [[C_IMAGP]], align 4
 // FULL_FAST-NEXT:    [[CONV:%.*]] = fpext float [[C_REAL]] to x86_fp80
 // FULL_FAST-NEXT:    [[CONV1:%.*]] = fpext float [[C_IMAG]] to x86_fp80
-// FULL_FAST-NEXT:    [[CALL:%.*]] = call { x86_fp80, x86_fp80 } @__divxc3(x86_fp80 noundef nofpclass(nan inf) [[B_REAL]], x86_fp80 noundef nofpclass(nan inf) [[B_IMAG]], x86_fp80 noundef nofpclass(nan inf) [[CONV]], x86_fp80 noundef nofpclass(nan inf) [[CONV1]]) #[[ATTR2]]
+// FULL_FAST-NEXT:    [[CALL:%.*]] = call reassoc nnan ninf nsz arcp afn nofpclass(nan inf) { x86_fp80, x86_fp80 } @__divxc3(x86_fp80 noundef nofpclass(nan inf) [[B_REAL]], x86_fp80 noundef nofpclass(nan inf) [[B_IMAG]], x86_fp80 noundef nofpclass(nan inf) [[CONV]], x86_fp80 noundef nofpclass(nan inf) [[CONV1]]) #[[ATTR2]]
 // FULL_FAST-NEXT:    [[TMP0:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 0
 // FULL_FAST-NEXT:    [[TMP1:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 1
 // FULL_FAST-NEXT:    [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP0]] to float
diff --git a/clang/test/CodeGen/nofpclass.c b/clang/test/CodeGen/nofpclass.c
index 16a3823a5c02d..da7cdff0b1a46 100644
--- a/clang/test/CodeGen/nofpclass.c
+++ b/clang/test/CodeGen/nofpclass.c
@@ -519,7 +519,7 @@ _Complex float defined_complex_func(_Complex float a, _Complex double b, _Comple
 }
 
 // CFINITEONLY: Function Attrs: noinline nounwind optnone
-// CFINITEONLY-LABEL: define dso_local { double, double } @defined_complex_func_f64_ret
+// CFINITEONLY-LABEL: define dso_local nofpclass(nan inf) { double, double } @defined_complex_func_f64_ret
 // CFINITEONLY-SAME: (double noundef nofpclass(nan inf) [[C_COERCE0:%.*]], double noundef nofpclass(nan inf) [[C_COERCE1:%.*]]) #[[ATTR0]] {
 // CFINITEONLY-NEXT:  entry:
 // CFINITEONLY-NEXT:    [[RETVAL:%.*]] = alloca { double, double }, align 8
@@ -548,7 +548,7 @@ _Complex float defined_complex_func(_Complex float a, _Complex double b, _Comple
 // CFINITEONLY-NEXT:    [[ISNAN_CMP5:%.*]] = fcmp nnan ninf uno double [[MUL_I]], [[MUL_I]]
 // CFINITEONLY-NEXT:    br i1 [[ISNAN_CMP5]], label [[COMPLEX_MUL_LIBCALL:%.*]], label [[COMPLEX_MUL_CONT]], !prof [[PROF2]]
 // CFINITEONLY:       complex_mul_libcall:
-// CFINITEONLY-NEXT:    [[CALL:%.*]] = call { double, double } @__muldc3(double noundef nofpclass(nan inf) [[C_REAL]], double noundef nofpclass(nan inf) [[C_IMAG]], double noundef nofpclass(nan inf) [[C_REAL2]], double noundef nofpclass(nan inf) [[C_IMAG4]]) #[[ATTR7:[0-9]+]]
+// CFINITEONLY-NEXT:    [[CALL:%.*]] = call nnan ninf nofpclass(nan inf) { double, double } @__muldc3(double noundef nofpclass(nan inf) [[C_REAL]], double noundef nofpclass(nan inf) [[C_IMAG]], double noundef nofpclass(nan inf) [[C_REAL2]], double noundef nofpclass(nan inf) [[C_IMAG4]]) #[[ATTR7:[0-9]+]]
 // CFINITEONLY-NEXT:    [[TMP2:%.*]] = extractvalue { double, double } [[CALL]], 0
 // CFINITEONLY-NEXT:    [[TMP3:%.*]] = extractvalue { double, double } [[CALL]], 1
 // CFINITEONLY-NEXT:    br label [[COMPLEX_MUL_CONT]]
@@ -563,7 +563,7 @@ _Complex float defined_complex_func(_Complex float a, _Complex double b, _Comple
 // CFINITEONLY-NEXT:    ret { double, double } [[TMP4]]
 //
 // CLFINITEONLY: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
-// CLFINITEONLY-LABEL: define dso_local { double, double } @defined_complex_func_f64_ret
+// CLFINITEONLY-LABEL: define dso_local nofpclass(nan inf) { double, double } @defined_complex_func_f64_ret
 // CLFINITEONLY-SAME: (double noundef nofpclass(nan inf) [[C_COERCE0:%.*]], double noundef nofpclass(nan inf) [[C_COERCE1:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CLFINITEONLY-NEXT:  entry:
 // CLFINITEONLY-NEXT:    [[MUL_AD:%.*]] = fmul nnan ninf double [[C_COERCE0]], [[C_COERCE1]]
@@ -576,7 +576,7 @@ _Complex float defined_complex_func(_Complex float a, _Complex double b, _Comple
 // CLFINITEONLY-NEXT:    ret { double, double } [[DOTFCA_1_INSERT]]
 //
 // NONANS: Function Attrs: noinline nounwind optnone
-// NONANS-LABEL: define dso_local { double, double } @defined_complex_func_f64_ret
+// NONANS-LABEL: define dso_local nofpclass(nan) { double, double } @defined_complex_func_f64_ret
 // NONANS-SAME: (double noundef nofpclass(nan) [[C_COERCE0:%.*]], double noundef nofpclass(nan) [[C_COERCE1:%.*]]) #[[ATTR0]] {
 // NONANS-NEXT:  entry:
 // NONANS-NEXT:    [[RETVAL:%.*]] = alloca { double, double }, align 8
@@ -605,7 +605,7 @@ _Complex float defined_complex_func(_Complex float a, _Complex double b, _Comple
 // NONANS-NEXT:    [[ISNAN_CMP5:%.*]] = fcmp nnan uno double [[MUL_I]], [[MUL_I]]
 // NONANS-NEXT:    br i1 [[ISNAN_CMP5]], label [[COMPLEX_MUL_LIBCALL:%.*]], label [[COMPLEX_MUL_CONT]], !prof [[PROF2]]
 // NONANS:       complex_mul_libcall:
-// NONANS-NEXT:    [[CALL:%.*]] = call { double, double } @__muldc3(double noundef nofpclass(nan) [[C_REAL]], double noundef nofpclass(nan) [[C_IMAG]], double noundef nofpclass(nan) [[C_REAL2]], double noundef nofpclass(nan) [[C_IMAG4]]) #[[ATTR7:[0-9]+]]
+// NONANS-NEXT:    [[CALL:%.*]] = call nnan nofpclass(nan) { double, double } @__muldc3(double noundef nofpclass(nan) [[C_REAL]], double noundef nofpclass(nan) [[C_IMAG]], double noundef nofpclass(nan) [[C_REAL2]], double noundef nofpclass(nan) [[C_IMAG4]]) #[[ATTR7:[0-9]+]]
 // NONANS-NEXT:    [[TMP2:%.*]] = extractvalue { double, double } [[CALL]], 0
 // NONANS-NEXT:    [[TMP3:%.*]] = extractvalue { double, double } [[CALL]], 1
 // NONANS-NEXT:    br label [[COMPLEX_MUL_CONT]]
@@ -620,7 +620,7 @@ _Complex float defined_complex_func(_Complex float a, _Complex double b, _Comple
 // NONANS-NEXT:    ret { double, double } [[TMP4]]
 //
 // NOINFS: Function Attrs: noinline nounwind optnone
-// NOINFS-LABEL: define dso_local { double, double } @defined_complex_func_f64_ret
+// NOINFS-LABEL: define dso_local nofpclass(inf) { double, double } @defined_complex_func_f64_ret
 // NOINFS-SAME: (double noundef nofpclass(inf) [[C_COERCE0:%.*]], double noundef nofpclass(inf) [[C_COERCE1:%.*]]) #[[ATTR0]] {
 // NOINFS-NEXT:  entry:
 // NOINFS-NEXT:    [[RETVAL:%.*]] = alloca { double, double }, align 8
@@ -649,7 +649,7 @@ _Complex float defined_complex_func(_Complex float a, _Complex double b, _Comple
 // NOINFS-NEXT:    [[ISNAN_CMP5:%.*]] = fcmp ninf uno double [[MUL_I]], [[MUL_I]]
 // NOINFS-NEXT:    br i1 [[ISNAN_CMP5]], label [[COMPLEX_MUL_LIBCALL:%.*]], label [[COMPLEX_MUL_CONT]], !prof [[PROF2]]
 // NOINFS:       complex_mul_libcall:
-// NOINFS-NEXT:    [[CALL:%.*]] = call { double, double } @__muldc3(double noundef nofpclass(inf) [[C_REAL]], double noundef nofpclass(inf) [[C_IMAG]], double noundef nofpclass(inf) [[C_REAL2]], double noundef nofpclass(inf) [[C_IMAG4]]) #[[ATTR7:[0-9]+]]
+// NOINFS-NEXT:    [[CALL:%.*]] = call ninf nofpclass(inf) { double, double } @__muldc3(double noundef nofpclass(inf) [[C_REAL]], double noundef nofpclass(inf) [[C_IMAG]], double noundef nofpclass(inf) [[C_REAL2]], double noundef nofpclass(inf) [[C_IMAG4]]) #[[ATTR7:[0-9]+]]
 // NOINFS-NEXT:    [[TMP2:%.*]] = extractvalue { double, double } [[CALL]], 0
 // NOINFS-NEXT:    [[TMP3:%.*]] = extractvalue { double, double } [[CALL]], 1
 // NOINFS-NEXT:    br label [[COMPLEX_MUL_CONT]]
diff --git a/clang/test/CodeGen/nvptx_attributes.c b/clang/test/CodeGen/nvptx_attributes.c
index dcf79ad4e0e6a..7dbd9f1321e28 100644
--- a/clang/test/CodeGen/nvptx_attributes.c
+++ b/clang/test/CodeGen/nvptx_attributes.c
@@ -1,7 +1,7 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --check-attributes --check-globals
 // RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -target-cpu sm_61 -emit-llvm %s -o - | FileCheck %s
 
-// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK: Function Attrs: convergent noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@foo
 // CHECK-SAME: (ptr noundef [[RET:%.*]]) #[[ATTR0:[0-9]+]] {
 // CHECK-NEXT:  entry:
diff --git a/clang/test/CodeGen/rtsan_attribute_inserted.c b/clang/test/CodeGen/rtsan_attribute_inserted.c
index 05a1d9a8c2047..b21ecb6b6b06a 100644
--- a/clang/test/CodeGen/rtsan_attribute_inserted.c
+++ b/clang/test/CodeGen/rtsan_attribute_inserted.c
@@ -1,7 +1,11 @@
 // RUN: %clang_cc1  -triple x86_64-unknown-linux -fsanitize=realtime %s -emit-llvm -o - %s | FileCheck %s
 
 float process(float *a) [[clang::nonblocking]] { return *a; }
-
-// CHECK-LABEL: @process{{.*}}#0 {
+// CHECK: @process{{.*}} #0 {
 // CHECK: attributes #0 = {
-// CHECK-SAME: {{.*sanitize_realtime.*}}
+// CHECK-SAME: {{.*sanitize_realtime .*}}
+
+int spinlock(int *a) [[clang::blocking]] { return *a; }
+// CHECK: @spinlock{{.*}} #1 {
+// CHECK: attributes #1 = {
+// CHECK-SAME: {{.*sanitize_realtime_unsafe .*}}
diff --git a/clang/test/CodeGen/rtsan_no_attribute_sanitizer_disabled.c b/clang/test/CodeGen/rtsan_no_attribute_sanitizer_disabled.c
index 43ad6ed1a429e..0f43007c5e4c1 100644
--- a/clang/test/CodeGen/rtsan_no_attribute_sanitizer_disabled.c
+++ b/clang/test/CodeGen/rtsan_no_attribute_sanitizer_disabled.c
@@ -1,6 +1,8 @@
 // RUN: %clang_cc1 -triple aarch64-unknown-linux-gnu -emit-llvm -o - %s | FileCheck %s
 
 float process(float *a) [[clang::nonblocking]] { return *a; }
+int spinlock(int *a) [[clang::blocking]] { return *a; }
 
-// Without the -fsanitize=realtime flag, we shouldn't attach the attribute.
-// CHECK-NOT: {{.*sanitize_realtime.*}}
+// Without the -fsanitize=realtime flag, we shouldn't attach the attributes.
+// CHECK-NOT: {{.*sanitize_realtime .*}}
+// CHECK-NOT: {{.*sanitize_realtime_unsafe .*}}
diff --git a/clang/test/CodeGen/stack-protector-guard.c b/clang/test/CodeGen/stack-protector-guard.c
index 5839ab06033a1..4777367c94e73 100644
--- a/clang/test/CodeGen/stack-protector-guard.c
+++ b/clang/test/CodeGen/stack-protector-guard.c
@@ -9,6 +9,9 @@
 // RUN: %clang_cc1 -mstack-protector-guard=sysreg -triple aarch64-linux-gnu \
 // RUN:   -mstack-protector-guard-offset=1024 -mstack-protector-guard-reg=sp_el0 \
 // RUN:   -emit-llvm %s -o - | FileCheck %s --check-prefix=AARCH64
+// RUN: %clang_cc1 -mstack-protector-guard=tls -triple riscv64-unknown-elf \
+// RUN:   -mstack-protector-guard-offset=44 -mstack-protector-guard-reg=tp \
+// RUN:   -emit-llvm %s -o - | FileCheck %s --check-prefix=RISCV
 void foo(int*);
 void bar(int x) {
   int baz[x];
@@ -23,3 +26,8 @@ void bar(int x) {
 // AARCH64: [[ATTR1]] = !{i32 1, !"stack-protector-guard", !"sysreg"}
 // AARCH64: [[ATTR2]] = !{i32 1, !"stack-protector-guard-reg", !"sp_el0"}
 // AARCH64: [[ATTR3]] = !{i32 1, !"stack-protector-guard-offset", i32 1024}
+
+// RISCV: !llvm.module.flags = !{{{.*}}[[ATTR1:![0-9]+]], [[ATTR2:![0-9]+]], [[ATTR3:![0-9]+]], [[ATTR4:![0-9]+]]}
+// RISCV: [[ATTR1]] = !{i32 1, !"stack-protector-guard", !"tls"}
+// RISCV: [[ATTR2]] = !{i32 1, !"stack-protector-guard-reg", !"tp"}
+// RISCV: [[ATTR3]] = !{i32 1, !"stack-protector-guard-offset", i32 44}
diff --git a/clang/test/CodeGen/strictfp-elementwise-bulitins.cpp b/clang/test/CodeGen/strictfp-elementwise-bulitins.cpp
index 651f5bfc94c6c..175ad22601839 100644
--- a/clang/test/CodeGen/strictfp-elementwise-bulitins.cpp
+++ b/clang/test/CodeGen/strictfp-elementwise-bulitins.cpp
@@ -47,6 +47,26 @@ float4 strict_elementwise_min(float4 a, float4 b) {
   return __builtin_elementwise_min(a, b);
 }
 
+// CHECK-LABEL: define dso_local noundef <4 x float> @_Z26strict_elementwise_maximumDv4_fS_
+// CHECK-SAME: (<4 x float> noundef [[A:%.*]], <4 x float> noundef [[B:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[ELT_MAXIMUM:%.*]] = tail call <4 x float> @llvm.maximum.v4f32(<4 x float> [[A]], <4 x float> [[B]]) #[[ATTR4]]
+// CHECK-NEXT:    ret <4 x float> [[ELT_MAXIMUM]]
+//
+float4 strict_elementwise_maximum(float4 a, float4 b) {
+  return __builtin_elementwise_maximum(a, b);
+}
+
+// CHECK-LABEL: define dso_local noundef <4 x float> @_Z26strict_elementwise_minimumDv4_fS_
+// CHECK-SAME: (<4 x float> noundef [[A:%.*]], <4 x float> noundef [[B:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[ELT_MINIMUM:%.*]] = tail call <4 x float> @llvm.minimum.v4f32(<4 x float> [[A]], <4 x float> [[B]]) #[[ATTR4]]
+// CHECK-NEXT:    ret <4 x float> [[ELT_MINIMUM]]
+//
+float4 strict_elementwise_minimum(float4 a, float4 b) {
+  return __builtin_elementwise_minimum(a, b);
+}
+
 // CHECK-LABEL: define dso_local noundef <4 x float> @_Z23strict_elementwise_ceilDv4_f
 // CHECK-SAME: (<4 x float> noundef [[A:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // CHECK-NEXT:  entry:
@@ -257,6 +277,16 @@ float4 strict_elementwise_tanh(float4 a) {
   return __builtin_elementwise_tanh(a);
 }
 
+// CHECK-LABEL: define dso_local noundef <4 x float> @_Z24strict_elementwise_atan2Dv4_fS_
+// CHECK-SAME: (<4 x float> noundef [[A:%.*]], <4 x float> noundef [[B:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[ELT_ATAN2:%.*]] = tail call <4 x float> @llvm.atan2.v4f32(<4 x float> [[A]], <4 x float> [[B]]) #[[ATTR4]]
+// CHECK-NEXT:    ret <4 x float> [[ELT_ATAN2]]
+//
+float4 strict_elementwise_atan2(float4 a, float4 b) {
+  return __builtin_elementwise_atan2(a, b);
+}
+
 // CHECK-LABEL: define dso_local noundef <4 x float> @_Z24strict_elementwise_truncDv4_f
 // CHECK-SAME: (<4 x float> noundef [[A:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // CHECK-NEXT:  entry:
diff --git a/clang/test/CodeGenCXX/debug-info-alias.cpp b/clang/test/CodeGenCXX/debug-info-alias.cpp
index bf2dbee465959..fbb2ad6e5ff08 100644
--- a/clang/test/CodeGenCXX/debug-info-alias.cpp
+++ b/clang/test/CodeGenCXX/debug-info-alias.cpp
@@ -24,7 +24,7 @@ x::bar<int> bi;
 // CHECK: [[BFLOAT]] = !DIDerivedType(tag: DW_TAG_typedef, name: "bar<float>"
 x::bar<float> bf;
 // CHECK: !DIGlobalVariable(name: "bz",{{.*}} type: [[BBAZ:![0-9]+]]
-// CHECK: [[BBAZ]] = !DIDerivedType(tag: DW_TAG_typedef, name: "bar<baz<int> >"
+// CHECK: [[BBAZ]] = !DIDerivedType(tag: DW_TAG_typedef, name: "bar<baz<int,{{ *}}int> >"
 x::bar<baz<int>> bz;
 
 using
diff --git a/clang/test/CodeGenCXX/dynamic-cast-address-space.cpp b/clang/test/CodeGenCXX/dynamic-cast-address-space.cpp
index b967701ca1fa9..d0c87d9dfda5f 100644
--- a/clang/test/CodeGenCXX/dynamic-cast-address-space.cpp
+++ b/clang/test/CodeGenCXX/dynamic-cast-address-space.cpp
@@ -106,9 +106,9 @@ const B& f(A *a) {
 
 
 //.
-// CHECK: attributes #[[ATTR0]] = { mustprogress noinline optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+// CHECK: attributes #[[ATTR0]] = { convergent mustprogress noinline optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
 // CHECK: attributes #[[ATTR1:[0-9]+]] = { nounwind willreturn memory(read) }
-// CHECK: attributes #[[ATTR2:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+// CHECK: attributes #[[ATTR2:[0-9]+]] = { convergent "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
 // CHECK: attributes #[[ATTR3]] = { nounwind }
 // CHECK: attributes #[[ATTR4]] = { noreturn }
 //.
diff --git a/clang/test/CodeGenCXX/mangle-concept.cpp b/clang/test/CodeGenCXX/mangle-concept.cpp
index 6053511c00a7b..aa0940c35b237 100644
--- a/clang/test/CodeGenCXX/mangle-concept.cpp
+++ b/clang/test/CodeGenCXX/mangle-concept.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -verify -std=c++20 -emit-llvm -triple %itanium_abi_triple -o - %s -fclang-abi-compat=latest | FileCheck %s
+// RUN: %clang_cc1 -verify -std=c++20 -emit-llvm -triple %itanium_abi_triple -o - %s -fclang-abi-compat=19 | FileCheck %s --check-prefix=CLANG19
 // RUN: %clang_cc1 -verify -std=c++20 -emit-llvm -triple %itanium_abi_triple -o - %s -fclang-abi-compat=17 | FileCheck %s --check-prefix=CLANG17
 // expected-no-diagnostics
 
@@ -59,18 +60,21 @@ namespace test2 {
     // CLANG17: call {{.*}}@_ZN5test21fEz(
     f(ai);
     // CHECK: call {{.*}}@_ZN5test21AIiEF1gIvEEvzQaa4TrueIT_E4TrueITL0__E(
+    // CLANG19: call {{.*}}@_ZN5test2F1gIvEEvzQaa4TrueIT_E4TrueITL0__E(
     // CLANG17: call {{.*}}@_ZN5test21gIvEEvz(
     g(ai);
     // CHECK: call {{.*}}@_ZN5test21hIvEEvzQ4TrueITL0__E(
     // CLANG17: call {{.*}}@_ZN5test21hIvEEvz(
     h(ai);
     // CHECK: call {{.*}}@_ZN5test21AIiEF1iIvQaa4TrueIT_E4TrueITL0__EEEvz(
+    // CLANG19: call {{.*}}@_ZN5test2F1iIvQaa4TrueIT_E4TrueITL0__EEEvz(
     // CLANG17: call {{.*}}@_ZN5test21iIvEEvz(
     i(ai);
     // CHECK: call {{.*}}@_ZN5test21jIvQ4TrueITL0__EEEvz(
     // CLANG17: call {{.*}}@_ZN5test21jIvEEvz(
     j(ai);
     // CHECK: call {{.*}}@_ZN5test21AIiEF1kITk4TruevQ4TrueIT_EEEvz(
+    // CLANG19: call {{.*}}@_ZN5test2F1kITk4TruevQ4TrueIT_EEEvz(
     // CLANG17: call {{.*}}@_ZN5test21kIvEEvz(
     k(ai);
     // CHECK: call {{.*}}@_ZN5test21lITk4TruevEEvz(
diff --git a/clang/test/CodeGenHLSL/ArrayAssignable.hlsl b/clang/test/CodeGenHLSL/ArrayAssignable.hlsl
new file mode 100644
index 0000000000000..a0dfe26e5d147
--- /dev/null
+++ b/clang/test/CodeGenHLSL/ArrayAssignable.hlsl
@@ -0,0 +1,120 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s --enable-var-scope
+
+// CHECK-LABEL: define void {{.*}}arr_assign1
+// CHECK: [[Arr:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: [[Arr2:%.*]] = alloca [2 x i32], align 4
+// CHECK-NOT: alloca
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr]], ptr align 4 {{@.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memset.p0.i32(ptr align 4 [[Arr2]], i8 0, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr]], ptr align 4 [[Arr2]], i32 8, i1 false)
+// CHECK-NEXT: ret void
+void arr_assign1() {
+  int Arr[2] = {0, 1};
+  int Arr2[2] = {0, 0};
+  Arr = Arr2;
+}
+
+// CHECK-LABEL: define void {{.*}}arr_assign2
+// CHECK: [[Arr:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: [[Arr2:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: [[Arr3:%.*]] = alloca [2 x i32], align 4
+// CHECK-NOT: alloca
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr]], ptr align 4 {{@.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memset.p0.i32(ptr align 4 [[Arr2]], i8 0, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr3]], ptr align 4 {{@.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr2]], ptr align 4 [[Arr3]], i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr]], ptr align 4 [[Arr2]], i32 8, i1 false)
+// CHECK-NEXT: ret void
+void arr_assign2() {
+  int Arr[2] = {0, 1};
+  int Arr2[2] = {0, 0};
+  int Arr3[2] = {3, 4};
+  Arr = Arr2 = Arr3;
+}
+
+// CHECK-LABEL: define void {{.*}}arr_assign3
+// CHECK: [[Arr3:%.*]] = alloca [2 x [2 x i32]], align 4
+// CHECK-NEXT: [[Arr4:%.*]] = alloca [2 x [2 x i32]], align 4
+// CHECK-NOT: alloca
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr3]], ptr align 4 {{@.*}}, i32 16, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr4]], ptr align 4 {{@.*}}, i32 16, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr3]], ptr align 4 [[Arr4]], i32 16, i1 false)
+// CHECK-NEXT: ret void
+void arr_assign3() {
+  int Arr2[2][2] = {{0, 0}, {1, 1}};
+  int Arr3[2][2] = {{1, 1}, {0, 0}};
+  Arr2 = Arr3;
+}
+
+// CHECK-LABEL: define void {{.*}}arr_assign4
+// CHECK: [[Arr:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: [[Arr2:%.*]] = alloca [2 x i32], align 4
+// CHECK-NOT: alloca
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr]], ptr align 4 {{@.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memset.p0.i32(ptr align 4 [[Arr2]], i8 0, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr]], ptr align 4 [[Arr2]], i32 8, i1 false)
+// CHECK-NEXT: [[Idx:%.*]] = getelementptr inbounds [2 x i32], ptr [[Arr]], i32 0, i32 0
+// CHECK-NEXT: store i32 6, ptr [[Idx]], align 4
+// CHECK-NEXT: ret void
+void arr_assign4() {
+  int Arr[2] = {0, 1};
+  int Arr2[2] = {0, 0};
+  (Arr = Arr2)[0] = 6;
+}
+
+// CHECK-LABEL: define void {{.*}}arr_assign5
+// CHECK: [[Arr:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: [[Arr2:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT: [[Arr3:%.*]] = alloca [2 x i32], align 4
+// CHECK-NOT: alloca
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr]], ptr align 4 {{@.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memset.p0.i32(ptr align 4 [[Arr2]], i8 0, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr3]], ptr align 4 {{@.*}}, i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr2]], ptr align 4 [[Arr3]], i32 8, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr]], ptr align 4 [[Arr2]], i32 8, i1 false)
+// CHECK-NEXT: [[Idx:%.*]] = getelementptr inbounds [2 x i32], ptr [[Arr]], i32 0, i32 0
+// CHECK-NEXT: store i32 6, ptr [[Idx]], align 4
+// CHECK-NEXT: ret void
+void arr_assign5() {
+  int Arr[2] = {0, 1};
+  int Arr2[2] = {0, 0};
+  int Arr3[2] = {3, 4};
+  (Arr = Arr2 = Arr3)[0] = 6;
+}
+
+// CHECK-LABEL: define void {{.*}}arr_assign6
+// CHECK: [[Arr3:%.*]] = alloca [2 x [2 x i32]], align 4
+// CHECK-NEXT: [[Arr4:%.*]] = alloca [2 x [2 x i32]], align 4
+// CHECK-NOT: alloca
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr3]], ptr align 4 {{@.*}}, i32 16, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr4]], ptr align 4 {{@.*}}, i32 16, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr3]], ptr align 4 [[Arr4]], i32 16, i1 false)
+// CHECK-NEXT: [[Idx:%.*]] = getelementptr inbounds [2 x [2 x i32]], ptr [[Arr3]], i32 0, i32 0
+// CHECK-NEXT: [[Idx2:%.*]] = getelementptr inbounds [2 x i32], ptr [[Idx]], i32 0, i32 0
+// CHECK-NEXT: store i32 6, ptr [[Idx2]], align 4
+// CHECK-NEXT: ret void
+void arr_assign6() {
+  int Arr[2][2] = {{0, 0}, {1, 1}};
+  int Arr2[2][2] = {{1, 1}, {0, 0}};
+  (Arr = Arr2)[0][0] = 6;
+}
+
+// CHECK-LABEL: define void {{.*}}arr_assign7
+// CHECK: [[Arr3:%.*]] = alloca [2 x [2 x i32]], align 4
+// CHECK-NEXT: [[Arr4:%.*]] = alloca [2 x [2 x i32]], align 4
+// CHECK-NEXT: [[Tmp:%.*]] = alloca [2 x i32], align 4
+// CHECK-NOT: alloca
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr3]], ptr align 4 {{@.*}}, i32 16, i1 false)
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr4]], ptr align 4 {{@.*}}, i32 16, i1 false)
+// CHECK-NEXT: store i32 6, ptr [[Tmp]], align 4
+// CHECK-NEXT: [[AIE:%.*]] = getelementptr inbounds i32, ptr [[Tmp]], i32 1
+// CHECK-NEXT: store i32 6, ptr [[AIE]], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Arr3]], ptr align 4 [[Arr4]], i32 16, i1 false)
+// CHECK-NEXT: [[Idx:%.*]] = getelementptr inbounds [2 x [2 x i32]], ptr [[Arr3]], i32 0, i32 0
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[Idx]], ptr align 4 [[Tmp]], i32 8, i1 false)
+// CHECK-NEXT: ret void
+void arr_assign7() {
+  int Arr[2][2] = {{0, 1}, {2, 3}};
+  int Arr2[2][2] = {{0, 0}, {1, 1}};
+  (Arr = Arr2)[0] = {6, 6};
+}
diff --git a/clang/test/CodeGenHLSL/builtins/atan2.hlsl b/clang/test/CodeGenHLSL/builtins/atan2.hlsl
new file mode 100644
index 0000000000000..40796052e608f
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/atan2.hlsl
@@ -0,0 +1,59 @@
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   dxil-pc-shadermodel6.3-library %s -fnative-half-type \
+// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s \ 
+// RUN:   --check-prefixes=CHECK,NATIVE_HALF
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-unknown-vulkan-compute %s -emit-llvm -disable-llvm-passes \
+// RUN:   -o - | FileCheck %s --check-prefixes=CHECK,NO_HALF
+
+// CHECK-LABEL: test_atan2_half
+// NATIVE_HALF: call half @llvm.atan2.f16
+// NO_HALF: call float @llvm.atan2.f32
+half test_atan2_half (half p0, half p1) {
+  return atan2(p0, p1);
+}
+
+// CHECK-LABEL: test_atan2_half2
+// NATIVE_HALF: call <2 x half> @llvm.atan2.v2f16
+// NO_HALF: call <2 x float> @llvm.atan2.v2f32
+half2 test_atan2_half2 (half2 p0, half2 p1) {
+  return atan2(p0, p1);
+}
+
+// CHECK-LABEL: test_atan2_half3
+// NATIVE_HALF: call <3 x half> @llvm.atan2.v3f16
+// NO_HALF: call <3 x float> @llvm.atan2.v3f32
+half3 test_atan2_half3 (half3 p0, half3 p1) {
+  return atan2(p0, p1);
+}
+
+// CHECK-LABEL: test_atan2_half4
+// NATIVE_HALF: call <4 x half> @llvm.atan2.v4f16
+// NO_HALF: call <4 x float> @llvm.atan2.v4f32
+half4 test_atan2_half4 (half4 p0, half4 p1) {
+  return atan2(p0, p1);
+}
+
+// CHECK-LABEL: test_atan2_float
+// CHECK: call float @llvm.atan2.f32
+float test_atan2_float (float p0, float p1) {
+  return atan2(p0, p1);
+}
+
+// CHECK-LABEL: test_atan2_float2
+// CHECK: call <2 x float> @llvm.atan2.v2f32
+float2 test_atan2_float2 (float2 p0, float2 p1) {
+  return atan2(p0, p1);
+}
+
+// CHECK-LABEL: test_atan2_float3
+// CHECK: call <3 x float> @llvm.atan2.v3f32
+float3 test_atan2_float3 (float3 p0, float3 p1) {
+  return atan2(p0, p1);
+}
+
+// CHECK-LABEL: test_atan2_float4
+// CHECK: call <4 x float> @llvm.atan2.v4f32
+float4 test_atan2_float4 (float4 p0, float4 p1) {
+  return atan2(p0, p1);
+}
diff --git a/clang/test/CodeGenHLSL/builtins/cross.hlsl b/clang/test/CodeGenHLSL/builtins/cross.hlsl
new file mode 100644
index 0000000000000..514e57d36b201
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/cross.hlsl
@@ -0,0 +1,37 @@
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   dxil-pc-shadermodel6.3-library %s -fnative-half-type \
+// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s \
+// RUN:   --check-prefixes=CHECK,NATIVE_HALF \
+// RUN:   -DFNATTRS=noundef -DTARGET=dx
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   dxil-pc-shadermodel6.3-library %s -emit-llvm -disable-llvm-passes \
+// RUN:   -o - | FileCheck %s --check-prefixes=CHECK,NO_HALF \
+// RUN:   -DFNATTRS=noundef -DTARGET=dx
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-unknown-vulkan-compute %s -fnative-half-type \
+// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s \
+// RUN:   --check-prefixes=CHECK,NATIVE_HALF \
+// RUN:   -DFNATTRS="spir_func noundef" -DTARGET=spv
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-unknown-vulkan-compute %s -emit-llvm -disable-llvm-passes \
+// RUN:   -o - | FileCheck %s --check-prefixes=CHECK,NO_HALF \
+// RUN:   -DFNATTRS="spir_func noundef" -DTARGET=spv
+
+// NATIVE_HALF: define [[FNATTRS]] <3 x half> @
+// NATIVE_HALF: call <3 x half> @llvm.[[TARGET]].cross.v3f16(<3 x half>
+// NATIVE_HALF: ret <3 x half> %hlsl.cross
+// NO_HALF: define [[FNATTRS]] <3 x float> @
+// NO_HALF: call <3 x float> @llvm.[[TARGET]].cross.v3f32(<3 x float>
+// NO_HALF: ret <3 x float> %hlsl.cross
+half3 test_cross_half3(half3 p0, half3 p1)
+{
+    return cross(p0, p1);
+}
+
+// CHECK: define [[FNATTRS]] <3 x float> @
+// CHECK: %hlsl.cross = call <3 x float> @llvm.[[TARGET]].cross.v3f32(
+// CHECK: ret <3 x float> %hlsl.cross
+float3 test_cross_float3(float3 p0, float3 p1)
+{
+    return cross(p0, p1);
+}
diff --git a/clang/test/Driver/aarch64-v96a.c b/clang/test/Driver/aarch64-v96a.c
new file mode 100644
index 0000000000000..0aaadddb2842f
--- /dev/null
+++ b/clang/test/Driver/aarch64-v96a.c
@@ -0,0 +1,19 @@
+// ===== Base v9.6a architecture =====
+
+// RUN: %clang -target aarch64 -march=armv9.6a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV96A %s
+// RUN: %clang -target aarch64 -march=armv9.6-a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV96A %s
+// RUN: %clang -target aarch64 -mlittle-endian -march=armv9.6a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV96A %s
+// RUN: %clang -target aarch64 -mlittle-endian -march=armv9.6-a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV96A %s
+// RUN: %clang -target aarch64_be -mlittle-endian -march=armv9.6a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV96A %s
+// RUN: %clang -target aarch64_be -mlittle-endian -march=armv9.6-a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV96A %s
+// GENERICV96A: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "generic" "-target-feature" "+v9.6a"
+
+// RUN: %clang -target aarch64_be -march=armv9.6a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV96A-BE %s
+// RUN: %clang -target aarch64_be -march=armv9.6-a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV96A-BE %s
+// RUN: %clang -target aarch64 -mbig-endian -march=armv9.6a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV96A-BE %s
+// RUN: %clang -target aarch64 -mbig-endian -march=armv9.6-a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV96A-BE %s
+// RUN: %clang -target aarch64_be -mbig-endian -march=armv9.6a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV96A-BE %s
+// RUN: %clang -target aarch64_be -mbig-endian -march=armv9.6-a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV96A-BE %s
+// GENERICV96A-BE: "-cc1"{{.*}} "-triple" "aarch64_be{{.*}}" "-target-cpu" "generic" "-target-feature" "+v9.6a"
+//
+// ===== Features supported on aarch64 =====
diff --git a/clang/test/Driver/arm-cortex-cpus-1.c b/clang/test/Driver/arm-cortex-cpus-1.c
index 6f0b64910f9b0..5d3169ff3d503 100644
--- a/clang/test/Driver/arm-cortex-cpus-1.c
+++ b/clang/test/Driver/arm-cortex-cpus-1.c
@@ -495,3 +495,20 @@
 // RUN: %clang -target arm -march=armebv9.5a -mbig-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-V95A %s
 // RUN: %clang -target arm -march=armebv9.5-a -mbig-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-V95A %s
 // CHECK-BE-V95A: "-cc1"{{.*}} "-triple" "armebv9.5{{.*}}" "-target-cpu" "generic"
+//
+// RUN: %clang -target armv9.6a -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V96A %s
+// RUN: %clang -target arm -march=armv9.6a -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V96A %s
+// RUN: %clang -target arm -march=armv9.6-a -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V96A %s
+// RUN: %clang -target arm -march=armv9.6a -mlittle-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V96A %s
+// RUN: %clang -target armv9.6a -mlittle-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V96A %s
+// RUN: %clang -target arm -march=armv9.6a -mlittle-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V96A %s
+// RUN: %clang -target arm -mlittle-endian -march=armv9.6-a -mlittle-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V96A %s
+// CHECK-V96A: "-cc1"{{.*}} "-triple" "armv9.6{{.*}}" "-target-cpu" "generic"
+
+// RUN: %clang -target armebv9.6a -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-V96A %s
+// RUN: %clang -target armv9.6a -mbig-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-V96A %s
+// RUN: %clang -target armeb -march=armebv9.6a -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-V96A %s
+// RUN: %clang -target armeb -march=armebv9.6-a -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-V96A %s
+// RUN: %clang -target arm -march=armebv9.6a -mbig-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-V96A %s
+// RUN: %clang -target arm -march=armebv9.6-a -mbig-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-V96A %s
+// CHECK-BE-V96A: "-cc1"{{.*}} "-triple" "armebv9.6{{.*}}" "-target-cpu" "generic"
diff --git a/clang/test/Driver/cl-options.c b/clang/test/Driver/cl-options.c
index 07a25da0269fd..48d281bcd447e 100644
--- a/clang/test/Driver/cl-options.c
+++ b/clang/test/Driver/cl-options.c
@@ -626,6 +626,8 @@
 // LTO-THIN: -flto=thin
 
 // RUN: not %clang_cl -### -Fe%t.exe -entry:main -flto -- %s 2>&1 | FileCheck -check-prefix=LTO-WITHOUT-LLD %s
+// RUN: not %clang_cl -### -fuse-ld=link -Fe%t.exe -entry:main -flto -- %s 2>&1 | FileCheck -check-prefix=LTO-WITHOUT-LLD %s
+// RUN: not %clang -### --target=x86_64-windows-msvc -fuse-ld=link -Fe%t.exe -entry:main -flto -- %s 2>&1 | FileCheck -check-prefix=LTO-WITHOUT-LLD %s
 // LTO-WITHOUT-LLD: LTO requires -fuse-ld=lld
 
 // RUN: %clang_cl  -### -- %s 2>&1 | FileCheck -check-prefix=NOCFGUARD %s
diff --git a/clang/test/Driver/clang_f_opts.c b/clang/test/Driver/clang_f_opts.c
index 2e1736ca72aa7..fd15552715cb3 100644
--- a/clang/test/Driver/clang_f_opts.c
+++ b/clang/test/Driver/clang_f_opts.c
@@ -605,7 +605,7 @@
 // RUN: %clang -### -S -fjmc -g -fno-jmc --target=x86_64-pc-windows-msvc %s 2>&1 | FileCheck -check-prefix=CHECK_NOJMC %s
 // RUN: %clang -### -S -fjmc -g --target=x86_64-unknown-linux %s 2>&1 | FileCheck -check-prefix=CHECK_JMC %s
 // RUN: %clang -### -S -fjmc -g -fno-jmc --target=x86_64-unknown-linux %s 2>&1 | FileCheck -check-prefix=CHECK_NOJMC %s
-// RUN: %clang -### -fjmc -g -flto --target=x86_64-pc-windows-msvc %s 2>&1 | FileCheck -check-prefix=CHECK_NOJMC_LTO %s
+// RUN: %clang -### -fjmc -g -flto -fuse-ld=lld --target=x86_64-pc-windows-msvc %s 2>&1 | FileCheck -check-prefix=CHECK_NOJMC_LTO %s
 // RUN: %clang -### -fjmc -g -flto --target=x86_64-unknown-linux %s 2>&1 | FileCheck -check-prefix=CHECK_JMC_LTO %s
 // RUN: %clang -### -fjmc -g -flto -fno-jmc --target=x86_64-unknown-linux %s 2>&1 | FileCheck -check-prefix=CHECK_NOJMC_LTO %s
 // CHECK_JMC_WARN: -fjmc requires debug info. Use -g or debug options that enable debugger's stepping function; option ignored
diff --git a/clang/test/Driver/lto.c b/clang/test/Driver/lto.c
index b6c89eb99e274..5be95013f00d7 100644
--- a/clang/test/Driver/lto.c
+++ b/clang/test/Driver/lto.c
@@ -5,7 +5,7 @@
 // CHECK-COMPILE-ACTIONS: 2: compiler, {1}, ir
 // CHECK-COMPILE-ACTIONS: 3: backend, {2}, lto-bc
 
-// RUN: %clang -ccc-print-phases %s -flto 2> %t
+// RUN: %clang -ccc-print-phases %if target={{.*-windows-msvc.*}} %{ -fuse-ld=lld %} -flto %s 2> %t
 // RUN: FileCheck -check-prefix=CHECK-COMPILELINK-ACTIONS < %t %s
 //
 // CHECK-COMPILELINK-ACTIONS: 0: input, "{{.*}}lto.c", c
diff --git a/clang/test/Driver/print-multi-selection-flags.c b/clang/test/Driver/print-multi-selection-flags.c
index 0116c7f5a03b9..b490ff13d30eb 100644
--- a/clang/test/Driver/print-multi-selection-flags.c
+++ b/clang/test/Driver/print-multi-selection-flags.c
@@ -19,6 +19,11 @@
 // CHECK-HARD: -mfloat-abi=hard
 // CHECK-HARD: -mfpu=fpv5-d16
 
+// RUN: %clang -print-multi-flags-experimental --target=aarch64-none-elf -mabi=aapcs | FileCheck --check-prefix=CHECK-ABI-AAPCS %s
+// RUN: %clang -print-multi-flags-experimental --target=aarch64-none-elf -mabi=aapcs-soft | FileCheck --check-prefix=CHECK-ABI-AAPCS-SOFT %s
+// CHECK-ABI-AAPCS: -mabi=aapcs
+// CHECK-ABI-AAPCS-SOFT: -mabi=aapcs-soft
+
 // RUN: %clang -print-multi-flags-experimental --target=arm-none-eabi -mfloat-abi=soft -march=armv8-m.main+nofp | FileCheck --check-prefix=CHECK-V8MMAIN-NOFP %s
 // CHECK-V8MMAIN-NOFP: --target=thumbv8m.main-unknown-none-eabi
 // CHECK-V8MMAIN-NOFP: -mfloat-abi=soft
diff --git a/clang/test/Driver/stack-protector-guard.c b/clang/test/Driver/stack-protector-guard.c
index af4e11f8eaabc..d8475a70e3709 100644
--- a/clang/test/Driver/stack-protector-guard.c
+++ b/clang/test/Driver/stack-protector-guard.c
@@ -85,3 +85,22 @@
 // CHECK-AARCH64: "-cc1" {{.*}}"-mstack-protector-guard=sysreg" "-mstack-protector-guard-offset=0" "-mstack-protector-guard-reg=sp_el0"
 // INVALID-VALUE-AARCH64: error: invalid value 'tls' in 'mstack-protector-guard=', expected one of: sysreg global
 // INVALID-REG-AARCH64: error: invalid value 'foo' in 'mstack-protector-guard-reg='
+
+// RUN: %clang -### -target riscv64-unknown-elf -mstack-protector-guard=tls -mstack-protector-guard-offset=24 -mstack-protector-guard-reg=tp %s 2>&1 | \
+// RUN:   FileCheck -v -check-prefix=CHECK-TLS-RISCV %s
+// RUN: %clang -### -target riscv64-unknown-elf -mstack-protector-guard=global %s 2>&1 | \
+// RUN:   FileCheck -check-prefix=CHECK-GLOBAL %s
+
+// CHECK-TLS-RISCV: "-cc1" {{.*}}"-mstack-protector-guard=tls" "-mstack-protector-guard-offset=24" "-mstack-protector-guard-reg=tp"
+
+// RUN: not %clang -target riscv64-unknown-elf -mstack-protector-guard=tls %s 2>&1 | \
+// RUN:   FileCheck -check-prefix=MISSING-OFFSET %s
+
+// RUN: not %clang -target riscv64-unknown-elf -mstack-protector-guard=sysreg %s 2>&1 | \
+// RUN:   FileCheck -check-prefix=INVALID-VALUE2 %s
+
+// RUN: not %clang -target riscv64-unknown-elf -mstack-protector-guard=tls \
+// RUN: -mstack-protector-guard-offset=20 -mstack-protector-guard-reg=sp %s 2>&1 | \
+// RUN:   FileCheck -check-prefix=INVALID-REG-RISCV %s
+
+// INVALID-REG-RISCV: error: invalid value 'sp' in 'mstack-protector-guard-reg=', expected one of: tp
diff --git a/clang/test/Driver/thinlto.c b/clang/test/Driver/thinlto.c
index e08435f71cc0b..06424524b5dca 100644
--- a/clang/test/Driver/thinlto.c
+++ b/clang/test/Driver/thinlto.c
@@ -5,7 +5,7 @@
 // CHECK-COMPILE-ACTIONS: 2: compiler, {1}, ir
 // CHECK-COMPILE-ACTIONS: 3: backend, {2}, lto-bc
 
-// RUN: %clang -ccc-print-phases %s -flto=thin 2> %t
+// RUN: %clang -ccc-print-phases %if target={{.*-windows-msvc.*}} %{ -fuse-ld=lld %} -flto=thin %s 2> %t
 // RUN: FileCheck -check-prefix=CHECK-COMPILELINK-ACTIONS < %t %s
 //
 // CHECK-COMPILELINK-ACTIONS: 0: input, "{{.*}}thinlto.c", c
diff --git a/clang/test/Driver/thinlto.cu b/clang/test/Driver/thinlto.cu
index c175aae37c718..604a872786f70 100644
--- a/clang/test/Driver/thinlto.cu
+++ b/clang/test/Driver/thinlto.cu
@@ -6,7 +6,7 @@
 // CHECK-COMPILE-ACTIONS-NOT: lto-bc
 // CHECK-COMPILE-ACTIONS: 12: backend, {11}, lto-bc, (host-cuda)
 
-// RUN: %clangxx -ccc-print-phases --no-offload-new-driver -nocudainc -nocudalib %s -flto=thin 2> %t
+// RUN: %clangxx -ccc-print-phases %if target={{.*-windows-msvc.*}} %{ -fuse-ld=lld %} --no-offload-new-driver -nocudainc -nocudalib %s -flto=thin 2> %t
 // RUN: FileCheck -check-prefix=CHECK-COMPILELINK-ACTIONS < %t %s
 //
 // CHECK-COMPILELINK-ACTIONS: 0: input, "{{.*}}thinlto.cu", cuda, (host-cuda)
diff --git a/clang/test/Driver/zos-pragma-pack.c b/clang/test/Driver/zos-pragma-pack.c
new file mode 100644
index 0000000000000..0e04878daba4c
--- /dev/null
+++ b/clang/test/Driver/zos-pragma-pack.c
@@ -0,0 +1,8 @@
+// REQUIRES: systemz-registered-target
+
+// RUN: %clang -### -target s390x-ibm-zos -c %s -o /dev/null 2>&1 | FileCheck %s
+// CHECK: "-fxl-pragma-pack"
+
+// RUN: %clang -### -fno-xl-pragma-pack -target s390x-ibm-zos -c %s -o /dev/null 2>&1 | FileCheck %s -check-prefix=NOOPT
+// NOOPT-NOT: "-fxl-pragma-pack"
+
diff --git a/clang/test/ExtractAPI/typedef_anonymous_record.c b/clang/test/ExtractAPI/typedef_anonymous_record.c
index 9c03e9e190ed6..8e298f8d9ce82 100644
--- a/clang/test/ExtractAPI/typedef_anonymous_record.c
+++ b/clang/test/ExtractAPI/typedef_anonymous_record.c
@@ -35,7 +35,21 @@ typedef struct { } MyStruct;
 // MYSTRUCT:      "kind": {
 // MYSTRUCT-NEXT:   "displayName": "Structure",
 // MYSTRUCT-NEXT:   "identifier": "c.struct"
-// MYSTRUCT: "title": "MyStruct"
+// MYSTRUCT:           "names": {
+// MYSTRUCT-NEXT:        "navigator": [
+// MYSTRUCT-NEXT:          {
+// MYSTRUCT-NEXT:            "kind": "identifier",
+// MYSTRUCT-NEXT:            "spelling": "MyStruct"
+// MYSTRUCT-NEXT:          }
+// MYSTRUCT-NEXT:        ],
+// MYSTRUCT-NEXT:        "subHeading": [
+// MYSTRUCT-NEXT:          {
+// MYSTRUCT-NEXT:            "kind": "identifier",
+// MYSTRUCT-NEXT:            "spelling": "MyStruct"
+// MYSTRUCT-NEXT:          }
+// MYSTRUCT-NEXT:        ],
+// MYSTRUCT-NEXT:        "title": "MyStruct"
+// MYSTRUCT-NEXT:      },
 // MYSTRUCT:      "pathComponents": [
 // MYSTRUCT-NEXT:    "MyStruct"
 // MYSTRUCT-NEXT:  ]
@@ -111,7 +125,21 @@ typedef enum { Case } MyEnum;
 // MYENUM:     "kind": {
 // MYENUM-NEXT:  "displayName": "Enumeration",
 // MYENUM-NEXT:  "identifier": "c.enum"
-// MYENUM: "title": "MyEnum"
+// MYENUM:           "names": {
+// MYENUM-NEXT:        "navigator": [
+// MYENUM-NEXT:          {
+// MYENUM-NEXT:            "kind": "identifier",
+// MYENUM-NEXT:            "spelling": "MyEnum"
+// MYENUM-NEXT:          }
+// MYENUM-NEXT:        ],
+// MYENUM-NEXT:        "subHeading": [
+// MYENUM-NEXT:          {
+// MYENUM-NEXT:            "kind": "identifier",
+// MYENUM-NEXT:            "spelling": "MyEnum"
+// MYENUM-NEXT:          }
+// MYENUM-NEXT:        ],
+// MYENUM-NEXT:        "title": "MyEnum"
+// MYENUM-NEXT:      },
 
 // CASE-LABEL: "!testLabel": "c:@EA@MyEnum@Case"
 // CASE:      "pathComponents": [
diff --git a/clang/test/FixIt/member-mismatch.cpp b/clang/test/FixIt/member-mismatch.cpp
index 2d2aec2e176b2..e9c649d85c5c7 100644
--- a/clang/test/FixIt/member-mismatch.cpp
+++ b/clang/test/FixIt/member-mismatch.cpp
@@ -1,12 +1,14 @@
 // RUN: %clang_cc1 -verify %s
 // RUN: not %clang_cc1 -fdiagnostics-parseable-fixits %s 2>&1 | FileCheck %s
 
-class Foo {
+class Foo { // #defined-here
   int get() const; // expected-note {{because it is const qualified}}
   void set(int); // expected-note {{because it is not const qualified}}
 };
 
 // CHECK: fix-it:"{{.*}}":{[[@LINE+1]]:15-[[@LINE+1]]:15}:" const"
 int Foo::get() {} // expected-error {{does not match any declaration}}
+                  // expected-note@#defined-here {{defined here}}
 // CHECK: fix-it:"{{.*}}":{[[@LINE+1]]:20-[[@LINE+1]]:26}:""
 void Foo::set(int) const {} // expected-error {{does not match any declaration}}
+                            // expected-note@#defined-here {{defined here}}
diff --git a/clang/test/Headers/Inputs/include/setjmp.h b/clang/test/Headers/Inputs/include/setjmp.h
index 3d5e903eff6fc..c24951e92501f 100644
--- a/clang/test/Headers/Inputs/include/setjmp.h
+++ b/clang/test/Headers/Inputs/include/setjmp.h
@@ -5,4 +5,7 @@ typedef struct {
   int x[42];
 } jmp_buf;
 
+ __attribute__((noreturn))
+void longjmp(jmp_buf, int);
+
 #endif
diff --git a/clang/test/Misc/warning-wall.c b/clang/test/Misc/warning-wall.c
index 4909ab034ef30..91de843f88c91 100644
--- a/clang/test/Misc/warning-wall.c
+++ b/clang/test/Misc/warning-wall.c
@@ -108,5 +108,6 @@ CHECK-NEXT:  -Wmisleading-indentation
 CHECK-NEXT:  -Wpacked-non-pod
 CHECK-NEXT:  -Wvla-cxx-extension
 CHECK-NEXT:    -Wvla-extension-static-assert
+CHECK-NEXT:  -Wperf-constraint-implies-noexcept
 
 CHECK-NOT:-W
diff --git a/clang/test/Modules/cxx-templates.cpp b/clang/test/Modules/cxx-templates.cpp
index b7d5741e69af6..953fc0d9e867c 100644
--- a/clang/test/Modules/cxx-templates.cpp
+++ b/clang/test/Modules/cxx-templates.cpp
@@ -40,23 +40,14 @@ void g() {
 
   template_param_kinds_1<0>(); // ok, from cxx-templates-a.h
   template_param_kinds_1<int>(); // ok, from cxx-templates-b.h
-
-  template_param_kinds_2<Tmpl_T_C>(); // expected-error {{no matching function}}
-  // expected-note@Inputs/cxx-templates-a.h:11 {{invalid explicitly-specified argument}}
-  // expected-note@Inputs/cxx-templates-b.h:11 {{invalid explicitly-specified argument}}
+  template_param_kinds_2<Tmpl_T_C>(); // ok, from cxx-templates-b.h
 
   template_param_kinds_2<Tmpl_T_I_I>(); // expected-error {{ambiguous}}
   // expected-note@Inputs/cxx-templates-a.h:11 {{candidate}}
   // expected-note@Inputs/cxx-templates-b.h:11 {{candidate}}
 
-  // FIXME: This should be valid, but we incorrectly match the template template
-  // argument against both template template parameters.
-  template_param_kinds_3<Tmpl_T_T_A>(); // expected-error {{ambiguous}}
-  // expected-note@Inputs/cxx-templates-a.h:12 {{candidate}}
-  // expected-note@Inputs/cxx-templates-b.h:12 {{candidate}}
-  template_param_kinds_3<Tmpl_T_T_B>(); // expected-error {{ambiguous}}
-  // expected-note@Inputs/cxx-templates-a.h:12 {{candidate}}
-  // expected-note@Inputs/cxx-templates-b.h:12 {{candidate}}
+  template_param_kinds_3<Tmpl_T_T_A>();
+  template_param_kinds_3<Tmpl_T_T_B>();
 
   // Trigger the instantiation of a template in 'a' that uses a type defined in
   // 'common'. That type is not visible here.
diff --git a/clang/test/OpenMP/target_parallel_for_codegen.cpp b/clang/test/OpenMP/target_parallel_for_codegen.cpp
index 8a8c8a02aaef0..5378ff18b34c9 100644
--- a/clang/test/OpenMP/target_parallel_for_codegen.cpp
+++ b/clang/test/OpenMP/target_parallel_for_codegen.cpp
@@ -3782,7 +3782,7 @@ int bar(int n){
 // CHECK9-NEXT:    store i32 [[TMP0]], ptr [[DOTLINEAR_START]], align 4
 // CHECK9-NEXT:    [[TMP1:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK9-NEXT:    store i32 [[TMP1]], ptr [[DOTLINEAR_START1]], align 4
-// CHECK9-NEXT:    [[CALL:%.*]] = call noundef i64 @_Z7get_valv() #[[ATTR4:[0-9]+]]
+// CHECK9-NEXT:    [[CALL:%.*]] = call noundef i64 @_Z7get_valv()
 // CHECK9-NEXT:    store i64 [[CALL]], ptr [[DOTLINEAR_STEP]], align 8
 // CHECK9-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
 // CHECK9-NEXT:    store i64 3, ptr [[DOTOMP_UB]], align 8
@@ -4561,7 +4561,7 @@ int bar(int n){
 // CHECK11-NEXT:    store i32 [[TMP0]], ptr [[DOTLINEAR_START]], align 4
 // CHECK11-NEXT:    [[TMP1:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK11-NEXT:    store i32 [[TMP1]], ptr [[DOTLINEAR_START1]], align 4
-// CHECK11-NEXT:    [[CALL:%.*]] = call noundef i64 @_Z7get_valv() #[[ATTR4:[0-9]+]]
+// CHECK11-NEXT:    [[CALL:%.*]] = call noundef i64 @_Z7get_valv()
 // CHECK11-NEXT:    store i64 [[CALL]], ptr [[DOTLINEAR_STEP]], align 8
 // CHECK11-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
 // CHECK11-NEXT:    store i64 3, ptr [[DOTOMP_UB]], align 8
diff --git a/clang/test/OpenMP/target_parallel_for_simd_codegen.cpp b/clang/test/OpenMP/target_parallel_for_simd_codegen.cpp
index 4bde6a4cfda3f..e4570dbb1efcd 100644
--- a/clang/test/OpenMP/target_parallel_for_simd_codegen.cpp
+++ b/clang/test/OpenMP/target_parallel_for_simd_codegen.cpp
@@ -9131,7 +9131,7 @@ int bar(int n){
 // CHECK17-NEXT:    store i32 [[TMP0]], ptr [[DOTLINEAR_START]], align 4
 // CHECK17-NEXT:    [[TMP1:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK17-NEXT:    store i32 [[TMP1]], ptr [[DOTLINEAR_START1]], align 4
-// CHECK17-NEXT:    [[CALL:%.*]] = call noundef i64 @_Z7get_valv() #[[ATTR4:[0-9]+]]
+// CHECK17-NEXT:    [[CALL:%.*]] = call noundef i64 @_Z7get_valv()
 // CHECK17-NEXT:    store i64 [[CALL]], ptr [[DOTLINEAR_STEP]], align 8
 // CHECK17-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
 // CHECK17-NEXT:    store i64 3, ptr [[DOTOMP_UB]], align 8
@@ -9935,7 +9935,7 @@ int bar(int n){
 // CHECK19-NEXT:    store i32 [[TMP0]], ptr [[DOTLINEAR_START]], align 4
 // CHECK19-NEXT:    [[TMP1:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK19-NEXT:    store i32 [[TMP1]], ptr [[DOTLINEAR_START1]], align 4
-// CHECK19-NEXT:    [[CALL:%.*]] = call noundef i64 @_Z7get_valv() #[[ATTR4:[0-9]+]]
+// CHECK19-NEXT:    [[CALL:%.*]] = call noundef i64 @_Z7get_valv()
 // CHECK19-NEXT:    store i64 [[CALL]], ptr [[DOTLINEAR_STEP]], align 8
 // CHECK19-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
 // CHECK19-NEXT:    store i64 3, ptr [[DOTOMP_UB]], align 8
@@ -10739,7 +10739,7 @@ int bar(int n){
 // CHECK21-NEXT:    store i32 [[TMP0]], ptr [[DOTLINEAR_START]], align 4
 // CHECK21-NEXT:    [[TMP1:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK21-NEXT:    store i32 [[TMP1]], ptr [[DOTLINEAR_START1]], align 4
-// CHECK21-NEXT:    [[CALL:%.*]] = call noundef i64 @_Z7get_valv() #[[ATTR4:[0-9]+]]
+// CHECK21-NEXT:    [[CALL:%.*]] = call noundef i64 @_Z7get_valv()
 // CHECK21-NEXT:    store i64 [[CALL]], ptr [[DOTLINEAR_STEP]], align 8
 // CHECK21-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
 // CHECK21-NEXT:    store i64 3, ptr [[DOTOMP_UB]], align 8
@@ -11629,7 +11629,7 @@ int bar(int n){
 // CHECK23-NEXT:    store i32 [[TMP0]], ptr [[DOTLINEAR_START]], align 4
 // CHECK23-NEXT:    [[TMP1:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK23-NEXT:    store i32 [[TMP1]], ptr [[DOTLINEAR_START1]], align 4
-// CHECK23-NEXT:    [[CALL:%.*]] = call noundef i64 @_Z7get_valv() #[[ATTR4:[0-9]+]]
+// CHECK23-NEXT:    [[CALL:%.*]] = call noundef i64 @_Z7get_valv()
 // CHECK23-NEXT:    store i64 [[CALL]], ptr [[DOTLINEAR_STEP]], align 8
 // CHECK23-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
 // CHECK23-NEXT:    store i64 3, ptr [[DOTOMP_UB]], align 8
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp
index 00a900d441b03..78e40e54671ac 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp
@@ -2718,17 +2718,17 @@ int main() {
 // CHECK13:       omp.arraycpy.body:
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_SRCELEMENTPAST:%.*]] = phi ptr [ [[TMP1]], [[ENTRY:%.*]] ], [ [[OMP_ARRAYCPY_SRC_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DESTELEMENTPAST:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY]] ], [ [[OMP_ARRAYCPY_DEST_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
-// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR4:[0-9]+]]
-// CHECK13-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR5:[0-9]+]]
+// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]])
+// CHECK13-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]])
+// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR3:[0-9]+]]
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DEST_ELEMENT]] = getelementptr [[STRUCT_S]], ptr [[OMP_ARRAYCPY_DESTELEMENTPAST]], i32 1
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_SRC_ELEMENT]] = getelementptr [[STRUCT_S]], ptr [[OMP_ARRAYCPY_SRCELEMENTPAST]], i32 1
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK13-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE3]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK13:       omp.arraycpy.done3:
-// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP2]], ptr noundef [[AGG_TMP5]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]])
+// CHECK13-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP2]], ptr noundef [[AGG_TMP5]])
+// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
 // CHECK13-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1:[0-9]+]], i32 [[TMP5]], i32 92, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_COMB_LB]], ptr [[DOTOMP_COMB_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -2778,14 +2778,14 @@ int main() {
 // CHECK13-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP22:%.*]] = load i32, ptr [[TMP21]], align 4
 // CHECK13-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP22]])
-// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[ARRAY_BEGIN7:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP23:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN7]], i64 2
 // CHECK13-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK13:       arraydestroy.body:
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP23]], [[OMP_LOOP_EXIT]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1
-// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN7]]
 // CHECK13-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE8:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK13:       arraydestroy.done8:
@@ -2798,7 +2798,7 @@ int main() {
 // CHECK13-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK13-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN2StC2Ev(ptr noundef nonnull align 4 dereferenceable(8) [[THIS1]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN2StC2Ev(ptr noundef nonnull align 4 dereferenceable(8) [[THIS1]])
 // CHECK13-NEXT:    ret void
 //
 //
@@ -2813,7 +2813,7 @@ int main() {
 // CHECK13-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN1SIfEC2ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP0]], ptr noundef [[T]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIfEC2ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP0]], ptr noundef [[T]])
 // CHECK13-NEXT:    ret void
 //
 //
@@ -2823,7 +2823,7 @@ int main() {
 // CHECK13-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK13-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN2StD2Ev(ptr noundef nonnull align 4 dereferenceable(8) [[THIS1]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN2StD2Ev(ptr noundef nonnull align 4 dereferenceable(8) [[THIS1]]) #[[ATTR3]]
 // CHECK13-NEXT:    ret void
 //
 //
@@ -2881,17 +2881,17 @@ int main() {
 // CHECK13:       omp.arraycpy.body:
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_SRCELEMENTPAST:%.*]] = phi ptr [ [[TMP1]], [[ENTRY:%.*]] ], [ [[OMP_ARRAYCPY_SRC_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DESTELEMENTPAST:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY]] ], [ [[OMP_ARRAYCPY_DEST_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
-// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]])
+// CHECK13-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]])
+// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DEST_ELEMENT]] = getelementptr [[STRUCT_S]], ptr [[OMP_ARRAYCPY_DESTELEMENTPAST]], i32 1
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_SRC_ELEMENT]] = getelementptr [[STRUCT_S]], ptr [[OMP_ARRAYCPY_SRCELEMENTPAST]], i32 1
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK13-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK13:       omp.arraycpy.done4:
-// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP2]], ptr noundef [[AGG_TMP6]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
+// CHECK13-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP2]], ptr noundef [[AGG_TMP6]])
+// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 // CHECK13-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB2:[0-9]+]], i32 [[TMP7]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -2948,14 +2948,14 @@ int main() {
 // CHECK13-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4
 // CHECK13-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB2]], i32 [[TMP21]])
-// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[ARRAY_BEGIN12:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR3]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP22:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN12]], i64 2
 // CHECK13-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK13:       arraydestroy.body:
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP22]], [[OMP_LOOP_EXIT]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1
-// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN12]]
 // CHECK13-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE13:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK13:       arraydestroy.done13:
@@ -2968,7 +2968,7 @@ int main() {
 // CHECK13-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK13-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN1SIfED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIfED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR3]]
 // CHECK13-NEXT:    ret void
 //
 //
@@ -3045,18 +3045,18 @@ int main() {
 // CHECK13:       omp.arraycpy.body:
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_SRCELEMENTPAST:%.*]] = phi ptr [ [[TMP1]], [[ENTRY:%.*]] ], [ [[OMP_ARRAYCPY_SRC_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DESTELEMENTPAST:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY]] ], [ [[OMP_ARRAYCPY_DEST_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
-// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]])
+// CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]])
+// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DEST_ELEMENT]] = getelementptr [[STRUCT_S_0]], ptr [[OMP_ARRAYCPY_DESTELEMENTPAST]], i32 1
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_SRC_ELEMENT]] = getelementptr [[STRUCT_S_0]], ptr [[OMP_ARRAYCPY_SRCELEMENTPAST]], i32 1
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK13-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK13:       omp.arraycpy.done4:
 // CHECK13-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
+// CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
+// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR3]]
 // CHECK13-NEXT:    store ptr [[VAR5]], ptr [[_TMP7]], align 8
 // CHECK13-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
@@ -3105,14 +3105,14 @@ int main() {
 // CHECK13-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP22:%.*]] = load i32, ptr [[TMP21]], align 4
 // CHECK13-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP22]])
-// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[ARRAY_BEGIN9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP23:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN9]], i64 2
 // CHECK13-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK13:       arraydestroy.body:
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP23]], [[OMP_LOOP_EXIT]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1
-// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN9]]
 // CHECK13-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE10:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK13:       arraydestroy.done10:
@@ -3130,7 +3130,7 @@ int main() {
 // CHECK13-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN1SIiEC2ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP0]], ptr noundef [[T]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIiEC2ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP0]], ptr noundef [[T]])
 // CHECK13-NEXT:    ret void
 //
 //
@@ -3189,18 +3189,18 @@ int main() {
 // CHECK13:       omp.arraycpy.body:
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_SRCELEMENTPAST:%.*]] = phi ptr [ [[TMP1]], [[ENTRY:%.*]] ], [ [[OMP_ARRAYCPY_SRC_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DESTELEMENTPAST:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY]] ], [ [[OMP_ARRAYCPY_DEST_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
-// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]])
+// CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]])
+// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DEST_ELEMENT]] = getelementptr [[STRUCT_S_0]], ptr [[OMP_ARRAYCPY_DESTELEMENTPAST]], i32 1
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_SRC_ELEMENT]] = getelementptr [[STRUCT_S_0]], ptr [[OMP_ARRAYCPY_SRCELEMENTPAST]], i32 1
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK13-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE5]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK13:       omp.arraycpy.done5:
 // CHECK13-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR6]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP7]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]])
+// CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR6]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP7]])
+// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]]) #[[ATTR3]]
 // CHECK13-NEXT:    store ptr [[VAR6]], ptr [[_TMP8]], align 8
 // CHECK13-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
@@ -3255,14 +3255,14 @@ int main() {
 // CHECK13-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4
 // CHECK13-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB2]], i32 [[TMP21]])
-// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR6]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR6]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[ARRAY_BEGIN13:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR4]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP22:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN13]], i64 2
 // CHECK13-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK13:       arraydestroy.body:
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP22]], [[OMP_LOOP_EXIT]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1
-// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN13]]
 // CHECK13-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE14:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK13:       arraydestroy.done14:
@@ -3275,7 +3275,7 @@ int main() {
 // CHECK13-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK13-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN1SIiED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR3]]
 // CHECK13-NEXT:    ret void
 //
 //
@@ -3438,17 +3438,17 @@ int main() {
 // CHECK15:       omp.arraycpy.body:
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_SRCELEMENTPAST:%.*]] = phi ptr [ [[TMP1]], [[ENTRY:%.*]] ], [ [[OMP_ARRAYCPY_SRC_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DESTELEMENTPAST:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY]] ], [ [[OMP_ARRAYCPY_DEST_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
-// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR4:[0-9]+]]
-// CHECK15-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR5:[0-9]+]]
+// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]])
+// CHECK15-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]])
+// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR3:[0-9]+]]
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DEST_ELEMENT]] = getelementptr [[STRUCT_S]], ptr [[OMP_ARRAYCPY_DESTELEMENTPAST]], i32 1
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_SRC_ELEMENT]] = getelementptr [[STRUCT_S]], ptr [[OMP_ARRAYCPY_SRCELEMENTPAST]], i32 1
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK15-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE3]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK15:       omp.arraycpy.done3:
-// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP2]], ptr noundef [[AGG_TMP5]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]])
+// CHECK15-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP2]], ptr noundef [[AGG_TMP5]])
+// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
 // CHECK15-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1:[0-9]+]], i32 [[TMP5]], i32 92, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_COMB_LB]], ptr [[DOTOMP_COMB_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -3496,14 +3496,14 @@ int main() {
 // CHECK15-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP20:%.*]] = load i32, ptr [[TMP19]], align 4
 // CHECK15-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP20]])
-// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[ARRAY_BEGIN7:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP21:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN7]], i32 2
 // CHECK15-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK15:       arraydestroy.body:
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP21]], [[OMP_LOOP_EXIT]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i32 -1
-// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN7]]
 // CHECK15-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE8:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK15:       arraydestroy.done8:
@@ -3516,7 +3516,7 @@ int main() {
 // CHECK15-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 4
 // CHECK15-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN2StC2Ev(ptr noundef nonnull align 4 dereferenceable(8) [[THIS1]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN2StC2Ev(ptr noundef nonnull align 4 dereferenceable(8) [[THIS1]])
 // CHECK15-NEXT:    ret void
 //
 //
@@ -3531,7 +3531,7 @@ int main() {
 // CHECK15-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN1SIfEC2ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP0]], ptr noundef [[T]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIfEC2ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP0]], ptr noundef [[T]])
 // CHECK15-NEXT:    ret void
 //
 //
@@ -3541,7 +3541,7 @@ int main() {
 // CHECK15-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 4
 // CHECK15-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN2StD2Ev(ptr noundef nonnull align 4 dereferenceable(8) [[THIS1]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN2StD2Ev(ptr noundef nonnull align 4 dereferenceable(8) [[THIS1]]) #[[ATTR3]]
 // CHECK15-NEXT:    ret void
 //
 //
@@ -3597,17 +3597,17 @@ int main() {
 // CHECK15:       omp.arraycpy.body:
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_SRCELEMENTPAST:%.*]] = phi ptr [ [[TMP1]], [[ENTRY:%.*]] ], [ [[OMP_ARRAYCPY_SRC_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DESTELEMENTPAST:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY]] ], [ [[OMP_ARRAYCPY_DEST_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
-// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]])
+// CHECK15-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]])
+// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DEST_ELEMENT]] = getelementptr [[STRUCT_S]], ptr [[OMP_ARRAYCPY_DESTELEMENTPAST]], i32 1
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_SRC_ELEMENT]] = getelementptr [[STRUCT_S]], ptr [[OMP_ARRAYCPY_SRCELEMENTPAST]], i32 1
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK15-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE3]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK15:       omp.arraycpy.done3:
-// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP2]], ptr noundef [[AGG_TMP5]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]])
+// CHECK15-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP2]], ptr noundef [[AGG_TMP5]])
+// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 // CHECK15-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB2:[0-9]+]], i32 [[TMP7]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -3662,14 +3662,14 @@ int main() {
 // CHECK15-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4
 // CHECK15-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB2]], i32 [[TMP21]])
-// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[ARRAY_BEGIN10:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP22:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN10]], i32 2
 // CHECK15-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK15:       arraydestroy.body:
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP22]], [[OMP_LOOP_EXIT]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i32 -1
-// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN10]]
 // CHECK15-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE11:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK15:       arraydestroy.done11:
@@ -3682,7 +3682,7 @@ int main() {
 // CHECK15-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 4
 // CHECK15-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN1SIfED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIfED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR3]]
 // CHECK15-NEXT:    ret void
 //
 //
@@ -3759,18 +3759,18 @@ int main() {
 // CHECK15:       omp.arraycpy.body:
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_SRCELEMENTPAST:%.*]] = phi ptr [ [[TMP1]], [[ENTRY:%.*]] ], [ [[OMP_ARRAYCPY_SRC_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DESTELEMENTPAST:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY]] ], [ [[OMP_ARRAYCPY_DEST_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
-// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]])
+// CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]])
+// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DEST_ELEMENT]] = getelementptr [[STRUCT_S_0]], ptr [[OMP_ARRAYCPY_DESTELEMENTPAST]], i32 1
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_SRC_ELEMENT]] = getelementptr [[STRUCT_S_0]], ptr [[OMP_ARRAYCPY_SRCELEMENTPAST]], i32 1
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK15-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK15:       omp.arraycpy.done4:
 // CHECK15-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
-// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
+// CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
+// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR3]]
 // CHECK15-NEXT:    store ptr [[VAR5]], ptr [[_TMP7]], align 4
 // CHECK15-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
@@ -3817,14 +3817,14 @@ int main() {
 // CHECK15-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP20:%.*]] = load i32, ptr [[TMP19]], align 4
 // CHECK15-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP20]])
-// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[ARRAY_BEGIN9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP21:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN9]], i32 2
 // CHECK15-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK15:       arraydestroy.body:
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP21]], [[OMP_LOOP_EXIT]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i32 -1
-// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN9]]
 // CHECK15-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE10:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK15:       arraydestroy.done10:
@@ -3842,7 +3842,7 @@ int main() {
 // CHECK15-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN1SIiEC2ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP0]], ptr noundef [[T]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIiEC2ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP0]], ptr noundef [[T]])
 // CHECK15-NEXT:    ret void
 //
 //
@@ -3899,18 +3899,18 @@ int main() {
 // CHECK15:       omp.arraycpy.body:
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_SRCELEMENTPAST:%.*]] = phi ptr [ [[TMP1]], [[ENTRY:%.*]] ], [ [[OMP_ARRAYCPY_SRC_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DESTELEMENTPAST:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY]] ], [ [[OMP_ARRAYCPY_DEST_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
-// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]])
+// CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]])
+// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DEST_ELEMENT]] = getelementptr [[STRUCT_S_0]], ptr [[OMP_ARRAYCPY_DESTELEMENTPAST]], i32 1
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_SRC_ELEMENT]] = getelementptr [[STRUCT_S_0]], ptr [[OMP_ARRAYCPY_SRCELEMENTPAST]], i32 1
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK15-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK15:       omp.arraycpy.done4:
 // CHECK15-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4
-// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP6]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
+// CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP6]])
+// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR3]]
 // CHECK15-NEXT:    store ptr [[VAR5]], ptr [[_TMP7]], align 4
 // CHECK15-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
@@ -3963,14 +3963,14 @@ int main() {
 // CHECK15-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4
 // CHECK15-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB2]], i32 [[TMP21]])
-// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[ARRAY_BEGIN11:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP22:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN11]], i32 2
 // CHECK15-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK15:       arraydestroy.body:
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP22]], [[OMP_LOOP_EXIT]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i32 -1
-// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN11]]
 // CHECK15-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE12:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK15:       arraydestroy.done12:
@@ -3983,7 +3983,7 @@ int main() {
 // CHECK15-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 4
 // CHECK15-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN1SIiED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR3]]
 // CHECK15-NEXT:    ret void
 //
 //
@@ -4256,7 +4256,7 @@ int main() {
 // CHECK17-NEXT:    store ptr [[TMP13]], ptr [[TMP12]], align 8
 // CHECK17-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw [[CLASS_ANON]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK17-NEXT:    store ptr [[SIVAR_ADDR]], ptr [[TMP14]], align 8
-// CHECK17-NEXT:    call void @"_ZZZ4mainENK3$_0clEvENKUlvE_clEv"(ptr noundef nonnull align 8 dereferenceable(24) [[REF_TMP]]) #[[ATTR3:[0-9]+]]
+// CHECK17-NEXT:    call void @"_ZZZ4mainENK3$_0clEvENKUlvE_clEv"(ptr noundef nonnull align 8 dereferenceable(24) [[REF_TMP]])
 // CHECK17-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK17:       omp.body.continue:
 // CHECK17-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_private_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_private_codegen.cpp
index 5283cb148edfd..c0260af74c74b 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_private_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_private_codegen.cpp
@@ -1965,12 +1965,12 @@ int main() {
 // CHECK13-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
 // CHECK13:       arrayctor.loop:
 // CHECK13-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
-// CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]]) #[[ATTR4:[0-9]+]]
+// CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]])
 // CHECK13-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYCTOR_CUR]], i64 1
 // CHECK13-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
 // CHECK13-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
 // CHECK13:       arrayctor.cont:
-// CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]])
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
 // CHECK13-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1:[0-9]+]], i32 [[TMP1]], i32 92, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_COMB_LB]], ptr [[DOTOMP_COMB_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -2014,14 +2014,14 @@ int main() {
 // CHECK13-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP14:%.*]] = load i32, ptr [[TMP13]], align 4
 // CHECK13-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP14]])
-// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR5:[0-9]+]]
+// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR2:[0-9]+]]
 // CHECK13-NEXT:    [[ARRAY_BEGIN2:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN2]], i64 2
 // CHECK13-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK13:       arraydestroy.body:
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP15]], [[OMP_LOOP_EXIT]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1
-// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR2]]
 // CHECK13-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN2]]
 // CHECK13-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE3:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK13:       arraydestroy.done3:
@@ -2034,7 +2034,7 @@ int main() {
 // CHECK13-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK13-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN1SIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]])
 // CHECK13-NEXT:    ret void
 //
 //
@@ -2076,12 +2076,12 @@ int main() {
 // CHECK13-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
 // CHECK13:       arrayctor.loop:
 // CHECK13-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
-// CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]])
 // CHECK13-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYCTOR_CUR]], i64 1
 // CHECK13-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
 // CHECK13-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
 // CHECK13:       arrayctor.cont:
-// CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]])
 // CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK13-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB2:[0-9]+]], i32 [[TMP3]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -2138,14 +2138,14 @@ int main() {
 // CHECK13-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
 // CHECK13-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB2]], i32 [[TMP17]])
-// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR2]]
 // CHECK13-NEXT:    [[ARRAY_BEGIN7:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP18:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN7]], i64 2
 // CHECK13-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK13:       arraydestroy.body:
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP18]], [[OMP_LOOP_EXIT]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1
-// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR2]]
 // CHECK13-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN7]]
 // CHECK13-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE8:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK13:       arraydestroy.done8:
@@ -2158,7 +2158,7 @@ int main() {
 // CHECK13-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK13-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN1SIfED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIfED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR2]]
 // CHECK13-NEXT:    ret void
 //
 //
@@ -2201,12 +2201,12 @@ int main() {
 // CHECK13-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
 // CHECK13:       arrayctor.loop:
 // CHECK13-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
-// CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]])
 // CHECK13-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYCTOR_CUR]], i64 1
 // CHECK13-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
 // CHECK13-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
 // CHECK13:       arrayctor.cont:
-// CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]])
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[_TMP2]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
@@ -2251,14 +2251,14 @@ int main() {
 // CHECK13-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP14:%.*]] = load i32, ptr [[TMP13]], align 4
 // CHECK13-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP14]])
-// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR2]]
 // CHECK13-NEXT:    [[ARRAY_BEGIN4:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN4]], i64 2
 // CHECK13-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK13:       arraydestroy.body:
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP15]], [[OMP_LOOP_EXIT]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1
-// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR2]]
 // CHECK13-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN4]]
 // CHECK13-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE5:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK13:       arraydestroy.done5:
@@ -2271,7 +2271,7 @@ int main() {
 // CHECK13-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK13-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN1SIiEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIiEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]])
 // CHECK13-NEXT:    ret void
 //
 //
@@ -2315,12 +2315,12 @@ int main() {
 // CHECK13-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
 // CHECK13:       arrayctor.loop:
 // CHECK13-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
-// CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]])
 // CHECK13-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYCTOR_CUR]], i64 1
 // CHECK13-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
 // CHECK13-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
 // CHECK13:       arrayctor.cont:
-// CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]])
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[_TMP3]], align 8
 // CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
@@ -2375,14 +2375,14 @@ int main() {
 // CHECK13-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4
 // CHECK13-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB2]], i32 [[TMP16]])
-// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR2]]
 // CHECK13-NEXT:    [[ARRAY_BEGIN8:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN8]], i64 2
 // CHECK13-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK13:       arraydestroy.body:
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP17]], [[OMP_LOOP_EXIT]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1
-// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR2]]
 // CHECK13-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN8]]
 // CHECK13-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE9:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK13:       arraydestroy.done9:
@@ -2395,7 +2395,7 @@ int main() {
 // CHECK13-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK13-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN1SIiED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR2]]
 // CHECK13-NEXT:    ret void
 //
 //
@@ -2479,12 +2479,12 @@ int main() {
 // CHECK15-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
 // CHECK15:       arrayctor.loop:
 // CHECK15-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
-// CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]]) #[[ATTR4:[0-9]+]]
+// CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]])
 // CHECK15-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYCTOR_CUR]], i32 1
 // CHECK15-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
 // CHECK15-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
 // CHECK15:       arrayctor.cont:
-// CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]])
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
 // CHECK15-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1:[0-9]+]], i32 [[TMP1]], i32 92, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_COMB_LB]], ptr [[DOTOMP_COMB_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -2526,14 +2526,14 @@ int main() {
 // CHECK15-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
 // CHECK15-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP12]])
-// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR5:[0-9]+]]
+// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR2:[0-9]+]]
 // CHECK15-NEXT:    [[ARRAY_BEGIN2:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN2]], i32 2
 // CHECK15-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK15:       arraydestroy.body:
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP13]], [[OMP_LOOP_EXIT]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i32 -1
-// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR2]]
 // CHECK15-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN2]]
 // CHECK15-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE3:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK15:       arraydestroy.done3:
@@ -2546,7 +2546,7 @@ int main() {
 // CHECK15-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 4
 // CHECK15-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN1SIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]])
 // CHECK15-NEXT:    ret void
 //
 //
@@ -2586,12 +2586,12 @@ int main() {
 // CHECK15-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
 // CHECK15:       arrayctor.loop:
 // CHECK15-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
-// CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]])
 // CHECK15-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYCTOR_CUR]], i32 1
 // CHECK15-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
 // CHECK15-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
 // CHECK15:       arrayctor.cont:
-// CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]])
 // CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK15-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB2:[0-9]+]], i32 [[TMP3]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -2646,14 +2646,14 @@ int main() {
 // CHECK15-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
 // CHECK15-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB2]], i32 [[TMP17]])
-// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR2]]
 // CHECK15-NEXT:    [[ARRAY_BEGIN5:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP18:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN5]], i32 2
 // CHECK15-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK15:       arraydestroy.body:
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP18]], [[OMP_LOOP_EXIT]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i32 -1
-// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR2]]
 // CHECK15-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN5]]
 // CHECK15-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE6:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK15:       arraydestroy.done6:
@@ -2666,7 +2666,7 @@ int main() {
 // CHECK15-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 4
 // CHECK15-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN1SIfED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIfED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR2]]
 // CHECK15-NEXT:    ret void
 //
 //
@@ -2709,12 +2709,12 @@ int main() {
 // CHECK15-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
 // CHECK15:       arrayctor.loop:
 // CHECK15-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
-// CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]])
 // CHECK15-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYCTOR_CUR]], i32 1
 // CHECK15-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
 // CHECK15-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
 // CHECK15:       arrayctor.cont:
-// CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]])
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[_TMP2]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
@@ -2757,14 +2757,14 @@ int main() {
 // CHECK15-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
 // CHECK15-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP12]])
-// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR2]]
 // CHECK15-NEXT:    [[ARRAY_BEGIN4:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN4]], i32 2
 // CHECK15-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK15:       arraydestroy.body:
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP13]], [[OMP_LOOP_EXIT]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i32 -1
-// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR2]]
 // CHECK15-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN4]]
 // CHECK15-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE5:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK15:       arraydestroy.done5:
@@ -2777,7 +2777,7 @@ int main() {
 // CHECK15-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 4
 // CHECK15-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN1SIiEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIiEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]])
 // CHECK15-NEXT:    ret void
 //
 //
@@ -2819,12 +2819,12 @@ int main() {
 // CHECK15-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
 // CHECK15:       arrayctor.loop:
 // CHECK15-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
-// CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]])
 // CHECK15-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYCTOR_CUR]], i32 1
 // CHECK15-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
 // CHECK15-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
 // CHECK15:       arrayctor.cont:
-// CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]])
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[_TMP2]], align 4
 // CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
@@ -2877,14 +2877,14 @@ int main() {
 // CHECK15-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4
 // CHECK15-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB2]], i32 [[TMP16]])
-// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR2]]
 // CHECK15-NEXT:    [[ARRAY_BEGIN6:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN6]], i32 2
 // CHECK15-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK15:       arraydestroy.body:
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP17]], [[OMP_LOOP_EXIT]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i32 -1
-// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR2]]
 // CHECK15-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN6]]
 // CHECK15-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE7:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK15:       arraydestroy.done7:
@@ -2897,7 +2897,7 @@ int main() {
 // CHECK15-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 4
 // CHECK15-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN1SIiED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR2]]
 // CHECK15-NEXT:    ret void
 //
 //
@@ -3094,7 +3094,7 @@ int main() {
 // CHECK17-NEXT:    store ptr [[TMP13]], ptr [[TMP12]], align 8
 // CHECK17-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw [[CLASS_ANON]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK17-NEXT:    store ptr [[SIVAR]], ptr [[TMP14]], align 8
-// CHECK17-NEXT:    call void @"_ZZZ4mainENK3$_0clEvENKUlvE_clEv"(ptr noundef nonnull align 8 dereferenceable(24) [[REF_TMP]]) #[[ATTR3:[0-9]+]]
+// CHECK17-NEXT:    call void @"_ZZZ4mainENK3$_0clEvENKUlvE_clEv"(ptr noundef nonnull align 8 dereferenceable(24) [[REF_TMP]])
 // CHECK17-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK17:       omp.body.continue:
 // CHECK17-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_firstprivate_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_firstprivate_codegen.cpp
index 66a13cf724c36..afc473c50452b 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_firstprivate_codegen.cpp
@@ -3558,17 +3558,17 @@ int main() {
 // CHECK13:       omp.arraycpy.body:
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_SRCELEMENTPAST:%.*]] = phi ptr [ [[TMP1]], [[ENTRY:%.*]] ], [ [[OMP_ARRAYCPY_SRC_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DESTELEMENTPAST:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY]] ], [ [[OMP_ARRAYCPY_DEST_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
-// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR4:[0-9]+]]
-// CHECK13-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR5:[0-9]+]]
+// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]])
+// CHECK13-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]])
+// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR3:[0-9]+]]
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DEST_ELEMENT]] = getelementptr [[STRUCT_S]], ptr [[OMP_ARRAYCPY_DESTELEMENTPAST]], i32 1
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_SRC_ELEMENT]] = getelementptr [[STRUCT_S]], ptr [[OMP_ARRAYCPY_SRCELEMENTPAST]], i32 1
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK13-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE3]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK13:       omp.arraycpy.done3:
-// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP2]], ptr noundef [[AGG_TMP5]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]])
+// CHECK13-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP2]], ptr noundef [[AGG_TMP5]])
+// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
 // CHECK13-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1:[0-9]+]], i32 [[TMP5]], i32 92, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_COMB_LB]], ptr [[DOTOMP_COMB_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -3625,14 +3625,14 @@ int main() {
 // CHECK13-NEXT:    store i32 2, ptr [[I]], align 4
 // CHECK13-NEXT:    br label [[DOTOMP_FINAL_DONE]]
 // CHECK13:       .omp.final.done:
-// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[ARRAY_BEGIN7:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP25:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN7]], i64 2
 // CHECK13-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK13:       arraydestroy.body:
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP25]], [[DOTOMP_FINAL_DONE]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1
-// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN7]]
 // CHECK13-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE8:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK13:       arraydestroy.done8:
@@ -3645,7 +3645,7 @@ int main() {
 // CHECK13-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK13-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN2StC2Ev(ptr noundef nonnull align 4 dereferenceable(8) [[THIS1]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN2StC2Ev(ptr noundef nonnull align 4 dereferenceable(8) [[THIS1]])
 // CHECK13-NEXT:    ret void
 //
 //
@@ -3660,7 +3660,7 @@ int main() {
 // CHECK13-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN1SIfEC2ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP0]], ptr noundef [[T]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIfEC2ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP0]], ptr noundef [[T]])
 // CHECK13-NEXT:    ret void
 //
 //
@@ -3670,7 +3670,7 @@ int main() {
 // CHECK13-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK13-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN2StD2Ev(ptr noundef nonnull align 4 dereferenceable(8) [[THIS1]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN2StD2Ev(ptr noundef nonnull align 4 dereferenceable(8) [[THIS1]]) #[[ATTR3]]
 // CHECK13-NEXT:    ret void
 //
 //
@@ -3728,17 +3728,17 @@ int main() {
 // CHECK13:       omp.arraycpy.body:
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_SRCELEMENTPAST:%.*]] = phi ptr [ [[TMP1]], [[ENTRY:%.*]] ], [ [[OMP_ARRAYCPY_SRC_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DESTELEMENTPAST:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY]] ], [ [[OMP_ARRAYCPY_DEST_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
-// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]])
+// CHECK13-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]])
+// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DEST_ELEMENT]] = getelementptr [[STRUCT_S]], ptr [[OMP_ARRAYCPY_DESTELEMENTPAST]], i32 1
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_SRC_ELEMENT]] = getelementptr [[STRUCT_S]], ptr [[OMP_ARRAYCPY_SRCELEMENTPAST]], i32 1
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK13-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK13:       omp.arraycpy.done4:
-// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP2]], ptr noundef [[AGG_TMP6]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
+// CHECK13-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP2]], ptr noundef [[AGG_TMP6]])
+// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 // CHECK13-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB2:[0-9]+]], i32 [[TMP7]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -3802,14 +3802,14 @@ int main() {
 // CHECK13-NEXT:    store i32 2, ptr [[I]], align 4
 // CHECK13-NEXT:    br label [[DOTOMP_FINAL_DONE]]
 // CHECK13:       .omp.final.done:
-// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[ARRAY_BEGIN12:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR3]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP24:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN12]], i64 2
 // CHECK13-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK13:       arraydestroy.body:
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP24]], [[DOTOMP_FINAL_DONE]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1
-// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN12]]
 // CHECK13-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE13:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK13:       arraydestroy.done13:
@@ -3822,7 +3822,7 @@ int main() {
 // CHECK13-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK13-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN1SIfED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIfED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR3]]
 // CHECK13-NEXT:    ret void
 //
 //
@@ -3899,18 +3899,18 @@ int main() {
 // CHECK13:       omp.arraycpy.body:
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_SRCELEMENTPAST:%.*]] = phi ptr [ [[TMP1]], [[ENTRY:%.*]] ], [ [[OMP_ARRAYCPY_SRC_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DESTELEMENTPAST:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY]] ], [ [[OMP_ARRAYCPY_DEST_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
-// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]])
+// CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]])
+// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DEST_ELEMENT]] = getelementptr [[STRUCT_S_0]], ptr [[OMP_ARRAYCPY_DESTELEMENTPAST]], i32 1
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_SRC_ELEMENT]] = getelementptr [[STRUCT_S_0]], ptr [[OMP_ARRAYCPY_SRCELEMENTPAST]], i32 1
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK13-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK13:       omp.arraycpy.done4:
 // CHECK13-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
+// CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
+// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR3]]
 // CHECK13-NEXT:    store ptr [[VAR5]], ptr [[_TMP7]], align 8
 // CHECK13-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
@@ -3966,14 +3966,14 @@ int main() {
 // CHECK13-NEXT:    store i32 2, ptr [[I]], align 4
 // CHECK13-NEXT:    br label [[DOTOMP_FINAL_DONE]]
 // CHECK13:       .omp.final.done:
-// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[ARRAY_BEGIN9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP25:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN9]], i64 2
 // CHECK13-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK13:       arraydestroy.body:
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP25]], [[DOTOMP_FINAL_DONE]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1
-// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN9]]
 // CHECK13-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE10:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK13:       arraydestroy.done10:
@@ -3991,7 +3991,7 @@ int main() {
 // CHECK13-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN1SIiEC2ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP0]], ptr noundef [[T]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIiEC2ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP0]], ptr noundef [[T]])
 // CHECK13-NEXT:    ret void
 //
 //
@@ -4050,18 +4050,18 @@ int main() {
 // CHECK13:       omp.arraycpy.body:
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_SRCELEMENTPAST:%.*]] = phi ptr [ [[TMP1]], [[ENTRY:%.*]] ], [ [[OMP_ARRAYCPY_SRC_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DESTELEMENTPAST:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY]] ], [ [[OMP_ARRAYCPY_DEST_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
-// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]])
+// CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]])
+// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DEST_ELEMENT]] = getelementptr [[STRUCT_S_0]], ptr [[OMP_ARRAYCPY_DESTELEMENTPAST]], i32 1
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_SRC_ELEMENT]] = getelementptr [[STRUCT_S_0]], ptr [[OMP_ARRAYCPY_SRCELEMENTPAST]], i32 1
 // CHECK13-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK13-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE5]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK13:       omp.arraycpy.done5:
 // CHECK13-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR6]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP7]]) #[[ATTR4]]
-// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]])
+// CHECK13-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR6]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP7]])
+// CHECK13-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]]) #[[ATTR3]]
 // CHECK13-NEXT:    store ptr [[VAR6]], ptr [[_TMP8]], align 8
 // CHECK13-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
@@ -4123,14 +4123,14 @@ int main() {
 // CHECK13-NEXT:    store i32 2, ptr [[I]], align 4
 // CHECK13-NEXT:    br label [[DOTOMP_FINAL_DONE]]
 // CHECK13:       .omp.final.done:
-// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR6]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR6]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[ARRAY_BEGIN13:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR4]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP24:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN13]], i64 2
 // CHECK13-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK13:       arraydestroy.body:
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP24]], [[DOTOMP_FINAL_DONE]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1
-// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR3]]
 // CHECK13-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN13]]
 // CHECK13-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE14:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK13:       arraydestroy.done14:
@@ -4143,7 +4143,7 @@ int main() {
 // CHECK13-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK13-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN1SIiED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR3]]
 // CHECK13-NEXT:    ret void
 //
 //
@@ -4306,17 +4306,17 @@ int main() {
 // CHECK15:       omp.arraycpy.body:
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_SRCELEMENTPAST:%.*]] = phi ptr [ [[TMP1]], [[ENTRY:%.*]] ], [ [[OMP_ARRAYCPY_SRC_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DESTELEMENTPAST:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY]] ], [ [[OMP_ARRAYCPY_DEST_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
-// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR4:[0-9]+]]
-// CHECK15-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR5:[0-9]+]]
+// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]])
+// CHECK15-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]])
+// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR3:[0-9]+]]
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DEST_ELEMENT]] = getelementptr [[STRUCT_S]], ptr [[OMP_ARRAYCPY_DESTELEMENTPAST]], i32 1
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_SRC_ELEMENT]] = getelementptr [[STRUCT_S]], ptr [[OMP_ARRAYCPY_SRCELEMENTPAST]], i32 1
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK15-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE3]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK15:       omp.arraycpy.done3:
-// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP2]], ptr noundef [[AGG_TMP5]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]])
+// CHECK15-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP2]], ptr noundef [[AGG_TMP5]])
+// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
 // CHECK15-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1:[0-9]+]], i32 [[TMP5]], i32 92, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_COMB_LB]], ptr [[DOTOMP_COMB_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -4371,14 +4371,14 @@ int main() {
 // CHECK15-NEXT:    store i32 2, ptr [[I]], align 4
 // CHECK15-NEXT:    br label [[DOTOMP_FINAL_DONE]]
 // CHECK15:       .omp.final.done:
-// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[ARRAY_BEGIN7:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP23:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN7]], i32 2
 // CHECK15-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK15:       arraydestroy.body:
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP23]], [[DOTOMP_FINAL_DONE]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i32 -1
-// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN7]]
 // CHECK15-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE8:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK15:       arraydestroy.done8:
@@ -4391,7 +4391,7 @@ int main() {
 // CHECK15-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 4
 // CHECK15-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN2StC2Ev(ptr noundef nonnull align 4 dereferenceable(8) [[THIS1]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN2StC2Ev(ptr noundef nonnull align 4 dereferenceable(8) [[THIS1]])
 // CHECK15-NEXT:    ret void
 //
 //
@@ -4406,7 +4406,7 @@ int main() {
 // CHECK15-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN1SIfEC2ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP0]], ptr noundef [[T]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIfEC2ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP0]], ptr noundef [[T]])
 // CHECK15-NEXT:    ret void
 //
 //
@@ -4416,7 +4416,7 @@ int main() {
 // CHECK15-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 4
 // CHECK15-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN2StD2Ev(ptr noundef nonnull align 4 dereferenceable(8) [[THIS1]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN2StD2Ev(ptr noundef nonnull align 4 dereferenceable(8) [[THIS1]]) #[[ATTR3]]
 // CHECK15-NEXT:    ret void
 //
 //
@@ -4472,17 +4472,17 @@ int main() {
 // CHECK15:       omp.arraycpy.body:
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_SRCELEMENTPAST:%.*]] = phi ptr [ [[TMP1]], [[ENTRY:%.*]] ], [ [[OMP_ARRAYCPY_SRC_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DESTELEMENTPAST:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY]] ], [ [[OMP_ARRAYCPY_DEST_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
-// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]])
+// CHECK15-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]])
+// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DEST_ELEMENT]] = getelementptr [[STRUCT_S]], ptr [[OMP_ARRAYCPY_DESTELEMENTPAST]], i32 1
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_SRC_ELEMENT]] = getelementptr [[STRUCT_S]], ptr [[OMP_ARRAYCPY_SRCELEMENTPAST]], i32 1
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK15-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE3]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK15:       omp.arraycpy.done3:
-// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP2]], ptr noundef [[AGG_TMP5]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]])
+// CHECK15-NEXT:    call void @_ZN1SIfEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP2]], ptr noundef [[AGG_TMP5]])
+// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP5]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 // CHECK15-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB2:[0-9]+]], i32 [[TMP7]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -4544,14 +4544,14 @@ int main() {
 // CHECK15-NEXT:    store i32 2, ptr [[I]], align 4
 // CHECK15-NEXT:    br label [[DOTOMP_FINAL_DONE]]
 // CHECK15:       .omp.final.done:
-// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR4]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[ARRAY_BEGIN10:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP24:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN10]], i32 2
 // CHECK15-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK15:       arraydestroy.body:
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP24]], [[DOTOMP_FINAL_DONE]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i32 -1
-// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN10]]
 // CHECK15-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE11:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK15:       arraydestroy.done11:
@@ -4564,7 +4564,7 @@ int main() {
 // CHECK15-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 4
 // CHECK15-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN1SIfED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIfED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR3]]
 // CHECK15-NEXT:    ret void
 //
 //
@@ -4641,18 +4641,18 @@ int main() {
 // CHECK15:       omp.arraycpy.body:
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_SRCELEMENTPAST:%.*]] = phi ptr [ [[TMP1]], [[ENTRY:%.*]] ], [ [[OMP_ARRAYCPY_SRC_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DESTELEMENTPAST:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY]] ], [ [[OMP_ARRAYCPY_DEST_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
-// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]])
+// CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]])
+// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DEST_ELEMENT]] = getelementptr [[STRUCT_S_0]], ptr [[OMP_ARRAYCPY_DESTELEMENTPAST]], i32 1
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_SRC_ELEMENT]] = getelementptr [[STRUCT_S_0]], ptr [[OMP_ARRAYCPY_SRCELEMENTPAST]], i32 1
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
 // CHECK15-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK15:       omp.arraycpy.done4:
 // CHECK15-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
-// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
+// CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
+// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR3]]
 // CHECK15-NEXT:    store ptr [[VAR5]], ptr [[_TMP7]], align 4
 // CHECK15-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
@@ -4706,14 +4706,14 @@ int main() {
 // CHECK15-NEXT:    store i32 2, ptr [[I]], align 4
 // CHECK15-NEXT:    br label [[DOTOMP_FINAL_DONE]]
 // CHECK15:       .omp.final.done:
-// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[ARRAY_BEGIN9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP23:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN9]], i32 2
 // CHECK15-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK15:       arraydestroy.body:
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP23]], [[DOTOMP_FINAL_DONE]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i32 -1
-// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN9]]
 // CHECK15-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE10:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK15:       arraydestroy.done10:
@@ -4731,7 +4731,7 @@ int main() {
 // CHECK15-NEXT:    store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN1SIiEC2ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP0]], ptr noundef [[T]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIiEC2ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP0]], ptr noundef [[T]])
 // CHECK15-NEXT:    ret void
 //
 //
@@ -4788,18 +4788,18 @@ int main() {
 // CHECK15:       omp.arraycpy.body:
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_SRCELEMENTPAST:%.*]] = phi ptr [ [[TMP1]], [[ENTRY:%.*]] ], [ [[OMP_ARRAYCPY_SRC_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DESTELEMENTPAST:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY]] ], [ [[OMP_ARRAYCPY_DEST_ELEMENT:%.*]], [[OMP_ARRAYCPY_BODY]] ]
-// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]])
+// CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_DESTELEMENTPAST]], ptr noundef nonnull align 4 dereferenceable(4) [[OMP_ARRAYCPY_SRCELEMENTPAST]], ptr noundef [[AGG_TMP]])
+// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DEST_ELEMENT]] = getelementptr [[STRUCT_S_0]], ptr [[OMP_ARRAYCPY_DESTELEMENTPAST]], i32 1
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_SRC_ELEMENT]] = getelementptr [[STRUCT_S_0]], ptr [[OMP_ARRAYCPY_SRCELEMENTPAST]], i32 1
 // CHECK15-NEXT:    [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
 // CHECK15-NEXT:    br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
 // CHECK15:       omp.arraycpy.done4:
 // CHECK15-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4
-// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP6]]) #[[ATTR4]]
-// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
+// CHECK15-NEXT:    call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP6]])
+// CHECK15-NEXT:    call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR3]]
 // CHECK15-NEXT:    store ptr [[VAR5]], ptr [[_TMP7]], align 4
 // CHECK15-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
@@ -4859,14 +4859,14 @@ int main() {
 // CHECK15-NEXT:    store i32 2, ptr [[I]], align 4
 // CHECK15-NEXT:    br label [[DOTOMP_FINAL_DONE]]
 // CHECK15:       .omp.final.done:
-// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[ARRAY_BEGIN11:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP24:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN11]], i32 2
 // CHECK15-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK15:       arraydestroy.body:
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP24]], [[DOTOMP_FINAL_DONE]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i32 -1
-// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR3]]
 // CHECK15-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN11]]
 // CHECK15-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE12:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK15:       arraydestroy.done12:
@@ -4879,7 +4879,7 @@ int main() {
 // CHECK15-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 4
 // CHECK15-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN1SIiED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR3]]
 // CHECK15-NEXT:    ret void
 //
 //
@@ -5159,7 +5159,7 @@ int main() {
 // CHECK17-NEXT:    store ptr [[TMP13]], ptr [[TMP12]], align 8, !llvm.access.group [[ACC_GRP9]]
 // CHECK17-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw [[CLASS_ANON]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK17-NEXT:    store ptr [[SIVAR_ADDR]], ptr [[TMP14]], align 8, !llvm.access.group [[ACC_GRP9]]
-// CHECK17-NEXT:    call void @"_ZZZ4mainENK3$_0clEvENKUlvE_clEv"(ptr noundef nonnull align 8 dereferenceable(24) [[REF_TMP]]) #[[ATTR3:[0-9]+]], !llvm.access.group [[ACC_GRP9]]
+// CHECK17-NEXT:    call void @"_ZZZ4mainENK3$_0clEvENKUlvE_clEv"(ptr noundef nonnull align 8 dereferenceable(24) [[REF_TMP]]), !llvm.access.group [[ACC_GRP9]]
 // CHECK17-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK17:       omp.body.continue:
 // CHECK17-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_private_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_private_codegen.cpp
index 2b76d9032036c..cbf3162706d77 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_private_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_private_codegen.cpp
@@ -2917,12 +2917,12 @@ int main() {
 // CHECK13-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
 // CHECK13:       arrayctor.loop:
 // CHECK13-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
-// CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]]) #[[ATTR4:[0-9]+]]
+// CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]])
 // CHECK13-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYCTOR_CUR]], i64 1
 // CHECK13-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
 // CHECK13-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
 // CHECK13:       arrayctor.cont:
-// CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]])
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
 // CHECK13-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1:[0-9]+]], i32 [[TMP1]], i32 92, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_COMB_LB]], ptr [[DOTOMP_COMB_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -2973,14 +2973,14 @@ int main() {
 // CHECK13-NEXT:    store i32 2, ptr [[I]], align 4
 // CHECK13-NEXT:    br label [[DOTOMP_FINAL_DONE]]
 // CHECK13:       .omp.final.done:
-// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR5:[0-9]+]]
+// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR2:[0-9]+]]
 // CHECK13-NEXT:    [[ARRAY_BEGIN2:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN2]], i64 2
 // CHECK13-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK13:       arraydestroy.body:
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP17]], [[DOTOMP_FINAL_DONE]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1
-// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR2]]
 // CHECK13-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN2]]
 // CHECK13-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE3:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK13:       arraydestroy.done3:
@@ -2993,7 +2993,7 @@ int main() {
 // CHECK13-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK13-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN1SIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]])
 // CHECK13-NEXT:    ret void
 //
 //
@@ -3035,12 +3035,12 @@ int main() {
 // CHECK13-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
 // CHECK13:       arrayctor.loop:
 // CHECK13-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
-// CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]])
 // CHECK13-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYCTOR_CUR]], i64 1
 // CHECK13-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
 // CHECK13-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
 // CHECK13:       arrayctor.cont:
-// CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]])
 // CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK13-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB2:[0-9]+]], i32 [[TMP3]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -3104,14 +3104,14 @@ int main() {
 // CHECK13-NEXT:    store i32 2, ptr [[I]], align 4
 // CHECK13-NEXT:    br label [[DOTOMP_FINAL_DONE]]
 // CHECK13:       .omp.final.done:
-// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR2]]
 // CHECK13-NEXT:    [[ARRAY_BEGIN7:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP20:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN7]], i64 2
 // CHECK13-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK13:       arraydestroy.body:
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP20]], [[DOTOMP_FINAL_DONE]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1
-// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR2]]
 // CHECK13-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN7]]
 // CHECK13-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE8:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK13:       arraydestroy.done8:
@@ -3124,7 +3124,7 @@ int main() {
 // CHECK13-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK13-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN1SIfED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIfED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR2]]
 // CHECK13-NEXT:    ret void
 //
 //
@@ -3167,12 +3167,12 @@ int main() {
 // CHECK13-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
 // CHECK13:       arrayctor.loop:
 // CHECK13-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
-// CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]])
 // CHECK13-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYCTOR_CUR]], i64 1
 // CHECK13-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
 // CHECK13-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
 // CHECK13:       arrayctor.cont:
-// CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]])
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[_TMP2]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
@@ -3224,14 +3224,14 @@ int main() {
 // CHECK13-NEXT:    store i32 2, ptr [[I]], align 4
 // CHECK13-NEXT:    br label [[DOTOMP_FINAL_DONE]]
 // CHECK13:       .omp.final.done:
-// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR2]]
 // CHECK13-NEXT:    [[ARRAY_BEGIN4:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN4]], i64 2
 // CHECK13-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK13:       arraydestroy.body:
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP17]], [[DOTOMP_FINAL_DONE]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1
-// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR2]]
 // CHECK13-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN4]]
 // CHECK13-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE5:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK13:       arraydestroy.done5:
@@ -3244,7 +3244,7 @@ int main() {
 // CHECK13-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK13-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN1SIiEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIiEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]])
 // CHECK13-NEXT:    ret void
 //
 //
@@ -3288,12 +3288,12 @@ int main() {
 // CHECK13-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
 // CHECK13:       arrayctor.loop:
 // CHECK13-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
-// CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]])
 // CHECK13-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYCTOR_CUR]], i64 1
 // CHECK13-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
 // CHECK13-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
 // CHECK13:       arrayctor.cont:
-// CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]])
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[_TMP3]], align 8
 // CHECK13-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
@@ -3355,14 +3355,14 @@ int main() {
 // CHECK13-NEXT:    store i32 2, ptr [[I]], align 4
 // CHECK13-NEXT:    br label [[DOTOMP_FINAL_DONE]]
 // CHECK13:       .omp.final.done:
-// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR2]]
 // CHECK13-NEXT:    [[ARRAY_BEGIN8:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP19:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN8]], i64 2
 // CHECK13-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK13:       arraydestroy.body:
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP19]], [[DOTOMP_FINAL_DONE]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1
-// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR2]]
 // CHECK13-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN8]]
 // CHECK13-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE9:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK13:       arraydestroy.done9:
@@ -3375,7 +3375,7 @@ int main() {
 // CHECK13-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK13-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN1SIiED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR2]]
 // CHECK13-NEXT:    ret void
 //
 //
@@ -3459,12 +3459,12 @@ int main() {
 // CHECK15-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
 // CHECK15:       arrayctor.loop:
 // CHECK15-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
-// CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]]) #[[ATTR4:[0-9]+]]
+// CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]])
 // CHECK15-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYCTOR_CUR]], i32 1
 // CHECK15-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
 // CHECK15-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
 // CHECK15:       arrayctor.cont:
-// CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]])
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
 // CHECK15-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1:[0-9]+]], i32 [[TMP1]], i32 92, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_COMB_LB]], ptr [[DOTOMP_COMB_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -3513,14 +3513,14 @@ int main() {
 // CHECK15-NEXT:    store i32 2, ptr [[I]], align 4
 // CHECK15-NEXT:    br label [[DOTOMP_FINAL_DONE]]
 // CHECK15:       .omp.final.done:
-// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR5:[0-9]+]]
+// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR2:[0-9]+]]
 // CHECK15-NEXT:    [[ARRAY_BEGIN2:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN2]], i32 2
 // CHECK15-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK15:       arraydestroy.body:
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP15]], [[DOTOMP_FINAL_DONE]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i32 -1
-// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR2]]
 // CHECK15-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN2]]
 // CHECK15-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE3:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK15:       arraydestroy.done3:
@@ -3533,7 +3533,7 @@ int main() {
 // CHECK15-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 4
 // CHECK15-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN1SIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]])
 // CHECK15-NEXT:    ret void
 //
 //
@@ -3573,12 +3573,12 @@ int main() {
 // CHECK15-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
 // CHECK15:       arrayctor.loop:
 // CHECK15-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
-// CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]])
 // CHECK15-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYCTOR_CUR]], i32 1
 // CHECK15-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
 // CHECK15-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
 // CHECK15:       arrayctor.cont:
-// CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]])
 // CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK15-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB2:[0-9]+]], i32 [[TMP3]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -3640,14 +3640,14 @@ int main() {
 // CHECK15-NEXT:    store i32 2, ptr [[I]], align 4
 // CHECK15-NEXT:    br label [[DOTOMP_FINAL_DONE]]
 // CHECK15:       .omp.final.done:
-// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR2]]
 // CHECK15-NEXT:    [[ARRAY_BEGIN5:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP20:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN5]], i32 2
 // CHECK15-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK15:       arraydestroy.body:
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP20]], [[DOTOMP_FINAL_DONE]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i32 -1
-// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR2]]
 // CHECK15-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN5]]
 // CHECK15-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE6:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK15:       arraydestroy.done6:
@@ -3660,7 +3660,7 @@ int main() {
 // CHECK15-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 4
 // CHECK15-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN1SIfED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIfED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR2]]
 // CHECK15-NEXT:    ret void
 //
 //
@@ -3703,12 +3703,12 @@ int main() {
 // CHECK15-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
 // CHECK15:       arrayctor.loop:
 // CHECK15-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
-// CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]])
 // CHECK15-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYCTOR_CUR]], i32 1
 // CHECK15-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
 // CHECK15-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
 // CHECK15:       arrayctor.cont:
-// CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]])
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[_TMP2]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
@@ -3758,14 +3758,14 @@ int main() {
 // CHECK15-NEXT:    store i32 2, ptr [[I]], align 4
 // CHECK15-NEXT:    br label [[DOTOMP_FINAL_DONE]]
 // CHECK15:       .omp.final.done:
-// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR2]]
 // CHECK15-NEXT:    [[ARRAY_BEGIN4:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN4]], i32 2
 // CHECK15-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK15:       arraydestroy.body:
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP15]], [[DOTOMP_FINAL_DONE]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i32 -1
-// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR2]]
 // CHECK15-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN4]]
 // CHECK15-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE5:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK15:       arraydestroy.done5:
@@ -3778,7 +3778,7 @@ int main() {
 // CHECK15-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 4
 // CHECK15-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN1SIiEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIiEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]])
 // CHECK15-NEXT:    ret void
 //
 //
@@ -3820,12 +3820,12 @@ int main() {
 // CHECK15-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
 // CHECK15:       arrayctor.loop:
 // CHECK15-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
-// CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]])
 // CHECK15-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYCTOR_CUR]], i32 1
 // CHECK15-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
 // CHECK15-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
 // CHECK15:       arrayctor.cont:
-// CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]])
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[_TMP2]], align 4
 // CHECK15-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
@@ -3885,14 +3885,14 @@ int main() {
 // CHECK15-NEXT:    store i32 2, ptr [[I]], align 4
 // CHECK15-NEXT:    br label [[DOTOMP_FINAL_DONE]]
 // CHECK15:       .omp.final.done:
-// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR2]]
 // CHECK15-NEXT:    [[ARRAY_BEGIN6:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP19:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN6]], i32 2
 // CHECK15-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK15:       arraydestroy.body:
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP19]], [[DOTOMP_FINAL_DONE]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i32 -1
-// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR2]]
 // CHECK15-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN6]]
 // CHECK15-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE7:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK15:       arraydestroy.done7:
@@ -3905,7 +3905,7 @@ int main() {
 // CHECK15-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 4
 // CHECK15-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN1SIiED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR2]]
 // CHECK15-NEXT:    ret void
 //
 //
@@ -4109,7 +4109,7 @@ int main() {
 // CHECK17-NEXT:    store ptr [[TMP13]], ptr [[TMP12]], align 8, !llvm.access.group [[ACC_GRP9]]
 // CHECK17-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw [[CLASS_ANON]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK17-NEXT:    store ptr [[SIVAR]], ptr [[TMP14]], align 8, !llvm.access.group [[ACC_GRP9]]
-// CHECK17-NEXT:    call void @"_ZZZ4mainENK3$_0clEvENKUlvE_clEv"(ptr noundef nonnull align 8 dereferenceable(24) [[REF_TMP]]) #[[ATTR3:[0-9]+]], !llvm.access.group [[ACC_GRP9]]
+// CHECK17-NEXT:    call void @"_ZZZ4mainENK3$_0clEvENKUlvE_clEv"(ptr noundef nonnull align 8 dereferenceable(24) [[REF_TMP]]), !llvm.access.group [[ACC_GRP9]]
 // CHECK17-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK17:       omp.body.continue:
 // CHECK17-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
diff --git a/clang/test/OpenMP/target_teams_generic_loop_private_codegen.cpp b/clang/test/OpenMP/target_teams_generic_loop_private_codegen.cpp
index e87f3c82179ce..6b4e4d42ab9c9 100644
--- a/clang/test/OpenMP/target_teams_generic_loop_private_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_generic_loop_private_codegen.cpp
@@ -1486,12 +1486,12 @@ int main() {
 // CHECK13-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
 // CHECK13:       arrayctor.loop:
 // CHECK13-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
-// CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]]) #[[ATTR4:[0-9]+]]
+// CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]])
 // CHECK13-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYCTOR_CUR]], i64 1
 // CHECK13-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
 // CHECK13-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
 // CHECK13:       arrayctor.cont:
-// CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]])
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
 // CHECK13-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1:[0-9]+]], i32 [[TMP1]], i32 92, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_COMB_LB]], ptr [[DOTOMP_COMB_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -1548,14 +1548,14 @@ int main() {
 // CHECK13-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4
 // CHECK13-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP15]])
-// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR5:[0-9]+]]
+// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR2:[0-9]+]]
 // CHECK13-NEXT:    [[ARRAY_BEGIN6:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN6]], i64 2
 // CHECK13-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK13:       arraydestroy.body:
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP16]], [[OMP_LOOP_EXIT]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1
-// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR2]]
 // CHECK13-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN6]]
 // CHECK13-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE7:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK13:       arraydestroy.done7:
@@ -1568,7 +1568,7 @@ int main() {
 // CHECK13-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK13-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN1SIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]])
 // CHECK13-NEXT:    ret void
 //
 //
@@ -1578,7 +1578,7 @@ int main() {
 // CHECK13-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK13-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN1SIfED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIfED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR2]]
 // CHECK13-NEXT:    ret void
 //
 //
@@ -1621,12 +1621,12 @@ int main() {
 // CHECK13-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
 // CHECK13:       arrayctor.loop:
 // CHECK13-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
-// CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]])
 // CHECK13-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYCTOR_CUR]], i64 1
 // CHECK13-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
 // CHECK13-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
 // CHECK13:       arrayctor.cont:
-// CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]])
 // CHECK13-NEXT:    store ptr [[VAR]], ptr [[_TMP2]], align 8
 // CHECK13-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
@@ -1681,14 +1681,14 @@ int main() {
 // CHECK13-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK13-NEXT:    [[TMP14:%.*]] = load i32, ptr [[TMP13]], align 4
 // CHECK13-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP14]])
-// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR2]]
 // CHECK13-NEXT:    [[ARRAY_BEGIN7:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
 // CHECK13-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN7]], i64 2
 // CHECK13-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK13:       arraydestroy.body:
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP15]], [[OMP_LOOP_EXIT]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK13-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i64 -1
-// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR2]]
 // CHECK13-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN7]]
 // CHECK13-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE8:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK13:       arraydestroy.done8:
@@ -1701,7 +1701,7 @@ int main() {
 // CHECK13-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK13-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN1SIiEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR4]]
+// CHECK13-NEXT:    call void @_ZN1SIiEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]])
 // CHECK13-NEXT:    ret void
 //
 //
@@ -1711,7 +1711,7 @@ int main() {
 // CHECK13-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK13-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK13-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK13-NEXT:    call void @_ZN1SIiED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR5]]
+// CHECK13-NEXT:    call void @_ZN1SIiED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR2]]
 // CHECK13-NEXT:    ret void
 //
 //
@@ -1795,12 +1795,12 @@ int main() {
 // CHECK15-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
 // CHECK15:       arrayctor.loop:
 // CHECK15-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
-// CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]]) #[[ATTR4:[0-9]+]]
+// CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]])
 // CHECK15-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYCTOR_CUR]], i32 1
 // CHECK15-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
 // CHECK15-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
 // CHECK15:       arrayctor.cont:
-// CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIfEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]])
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
 // CHECK15-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1:[0-9]+]], i32 [[TMP1]], i32 92, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_COMB_LB]], ptr [[DOTOMP_COMB_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -1855,14 +1855,14 @@ int main() {
 // CHECK15-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4
 // CHECK15-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP15]])
-// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR5:[0-9]+]]
+// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR2:[0-9]+]]
 // CHECK15-NEXT:    [[ARRAY_BEGIN5:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN5]], i32 2
 // CHECK15-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK15:       arraydestroy.body:
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP16]], [[OMP_LOOP_EXIT]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i32 -1
-// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIfED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR2]]
 // CHECK15-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN5]]
 // CHECK15-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE6:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK15:       arraydestroy.done6:
@@ -1875,7 +1875,7 @@ int main() {
 // CHECK15-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 4
 // CHECK15-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN1SIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIfEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]])
 // CHECK15-NEXT:    ret void
 //
 //
@@ -1885,7 +1885,7 @@ int main() {
 // CHECK15-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 4
 // CHECK15-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN1SIfED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIfED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR2]]
 // CHECK15-NEXT:    ret void
 //
 //
@@ -1928,12 +1928,12 @@ int main() {
 // CHECK15-NEXT:    br label [[ARRAYCTOR_LOOP:%.*]]
 // CHECK15:       arrayctor.loop:
 // CHECK15-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi ptr [ [[ARRAY_BEGIN]], [[ENTRY:%.*]] ], [ [[ARRAYCTOR_NEXT:%.*]], [[ARRAYCTOR_LOOP]] ]
-// CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYCTOR_CUR]])
 // CHECK15-NEXT:    [[ARRAYCTOR_NEXT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYCTOR_CUR]], i32 1
 // CHECK15-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq ptr [[ARRAYCTOR_NEXT]], [[ARRAYCTOR_END]]
 // CHECK15-NEXT:    br i1 [[ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP]]
 // CHECK15:       arrayctor.cont:
-// CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIiEC1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]])
 // CHECK15-NEXT:    store ptr [[VAR]], ptr [[_TMP2]], align 4
 // CHECK15-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
@@ -1986,14 +1986,14 @@ int main() {
 // CHECK15-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 4
 // CHECK15-NEXT:    [[TMP14:%.*]] = load i32, ptr [[TMP13]], align 4
 // CHECK15-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP14]])
-// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[VAR]]) #[[ATTR2]]
 // CHECK15-NEXT:    [[ARRAY_BEGIN6:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
 // CHECK15-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN6]], i32 2
 // CHECK15-NEXT:    br label [[ARRAYDESTROY_BODY:%.*]]
 // CHECK15:       arraydestroy.body:
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENTPAST:%.*]] = phi ptr [ [[TMP15]], [[OMP_LOOP_EXIT]] ], [ [[ARRAYDESTROY_ELEMENT:%.*]], [[ARRAYDESTROY_BODY]] ]
 // CHECK15-NEXT:    [[ARRAYDESTROY_ELEMENT]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAYDESTROY_ELEMENTPAST]], i32 -1
-// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED1Ev(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYDESTROY_ELEMENT]]) #[[ATTR2]]
 // CHECK15-NEXT:    [[ARRAYDESTROY_DONE:%.*]] = icmp eq ptr [[ARRAYDESTROY_ELEMENT]], [[ARRAY_BEGIN6]]
 // CHECK15-NEXT:    br i1 [[ARRAYDESTROY_DONE]], label [[ARRAYDESTROY_DONE7:%.*]], label [[ARRAYDESTROY_BODY]]
 // CHECK15:       arraydestroy.done7:
@@ -2006,7 +2006,7 @@ int main() {
 // CHECK15-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 4
 // CHECK15-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN1SIiEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR4]]
+// CHECK15-NEXT:    call void @_ZN1SIiEC2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]])
 // CHECK15-NEXT:    ret void
 //
 //
@@ -2016,7 +2016,7 @@ int main() {
 // CHECK15-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 4
 // CHECK15-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 4
 // CHECK15-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK15-NEXT:    call void @_ZN1SIiED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR5]]
+// CHECK15-NEXT:    call void @_ZN1SIiED2Ev(ptr noundef nonnull align 4 dereferenceable(4) [[THIS1]]) #[[ATTR2]]
 // CHECK15-NEXT:    ret void
 //
 //
@@ -2136,7 +2136,7 @@ int main() {
 // CHECK17-NEXT:    store ptr [[TMP11]], ptr [[TMP10]], align 8
 // CHECK17-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON]], ptr [[REF_TMP]], i32 0, i32 2
 // CHECK17-NEXT:    store ptr [[SIVAR]], ptr [[TMP12]], align 8
-// CHECK17-NEXT:    call void @"_ZZZ4mainENK3$_0clEvENKUlvE_clEv"(ptr noundef nonnull align 8 dereferenceable(24) [[REF_TMP]]) #[[ATTR3:[0-9]+]]
+// CHECK17-NEXT:    call void @"_ZZZ4mainENK3$_0clEvENKUlvE_clEv"(ptr noundef nonnull align 8 dereferenceable(24) [[REF_TMP]])
 // CHECK17-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK17:       omp.body.continue:
 // CHECK17-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
diff --git a/clang/test/OpenMP/target_teams_map_codegen.cpp b/clang/test/OpenMP/target_teams_map_codegen.cpp
index 3db8cb949822a..24f3cca899b1d 100644
--- a/clang/test/OpenMP/target_teams_map_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_map_codegen.cpp
@@ -2759,7 +2759,7 @@ void mapInt128() {
 // CHECK5-NEXT:    br label [[DOTOMP_REDUCTION_DEFAULT]]
 // CHECK5:       .omp.reduction.case2:
 // CHECK5-NEXT:    [[TMP9:%.*]] = load i128, ptr [[Z2]], align 16
-// CHECK5-NEXT:    call void @__atomic_load(i64 noundef 16, ptr noundef [[TMP1]], ptr noundef [[ATOMIC_TEMP]], i32 noundef signext 0) #[[ATTR6:[0-9]+]]
+// CHECK5-NEXT:    call void @__atomic_load(i64 noundef 16, ptr noundef [[TMP1]], ptr noundef [[ATOMIC_TEMP]], i32 noundef signext 0)
 // CHECK5-NEXT:    br label [[ATOMIC_CONT:%.*]]
 // CHECK5:       atomic_cont:
 // CHECK5-NEXT:    [[TMP10:%.*]] = load i128, ptr [[ATOMIC_TEMP]], align 16
@@ -2768,7 +2768,7 @@ void mapInt128() {
 // CHECK5-NEXT:    [[TMP12:%.*]] = load i128, ptr [[Z2]], align 16
 // CHECK5-NEXT:    [[ADD4:%.*]] = add nsw i128 [[TMP11]], [[TMP12]]
 // CHECK5-NEXT:    store i128 [[ADD4]], ptr [[ATOMIC_TEMP3]], align 16
-// CHECK5-NEXT:    [[CALL:%.*]] = call noundef zeroext i1 @__atomic_compare_exchange(i64 noundef 16, ptr noundef [[TMP1]], ptr noundef [[ATOMIC_TEMP]], ptr noundef [[ATOMIC_TEMP3]], i32 noundef signext 0, i32 noundef signext 0) #[[ATTR6]]
+// CHECK5-NEXT:    [[CALL:%.*]] = call noundef zeroext i1 @__atomic_compare_exchange(i64 noundef 16, ptr noundef [[TMP1]], ptr noundef [[ATOMIC_TEMP]], ptr noundef [[ATOMIC_TEMP3]], i32 noundef signext 0, i32 noundef signext 0)
 // CHECK5-NEXT:    br i1 [[CALL]], label [[ATOMIC_EXIT:%.*]], label [[ATOMIC_CONT]]
 // CHECK5:       atomic_exit:
 // CHECK5-NEXT:    call void @__kmpc_end_reduce(ptr @[[GLOB2]], i32 [[TMP5]], ptr @.gomp_critical_user_.reduction.var)
@@ -2853,7 +2853,7 @@ void mapInt128() {
 // CHECK5-NEXT:    br label [[DOTOMP_REDUCTION_DEFAULT]]
 // CHECK5:       .omp.reduction.case2:
 // CHECK5-NEXT:    [[TMP9:%.*]] = load i128, ptr [[Z2]], align 16
-// CHECK5-NEXT:    call void @__atomic_load(i64 noundef 16, ptr noundef [[TMP1]], ptr noundef [[ATOMIC_TEMP]], i32 noundef signext 0) #[[ATTR6]]
+// CHECK5-NEXT:    call void @__atomic_load(i64 noundef 16, ptr noundef [[TMP1]], ptr noundef [[ATOMIC_TEMP]], i32 noundef signext 0)
 // CHECK5-NEXT:    br label [[ATOMIC_CONT:%.*]]
 // CHECK5:       atomic_cont:
 // CHECK5-NEXT:    [[TMP10:%.*]] = load i128, ptr [[ATOMIC_TEMP]], align 16
@@ -2862,7 +2862,7 @@ void mapInt128() {
 // CHECK5-NEXT:    [[TMP12:%.*]] = load i128, ptr [[Z2]], align 16
 // CHECK5-NEXT:    [[ADD4:%.*]] = add nsw i128 [[TMP11]], [[TMP12]]
 // CHECK5-NEXT:    store i128 [[ADD4]], ptr [[ATOMIC_TEMP3]], align 16
-// CHECK5-NEXT:    [[CALL:%.*]] = call noundef zeroext i1 @__atomic_compare_exchange(i64 noundef 16, ptr noundef [[TMP1]], ptr noundef [[ATOMIC_TEMP]], ptr noundef [[ATOMIC_TEMP3]], i32 noundef signext 0, i32 noundef signext 0) #[[ATTR6]]
+// CHECK5-NEXT:    [[CALL:%.*]] = call noundef zeroext i1 @__atomic_compare_exchange(i64 noundef 16, ptr noundef [[TMP1]], ptr noundef [[ATOMIC_TEMP]], ptr noundef [[ATOMIC_TEMP3]], i32 noundef signext 0, i32 noundef signext 0)
 // CHECK5-NEXT:    br i1 [[CALL]], label [[ATOMIC_EXIT:%.*]], label [[ATOMIC_CONT]]
 // CHECK5:       atomic_exit:
 // CHECK5-NEXT:    call void @__kmpc_end_reduce(ptr @[[GLOB2]], i32 [[TMP5]], ptr @.gomp_critical_user_.reduction.var)
diff --git a/clang/test/Parser/colon-colon-parentheses.cpp b/clang/test/Parser/colon-colon-parentheses.cpp
index b8e203c935990..018fa7f20e521 100644
--- a/clang/test/Parser/colon-colon-parentheses.cpp
+++ b/clang/test/Parser/colon-colon-parentheses.cpp
@@ -11,7 +11,7 @@ int S::(*e;  // expected-error{{expected unqualified-id}}
 int S::*f;
 int g = S::(a);  // expected-error {{expected unqualified-id}} expected-error {{use of undeclared identifier 'a'}}
 int h = S::(b;  // expected-error {{expected unqualified-id}} expected-error {{use of undeclared identifier 'b'}}
-            );
+            );  // expected-error {{expected unqualified-id}}
 int i = S::c;
 
 void foo() {
diff --git a/clang/test/Parser/cxx-ambig-paren-expr-asan.cpp b/clang/test/Parser/cxx-ambig-paren-expr-asan.cpp
index ec9d6b9da39d0..7b9e51f12c95e 100644
--- a/clang/test/Parser/cxx-ambig-paren-expr-asan.cpp
+++ b/clang/test/Parser/cxx-ambig-paren-expr-asan.cpp
@@ -6,4 +6,3 @@ int H((int()[)]);
 // expected-error@-1 {{expected expression}}
 // expected-error@-2 {{expected ']'}}
 // expected-note@-3 {{to match this '['}}
-// expected-error@-4 {{expected ';' after top level declarator}}
diff --git a/clang/test/Parser/cxx-class.cpp b/clang/test/Parser/cxx-class.cpp
index c90c7e030a8bd..a6757f39146a1 100644
--- a/clang/test/Parser/cxx-class.cpp
+++ b/clang/test/Parser/cxx-class.cpp
@@ -100,9 +100,10 @@ class F {
 #endif
 
 namespace ctor_error {
-  class Foo {};
+  class Foo {}; // #defined-here
   // By [class.qual]p2, this is a constructor declaration.
   Foo::Foo (F) = F(); // expected-error{{does not match any declaration in 'ctor_error::Foo'}}
+                      // expected-note@#defined-here{{defined here}}
 
   class Ctor { // expected-note{{not complete until the closing '}'}}
     Ctor(f)(int); // ok
diff --git a/clang/test/ParserOpenACC/parse-clauses.c b/clang/test/ParserOpenACC/parse-clauses.c
index 6c9ce4ad5e196..6c382379a8a7e 100644
--- a/clang/test/ParserOpenACC/parse-clauses.c
+++ b/clang/test/ParserOpenACC/parse-clauses.c
@@ -1153,8 +1153,7 @@ void Tile() {
   // expected-note@+1{{to match this '('}}
 #pragma acc loop tile(
   for(;;){}
-  // expected-error@+2{{expected expression}}
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected expression}}
 #pragma acc loop tile()
   for(;;){}
   // expected-error@+3{{expected expression}}
@@ -1162,43 +1161,44 @@ void Tile() {
   // expected-note@+1{{to match this '('}}
 #pragma acc loop tile(,
   for(;;){}
-  // expected-error@+2{{expected expression}}
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected expression}}
 #pragma acc loop tile(,)
   for(;;){}
-  // expected-error@+2{{use of undeclared identifier 'invalid'}}
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}}
+  // expected-error@+1{{use of undeclared identifier 'invalid'}}
 #pragma acc loop tile(returns_int(), *, invalid, *)
   for(;;){}
 
-  // expected-error@+2{{expected expression}}
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected expression}}
 #pragma acc loop tile(returns_int() *, Foo, *)
   for(;;){}
 
-  // expected-error@+2{{indirection requires pointer operand ('int' invalid)}}
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}}
+  // expected-error@+1{{indirection requires pointer operand ('int' invalid)}}
 #pragma acc loop tile(* returns_int() , *)
-  for(;;){}
+  for(;;){
+    for(;;);
+  }
 
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}}
 #pragma acc loop tile(*)
   for(;;){}
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}}
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be an asterisk or a constant expression}}
 #pragma acc loop tile(*Foo, *Foo)
   for(;;){}
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}}
 #pragma acc loop tile(5)
   for(;;){}
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}}
 #pragma acc loop tile(*, 5)
-  for(;;){}
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}}
+  for(;;){
+    for(;;);
+  }
 #pragma acc loop tile(5, *)
-  for(;;){}
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}}
+  for(;;){
+    for(;;);
+  }
 #pragma acc loop tile(5, *, 3, *)
-  for(;;){}
+  for(;;){
+    for(;;)
+      for(;;)
+        for(;;);
+  }
 }
 
 void Gang() {
diff --git a/clang/test/Preprocessor/aarch64-target-features.c b/clang/test/Preprocessor/aarch64-target-features.c
index bcc0640c55e35..418430b0b19b8 100644
--- a/clang/test/Preprocessor/aarch64-target-features.c
+++ b/clang/test/Preprocessor/aarch64-target-features.c
@@ -213,6 +213,7 @@
 // RUN: %clang -target aarch64-none-linux-gnu -march=armv9.3-a -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SVE2 %s
 // RUN: %clang -target aarch64-none-linux-gnu -march=armv9.4-a -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SVE2 %s
 // RUN: %clang -target aarch64-none-linux-gnu -march=armv9.5-a -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SVE2 %s
+// RUN: %clang -target aarch64-none-linux-gnu -march=armv9.6-a -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SVE2 %s
 // RUN: %clang -target aarch64-none-linux-gnu -march=armv9-a+sve2 -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SVE2 %s
 // CHECK-SVE2: __ARM_FEATURE_FP16_SCALAR_ARITHMETIC 1
 // CHECK-SVE2: __ARM_FEATURE_FP16_VECTOR_ARITHMETIC 1
@@ -671,6 +672,7 @@
 // RUN: %clang -target aarch64-none-elf -march=armv9.3-a -x c -E -dM %s -o - | FileCheck --check-prefixes=CHECK-V81-OR-LATER,CHECK-V83-OR-LATER,CHECK-V85-OR-LATER %s
 // RUN: %clang -target aarch64-none-elf -march=armv9.4-a -x c -E -dM %s -o - | FileCheck --check-prefixes=CHECK-V81-OR-LATER,CHECK-V83-OR-LATER,CHECK-V85-OR-LATER %s
 // RUN: %clang -target aarch64-none-elf -march=armv9.5-a -x c -E -dM %s -o - | FileCheck --check-prefixes=CHECK-V81-OR-LATER,CHECK-V83-OR-LATER,CHECK-V85-OR-LATER %s
+// RUN: %clang -target aarch64-none-elf -march=armv9.6-a -x c -E -dM %s -o - | FileCheck --check-prefixes=CHECK-V81-OR-LATER,CHECK-V83-OR-LATER,CHECK-V85-OR-LATER %s
 // CHECK-V81-OR-LATER: __ARM_FEATURE_ATOMICS 1
 // CHECK-V85-OR-LATER: __ARM_FEATURE_BTI 1
 // CHECK-V83-OR-LATER: __ARM_FEATURE_COMPLEX 1
diff --git a/clang/test/Preprocessor/arm-target-features.c b/clang/test/Preprocessor/arm-target-features.c
index 2d65bfd4f4399..2999ee0d9e4d8 100644
--- a/clang/test/Preprocessor/arm-target-features.c
+++ b/clang/test/Preprocessor/arm-target-features.c
@@ -902,6 +902,11 @@
 // CHECK-V95A: #define __ARM_ARCH_9_5A__ 1
 // CHECK-V95A: #define __ARM_ARCH_PROFILE 'A'
 
+// RUN: %clang -target armv9.6a-none-none-eabi -x c -E -dM %s -o - | FileCheck -match-full-lines --check-prefix=CHECK-V96A %s
+// CHECK-V96A: #define __ARM_ARCH 9
+// CHECK-V96A: #define __ARM_ARCH_9_6A__ 1
+// CHECK-V96A: #define __ARM_ARCH_PROFILE 'A'
+
 // RUN: %clang -target arm-none-none-eabi -march=armv7-m -mfpu=softvfp -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SOFTVFP %s
 // CHECK-SOFTVFP-NOT: #define __ARM_FP 0x
 
diff --git a/clang/test/Sema/aarch64-sve-vector-pow-ops.c b/clang/test/Sema/aarch64-sve-vector-pow-ops.c
index e61d752911fc3..16dbe33c1afe7 100644
--- a/clang/test/Sema/aarch64-sve-vector-pow-ops.c
+++ b/clang/test/Sema/aarch64-sve-vector-pow-ops.c
@@ -7,5 +7,5 @@
 svfloat32_t test_pow_vv_i8mf8(svfloat32_t v) {
 
   return __builtin_elementwise_pow(v, v);
-  // expected-error@-1 {{1st argument must be a vector, integer or floating point type}}
+  // expected-error@-1 {{1st argument must be a floating point type}}
 }
diff --git a/clang/test/Sema/aarch64-sve-vector-trig-ops.c b/clang/test/Sema/aarch64-sve-vector-trig-ops.c
index dfa77d20e949f..3fe6834be2e0b 100644
--- a/clang/test/Sema/aarch64-sve-vector-trig-ops.c
+++ b/clang/test/Sema/aarch64-sve-vector-trig-ops.c
@@ -22,6 +22,12 @@ svfloat32_t test_atan_vv_i8mf8(svfloat32_t v) {
   // expected-error@-1 {{1st argument must be a vector, integer or floating point type}}
 }
 
+svfloat32_t test_atan2_vv_i8mf8(svfloat32_t v) {
+
+  return __builtin_elementwise_atan2(v, v);
+  // expected-error@-1 {{1st argument must be a floating point type}}
+}
+
 svfloat32_t test_sin_vv_i8mf8(svfloat32_t v) {
 
   return __builtin_elementwise_sin(v);
diff --git a/clang/test/Sema/attr-nonblocking-constraints-ms.cpp b/clang/test/Sema/attr-nonblocking-constraints-ms.cpp
new file mode 100644
index 0000000000000..dcbcef410987f
--- /dev/null
+++ b/clang/test/Sema/attr-nonblocking-constraints-ms.cpp
@@ -0,0 +1,26 @@
+// RUN: %clang_cc1 -triple=x86_64-pc-win32 -fsyntax-only -fblocks -fcxx-exceptions -fms-extensions -verify -Wfunction-effects %s
+
+#pragma clang diagnostic ignored "-Wperf-constraint-implies-noexcept"
+
+// These need '-fms-extensions' (and maybe '-fdeclspec')
+void f1() [[clang::nonblocking]] {
+    __try {} __except (1) {} // expected-warning {{function with 'nonblocking' attribute must not throw or catch exceptions}}
+}
+
+struct S {
+    int get_x(); // expected-note {{declaration cannot be inferred 'nonblocking' because it has no definition in this translation unit}}
+    __declspec(property(get = get_x)) int x;
+
+    int get_nb() { return 42; }
+    __declspec(property(get = get_nb)) int nb;
+
+    int get_nb2() [[clang::nonblocking]];
+    __declspec(property(get = get_nb2)) int nb2;
+};
+
+void f2() [[clang::nonblocking]] {
+    S a;
+    a.x; // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function 'S::get_x'}}
+    a.nb;
+    a.nb2;
+}
diff --git a/clang/test/Sema/attr-nonblocking-constraints.c b/clang/test/Sema/attr-nonblocking-constraints.c
new file mode 100644
index 0000000000000..e7ab661b3125a
--- /dev/null
+++ b/clang/test/Sema/attr-nonblocking-constraints.c
@@ -0,0 +1,27 @@
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c89 -Wfunction-effects -internal-isystem %S/../Headers/Inputs/include %s
+
+// Tests for a few cases involving C functions without prototypes.
+
+void hasproto(void) __attribute__((blocking)); // expected-note {{function does not permit inference of 'nonblocking' because it is declared 'blocking'}}
+
+// Has no prototype, inferably safe.
+void nb1() {}
+
+// Has no prototype, noreturn.
+[[noreturn]]
+void aborts();
+
+void nb2(void) __attribute__((nonblocking)) {
+  hasproto(); // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function 'hasproto'}}
+  nb1();
+  aborts(); // no diagnostic because it's noreturn.
+}
+
+#include <setjmp.h>
+
+void nb3(int x, int y) __attribute__((nonblocking)) {
+  if (x != y) {
+    jmp_buf jb;
+    longjmp(jb, 0); // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function 'longjmp'}}
+  }
+}
diff --git a/clang/test/Sema/attr-nonblocking-constraints.cpp b/clang/test/Sema/attr-nonblocking-constraints.cpp
new file mode 100644
index 0000000000000..c694860069c96
--- /dev/null
+++ b/clang/test/Sema/attr-nonblocking-constraints.cpp
@@ -0,0 +1,386 @@
+// RUN: %clang_cc1 -fsyntax-only -fblocks -fcxx-exceptions -std=c++20 -verify -Wfunction-effects %s
+// These are in a separate file because errors (e.g. incompatible attributes) currently prevent
+// the FXAnalysis pass from running at all.
+
+// This diagnostic is re-enabled and exercised in isolation later in this file.
+#pragma clang diagnostic ignored "-Wperf-constraint-implies-noexcept"
+
+// --- CONSTRAINTS ---
+
+void nb1() [[clang::nonblocking]]
+{
+	int *pInt = new int; // expected-warning {{function with 'nonblocking' attribute must not allocate or deallocate memory}}
+	delete pInt; // expected-warning {{function with 'nonblocking' attribute must not allocate or deallocate memory}}
+}
+
+void nb2() [[clang::nonblocking]]
+{
+	static int global; // expected-warning {{function with 'nonblocking' attribute must not have static local variables}}
+}
+
+void nb3() [[clang::nonblocking]]
+{
+	try {
+		throw 42; // expected-warning {{function with 'nonblocking' attribute must not throw or catch exceptions}}
+	}
+	catch (...) { // expected-warning {{function with 'nonblocking' attribute must not throw or catch exceptions}}
+	}
+}
+
+void nb4_inline() {}
+void nb4_not_inline(); // expected-note {{declaration cannot be inferred 'nonblocking' because it has no definition in this translation unit}}
+
+void nb4() [[clang::nonblocking]]
+{
+	nb4_inline(); // OK
+	nb4_not_inline(); // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function}}
+}
+
+
+struct HasVirtual {
+	virtual void unsafe(); // expected-note {{virtual method cannot be inferred 'nonblocking'}}
+};
+
+void nb5() [[clang::nonblocking]]
+{
+ 	HasVirtual hv;
+ 	hv.unsafe(); // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function}}
+}
+
+void nb6_unsafe(); // expected-note {{declaration cannot be inferred 'nonblocking' because it has no definition in this translation unit}}
+void nb6_transitively_unsafe()
+{
+	nb6_unsafe(); // expected-note {{function cannot be inferred 'nonblocking' because it calls non-'nonblocking' function}}
+}
+
+void nb6() [[clang::nonblocking]]
+{
+	nb6_transitively_unsafe(); // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function}}
+}
+
+thread_local int tl_var{ 42 };
+
+bool tl_test() [[clang::nonblocking]]
+{
+	return tl_var > 0; // expected-warning {{function with 'nonblocking' attribute must not use thread-local variables}}
+}
+
+void nb7()
+{
+	// Make sure we verify blocks
+	auto blk = ^() [[clang::nonblocking]] {
+		throw 42; // expected-warning {{block with 'nonblocking' attribute must not throw or catch exceptions}}
+	};
+}
+
+void nb8()
+{
+	// Make sure we verify lambdas
+	auto lambda = []() [[clang::nonblocking]] {
+		throw 42; // expected-warning {{lambda with 'nonblocking' attribute must not throw or catch exceptions}}
+	};
+}
+
+void nb8a() [[clang::nonblocking]]
+{
+	// A blocking lambda shouldn't make the outer function unsafe.
+	auto unsafeLambda = []() {
+		throw 42;
+	};
+}
+
+void nb8b() [[clang::nonblocking]]
+{
+	// An unsafe lambda capture makes the outer function unsafe.
+	auto unsafeCapture = [foo = new int]() { // expected-warning {{function with 'nonblocking' attribute must not allocate or deallocate memory}}
+		delete foo;
+	};
+}
+
+void nb8c()
+{
+	// An unsafe lambda capture does not make the lambda unsafe.
+	auto unsafeCapture = [foo = new int]() [[clang::nonblocking]] {
+	};
+}
+
+// Make sure template expansions are found and verified.
+	template <typename T>
+	struct Adder {
+		static T add_explicit(T x, T y) [[clang::nonblocking]]
+		{
+			return x + y; // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function}}
+		}
+		static T add_implicit(T x, T y)
+		{
+			return x + y; // expected-note {{function cannot be inferred 'nonblocking' because it calls non-'nonblocking' function}}
+		}
+	};
+
+	struct Stringy {
+		friend Stringy operator+(const Stringy& x, const Stringy& y)
+		{
+			// Do something inferably unsafe
+			auto* z = new char[42]; // expected-note {{function cannot be inferred 'nonblocking' because it allocates or deallocates memory}}
+			return {};
+		}
+	};
+
+	struct Stringy2 {
+		friend Stringy2 operator+(const Stringy2& x, const Stringy2& y)
+		{
+			// Do something inferably unsafe
+			throw 42; // expected-note {{function cannot be inferred 'nonblocking' because it throws or catches exceptions}}
+		}
+	};
+
+void nb9() [[clang::nonblocking]]
+{
+	Adder<int>::add_explicit(1, 2);
+	Adder<int>::add_implicit(1, 2);
+
+	Adder<Stringy>::add_explicit({}, {}); // expected-note {{in template expansion here}}
+	Adder<Stringy2>::add_implicit({}, {}); // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function}} \
+		expected-note {{in template expansion here}}
+}
+
+void nb10(
+	void (*fp1)(), // expected-note {{function pointer cannot be inferred 'nonblocking'}}
+	void (*fp2)() [[clang::nonblocking]]
+	) [[clang::nonblocking]]
+{
+	fp1(); // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function}}
+	fp2();
+
+	// When there's a cast, there's a separate diagnostic.
+	static_cast<void (*)()>(fp1)(); // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' expression}}
+}
+
+// Interactions with nonblocking(false)
+void nb11_no_inference_1() [[clang::nonblocking(false)]] // expected-note {{function does not permit inference of 'nonblocking'}}
+{
+}
+void nb11_no_inference_2() [[clang::nonblocking(false)]]; // expected-note {{function does not permit inference of 'nonblocking'}}
+
+template <bool V>
+struct ComputedNB {
+	void method() [[clang::nonblocking(V)]]; // expected-note {{function does not permit inference of 'nonblocking' because it is declared 'blocking'}}
+};
+
+void nb11() [[clang::nonblocking]]
+{
+	nb11_no_inference_1(); // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function}}
+	nb11_no_inference_2(); // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function}}
+
+	ComputedNB<true> CNB_true;
+	CNB_true.method();
+	
+	ComputedNB<false> CNB_false;
+	CNB_false.method(); // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function}}
+}
+
+// Verify that when attached to a redeclaration, the attribute successfully attaches.
+void nb12() {
+	static int x; // expected-warning {{function with 'nonblocking' attribute must not have static local variables}}
+}
+void nb12() [[clang::nonblocking]];
+void nb13() [[clang::nonblocking]] { nb12(); }
+
+// C++ member function pointers
+struct PTMFTester {
+	typedef void (PTMFTester::*ConvertFunction)() [[clang::nonblocking]];
+
+	void convert() [[clang::nonblocking]];
+
+	ConvertFunction mConvertFunc;
+};
+
+void PTMFTester::convert() [[clang::nonblocking]]
+{
+	(this->*mConvertFunc)();
+}
+
+// Block variables
+void nb17(void (^blk)() [[clang::nonblocking]]) [[clang::nonblocking]] {
+	blk();
+}
+
+// References to blocks
+void nb18(void (^block)() [[clang::nonblocking]]) [[clang::nonblocking]]
+{
+	auto &ref = block;
+	ref();
+}
+
+// Builtin functions
+void nb19() [[clang::nonblocking]] {
+	__builtin_assume(1);
+	void *ptr = __builtin_malloc(1); // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function '__builtin_malloc'}}
+	__builtin_free(ptr); // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function '__builtin_free'}}
+	
+	void *p2 = __builtin_operator_new(1); // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function '__builtin_operator_new'}}
+	__builtin_operator_delete(p2); // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function '__builtin_operator_delete'}}
+}
+
+// Function try-block
+void catches() try {} catch (...) {} // expected-note {{function cannot be inferred 'nonblocking' because it throws or catches exceptions}}
+
+void nb20() [[clang::nonblocking]] {
+	catches(); // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function 'catches'}}
+}
+
+struct S {
+    int x;
+    S(int x) try : x(x) {} catch (...) {} // expected-note {{constructor cannot be inferred 'nonblocking' because it throws or catches exceptions}}
+    S(double) : x((throw 3, 3)) {} // expected-note {{member initializer cannot be inferred 'nonblocking' because it throws or catches exceptions}} \
+                                      expected-note {{in constructor here}}
+};
+
+int badi(); // expected-note {{declaration cannot be inferred 'nonblocking' because it has no definition in this translation unit}} \
+            // expected-note {{declaration cannot be inferred 'nonblocking' because it has no definition in this translation unit}}
+
+struct A {                // expected-note {{in implicit constructor here}}
+    int x = (throw 3, 3); // expected-note {{member initializer cannot be inferred 'nonblocking' because it throws or catches exceptions}}
+};
+
+struct B {
+    int y = badi(); // expected-note {{member initializer cannot be inferred 'nonblocking' because it calls non-'nonblocking' function 'badi'}}
+};
+
+void f() [[clang::nonblocking]] {
+    S s1(3);   // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' constructor 'S::S'}}
+    S s2(3.0); // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' constructor 'S::S'}}
+    A a;       // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' constructor 'A::A'}}
+    B b;       // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' constructor 'B::B'}}
+}
+
+struct T {
+	int x = badi();               // expected-warning {{member initializer of constructor with 'nonblocking' attribute must not call non-'nonblocking' function 'badi'}}
+	T() [[clang::nonblocking]] {} // expected-note {{in constructor here}}
+	T(int x) [[clang::nonblocking]] : x(x) {} // OK
+};
+
+// Default arguments
+int badForDefaultArg(); // expected-note {{declaration cannot be inferred 'nonblocking' because it has no definition in this translation unit}} \
+                           expected-note {{declaration cannot be inferred 'nonblocking' because it has no definition in this translation unit}} \
+						   expected-note {{declaration cannot be inferred 'nonblocking' because it has no definition in this translation unit}}
+
+void hasDefaultArg(int param = badForDefaultArg()) { // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function 'badForDefaultArg'}} \
+                                                        expected-note {{function cannot be inferred 'nonblocking' because it calls non-'nonblocking' function 'badForDefaultArg'}}
+}
+
+void nb21() [[clang::nonblocking]] {
+	hasDefaultArg(); // expected-note {{in evaluating default argument here}} \
+	                    expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function 'hasDefaultArg'}}
+}
+
+void nb22(int param = badForDefaultArg()) [[clang::nonblocking]] { // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function 'badForDefaultArg'}}
+}
+
+// Verify traversal of implicit code paths - constructors and destructors.
+struct Unsafe {
+  static void problem1();   // expected-note {{declaration cannot be inferred 'nonblocking' because it has no definition in this translation unit}}
+  static void problem2();   // expected-note {{declaration cannot be inferred 'nonblocking' because it has no definition in this translation unit}}
+
+  Unsafe() { problem1(); }  // expected-note {{constructor cannot be inferred 'nonblocking' because it calls non-'nonblocking' function 'Unsafe::problem1'}}
+  ~Unsafe() { problem2(); } // expected-note {{destructor cannot be inferred 'nonblocking' because it calls non-'nonblocking' function 'Unsafe::problem2'}}
+
+  Unsafe(int x); // expected-note {{declaration cannot be inferred 'nonblocking' because it has no definition in this translation unit}} expected-note {{declaration cannot be inferred 'nonblocking' because it has no definition in this translation unit}}
+
+  // Delegating initializer.
+  Unsafe(float y) [[clang::nonblocking]] : Unsafe(int(y)) {} // expected-warning {{constructor with 'nonblocking' attribute must not call non-'nonblocking' constructor 'Unsafe::Unsafe'}}
+};
+
+struct DerivedFromUnsafe : public Unsafe {
+  DerivedFromUnsafe() [[clang::nonblocking]] {} // expected-warning {{constructor with 'nonblocking' attribute must not call non-'nonblocking' constructor 'Unsafe::Unsafe'}}
+  DerivedFromUnsafe(int x) [[clang::nonblocking]] : Unsafe(x) {} // expected-warning {{constructor with 'nonblocking' attribute must not call non-'nonblocking' constructor 'Unsafe::Unsafe'}}
+  ~DerivedFromUnsafe() [[clang::nonblocking]] {} // expected-warning {{destructor with 'nonblocking' attribute must not call non-'nonblocking' destructor 'Unsafe::~Unsafe'}}
+};
+
+// Don't try to follow a deleted destructor, as with std::optional<T>.
+struct HasDtor {
+	~HasDtor() {}
+};
+
+template <typename T>
+struct Optional {
+	union {
+		char __null_state_;
+		T __val_;
+	};
+	bool engaged = false;
+
+	~Optional() {
+		if (engaged)
+			__val_.~T();
+	}
+};
+
+void nb_opt() [[clang::nonblocking]] {
+	Optional<HasDtor> x;
+}
+
+// Virtual inheritance
+struct VBase {
+  int *Ptr;
+
+  VBase() { Ptr = new int; }       // expected-note {{constructor cannot be inferred 'nonblocking' because it allocates or deallocates memory}}
+  virtual ~VBase() { delete Ptr; } // expected-note {{virtual method cannot be inferred 'nonblocking'}}
+};
+
+struct VDerived : virtual VBase {
+  VDerived() [[clang::nonblocking]] {} // expected-warning {{constructor with 'nonblocking' attribute must not call non-'nonblocking' constructor 'VBase::VBase'}}
+
+  ~VDerived() [[clang::nonblocking]] {} // expected-warning {{destructor with 'nonblocking' attribute must not call non-'nonblocking' destructor 'VBase::~VBase'}}
+};
+
+// Contexts where there is no function call, no diagnostic.
+bool bad();
+
+template <bool>
+requires requires { bad(); }
+void g() [[clang::nonblocking]] {}
+
+void g() [[clang::nonblocking]] {
+    decltype(bad()) a; // doesn't generate a call so, OK
+    [[maybe_unused]] auto b = noexcept(bad());
+    [[maybe_unused]] auto c = sizeof(bad());
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wassume"
+    [[assume(bad())]]; // never evaluated, but maybe still semantically questionable?
+#pragma clang diagnostic pop
+}
+
+// Make sure we are skipping concept requirements -- they can trigger an unexpected
+// warning involving use of a function pointer (e.g. std::reverse_iterator::operator==
+struct HasFoo { int foo() const { return 0; } };
+
+template <class A, class B>
+inline bool compare(const A& a, const B& b)
+	requires requires { 
+		a.foo();
+	}
+{
+	return a.foo() == b.foo();
+}
+
+void nb25() [[clang::nonblocking]] {
+	HasFoo a, b;
+	compare(a, b);
+}
+
+// If the callee is both noreturn and noexcept, it presumably terminates.
+// Ignore it for the purposes of effect analysis.
+[[noreturn]] void abort_wrapper() noexcept;
+
+void nb26() [[clang::nonblocking]] {
+	abort_wrapper(); // no diagnostic
+}
+
+// --- nonblocking implies noexcept ---
+#pragma clang diagnostic warning "-Wperf-constraint-implies-noexcept"
+
+void needs_noexcept() [[clang::nonblocking]] // expected-warning {{function with 'nonblocking' attribute should be declared noexcept}}
+{
+	auto lambda = []() [[clang::nonblocking]] {}; // expected-warning {{lambda with 'nonblocking' attribute should be declared noexcept}}
+}
diff --git a/clang/test/Sema/attr-nonblocking-sema.cpp b/clang/test/Sema/attr-nonblocking-sema.cpp
index 38bf2ac8f8a4c..9056f81f5296b 100644
--- a/clang/test/Sema/attr-nonblocking-sema.cpp
+++ b/clang/test/Sema/attr-nonblocking-sema.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -fsyntax-only -fblocks -fcxx-exceptions -verify %s
-// RUN: %clang_cc1 -fsyntax-only -fblocks -verify -x c -std=c23 %s
+// RUN: %clang_cc1 -fsyntax-only -fblocks -fcxx-exceptions -verify -Wfunction-effects %s
+// RUN: %clang_cc1 -fsyntax-only -fblocks -verify -x c -std=c23 -Wfunction-effects %s
 
 #if !__has_attribute(nonblocking)
 #error "the 'nonblocking' attribute is not available"
@@ -77,7 +77,7 @@ void type_conversions()
   void (*fp_nonallocating)() [[clang::nonallocating]];
   fp_nonallocating = nullptr;
   fp_nonallocating = nonallocating;
-  fp_nonallocating = nonblocking; // no warning because nonblocking includes nonallocating fp_nonallocating = unannotated;
+  fp_nonallocating = nonblocking; // no warning because nonblocking includes nonallocating
   fp_nonallocating = unannotated; // expected-warning {{attribute 'nonallocating' should not be added via type conversion}}
 }
 
diff --git a/clang/test/Sema/attr-nonblocking-syntax.cpp b/clang/test/Sema/attr-nonblocking-syntax.cpp
index 644ed754b04da..90d074d01708f 100644
--- a/clang/test/Sema/attr-nonblocking-syntax.cpp
+++ b/clang/test/Sema/attr-nonblocking-syntax.cpp
@@ -3,6 +3,7 @@
 // Make sure that the attribute gets parsed and attached to the correct AST elements.
 
 #pragma clang diagnostic ignored "-Wunused-variable"
+#pragma clang diagnostic ignored "-Wperf-constraint-implies-noexcept"
 
 // =========================================================================================
 // Square brackets, true
diff --git a/clang/test/Sema/builtins-elementwise-math.c b/clang/test/Sema/builtins-elementwise-math.c
index 26b153dd5b210..6002e91f8ec6f 100644
--- a/clang/test/Sema/builtins-elementwise-math.c
+++ b/clang/test/Sema/builtins-elementwise-math.c
@@ -273,6 +273,88 @@ void test_builtin_elementwise_min(int i, short s, double d, float4 v, int3 iv, u
   // expected-error@-1 {{1st argument must be a vector, integer or floating point type (was '_Complex float')}}
 }
 
+void test_builtin_elementwise_maximum(int i, short s, float f, double d, float4 v, int3 iv, unsigned3 uv, int *p) {
+  i = __builtin_elementwise_maximum(p, d);
+  // expected-error@-1 {{arguments are of different types ('int *' vs 'double')}}
+
+  struct Foo foo = __builtin_elementwise_maximum(d, d);
+  // expected-error@-1 {{initializing 'struct Foo' with an expression of incompatible type 'double'}}
+
+  i = __builtin_elementwise_maximum(i);
+  // expected-error@-1 {{too few arguments to function call, expected 2, have 1}}
+
+  i = __builtin_elementwise_maximum();
+  // expected-error@-1 {{too few arguments to function call, expected 2, have 0}}
+
+  i = __builtin_elementwise_maximum(i, i, i);
+  // expected-error@-1 {{too many arguments to function call, expected 2, have 3}}
+
+  i = __builtin_elementwise_maximum(v, iv);
+  // expected-error@-1 {{arguments are of different types ('float4' (vector of 4 'float' values) vs 'int3' (vector of 3 'int' values))}}
+
+  i = __builtin_elementwise_maximum(uv, iv);
+  // expected-error@-1 {{arguments are of different types ('unsigned3' (vector of 3 'unsigned int' values) vs 'int3' (vector of 3 'int' values))}}
+
+  d = __builtin_elementwise_maximum(f, d);
+
+  v = __builtin_elementwise_maximum(v, v);
+
+  i = __builtin_elementwise_maximum(iv, iv);
+  // expected-error@-1 {{1st argument must be a floating point type (was 'int3' (vector of 3 'int' values))}}
+
+  i = __builtin_elementwise_maximum(i, i);
+  // expected-error@-1 {{1st argument must be a floating point type (was 'int')}}
+
+  int A[10];
+  A = __builtin_elementwise_maximum(A, A);
+  // expected-error@-1 {{1st argument must be a floating point type (was 'int *')}}
+
+  _Complex float c1, c2;
+  c1 = __builtin_elementwise_maximum(c1, c2);
+  // expected-error@-1 {{1st argument must be a floating point type (was '_Complex float')}}
+}
+
+void test_builtin_elementwise_minimum(int i, short s, float f, double d, float4 v, int3 iv, unsigned3 uv, int *p) {
+  i = __builtin_elementwise_minimum(p, d);
+  // expected-error@-1 {{arguments are of different types ('int *' vs 'double')}}
+
+  struct Foo foo = __builtin_elementwise_minimum(d, d);
+  // expected-error@-1 {{initializing 'struct Foo' with an expression of incompatible type 'double'}}
+
+  i = __builtin_elementwise_minimum(i);
+  // expected-error@-1 {{too few arguments to function call, expected 2, have 1}}
+
+  i = __builtin_elementwise_minimum();
+  // expected-error@-1 {{too few arguments to function call, expected 2, have 0}}
+
+  i = __builtin_elementwise_minimum(i, i, i);
+  // expected-error@-1 {{too many arguments to function call, expected 2, have 3}}
+
+  i = __builtin_elementwise_minimum(v, iv);
+  // expected-error@-1 {{arguments are of different types ('float4' (vector of 4 'float' values) vs 'int3' (vector of 3 'int' values))}}
+
+  i = __builtin_elementwise_minimum(uv, iv);
+  // expected-error@-1 {{arguments are of different types ('unsigned3' (vector of 3 'unsigned int' values) vs 'int3' (vector of 3 'int' values))}}
+
+  d = __builtin_elementwise_minimum(f, d);
+
+  v = __builtin_elementwise_minimum(v, v);
+
+  i = __builtin_elementwise_minimum(iv, iv);
+  // expected-error@-1 {{1st argument must be a floating point type (was 'int3' (vector of 3 'int' values))}}
+
+  i = __builtin_elementwise_minimum(i, i);
+  // expected-error@-1 {{1st argument must be a floating point type (was 'int')}}
+
+  int A[10];
+  A = __builtin_elementwise_minimum(A, A);
+  // expected-error@-1 {{1st argument must be a floating point type (was 'int *')}}
+
+  _Complex float c1, c2;
+  c1 = __builtin_elementwise_minimum(c1, c2);
+  // expected-error@-1 {{1st argument must be a floating point type (was '_Complex float')}}
+}
+
 void test_builtin_elementwise_bitreverse(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) {
 
   struct Foo s = __builtin_elementwise_bitreverse(i);
@@ -789,6 +871,30 @@ void test_builtin_elementwise_atan(int i, float f, double d, float4 v, int3 iv,
   // expected-error@-1 {{1st argument must be a floating point type (was 'unsigned4' (vector of 4 'unsigned int' values))}}
 }
 
+void test_builtin_elementwise_atan2(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) {
+
+  struct Foo s = __builtin_elementwise_atan2(f, f);
+  // expected-error@-1 {{initializing 'struct Foo' with an expression of incompatible type 'float'}}
+
+  i = __builtin_elementwise_atan2();
+  // expected-error@-1 {{too few arguments to function call, expected 2, have 0}}
+
+  i = __builtin_elementwise_atan2(f);
+  // expected-error@-1 {{too few arguments to function call, expected 2, have 1}}
+
+  i = __builtin_elementwise_atan2(i, i);
+  // expected-error@-1 {{1st argument must be a floating point type (was 'int')}}
+
+  i = __builtin_elementwise_atan2(f, f, f);
+  // expected-error@-1 {{too many arguments to function call, expected 2, have 3}}
+
+  u = __builtin_elementwise_atan2(u, u);
+  // expected-error@-1 {{1st argument must be a floating point type (was 'unsigned int')}}
+
+  uv = __builtin_elementwise_atan2(uv, uv);
+  // expected-error@-1 {{1st argument must be a floating point type (was 'unsigned4' (vector of 4 'unsigned int' values))}}
+}
+
 void test_builtin_elementwise_tan(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) {
 
   struct Foo s = __builtin_elementwise_tan(f);
diff --git a/clang/test/Sema/builtins-reduction-math.c b/clang/test/Sema/builtins-reduction-math.c
index 9d5eed75eb814..9b0d91bfd6e3d 100644
--- a/clang/test/Sema/builtins-reduction-math.c
+++ b/clang/test/Sema/builtins-reduction-math.c
@@ -120,3 +120,31 @@ void test_builtin_reduce_and(int i, float4 v, int3 iv) {
   i = __builtin_reduce_and(v);
   // expected-error@-1 {{1st argument must be a vector of integers (was 'float4' (vector of 4 'float' values))}}
 }
+
+void test_builtin_reduce_maximum(int i, float4 v, int3 iv) {
+  struct Foo s = __builtin_reduce_maximum(v);
+  // expected-error@-1 {{initializing 'struct Foo' with an expression of incompatible type 'float'}}
+
+  i = __builtin_reduce_maximum(v, v);
+  // expected-error@-1 {{too many arguments to function call, expected 1, have 2}}
+
+  i = __builtin_reduce_maximum();
+  // expected-error@-1 {{too few arguments to function call, expected 1, have 0}}
+
+  i = __builtin_reduce_maximum(i);
+  // expected-error@-1 {{1st argument must be a vector of floating points (was 'int')}}
+}
+
+void test_builtin_reduce_minimum(int i, float4 v, int3 iv) {
+  struct Foo s = __builtin_reduce_minimum(v);
+  // expected-error@-1 {{initializing 'struct Foo' with an expression of incompatible type 'float'}}
+
+  i = __builtin_reduce_minimum(v, v);
+  // expected-error@-1 {{too many arguments to function call, expected 1, have 2}}
+
+  i = __builtin_reduce_minimum();
+  // expected-error@-1 {{too few arguments to function call, expected 1, have 0}}
+
+  i = __builtin_reduce_minimum(i);
+  // expected-error@-1 {{1st argument must be a vector of floating points (was 'int')}}
+}
diff --git a/clang/test/Sema/riscv-rvv-vector-trig-ops.c b/clang/test/Sema/riscv-rvv-vector-trig-ops.c
index f0cd5ca4a1de1..0aed1b2a09986 100644
--- a/clang/test/Sema/riscv-rvv-vector-trig-ops.c
+++ b/clang/test/Sema/riscv-rvv-vector-trig-ops.c
@@ -23,6 +23,12 @@ vfloat32mf2_t test_asin_vv_i8mf8(vfloat32mf2_t v) {
     // expected-error@-1 {{1st argument must be a vector, integer or floating point type}}
   }
 
+vfloat32mf2_t test_atan2_vv_i8mf8(vfloat32mf2_t v) {
+
+  return __builtin_elementwise_atan2(v, v);
+  // expected-error@-1 {{1st argument must be a floating point type}}
+}
+
 vfloat32mf2_t test_sin_vv_i8mf8(vfloat32mf2_t v) {
 
   return __builtin_elementwise_sin(v);
@@ -58,4 +64,4 @@ vfloat32mf2_t test_sinh_vv_i8mf8(vfloat32mf2_t v) {
     return __builtin_elementwise_tanh(v);
     // expected-error@-1 {{1st argument must be a vector, integer or floating point type}}
   }
-  
\ No newline at end of file
+  
diff --git a/clang/test/Sema/riscv-sve-vector-pow-ops.c b/clang/test/Sema/riscv-sve-vector-pow-ops.c
index 78efb4b549f80..524e453441780 100644
--- a/clang/test/Sema/riscv-sve-vector-pow-ops.c
+++ b/clang/test/Sema/riscv-sve-vector-pow-ops.c
@@ -9,5 +9,5 @@
 vfloat32mf2_t test_pow_vv_i8mf8(vfloat32mf2_t v) {
 
   return __builtin_elementwise_pow(v, v);
-  // expected-error@-1 {{1st argument must be a vector, integer or floating point type}}
+  // expected-error@-1 {{1st argument must be a floating point type}}
 }
diff --git a/clang/test/SemaCXX/attr-target-mv.cpp b/clang/test/SemaCXX/attr-target-mv.cpp
index 5b2f0fc825f39..b8db6d371bd58 100644
--- a/clang/test/SemaCXX/attr-target-mv.cpp
+++ b/clang/test/SemaCXX/attr-target-mv.cpp
@@ -180,14 +180,15 @@ void constexpr_test() {
   static_assert(foo() == 2, "Should call 'default' in a constexpr context");
 }
 
-struct BadOutOfLine {
+struct BadOutOfLine { // #defined-here
   int __attribute__((target("sse4.2"))) foo(int);
   int __attribute__((target("default"))) foo(int);
 };
 
 int __attribute__((target("sse4.2"))) BadOutOfLine::foo(int) { return 0; }
 int __attribute__((target("default"))) BadOutOfLine::foo(int) { return 1; }
-// expected-error@+3 {{out-of-line definition of 'foo' does not match any declaration in 'BadOutOfLine'}}
-// expected-note@-3 {{member declaration nearly matches}}
-// expected-note@-3 {{member declaration nearly matches}}
+// expected-error@+4 {{out-of-line definition of 'foo' does not match any declaration in 'BadOutOfLine'}}
+// expected-note@#defined-here {{defined here}}
+// expected-note@-4 {{member declaration nearly matches}}
+// expected-note@-4 {{member declaration nearly matches}}
 int __attribute__((target("arch=atom"))) BadOutOfLine::foo(int) { return 1; }
diff --git a/clang/test/SemaCXX/attr-target-version.cpp b/clang/test/SemaCXX/attr-target-version.cpp
index b3385f043590f..14460756db17b 100644
--- a/clang/test/SemaCXX/attr-target-version.cpp
+++ b/clang/test/SemaCXX/attr-target-version.cpp
@@ -97,13 +97,14 @@ using ::Decl;
 __attribute__((target_version("jscvt"))) void Decl();
 } // namespace Nms
 
-class Out {
+class Out { // #defined-here
   int __attribute__((target_version("bti"))) func(void);
   int __attribute__((target_version("ssbs2"))) func(void);
 };
 int __attribute__((target_version("bti"))) Out::func(void) { return 1; }
 int __attribute__((target_version("ssbs2"))) Out::func(void) { return 2; }
-// expected-error@+3 {{out-of-line definition of 'func' does not match any declaration in 'Out'}}
-// expected-note@-3 {{member declaration nearly matches}}
-// expected-note@-3 {{member declaration nearly matches}}
+// expected-error@+4 {{out-of-line definition of 'func' does not match any declaration in 'Out'}}
+// expected-note@-2 {{member declaration nearly matches}}
+// expected-note@-4 {{member declaration nearly matches}}
+// expected-note@#defined-here {{defined here}}
 int __attribute__((target_version("rng"))) Out::func(void) { return 3; }
diff --git a/clang/test/SemaCXX/builtins-elementwise-math.cpp b/clang/test/SemaCXX/builtins-elementwise-math.cpp
index 5910796c5d298..0d891bdd661f1 100644
--- a/clang/test/SemaCXX/builtins-elementwise-math.cpp
+++ b/clang/test/SemaCXX/builtins-elementwise-math.cpp
@@ -76,6 +76,22 @@ void test_builtin_elementwise_min_fp() {
   static_assert(!is_const<decltype(__builtin_elementwise_min(a, a))>::value);
 }
 
+void test_builtin_elementwise_maximum() {
+  const float a = 2.0f;
+  float b = 1.0f;
+  static_assert(!is_const<decltype(__builtin_elementwise_maximum(a, b))>::value);
+  static_assert(!is_const<decltype(__builtin_elementwise_maximum(b, a))>::value);
+  static_assert(!is_const<decltype(__builtin_elementwise_maximum(a, a))>::value);
+}
+
+void test_builtin_elementwise_minimum() {
+  const float a = 2.0f;
+  float b = 1.0f;
+  static_assert(!is_const<decltype(__builtin_elementwise_minimum(a, b))>::value);
+  static_assert(!is_const<decltype(__builtin_elementwise_minimum(b, a))>::value);
+  static_assert(!is_const<decltype(__builtin_elementwise_minimum(a, a))>::value);
+}
+
 void test_builtin_elementwise_ceil() {
   const float a = 42.0;
   float b = 42.3;
@@ -146,6 +162,13 @@ void test_builtin_elementwise_atan() {
   static_assert(!is_const<decltype(__builtin_elementwise_atan(b))>::value);
 }
 
+void test_builtin_elementwise_atan2() {
+  const float a = 42.0;
+  float b = 42.3;
+  static_assert(!is_const<decltype(__builtin_elementwise_atan2(a, a))>::value);
+  static_assert(!is_const<decltype(__builtin_elementwise_atan2(b, b))>::value);
+}
+
 void test_builtin_elementwise_tan() {
   const float a = 42.0;
   float b = 42.3;
diff --git a/clang/test/SemaCXX/enable_if.cpp b/clang/test/SemaCXX/enable_if.cpp
index 5eec02a643340..1c307881e5d4a 100644
--- a/clang/test/SemaCXX/enable_if.cpp
+++ b/clang/test/SemaCXX/enable_if.cpp
@@ -5,7 +5,7 @@ int surrogate(int);
 struct Incomplete;  // expected-note{{forward declaration of 'Incomplete'}} \
                     // expected-note {{forward declaration of 'Incomplete'}}
 
-struct X {
+struct X { // expected-note{{defined here}}
   X() = default;  // expected-note{{candidate constructor not viable: requires 0 arguments, but 1 was provided}}
   X(const X&) = default;  // expected-note{{candidate constructor not viable: no known conversion from 'bool' to 'const X' for 1st argument}}
   X(bool b) __attribute__((enable_if(b, "chosen when 'b' is true")));  // expected-note{{candidate disabled: chosen when 'b' is true}}
diff --git a/clang/test/SemaCXX/function-redecl.cpp b/clang/test/SemaCXX/function-redecl.cpp
index 34d2acc75430c..8c0e9a880d070 100644
--- a/clang/test/SemaCXX/function-redecl.cpp
+++ b/clang/test/SemaCXX/function-redecl.cpp
@@ -61,7 +61,7 @@ void B::Notypocorrection(int) { // expected-error {{out-of-line definition of 'N
 }
 
 struct X { int f(); };
-struct Y : public X {};
+struct Y : public X {}; // expected-note {{defined here}}
 int Y::f() { return 3; } // expected-error {{out-of-line definition of 'f' does not match any declaration in 'Y'}}
 
 namespace test1 {
@@ -70,7 +70,7 @@ struct Foo {
 };
 }
 
-class Bar {
+class Bar { // expected-note {{defined here}}
   void f(test1::Foo::Inner foo) const; // expected-note {{member declaration does not match because it is const qualified}}
 };
 
@@ -80,7 +80,8 @@ void Bar::f(Foo::Inner foo) { // expected-error {{out-of-line definition of 'f'
   (void)foo;
 }
 
-class Crash {
+class Crash { // expected-note {{defined here}}
+              // expected-note@-1 {{defined here}}
  public:
   void GetCart(int count) const;
 };
@@ -89,7 +90,8 @@ void Crash::cart(int count) const {} // expected-error {{out-of-line definition
 // ...while this one crashed clang
 void Crash::chart(int count) const {} // expected-error {{out-of-line definition of 'chart' does not match any declaration in 'Crash'}}
 
-class TestConst {
+class TestConst { // expected-note {{defined here}}
+                  // expected-note@-1 {{defined here}}
  public:
   int getit() const; // expected-note {{member declaration does not match because it is const qualified}}
   void setit(int); // expected-note {{member declaration does not match because it is not const qualified}}
@@ -102,7 +104,7 @@ int TestConst::getit() { // expected-error {{out-of-line definition of 'getit' d
 void TestConst::setit(int) const { // expected-error {{out-of-line definition of 'setit' does not match any declaration in 'TestConst'}}
 }
 
-struct J { int typo() const; };
+struct J { int typo() const; }; // expected-note {{defined here}}
 int J::typo_() { return 3; } // expected-error {{out-of-line definition of 'typo_' does not match any declaration in 'J'}}
 
 // Ensure we correct the redecl of Foo::isGood to Bar::Foo::isGood and not
@@ -126,7 +128,7 @@ bool Foo::isGood() { // expected-error {{out-of-line definition of 'isGood' does
 void Foo::beEvil() {} // expected-error {{out-of-line definition of 'beEvil' does not match any declaration in namespace 'redecl_typo::Foo'; did you mean 'BeEvil'?}}
 }
 
-struct CVQualFun {
+struct CVQualFun { // expected-note {{defined here}}
   void func(int a, int &b); // expected-note {{type of 2nd parameter of member declaration does not match definition ('int &' vs 'int')}}
 };
 
diff --git a/clang/test/SemaCXX/lambda-unevaluated.cpp b/clang/test/SemaCXX/lambda-unevaluated.cpp
index 39ee89bc797f8..a9bcab58464e2 100644
--- a/clang/test/SemaCXX/lambda-unevaluated.cpp
+++ b/clang/test/SemaCXX/lambda-unevaluated.cpp
@@ -70,21 +70,23 @@ void use_g() { g<6>(&"hello"); } // expected-error {{ambiguous}}
 namespace GH51416 {
 
 template <class T>
-struct A {
+struct A { // #defined-here-A
   void spam(decltype([] {}));
 };
 
 template <class T>
 void A<T>::spam(decltype([] {})) // expected-error{{out-of-line definition of 'spam' does not match}}
+                                 // expected-note@#defined-here-A{{defined here}}
 {}
 
-struct B {
+struct B { // #defined-here-B
   template <class T>
   void spam(decltype([] {}));
 };
 
 template <class T>
 void B::spam(decltype([] {})) {} // expected-error{{out-of-line definition of 'spam' does not match}}
+                                 // expected-note@#defined-here-B{{defined here}}
 
 } // namespace GH51416
 
diff --git a/clang/test/SemaCXX/make_integer_seq.cpp b/clang/test/SemaCXX/make_integer_seq.cpp
index 8f72ce15eef47..71b7b8260d4ab 100644
--- a/clang/test/SemaCXX/make_integer_seq.cpp
+++ b/clang/test/SemaCXX/make_integer_seq.cpp
@@ -48,6 +48,5 @@ using illformed2 = ErrorSeq<int, -5>; // expected-note{{in instantiation}}
 template <typename T, T N> void f() {}
 __make_integer_seq<f, int, 0> x; // expected-error{{template template parameter must be a class template or type alias template}}
 
-__make_integer_seq<__make_integer_seq, int, 10> PR28494; // expected-error{{different template parameters}}
-// expected-note@make_integer_seq.cpp:* {{template parameter has a different kind}}
-// expected-note@make_integer_seq.cpp:* {{previous template template parameter is here}}
+__make_integer_seq<__make_integer_seq, int, 10> PR28494; // expected-note{{different template parameters}}
+// expected-error@make_integer_seq.cpp:* {{template argument for template template parameter must be a class template or type alias template}}
diff --git a/clang/test/SemaCXX/nested-name-spec.cpp b/clang/test/SemaCXX/nested-name-spec.cpp
index 920ef42bc1564..981152dda23f5 100644
--- a/clang/test/SemaCXX/nested-name-spec.cpp
+++ b/clang/test/SemaCXX/nested-name-spec.cpp
@@ -25,7 +25,7 @@ int A::C::cx = 17;
 
 static int A::C::cx2 = 17; // expected-error{{'static' can}}
 
-class C2 {
+class C2 { // #defined-here-C2
   void m(); // expected-note{{member declaration does not match because it is not const qualified}}
 
   void f(const int& parm); // expected-note{{type of 1st parameter of member declaration does not match definition ('const int &' vs 'int')}}
@@ -36,8 +36,10 @@ class C2 {
 };
 
 void C2::m() const { } // expected-error{{out-of-line definition of 'm' does not match any declaration in 'C2'}}
+                       // expected-note@#defined-here-C2{{defined here}}
 
 void C2::f(int) { } // expected-error{{out-of-line definition of 'f' does not match any declaration in 'C2'}}
+                    // expected-note@#defined-here-C2{{defined here}}
 
 void C2::m() {
   x = 0;
@@ -122,12 +124,13 @@ namespace E {
 }
 
 
-class Operators {
+class Operators { // #defined-here-Operators
   Operators operator+(const Operators&) const; // expected-note{{member declaration does not match because it is const qualified}}
   operator bool();
 };
 
 Operators Operators::operator+(const Operators&) { // expected-error{{out-of-line definition of 'operator+' does not match any declaration in 'Operators'}}
+                                                   // expected-note@#defined-here-Operators{{defined here}}
   Operators ops;
   return ops;
 }
@@ -149,9 +152,10 @@ void A::f() {} // expected-error-re{{out-of-line definition of 'f' does not matc
 
 void A::g(const int&) { } // expected-error{{out-of-line definition of 'g' does not match any declaration in namespace 'A'}}
 
-struct Struct { };
+struct Struct { }; // #defined-here-Struct
 
 void Struct::f() { } // expected-error{{out-of-line definition of 'f' does not match any declaration in 'Struct'}}
+                     // expected-note@#defined-here-Struct{{defined here}}
 
 void global_func(int);
 void global_func2(int);
diff --git a/clang/test/SemaCXX/out-of-line-def-mismatch.cpp b/clang/test/SemaCXX/out-of-line-def-mismatch.cpp
index a4e130d234353..1ce1cc0c7dfc0 100644
--- a/clang/test/SemaCXX/out-of-line-def-mismatch.cpp
+++ b/clang/test/SemaCXX/out-of-line-def-mismatch.cpp
@@ -6,7 +6,11 @@ namespace N2 {
   namespace N1 {
     class C1 {};
 
-    struct S2 {
+    struct S2 { // expected-note {{defined here}}
+                // expected-note@-1 {{defined here}}
+                // expected-note@-2 {{defined here}}
+                // expected-note@-3 {{defined here}}
+                // expected-note@-4 {{defined here}}
       void func(S1*); // expected-note {{type of 1st parameter of member declaration does not match definition ('S1 *' (aka 'N2::S1 *') vs 'S1 *' (aka 'N2::N1::S1 *'))}}
       void func(C1&, unsigned, const S1*); // expected-note {{type of 3rd parameter of member declaration does not match definition ('const S1 *' (aka 'const N2::S1 *') vs 'const S1 *' (aka 'const N2::N1::S1 *'))}}
       void func(const S1*, unsigned); //expected-note {{type of 1st parameter of member declaration does not match definition ('const S1 *' vs 'S1')}}
diff --git a/clang/test/SemaCXX/pragma-pack-packed-2.cpp b/clang/test/SemaCXX/pragma-pack-packed-2.cpp
new file mode 100644
index 0000000000000..3639addd6fe5f
--- /dev/null
+++ b/clang/test/SemaCXX/pragma-pack-packed-2.cpp
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -triple s390x-ibm-zos -fzos-extensions -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple s390x-ibm-zos -fzos-extensions -fxl-pragma-pack -fsyntax-only -verify %s
+// RUN: %clang -target s390x-ibm-zos -S -emit-llvm -Xclang -verify -fno-xl-pragma-pack %s
+
+#pragma pack(show) // expected-warning {{value of #pragma pack(show) == 8}}
+#pragma pack(twobyte)
+#pragma pack(packed)
+#pragma pack(show) // expected-warning {{value of #pragma pack(show) == 1}}
+#pragma pack(reset)
+#pragma pack(show) // expected-warning {{value of #pragma pack(show) == 2}}
+#pragma pack(pop)
+#pragma pack(show) // expected-warning {{value of #pragma pack(show) == 8}}
diff --git a/clang/test/SemaCXX/typo-correction.cpp b/clang/test/SemaCXX/typo-correction.cpp
index c262a62f0cb9c..45f42c4260358 100644
--- a/clang/test/SemaCXX/typo-correction.cpp
+++ b/clang/test/SemaCXX/typo-correction.cpp
@@ -173,6 +173,7 @@ class Parent {
 };
 class Child: public Parent {};
 void Child::add_types(int value) {} // expected-error{{out-of-line definition of 'add_types' does not match any declaration in 'Child'}}
+                                    // expected-note@-2{{defined here}}
 
 // Fix the callback based filtering of typo corrections within
 // Sema::ActOnIdExpression by Parser::ParseCastExpression to allow type names as
diff --git a/clang/test/SemaHLSL/ArrayAssignable_errors.hlsl b/clang/test/SemaHLSL/ArrayAssignable_errors.hlsl
new file mode 100644
index 0000000000000..1925032a93d48
--- /dev/null
+++ b/clang/test/SemaHLSL/ArrayAssignable_errors.hlsl
@@ -0,0 +1,29 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -verify
+
+void test_wrong_size1() {
+  int Arr[2] = {0, 1};
+  int Arr2[3] = {1, 2, 0};
+  Arr = Arr2;
+  // expected-error@-1 {{assigning to 'int[2]' from incompatible type 'int[3]'}}
+}
+
+void test_wrong_size2() {
+  int Arr[2] = {0, 1};
+  int Arr2[3] = {1, 2, 0};
+  Arr2 = Arr;
+  // expected-error@-1 {{assigning to 'int[3]' from incompatible type 'int[2]'}}
+}
+
+void test_wrong_size3() {
+  int Arr[2][2] = {{0, 1}, {2, 3}};
+  int Arr2[2] = {4, 5};
+  Arr = Arr2;
+  // expected-error@-1 {{assigning to 'int[2][2]' from incompatible type 'int[2]'}}
+}
+
+void test_wrong_size4() {
+  int Arr[2][2] = {{0, 1}, {2, 3}};
+  int Arr2[2] = {4, 5};
+  Arr2 = Arr;
+  // expected-error@-1 {{assigning to 'int[2]' from incompatible type 'int[2][2]'}}
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/cross-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/cross-errors.hlsl
new file mode 100644
index 0000000000000..423f5bac9471f
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/cross-errors.hlsl
@@ -0,0 +1,43 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -disable-llvm-passes -verify
+
+void test_too_few_arg()
+{
+  return __builtin_hlsl_cross();
+  // expected-error@-1 {{too few arguments to function call, expected 2, have 0}}
+}
+
+void test_too_many_arg(float3 p0)
+{
+  return __builtin_hlsl_cross(p0, p0, p0);
+  // expected-error@-1 {{too many arguments to function call, expected 2, have 3}}
+}
+
+bool builtin_bool_to_float_type_promotion(bool p1)
+{
+  return __builtin_hlsl_cross(p1, p1);
+  // expected-error@-1 {{passing 'bool' to parameter of incompatible type 'float'}}
+}
+
+bool builtin_cross_int_to_float_promotion(int p1)
+{
+  return __builtin_hlsl_cross(p1, p1);
+  // expected-error@-1 {{passing 'int' to parameter of incompatible type 'float'}}
+}
+
+bool2 builtin_cross_int2_to_float2_promotion(int2 p1)
+{
+  return __builtin_hlsl_cross(p1, p1);
+  // expected-error@-1 {{passing 'int2' (aka 'vector<int, 2>') to parameter of incompatible type '__attribute__((__vector_size__(2 * sizeof(float)))) float' (vector of 2 'float' values)}}
+}
+
+float2 builtin_cross_float2(float2 p1, float2 p2)
+{
+  return __builtin_hlsl_cross(p1, p2);
+  // expected-error@-1 {{too many elements in vector operand (expected 3 elements, have 2)}}
+}
+
+float3  builtin_cross_float3_int3(float3 p1, int3 p2)
+{
+  return __builtin_hlsl_cross(p1, p2);
+  // expected-error@-1 {{all arguments to '__builtin_hlsl_cross' must have the same type}}
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/fmod-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/fmod-errors.hlsl
deleted file mode 100644
index e4fa609dd6a05..0000000000000
--- a/clang/test/SemaHLSL/BuiltIns/fmod-errors.hlsl
+++ /dev/null
@@ -1,22 +0,0 @@
-
-// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -verify-ignore-unexpected
-
-float builtin_bool_to_float_type_promotion(bool p1, bool p2) {
-  return __builtin_elementwise_fmod(p1, p2);
-  // expected-error@-1 {{1st argument must be a vector, integer or floating point type (was 'bool')}}
-}
-
-float2 builtin_fmod_int2_to_float2_promotion(int2 p1, int2 p2) {
-  return __builtin_elementwise_fmod(p1, p2);
-  // expected-error@-1 {{1st argument must be a floating point type (was 'int2' (aka 'vector<int, 2>'))}}
-}
-
-half builtin_fmod_double_type (double p0, double p1) {
-  return __builtin_elementwise_fmod(p0, p1);
-  // expected-error@-1 {{passing 'double' to parameter of incompatible type 'float'}}
-}
-
-half builtin_fmod_double2_type (double2 p0, double2 p1) {
-  return __builtin_elementwise_fmod(p0, p1);
-  // expected-error@-1 {{passing 'double2' (aka 'vector<double, 2>') to parameter of incompatible type '__attribute__((__vector_size__(2 * sizeof(float)))) float' (vector of 2 'float' values)}}
-}
diff --git a/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl
index 1841f60568b56..0c1f7022bad60 100644
--- a/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl
@@ -18,7 +18,12 @@
 // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_tanh
 // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_trunc
 
-double2 test_double_builtin(double2 p0) {
+double test_double_builtin(double p0) {
+    return TEST_FUNC(p0);
+  // expected-error@-1 {{passing 'double' to parameter of incompatible type 'float'}}
+}
+
+double2 test_vec_double_builtin(double2 p0) {
     return TEST_FUNC(p0);
   // expected-error@-1 {{passing 'double2' (aka 'vector<double, 2>') to parameter of incompatible type '__attribute__((__vector_size__(2 * sizeof(float)))) float' (vector of 2 'float' values)}}
 }
diff --git a/clang/test/SemaHLSL/BuiltIns/half-float-only-errors2.hlsl b/clang/test/SemaHLSL/BuiltIns/half-float-only-errors2.hlsl
new file mode 100644
index 0000000000000..bfbd8b28257a3
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/half-float-only-errors2.hlsl
@@ -0,0 +1,13 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_atan2
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_fmod
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_pow
+
+double test_double_builtin(double p0, double p1) {
+    return TEST_FUNC(p0, p1);
+  // expected-error@-1 {{passing 'double' to parameter of incompatible type 'float'}}
+}
+
+double2 test_vec_double_builtin(double2 p0, double2 p1) {
+    return TEST_FUNC(p0, p1);
+  // expected-error@-1 {{passing 'double2' (aka 'vector<double, 2>') to parameter of incompatible type '__attribute__((__vector_size__(2 * sizeof(float)))) float' (vector of 2 'float' values)}}
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/pow-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/pow-errors.hlsl
deleted file mode 100644
index 5a2c9a3ef62ff..0000000000000
--- a/clang/test/SemaHLSL/BuiltIns/pow-errors.hlsl
+++ /dev/null
@@ -1,6 +0,0 @@
-// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify
-
-double2 test_double_builtin(double2 p0, double2 p1) {
-    return __builtin_elementwise_pow(p0,p1);
-  // expected-error@-1 {{passing 'double2' (aka 'vector<double, 2>') to parameter of incompatible type '__attribute__((__vector_size__(2 * sizeof(float)))) float' (vector of 2 'float' values)}}
-}
diff --git a/clang/test/SemaObjCXX/attr-nonblocking-constraints.mm b/clang/test/SemaObjCXX/attr-nonblocking-constraints.mm
new file mode 100644
index 0000000000000..abd0938ac321a
--- /dev/null
+++ b/clang/test/SemaObjCXX/attr-nonblocking-constraints.mm
@@ -0,0 +1,26 @@
+// RUN: %clang_cc1 -fsyntax-only -fblocks -fcxx-exceptions -fobjc-exceptions -verify -Wfunction-effects %s
+
+#pragma clang diagnostic ignored "-Wperf-constraint-implies-noexcept"
+
+// Objective-C
+@interface OCClass
+- (void)method;
+@end
+
+void nb1(OCClass *oc) [[clang::nonblocking]] {
+	[oc method]; // expected-warning {{function with 'nonblocking' attribute must not access ObjC methods or properties}}
+}
+void nb2(OCClass *oc) {
+	[oc method]; // expected-note {{function cannot be inferred 'nonblocking' because it accesses an ObjC method or property}}
+}
+void nb3(OCClass *oc) [[clang::nonblocking]] {
+	nb2(oc); // expected-warning {{function with 'nonblocking' attribute must not call non-'nonblocking' function 'nb2'}}
+}
+
+void nb4() [[clang::nonblocking]] {
+	@try {
+		@throw @"foo"; // expected-warning {{function with 'nonblocking' attribute must not throw or catch exceptions}}
+	}
+	@catch (...) { // expected-warning {{function with 'nonblocking' attribute must not throw or catch exceptions}}
+	}
+}
diff --git a/clang/test/SemaOpenACC/compute-construct-device_type-clause.c b/clang/test/SemaOpenACC/compute-construct-device_type-clause.c
index 26f0315fb86f1..d08497a7782ed 100644
--- a/clang/test/SemaOpenACC/compute-construct-device_type-clause.c
+++ b/clang/test/SemaOpenACC/compute-construct-device_type-clause.c
@@ -211,8 +211,7 @@ void uses() {
   while(1);
 #pragma acc kernels device_type(*) async
   while(1);
-  // expected-error@+2{{OpenACC clause 'tile' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
+  // expected-error@+1{{OpenACC 'tile' clause is not valid on 'kernels' directive}}
 #pragma acc kernels device_type(*) tile(Var, 1)
   while(1);
   // expected-error@+2{{OpenACC clause 'gang' may not follow a 'dtype' clause in a compute construct}}
diff --git a/clang/test/SemaOpenACC/loop-construct-auto_seq_independent-clauses.c b/clang/test/SemaOpenACC/loop-construct-auto_seq_independent-clauses.c
index 3212c19d089fc..3da7f0e9836be 100644
--- a/clang/test/SemaOpenACC/loop-construct-auto_seq_independent-clauses.c
+++ b/clang/test/SemaOpenACC/loop-construct-auto_seq_independent-clauses.c
@@ -165,9 +165,9 @@ void uses() {
   // expected-error@+1{{OpenACC 'async' clause is not valid on 'loop' directive}}
 #pragma acc loop auto async
   for(;;);
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented}}
-#pragma acc loop auto tile(Var, 1)
-  for(;;);
+#pragma acc loop auto tile(1+2, 1)
+  for(;;)
+    for(;;);
   // expected-warning@+1{{OpenACC clause 'gang' not yet implemented}}
 #pragma acc loop auto gang
   for(;;);
@@ -303,9 +303,9 @@ void uses() {
   // expected-error@+1{{OpenACC 'async' clause is not valid on 'loop' directive}}
 #pragma acc loop async auto
   for(;;);
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented}}
-#pragma acc loop tile(Var, 1) auto
-  for(;;);
+#pragma acc loop tile(1+2, 1) auto
+  for(;;)
+    for(;;);
   // expected-warning@+1{{OpenACC clause 'gang' not yet implemented}}
 #pragma acc loop gang auto
   for(;;);
@@ -442,9 +442,9 @@ void uses() {
   // expected-error@+1{{OpenACC 'async' clause is not valid on 'loop' directive}}
 #pragma acc loop independent async
   for(;;);
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented}}
-#pragma acc loop independent tile(Var, 1)
-  for(;;);
+#pragma acc loop independent tile(1+2, 1)
+  for(;;)
+    for(;;);
   // expected-warning@+1{{OpenACC clause 'gang' not yet implemented}}
 #pragma acc loop independent gang
   for(;;);
@@ -580,9 +580,9 @@ void uses() {
   // expected-error@+1{{OpenACC 'async' clause is not valid on 'loop' directive}}
 #pragma acc loop async independent
   for(;;);
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented}}
-#pragma acc loop tile(Var, 1) independent
-  for(;;);
+#pragma acc loop tile(1+2, 1) independent
+  for(;;)
+    for(;;);
   // expected-warning@+1{{OpenACC clause 'gang' not yet implemented}}
 #pragma acc loop gang independent
   for(;;);
@@ -728,9 +728,9 @@ void uses() {
   // expected-error@+1{{OpenACC 'async' clause is not valid on 'loop' directive}}
 #pragma acc loop seq async
   for(;;);
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented}}
-#pragma acc loop seq tile(Var, 1)
-  for(;;);
+#pragma acc loop seq tile(1+2, 1)
+  for(;;)
+    for(;;);
   // expected-error@+1{{OpenACC 'wait' clause is not valid on 'loop' directive}}
 #pragma acc loop seq wait
   for(;;);
@@ -875,9 +875,9 @@ void uses() {
   // expected-error@+1{{OpenACC 'async' clause is not valid on 'loop' directive}}
 #pragma acc loop async seq
   for(;;);
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented}}
-#pragma acc loop tile(Var, 1) seq
-  for(;;);
+#pragma acc loop tile(1+2, 1) seq
+  for(;;)
+    for(;;);
   // expected-error@+1{{OpenACC 'wait' clause is not valid on 'loop' directive}}
 #pragma acc loop wait seq
   for(;;);
diff --git a/clang/test/SemaOpenACC/loop-construct-collapse-clause.cpp b/clang/test/SemaOpenACC/loop-construct-collapse-clause.cpp
index 9c1e577773e8f..953775a423cdb 100644
--- a/clang/test/SemaOpenACC/loop-construct-collapse-clause.cpp
+++ b/clang/test/SemaOpenACC/loop-construct-collapse-clause.cpp
@@ -115,3 +115,367 @@ void negative_constexpr(int i) {
   negative_constexpr_templ<int, 1>(); // #NCET1
 }
 
+template<unsigned Val>
+void depth_too_high_templ() {
+  // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(Val)
+  for(;;)
+    for(;;);
+}
+
+void depth_too_high() {
+  depth_too_high_templ<3>(); // expected-note{{in instantiation of function template specialization}}
+
+  // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(3)
+  for(;;)
+    for(;;);
+
+  // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(three())
+  for(;;)
+    for(;;);
+
+  // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(ConvertsThree{})
+  for(;;)
+    for(;;);
+}
+
+template<typename T, unsigned Three>
+void not_single_loop_templ() {
+  T Arr[5];
+  // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1 2{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(3)
+  for(auto x : Arr) {
+    for(auto y : Arr){
+      do{}while(true); // expected-error{{do loop cannot appear in intervening code of a 'loop' with a 'collapse' clause}}
+    }
+  }
+
+  // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1 2{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(Three)
+  for(;;) {
+    for(;;){
+      do{}while(true); // expected-error{{do loop cannot appear in intervening code of a 'loop' with a 'collapse' clause}}
+    }
+  }
+
+#pragma acc loop collapse(Three)
+  for(;;) {
+    for(;;){
+      for(;;){
+        do{}while(true);
+      }
+    }
+  }
+  // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1 2{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(Three)
+  for(auto x : Arr) {
+    for(auto y: Arr) {
+      do{}while(true); // expected-error{{do loop cannot appear in intervening code of a 'loop' with a 'collapse' clause}}
+    }
+  }
+
+#pragma acc loop collapse(Three)
+  for(auto x : Arr) {
+    for(auto y: Arr) {
+      for(auto z: Arr) {
+        do{}while(true);
+      }
+    }
+  }
+}
+
+void not_single_loop() {
+  not_single_loop_templ<int, 3>(); // expected-note{{in instantiation of function template}}
+
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(3)
+  for(;;) {
+    for(;;){
+      for(;;);
+    }
+    while(true); // expected-error{{while loop cannot appear in intervening code of a 'loop' with a 'collapse' clause}}
+  }
+
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(3)
+  for(;;) {
+    for(;;){
+      for(;;);
+    }
+    do{}while(true); // expected-error{{do loop cannot appear in intervening code of a 'loop' with a 'collapse' clause}}
+  }
+
+  // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1 2{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(3)
+  for(;;) {
+    for(;;){
+      while(true); // expected-error{{while loop cannot appear in intervening code of a 'loop' with a 'collapse' clause}}
+    }
+  }
+  // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1 2{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(3)
+  for(;;) {
+    for(;;){
+      do{}while(true); // expected-error{{do loop cannot appear in intervening code of a 'loop' with a 'collapse' clause}}
+    }
+  }
+
+#pragma acc loop collapse(2)
+  for(;;) {
+    for(;;){
+      do{}while(true);
+    }
+  }
+#pragma acc loop collapse(2)
+  for(;;) {
+    for(;;){
+      while(true);
+    }
+  }
+
+  int Arr[5];
+  // expected-error@+2{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1 2{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(3)
+  for(auto x : Arr) {
+    for(auto y : Arr){
+      do{}while(true); // expected-error{{do loop cannot appear in intervening code of a 'loop' with a 'collapse' clause}}
+    }
+  }
+
+  // expected-note@+1 {{active 'collapse' clause defined here}}
+#pragma acc loop collapse(3)
+  for (;;) {
+    for (;;) {
+      for(;;);
+    }
+    // expected-error@+1{{more than one for-loop in a loop associated with OpenACC 'loop' construct with a 'collapse' clause}}
+    for(;;);
+  }
+
+  // expected-note@+1 {{active 'collapse' clause defined here}}
+#pragma acc loop collapse(3)
+  for (;;) {
+    for (;;) {
+      for(;;);
+    // expected-error@+1{{more than one for-loop in a loop associated with OpenACC 'loop' construct with a 'collapse' clause}}
+      for(;;);
+    }
+  }
+
+  for(;;);
+#pragma acc loop collapse(3)
+  for (;;) {
+    for (;;) {
+      for (;;);
+    }
+  }
+}
+
+template<unsigned Two, unsigned Three>
+void no_other_directives() {
+#pragma acc loop collapse(Two)
+  for(;;) {
+    for (;;) { // last loop associated with the top level.
+    // expected-error@+1{{'collapse' clause specifies a loop count greater than the number of available loops}}
+#pragma acc loop collapse(Three) // expected-note 2{{active 'collapse' clause defined here}}
+      for(;;) {
+        for(;;) {
+    // expected-error@+1{{OpenACC 'serial' construct cannot appear in intervening code of a 'loop' with a 'collapse' clause}}
+#pragma acc serial
+          ;
+        }
+      }
+    }
+  }
+#pragma acc loop collapse(Two)// expected-note{{active 'collapse' clause defined here}}
+  for(;;) {
+    for (;;) { // last loop associated with the top level.
+#pragma acc loop collapse(Three)
+      for(;;) {
+        for(;;) {
+          for(;;);
+        }
+      }
+    }
+    // expected-error@+1{{OpenACC 'serial' construct cannot appear in intervening code of a 'loop' with a 'collapse' clause}}
+#pragma acc serial
+          ;
+  }
+}
+
+void no_other_directives() {
+  no_other_directives<2,3>(); // expected-note{{in instantiation of function template specialization}}
+
+  // Ok, not inside the intervening list
+#pragma acc loop collapse(2)
+  for(;;) {
+    for(;;) {
+#pragma acc data // expected-warning{{OpenACC construct 'data' not yet implemented}}
+    }
+  }
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(2)
+  for(;;) {
+    // expected-error@+1{{OpenACC 'data' construct cannot appear in intervening code of a 'loop' with a 'collapse' clause}}
+#pragma acc data // expected-warning{{OpenACC construct 'data' not yet implemented}}
+    for(;;) {
+    }
+  }
+}
+
+void call();
+
+template<unsigned Two>
+void intervening_without_force_templ() {
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(2)
+  for(;;) {
+    // expected-error@+1{{inner loops must be tightly nested inside a 'collapse' clause on a 'loop' construct}}
+    call();
+    for(;;){}
+  }
+
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(Two)
+  for(;;) {
+    // expected-error@+1{{inner loops must be tightly nested inside a 'collapse' clause on a 'loop' construct}}
+    call();
+    for(;;){}
+  }
+
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(2)
+  for(;;) {
+    for(;;){}
+    // expected-error@+1{{inner loops must be tightly nested inside a 'collapse' clause on a 'loop' construct}}
+    call();
+  }
+
+#pragma acc loop collapse(force:2)
+  for(;;) {
+    call();
+    for(;;){}
+  }
+
+#pragma acc loop collapse(force:Two)
+  for(;;) {
+    call();
+    for(;;){}
+  }
+
+
+#pragma acc loop collapse(force:2)
+  for(;;) {
+    for(;;){}
+    call();
+  }
+
+#pragma acc loop collapse(force:Two)
+  for(;;) {
+    for(;;){}
+    call();
+  }
+
+#pragma acc loop collapse(Two)
+  for(;;) {
+    for(;;){
+    call();
+    }
+  }
+
+#pragma acc loop collapse(Two)
+  for(;;) {
+    {
+      {
+        for(;;){
+        call();
+        }
+      }
+    }
+  }
+
+#pragma acc loop collapse(force:Two)
+  for(;;) {
+    for(;;){
+    call();
+    }
+  }
+
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(Two)
+  for(;;) {
+    for(;;){}
+    // expected-error@+1{{inner loops must be tightly nested inside a 'collapse' clause on a 'loop' construct}}
+    call();
+  }
+}
+
+void intervening_without_force() {
+  intervening_without_force_templ<2>(); // expected-note{{in instantiation of function template specialization}}
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(2)
+  for(;;) {
+    // expected-error@+1{{inner loops must be tightly nested inside a 'collapse' clause on a 'loop' construct}}
+    call();
+    for(;;){}
+  }
+
+  // expected-note@+1{{active 'collapse' clause defined here}}
+#pragma acc loop collapse(2)
+  for(;;) {
+    for(;;){}
+    // expected-error@+1{{inner loops must be tightly nested inside a 'collapse' clause on a 'loop' construct}}
+    call();
+  }
+
+  // The below two are fine, as they use the 'force' tag.
+#pragma acc loop collapse(force:2)
+  for(;;) {
+    call();
+    for(;;){}
+  }
+
+#pragma acc loop collapse(force:2)
+  for(;;) {
+    for(;;){}
+    call();
+  }
+
+#pragma acc loop collapse(2)
+  for(;;) {
+    for(;;){
+    call();
+    }
+  }
+#pragma acc loop collapse(2)
+  for(;;) {
+    {
+      {
+        for(;;){
+        call();
+        }
+      }
+    }
+  }
+
+#pragma acc loop collapse(force:2)
+  for(;;) {
+    for(;;){
+    call();
+    }
+  }
+}
+
diff --git a/clang/test/SemaOpenACC/loop-construct-device_type-clause.c b/clang/test/SemaOpenACC/loop-construct-device_type-clause.c
index 47c9239f4f0e9..3d77c031f4263 100644
--- a/clang/test/SemaOpenACC/loop-construct-device_type-clause.c
+++ b/clang/test/SemaOpenACC/loop-construct-device_type-clause.c
@@ -188,9 +188,11 @@ void uses() {
   // expected-error@+1{{OpenACC 'async' clause is not valid on 'loop' directive}}
 #pragma acc loop device_type(*) async
   for(;;);
-  // expected-warning@+1{{OpenACC clause 'tile' not yet implemented, clause ignored}}
-#pragma acc loop device_type(*) tile(Var, 1)
-  for(;;);
+
+#pragma acc loop device_type(*) tile(*, 1)
+  for(;;)
+    for(;;);
+
   // expected-warning@+1{{OpenACC clause 'gang' not yet implemented, clause ignored}}
 #pragma acc loop dtype(*) gang
   for(;;);
diff --git a/clang/test/SemaOpenACC/loop-construct-tile-ast.cpp b/clang/test/SemaOpenACC/loop-construct-tile-ast.cpp
new file mode 100644
index 0000000000000..977c7e3b6d4cf
--- /dev/null
+++ b/clang/test/SemaOpenACC/loop-construct-tile-ast.cpp
@@ -0,0 +1,231 @@
+// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+#ifndef PCH_HELPER
+#define PCH_HELPER
+
+struct S {
+  constexpr S(){};
+  constexpr operator auto() {return 1;}
+};
+
+void NormalUses() {
+  // CHECK: FunctionDecl{{.*}}NormalUses
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc loop tile(S{}, 1, 2, *)
+  for(;;)
+    for(;;)
+      for(;;)
+        for(;;);
+  // CHECK-NEXT: OpenACCLoopConstruct
+  // CHECK-NEXT: tile clause
+  // CHECK-NEXT: ConstantExpr{{.*}} 'int'
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'int'
+  // CHECK-NEXT: MemberExpr{{.*}} .operator auto
+  // CHECK-NEXT: MaterializeTemporaryExpr{{.*}} 'S' lvalue
+  // CHECK-NEXT: CXXTemporaryObjectExpr{{.*}} 'S' 'void ()' list
+  // CHECK-NEXT: ConstantExpr{{.*}} 'int'
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 1
+  // CHECK-NEXT: ConstantExpr{{.*}} 'int'
+  // CHECK-NEXT: value: Int 2
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 2
+  // CHECK-NEXT: OpenACCAsteriskSizeExpr{{.*}}'int'
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: NullStmt
+}
+
+template<typename T, unsigned Value>
+void TemplUses() {
+  // CHECK: FunctionTemplateDecl{{.*}}TemplUses
+  // CHECK-NEXT: TemplateTypeParmDecl {{.*}} referenced typename depth 0 index 0 T
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} referenced 'unsigned int' depth 0 index 1 Value
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplUses 'void ()'
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc loop tile(S{}, T{}, *, T{} + S{}, Value, Value + 3)
+  for(;;)
+    for(;;)
+      for(;;)
+        for(;;)
+          for(;;)
+            for(;;);
+  // CHECK-NEXT: OpenACCLoopConstruct
+  // CHECK-NEXT: tile clause
+  //
+  // CHECK-NEXT: ConstantExpr{{.*}} 'int'
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'int'
+  // CHECK-NEXT: MemberExpr{{.*}} .operator auto
+  // CHECK-NEXT: MaterializeTemporaryExpr{{.*}} 'S' lvalue
+  // CHECK-NEXT: CXXTemporaryObjectExpr{{.*}} 'S' 'void ()' list
+  //
+  // CHECK-NEXT: CXXUnresolvedConstructExpr{{.*}} 'T' 'T' list
+  // CHECK-NEXT: InitListExpr{{.*}}'void'
+  //
+  // CHECK-NEXT: OpenACCAsteriskSizeExpr{{.*}}'int'
+  //
+  // CHECK-NEXT: BinaryOperator{{.*}}'<dependent type>' '+'
+  // CHECK-NEXT: CXXUnresolvedConstructExpr{{.*}} 'T' 'T' list
+  // CHECK-NEXT: InitListExpr{{.*}}'void'
+  // CHECK-NEXT: CXXTemporaryObjectExpr{{.*}} 'S' 'void ()' list
+  //
+  // CHECK-NEXT: DeclRefExpr{{.*}}'unsigned int' NonTypeTemplateParm{{.*}}'Value' 'unsigned int'
+  //
+  // CHECK-NEXT: BinaryOperator{{.*}}'unsigned int' '+'
+  // CHECK-NEXT: DeclRefExpr{{.*}}'unsigned int' NonTypeTemplateParm{{.*}}'Value' 'unsigned int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'unsigned int' <IntegralCast>
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 3
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: NullStmt
+
+  // Instantiation:
+  // CHECK-NEXT: FunctionDecl{{.*}} used TemplUses 'void ()' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument type 'S'
+  // CHECK-NEXT: RecordType{{.*}} 'S'
+  // CHECK-NEXT: CXXRecord{{.*}} 'S'
+  // CHECK-NEXT: TemplateArgument integral '2U'
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCLoopConstruct
+  // CHECK-NEXT: tile clause
+  //
+  // CHECK-NEXT: ConstantExpr{{.*}} 'int'
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'int'
+  // CHECK-NEXT: MemberExpr{{.*}} .operator auto
+  // CHECK-NEXT: MaterializeTemporaryExpr{{.*}} 'S' lvalue
+  // CHECK-NEXT: CXXTemporaryObjectExpr{{.*}} 'S' 'void ()' list
+  //
+  // CHECK-NEXT: ConstantExpr{{.*}} 'int'
+  // CHECK-NEXT: value: Int 1
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'int'
+  // CHECK-NEXT: MemberExpr{{.*}} .operator auto
+  // CHECK-NEXT: MaterializeTemporaryExpr{{.*}} 'S' lvalue
+  // CHECK-NEXT: CXXTemporaryObjectExpr{{.*}} 'S' 'void ()' list
+  //
+  // CHECK-NEXT: OpenACCAsteriskSizeExpr{{.*}}'int'
+  //
+  // CHECK-NEXT: ConstantExpr{{.*}} 'int'
+  // CHECK-NEXT: value: Int 2
+  // CHECK-NEXT: BinaryOperator{{.*}}'int' '+'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'int'
+  // CHECK-NEXT: MemberExpr{{.*}} .operator auto
+  // CHECK-NEXT: MaterializeTemporaryExpr{{.*}} 'S' lvalue
+  // CHECK-NEXT: CXXTemporaryObjectExpr{{.*}} 'S' 'void ()' list
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'int'
+  // CHECK-NEXT: MemberExpr{{.*}} .operator auto
+  // CHECK-NEXT: MaterializeTemporaryExpr{{.*}} 'S' lvalue
+  // CHECK-NEXT: CXXTemporaryObjectExpr{{.*}} 'S' 'void ()' list
+  //
+  // CHECK-NEXT: ConstantExpr{{.*}} 'unsigned int'
+  // CHECK-NEXT: value: Int 2
+  // CHECK-NEXT: SubstNonTypeTemplateParmExpr{{.*}} 'unsigned int'
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} 'unsigned int' depth 0 index 1 Value
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'unsigned int' 2
+  //
+  // CHECK-NEXT: ConstantExpr{{.*}} 'unsigned int'
+  // CHECK-NEXT: value: Int 5
+  // CHECK-NEXT: BinaryOperator{{.*}}'unsigned int' '+'
+  // CHECK-NEXT: SubstNonTypeTemplateParmExpr{{.*}} 'unsigned int'
+  // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} 'unsigned int' depth 0 index 1 Value
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'unsigned int' 2
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'unsigned int' <IntegralCast>
+  // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 3
+
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: NullStmt
+
+}
+
+void Inst() {
+  TemplUses<S, 2>();
+}
+#endif // PCH_HELPER
diff --git a/clang/test/SemaOpenACC/loop-construct-tile-clause.cpp b/clang/test/SemaOpenACC/loop-construct-tile-clause.cpp
new file mode 100644
index 0000000000000..ecf8dc1109fb7
--- /dev/null
+++ b/clang/test/SemaOpenACC/loop-construct-tile-clause.cpp
@@ -0,0 +1,376 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+constexpr int three() { return 3; }
+constexpr int one() { return 1; }
+constexpr int neg() { return -1; }
+constexpr int zero() { return 0; }
+
+struct NotConstexpr {
+  constexpr NotConstexpr(){};
+
+  operator int(){ return 1; }
+};
+struct ConvertsNegative {
+  constexpr ConvertsNegative(){};
+
+  constexpr operator int(){ return -1; }
+};
+struct ConvertsOne{
+  constexpr ConvertsOne(){};
+
+  constexpr operator int(){ return 1; }
+};
+
+struct ConvertsThree{
+  constexpr ConvertsThree(){};
+
+  constexpr operator int(){ return 3; }
+};
+
+template<typename T, int Val>
+void negative_zero_constexpr_templ() {
+  // expected-error@+1 2{{OpenACC 'tile' clause size expression must be positive integer value, evaluated to 0}}
+#pragma acc loop tile(*, T{})
+  for(;;)
+    for(;;);
+
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be positive integer value, evaluated to -1}}
+#pragma acc loop tile(Val, *)
+  for(;;)
+    for(;;);
+
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be positive integer value, evaluated to 0}}
+#pragma acc loop tile(zero(), *)
+  for(;;)
+    for(;;);
+}
+
+void negative_zero_constexpr() {
+  negative_zero_constexpr_templ<int, 1>(); // expected-note{{in instantiation of function template specialization}}
+  negative_zero_constexpr_templ<int, -1>(); // expected-note{{in instantiation of function template specialization}}
+
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be positive integer value, evaluated to 0}}
+#pragma acc loop tile(0, *)
+  for(;;)
+    for(;;);
+
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be positive integer value, evaluated to 0}}
+#pragma acc loop tile(1, 0)
+  for(;;)
+    for(;;);
+
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be positive integer value, evaluated to -1}}
+#pragma acc loop tile(1, -1)
+  for(;;)
+    for(;;);
+
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be positive integer value, evaluated to -1}}
+#pragma acc loop tile(-1, 0)
+  for(;;)
+    for(;;);
+
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be positive integer value, evaluated to 0}}
+#pragma acc loop tile(zero(), 0)
+  for(;;)
+    for(;;);
+
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be positive integer value, evaluated to -1}}
+#pragma acc loop tile(1, neg())
+  for(;;)
+    for(;;);
+
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be an asterisk or a constant expression}}
+#pragma acc loop tile(NotConstexpr{})
+  for(;;);
+
+  // expected-error@+1{{OpenACC 'tile' clause size expression must be positive integer value, evaluated to -1}}
+#pragma acc loop tile(1, ConvertsNegative{})
+  for(;;)
+    for(;;);
+
+#pragma acc loop tile(*, ConvertsOne{})
+  for(;;)
+    for(;;);
+}
+
+template<unsigned One>
+void only_for_loops_templ() {
+  // expected-note@+1{{'loop' construct is here}}
+#pragma acc loop tile(One)
+  // expected-error@+1{{OpenACC 'loop' construct can only be applied to a 'for' loop}}
+  while(true);
+
+  // expected-note@+1{{'loop' construct is here}}
+#pragma acc loop tile(One)
+  // expected-error@+1{{OpenACC 'loop' construct can only be applied to a 'for' loop}}
+  do {} while(true);
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc loop tile(One, 2) // expected-note 2{{active 'tile' clause defined here}}
+  for (;;)
+      // expected-error@+1{{while loop cannot appear in intervening code of a 'loop' with a 'tile' clause}}
+    while(true);
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc loop tile(One, 2) // expected-note 2{{active 'tile' clause defined here}}
+  for (;;)
+      // expected-error@+1{{do loop cannot appear in intervening code of a 'loop' with a 'tile' clause}}
+    do{}while(true);
+}
+
+
+void only_for_loops() {
+  // expected-note@+1{{'loop' construct is here}}
+#pragma acc loop tile(1)
+  // expected-error@+1{{OpenACC 'loop' construct can only be applied to a 'for' loop}}
+  while(true);
+
+  // expected-note@+1{{'loop' construct is here}}
+#pragma acc loop tile(1)
+  // expected-error@+1{{OpenACC 'loop' construct can only be applied to a 'for' loop}}
+  do {} while(true);
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc loop tile(1, 2) // expected-note 2{{active 'tile' clause defined here}}
+  for (;;)
+      // expected-error@+1{{while loop cannot appear in intervening code of a 'loop' with a 'tile' clause}}
+    while(true);
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc loop tile(1, 2) // expected-note 2{{active 'tile' clause defined here}}
+  for (;;)
+      // expected-error@+1{{do loop cannot appear in intervening code of a 'loop' with a 'tile' clause}}
+    do{}while(true);
+}
+
+void only_one_on_loop() {
+  // expected-error@+2{{OpenACC 'tile' clause cannot appear more than once on a 'loop' directive}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc loop tile(1) tile(1)
+  for(;;);
+}
+
+template<unsigned Val>
+void depth_too_high_templ() {
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc loop tile (Val, *, Val) // expected-note{{active 'tile' clause defined here}}
+  for(;;)
+    for(;;);
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc loop tile (Val, *, Val) // expected-note 2{{active 'tile' clause defined here}}
+  for(;;)
+    for(;;)
+      // expected-error@+1{{while loop cannot appear in intervening code of a 'loop' with a 'tile' clause}}
+      while(true);
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc loop tile (Val, *, Val) // expected-note 2{{active 'tile' clause defined here}}
+  for(;;)
+    for(;;)
+      // expected-error@+1{{do loop cannot appear in intervening code of a 'loop' with a 'tile' clause}}
+      do{}while(true);
+
+  int Arr[Val+5];
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc loop tile (Val, *, Val) // expected-note 2{{active 'tile' clause defined here}}
+  for(;;)
+    for(auto x : Arr)
+      // expected-error@+1{{while loop cannot appear in intervening code of a 'loop' with a 'tile' clause}}
+      while(true)
+        for(;;);
+
+#pragma acc loop tile (Val, *, Val)
+  for(;;)
+    for(auto x : Arr)
+      for(;;)
+        while(true);
+}
+
+void depth_too_high() {
+  depth_too_high_templ<3>();
+
+int Arr[5];
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc loop tile (1, *, 3) // expected-note{{active 'tile' clause defined here}}
+  for(;;)
+    for(;;);
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc loop tile (1, *, 3) // expected-note 2{{active 'tile' clause defined here}}
+  for(;;)
+    for(;;)
+      // expected-error@+1{{while loop cannot appear in intervening code of a 'loop' with a 'tile' clause}}
+      while(true);
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc loop tile (1, *, 3) // expected-note 2{{active 'tile' clause defined here}}
+  for(;;)
+    for(;;)
+      // expected-error@+1{{do loop cannot appear in intervening code of a 'loop' with a 'tile' clause}}
+      do{}while(true);
+
+  // expected-error@+1{{'tile' clause specifies a loop count greater than the number of available loops}}
+#pragma acc loop tile (1, *, 3) // expected-note 2{{active 'tile' clause defined here}}
+  for(;;)
+    for(;;)
+      // expected-error@+1{{while loop cannot appear in intervening code of a 'loop' with a 'tile' clause}}
+      while(true)
+        for(;;);
+
+#pragma acc loop tile (1, *, 3)
+  for(;;)
+    for(auto x : Arr)
+      for(;;)
+        while(true);
+}
+
+template<unsigned Val>
+void not_single_loop_templ() {
+
+  int Arr[Val];
+
+#pragma acc loop tile (Val, *, 3) // expected-note{{active 'tile' clause defined here}}
+  for(;;) {
+    for (auto x : Arr)
+      for(;;);
+  // expected-error@+1{{more than one for-loop in a loop associated with OpenACC 'loop' construct with a 'tile' clause}}
+    for(;;)
+      for(;;);
+  }
+}
+
+void not_single_loop() {
+  not_single_loop_templ<3>(); // no diagnostic, was diagnosed in phase 1.
+
+  int Arr[5];
+
+#pragma acc loop tile (1, *, 3)// expected-note{{active 'tile' clause defined here}}
+  for(;;) {
+    for (auto x : Arr)
+      for(;;);
+  // expected-error@+1{{more than one for-loop in a loop associated with OpenACC 'loop' construct with a 'tile' clause}}
+    for(;;)
+      for(;;);
+  }
+}
+
+template<unsigned Val>
+void no_other_directives_templ() {
+
+  int Arr[Val];
+
+#pragma acc loop tile (Val, *, 3) // expected-note{{active 'tile' clause defined here}}
+  for(;;) {
+    for (auto x : Arr) {
+  // expected-error@+1{{OpenACC 'serial' construct cannot appear in intervening code of a 'loop' with a 'tile' clause}}
+#pragma acc serial
+      ;
+      for(;;);
+    }
+  }
+
+  // OK, in innermost
+#pragma acc loop tile (Val, *, 3)
+  for(;;) {
+    for (;;) {
+      for (auto x : Arr) {
+#pragma acc serial
+      ;
+      }
+    }
+  }
+}
+
+void no_other_directives() {
+  no_other_directives_templ<3>();
+  int Arr[5];
+
+#pragma acc loop tile (1, *, 3) // expected-note{{active 'tile' clause defined here}}
+  for(;;) {
+    for (auto x : Arr) {
+  // expected-error@+1{{OpenACC 'serial' construct cannot appear in intervening code of a 'loop' with a 'tile' clause}}
+#pragma acc serial
+      ;
+      for(;;);
+    }
+  }
+
+  // OK, in innermost
+#pragma acc loop tile (3, *, 3)
+  for(;;) {
+    for (;;) {
+      for (auto x : Arr) {
+#pragma acc serial
+      ;
+      }
+    }
+  }
+}
+
+void call();
+template<unsigned Val>
+void intervening_templ() {
+#pragma acc loop tile(1, Val, *) // expected-note{{active 'tile' clause defined here}}
+  for(;;) {
+    //expected-error@+1{{inner loops must be tightly nested inside a 'tile' clause on a 'loop' construct}}
+    call();
+    for(;;)
+      for(;;);
+  }
+
+#pragma acc loop tile(1, Val, *) // expected-note{{active 'tile' clause defined here}}
+  for(;;) {
+    //expected-error@+1{{inner loops must be tightly nested inside a 'tile' clause on a 'loop' construct}}
+    unsigned I;
+    for(;;)
+      for(;;);
+  }
+
+#pragma acc loop tile(1, Val, *)
+  for(;;) {
+    for(;;)
+      for(;;)
+        call();
+  }
+}
+
+void intervening() {
+  intervening_templ<3>();
+
+#pragma acc loop tile(1, 2, *) // expected-note{{active 'tile' clause defined here}}
+  for(;;) {
+    //expected-error@+1{{inner loops must be tightly nested inside a 'tile' clause on a 'loop' construct}}
+    call();
+    for(;;)
+      for(;;);
+  }
+
+#pragma acc loop tile(1, 2, *) // expected-note{{active 'tile' clause defined here}}
+  for(;;) {
+    //expected-error@+1{{inner loops must be tightly nested inside a 'tile' clause on a 'loop' construct}}
+    unsigned I;
+    for(;;)
+      for(;;);
+  }
+
+#pragma acc loop tile(1, 2, *)
+  for(;;) {
+    for(;;)
+      for(;;)
+        call();
+  }
+}
+
+void collapse_tile_depth() {
+  // expected-error@+4{{'collapse' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+3{{active 'collapse' clause defined here}}
+  // expected-error@+2{{'tile' clause specifies a loop count greater than the number of available loops}}
+  // expected-note@+1{{active 'tile' clause defined here}}
+#pragma acc loop tile(1, 2, 3) collapse (3)
+  for(;;) {
+    for(;;);
+  }
+}
diff --git a/clang/test/SemaTemplate/concepts-lambda.cpp b/clang/test/SemaTemplate/concepts-lambda.cpp
index 9c5807bbabdcb..829a71bc703fe 100644
--- a/clang/test/SemaTemplate/concepts-lambda.cpp
+++ b/clang/test/SemaTemplate/concepts-lambda.cpp
@@ -279,3 +279,18 @@ void dependent_capture_packs() {
   L(V<0>{}, V<1>{}, V<2>{})(V<3>{}, V<4>{})(1);
 }
 } // namespace init_captures
+
+namespace GH110721 {
+
+template <int N> void connect() {
+  int x = N, y = N;
+  [x, y = y]()
+    requires requires { x; }
+  {}();
+}
+
+void foo() {
+  connect<42>();
+}
+
+} // namespace GH110721
diff --git a/clang/test/SemaTemplate/concepts-out-of-line-def.cpp b/clang/test/SemaTemplate/concepts-out-of-line-def.cpp
index 8ca399a0f729a..fe8f74928fc37 100644
--- a/clang/test/SemaTemplate/concepts-out-of-line-def.cpp
+++ b/clang/test/SemaTemplate/concepts-out-of-line-def.cpp
@@ -516,7 +516,7 @@ concept something_interesting = requires {
 };
 
 template <class T>
-struct X {
+struct X { // #defined-here
       void foo() requires requires { requires is_not_same_v<T, int>; };
       void bar(decltype(requires { requires is_not_same_v<T, int>; }));
 };
@@ -524,7 +524,8 @@ struct X {
 template <class T>
 void X<T>::foo() requires requires { requires something_interesting<T>; } {}
 // expected-error@-1{{definition of 'foo' does not match any declaration}}
-// expected-note@*{{}}
+// expected-note@#defined-here{{defined here}}
+// expected-note@-8{{member declaration nearly matches}}
 
 template <class T>
 void X<T>::foo() requires requires { requires is_not_same_v<T, int>; } {} // ok
@@ -532,6 +533,7 @@ void X<T>::foo() requires requires { requires is_not_same_v<T, int>; } {} // ok
 template <class T>
 void X<T>::bar(decltype(requires { requires something_interesting<T>; })) {}
 // expected-error@-1{{definition of 'bar' does not match any declaration}}
+// expected-note@#defined-here{{defined here}}
 
 template <class T>
 void X<T>::bar(decltype(requires { requires is_not_same_v<T, int>; })) {}
diff --git a/clang/test/SemaTemplate/cwg2398.cpp b/clang/test/SemaTemplate/cwg2398.cpp
index 6dc7af6ea1789..1b55af782fcb7 100644
--- a/clang/test/SemaTemplate/cwg2398.cpp
+++ b/clang/test/SemaTemplate/cwg2398.cpp
@@ -106,12 +106,10 @@ namespace type_pack3 {
   template<class T3> struct B;
 
   template<template<class T4              > class TT1, class T5              > struct B<TT1<T5        >>;
-  // new-note@-1 {{template is declared here}}
-  template<template<class T6, class ...T7s> class TT2, class T8, class ...T9s> struct B<TT2<T8, T9s...>>;
-  // old-note@-1 {{template is declared here}}
+
+  template<template<class T6, class ...T7s> class TT2, class T8, class ...T9s> struct B<TT2<T8, T9s...>> {};
 
   template struct B<A<int>>;
-  // expected-error@-1 {{explicit instantiation of undefined template}}
 } // namespace type_pack3
 
 namespace gcc_issue {
@@ -364,6 +362,73 @@ namespace classes {
   } // namespace defaulted
 } // namespace classes
 
+namespace packs {
+  namespace t1 {
+    // FIXME: This should be rejected
+    template<template<int, int...> class> struct A {};
+    // old-note@-1 {{previous non-type template parameter with type 'int' is here}}
+
+    template<char> struct B;
+    // old-note@-1 {{template non-type parameter has a different type 'char' in template argument}}
+
+    template struct A<B>;
+    // old-error@-1 {{has different template parameters}}
+  } // namespace t1
+  namespace t2 {
+    template<template<char, int...> class> struct A {};
+    // old-note@-1 {{previous non-type template parameter with type 'char' is here}}
+
+    template<int> struct B;
+    // old-note@-1 {{template non-type parameter has a different type 'int' in template argument}}
+
+    template struct A<B>;
+    // old-error@-1 {{has different template parameters}}
+  } // namespace t2
+  namespace t3 {
+    // FIXME: This should be rejected
+    template<template<int...> class> struct A {};
+    // old-note@-1 {{previous non-type template parameter with type 'int' is here}}
+
+    template<char> struct B;
+    // old-note@-1 {{template non-type parameter has a different type 'char' in template argument}}
+
+    template struct A<B>;
+    // old-error@-1 {{has different template parameters}}
+  } // namespace t3
+  namespace t4 {
+    template<template<char...> class> struct A {};
+    // old-note@-1 {{previous non-type template parameter with type 'char' is here}}
+
+    template<int> struct B;
+    // old-note@-1 {{template non-type parameter has a different type 'int' in template argument}}
+
+    template struct A<B>;
+    // old-error@-1 {{has different template parameters}}
+  } // namespace t4
+} // namespace packs
+
+namespace partial {
+  namespace t1 {
+    template<template<class... T1s> class TT1> struct A {};
+
+    template<template<class T2> class TT2> struct A<TT2>;
+    // new-note@-1 {{template is declared here}}
+
+    template<class... T3s> struct B;
+    template struct A<B>;
+    // new-error@-1 {{explicit instantiation of undefined template}}
+  } // namespace t1
+  namespace t2 {
+    template<template<class... T1s> class TT1> struct A;
+
+    template<template<class T2> class TT2> struct A<TT2> {};
+
+    template<class T3> struct B;
+    template struct A<B>;
+  } // namespace t1
+
+} // namespace partial
+
 namespace regression1 {
   template <typename T, typename Y> struct map {};
   template <typename T> class foo {};
@@ -380,6 +445,93 @@ namespace regression1 {
   }
 } // namespace regression1
 
+namespace constraints {
+  template <class T> concept C1 = true;
+  // new-note@-1 {{similar constraint expression here}}
+  // new-note@-2 2{{similar constraint expressions not considered equivalent}}
+
+  template <class T> concept C2 = C1<T> && true;
+  // new-note@-1 2{{similar constraint expression here}}
+
+  template <class T> concept D1 = true;
+  // new-note@-1 {{similar constraint expressions not considered equivalent}}
+
+  namespace t1 {
+    template<template<C1, class... T1s> class TT1> // new-note {{TT1' declared here}}
+    struct A {};
+    template<D1, class T2> struct B {}; // new-note {{'B' declared here}}
+    template struct A<B>;
+    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
+  } // namespace t1
+  namespace t2 {
+    template<template<C2, class... T1s> class TT1> struct A {};
+    template<C1, class T2> struct B {};
+    template struct A<B>;
+  } // namespace t2
+  namespace t3 {
+    template<template<C1, class... T1s> class TT1> // new-note {{'TT1' declared here}}
+    struct A {};
+    template<C2, class T2> struct B {}; // new-note {{'B' declared here}}
+    template struct A<B>;
+    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
+  } // namespace t2
+  namespace t4 {
+    // FIXME: This should be accepted.
+    template<template<C1... T1s> class TT1> // new-note {{'TT1' declared here}}
+    struct A {};
+    template<C1 T2> struct B {}; // new-note {{'B' declared here}}
+    template struct A<B>;
+    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
+  } // namespace t4
+  namespace t5 {
+    // FIXME: This should be accepted
+    template<template<C2... T1s> class TT1> // new-note {{'TT1' declared here}}
+    struct A {};
+    template<C1 T2> struct B {}; // new-note {{'B' declared here}}
+    template struct A<B>;
+    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
+  } // namespace t5
+  namespace t6 {
+    template<template<C1... T1s> class TT1> // new-note {{'TT1' declared here}}
+    struct A {};
+    template<C2 T2> struct B {}; // new-note {{'B' declared here}}
+    template struct A<B>;
+    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
+  } // namespace t6
+  namespace t7 {
+    template<template<class... T1s> class TT1>
+    struct A {};
+    template<C1 T2> struct B {};
+    template struct A<B>;
+  } // namespace t7
+  namespace t8 {
+    template<template<C1... T1s> class TT1>
+    struct A {};
+    template<class T2> struct B {};
+    template struct A<B>;
+  } // namespace t8
+  namespace t9 {
+    template<template<C1... T1s> class TT1> // new-note {{'TT1' declared here}}
+    struct A {};
+    template<D1 T2> struct B {}; // new-note {{'B' declared here}}
+    template struct A<B>;
+    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
+  } // namespace t9
+  namespace t10 {
+    template<template<class...> requires C1<int> class TT1> // new-note {{'TT1' declared here}}
+    struct A {};
+
+    template<class> requires C2<int> struct B {}; // new-note {{'B' declared here}}
+    template struct A<B>;
+    // new-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
+  } // namespace t10
+  namespace t11 {
+    template<template<class...> requires C2<int> class TT1> struct A {};
+    template<class> requires C1<int> struct B {};
+    template struct A<B>;
+  } // namespace t11
+} // namespace constraints
+
 namespace regression2 {
   template <class> struct D {};
 
@@ -389,3 +541,9 @@ namespace regression2 {
   template <typename, int> struct Matrix;
   template struct D<Matrix<double, 3>>;
 } // namespace regression2
+
+namespace regression3 {
+  template <template <auto...> class TT> struct A {};
+  template <auto, int> struct B;
+  template struct A<B>;
+} // namespace regression3
diff --git a/clang/test/SemaTemplate/recovery-crash.cpp b/clang/test/SemaTemplate/recovery-crash.cpp
index a451d9d108da5..88e989aeb4064 100644
--- a/clang/test/SemaTemplate/recovery-crash.cpp
+++ b/clang/test/SemaTemplate/recovery-crash.cpp
@@ -47,7 +47,7 @@ namespace PR16225 {
 namespace test1 {
   template <typename> class ArraySlice {};
   class Foo;
-  class NonTemplateClass {
+  class NonTemplateClass { // #defined-here
     void MemberFunction(ArraySlice<Foo>, int);
     template <class T> void MemberFuncTemplate(ArraySlice<T>, int);
   };
@@ -61,7 +61,8 @@ namespace test1 {
     // expected-error@+1 {{member 'UndeclaredMethod' used before its declaration}}
     UndeclaredMethod(resource_data);
   }
-  // expected-error@+2 {{out-of-line definition of 'UndeclaredMethod' does not match any declaration}}
-  // expected-note@+1 {{member is declared here}}
+  // expected-error@+3 {{out-of-line definition of 'UndeclaredMethod' does not match any declaration}}
+  // expected-note@+2 {{member is declared here}}
+  // expected-note@#defined-here {{defined here}}
   void NonTemplateClass::UndeclaredMethod() {}
 }
diff --git a/clang/test/SemaTemplate/temp_arg_nontype.cpp b/clang/test/SemaTemplate/temp_arg_nontype.cpp
index f360aa14950ed..8b270b22a12b4 100644
--- a/clang/test/SemaTemplate/temp_arg_nontype.cpp
+++ b/clang/test/SemaTemplate/temp_arg_nontype.cpp
@@ -387,12 +387,11 @@ namespace PR17696 {
 
 namespace partial_order_different_types {
   template<int, int, typename T, typename, T> struct A;
-  template<int N, typename T, typename U, T V> struct A<0, N, T, U, V>; // expected-note {{matches}}
-  // FIXME: It appears that this partial specialization should be ill-formed as
-  // it is not more specialized than the primary template. V is not deducible
-  // because it does not have the same type as the corresponding parameter.
-  template<int N, typename T, typename U, U V> struct A<0, N, T, U, V> {}; // expected-note {{matches}}
-  A<0, 0, int, int, 0> a; // expected-error {{ambiguous}}
+  // expected-note@-1 {{template is declared here}}
+  template<int N, typename T, typename U, T V> struct A<0, N, T, U, V> {};
+  template<int N, typename T, typename U, U V> struct A<0, N, T, U, V>;
+  // expected-error@-1 {{class template partial specialization is not more specialized than the primary template}}
+  A<0, 0, int, int, 0> a;
 }
 
 namespace partial_order_references {
@@ -458,13 +457,24 @@ namespace dependent_nested_partial_specialization {
 namespace nondependent_default_arg_ordering {
   int n, m;
   template<typename A, A B = &n> struct X {};
+
   template<typename A> void f(X<A>);
+  // expected-note@-1 {{candidate function}}
   template<typename A> void f(X<A, &m>);
+  // expected-note@-1 {{candidate function}}
   template<typename A, A B> void f(X<A, B>);
+  // expected-note@-1 2{{candidate function}}
   template<template<typename U, U> class T, typename A, int *B> void f(T<A, B>);
+  // expected-note@-1 2{{candidate function}}
+
+  // FIXME: When partial ordering, we get an inconsistent deduction between
+  // `A` (type-parameter-0-0) and `int *`, when deducing the first parameter.
+  // The deduction mechanism needs to be extended to be able to correctly
+  // handle these cases where the argument's template parameters appear in
+  // the result.
   void g() {
-    X<int *, &n> x; f(x);
-    X<int *, &m> y; f(y);
+    X<int *, &n> x; f(x); // expected-error {{call to 'f' is ambiguous}}
+    X<int *, &m> y; f(y); // expected-error {{call to 'f' is ambiguous}}
   }
 }
 
diff --git a/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp b/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp
index ae06055bf5265..8d9356bb3201c 100644
--- a/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp
+++ b/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp
@@ -613,3 +613,11 @@ struct {
 template<typename T>
 using a = s<f(T::x)>;
 }
+
+namespace GH73460 {
+  template <class T, T, T> struct A;
+  template <class T, T n> struct A<T, n, n> {};
+
+  int j;
+  template struct A<int&, j, j>;
+} // namespace GH73460
diff --git a/clang/test/SemaTemplate/temp_arg_template.cpp b/clang/test/SemaTemplate/temp_arg_template.cpp
index a7236669276aa..9908af5e78669 100644
--- a/clang/test/SemaTemplate/temp_arg_template.cpp
+++ b/clang/test/SemaTemplate/temp_arg_template.cpp
@@ -1,33 +1,40 @@
 // RUN: %clang_cc1 -fsyntax-only -verify=expected,precxx17 %std_cxx98-14 %s
 // RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx17 -std=c++17 %s
 
-template<template<typename T> class X> struct A; // expected-note 2{{previous template template parameter is here}}
+template<template<typename T> class X> struct A; // #A
+// expected-note@-1 2{{previous template template parameter is here}}
 
 template<template<typename T, int I> class X> struct B; // expected-note{{previous template template parameter is here}}
 
-template<template<int I> class X> struct C;  // expected-note {{previous non-type template parameter with type 'int' is here}}
+template<template<int I> class X> struct C;
+// precxx17-error@-1 {{deduced non-type template argument does not have the same type as the corresponding template parameter ('int' vs 'const int &')}}
+// cxx17-error@-2 {{conversion from 'int' to 'const int &' in converted constant expression would bind reference to a temporary}}
+// expected-note@-3 {{previous template template parameter is here}}
 
-template<class> struct X; // expected-note{{too few template parameters in template template argument}}
-template<int N> struct Y; // expected-note{{template parameter has a different kind in template argument}}
+template<class> struct X; // expected-note {{template is declared here}}
+template<int N> struct Y; // expected-note {{template parameter is declared here}}
 template<long N> struct Ylong;
-template<const int &N> struct Yref; // expected-note{{template non-type parameter has a different type 'const int &' in template argument}}
+template<const int &N> struct Yref; // precxx17-note {{template parameter is declared here}}
 
 namespace N {
   template<class> struct Z;
 }
-template<class, class> struct TooMany; // expected-note{{too many template parameters in template template argument}}
+template<class, class> struct TooMany; // expected-note{{template is declared here}}
 
 
 A<X> *a1;
 A<N::Z> *a2;
 A< ::N::Z> *a3;
 
-A<Y> *a4; // expected-error{{template template argument has different template parameters than its corresponding template template parameter}}
-A<TooMany> *a5; // expected-error{{template template argument has different template parameters than its corresponding template template parameter}}
-B<X> *a6; // expected-error{{template template argument has different template parameters than its corresponding template template parameter}}
+A<Y> *a4; // expected-error@#A {{template argument for non-type template parameter must be an expression}}
+          // expected-note@-1 {{different template parameters}}
+A<TooMany> *a5; // expected-error {{too few template arguments for class template 'TooMany'}}
+                // expected-note@-1 {{different template parameters}}
+B<X> *a6; // expected-error {{too many template arguments for class template 'X'}}
+          // expected-note@-1 {{different template parameters}}
 C<Y> *a7;
 C<Ylong> *a8;
-C<Yref> *a9; // expected-error{{template template argument has different template parameters than its corresponding template template parameter}}
+C<Yref> *a9; // expected-note {{different template parameters}}
 
 template<typename T> void f(int);
 
@@ -103,9 +110,9 @@ void foo() {
 
 namespace CheckDependentNonTypeParamTypes {
   template<template<typename T, typename U, T v> class X> struct A {
+    // expected-note@-1 {{previous template template parameter is here}}
     void f() {
-      X<int, void*, 3> x; // precxx17-error {{does not refer to any declaration}} \
-                             cxx17-error {{value of type 'int' is not implicitly convertible to 'void *'}}
+      X<int, void*, 3> x;
     }
     void g() {
       X<int, long, 3> x;
@@ -124,15 +131,16 @@ namespace CheckDependentNonTypeParamTypes {
     }
   };
 
-  template<typename T, typename U, U v> struct B { // precxx17-note {{parameter}}
+  template<typename T, typename U, U v> struct B {
+    // expected-error@-1 {{conflicting deduction 'U' against 'T' for parameter}}
     static const U value = v;
   };
 
   // FIXME: This should probably be rejected, but the rules are at best unclear.
-  A<B> ab;
+  A<B> ab; // expected-note {{different template parameters}}
 
   void use() {
-    ab.f(); // expected-note {{instantiation of}}
+    ab.f();
     ab.g();
     ab.h();
   }
diff --git a/clang/test/SemaTemplate/temp_arg_template_p0522.cpp b/clang/test/SemaTemplate/temp_arg_template_p0522.cpp
index 6f6568b9ab776..441b9e5af9d3d 100644
--- a/clang/test/SemaTemplate/temp_arg_template_p0522.cpp
+++ b/clang/test/SemaTemplate/temp_arg_template_p0522.cpp
@@ -1,14 +1,16 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++20 %s
 
-// expected-note@temp_arg_template_p0522.cpp:* 1+{{}}
+// expected-note@temp_arg_template_p0522.cpp:* 1+{{template is declared here}}
+// expected-note@temp_arg_template_p0522.cpp:* 1+{{template parameter is declared here}}
+// expected-note@temp_arg_template_p0522.cpp:* 1+{{previous template template parameter is here}}
 
-template<template<int> typename> struct Ti;
-template<template<int...> typename> struct TPi;
+template<template<int> typename> struct Ti; // #Ti
+template<template<int...> typename> struct TPi; // #TPi
 template<template<int, int...> typename> struct TiPi;
 template<template<int..., int...> typename> struct TPiPi; // FIXME: Why is this not ill-formed?
 
-template<typename T, template<T> typename> struct tT0;
-template<template<typename T, T> typename> struct Tt0;
+template<typename T, template<T> typename> struct tT0; // #tT0
+template<template<typename T, T> typename> struct Tt0; // #Tt0
 
 template<template<typename> typename> struct Tt;
 template<template<typename, typename...> typename> struct TtPt;
@@ -20,7 +22,7 @@ template<int...> struct Pi;
 template<int, int, int...> struct iiPi;
 
 template<int, typename = int> struct iDt;
-template<int, typename> struct it;
+template<int, typename> struct it; // #it
 
 template<typename T, T v> struct t0;
 
@@ -31,10 +33,14 @@ namespace IntParam {
         Ti<iDi>,
         Ti<Pi>,
         Ti<iDt>>;
-  using err1 = Ti<ii>; // expected-error {{different template parameters}}
-  using err2 = Ti<iiPi>; // expected-error {{different template parameters}}
-  using err3 = Ti<t0>; // expected-error {{different template parameters}}
-  using err4 = Ti<it>; // expected-error {{different template parameters}}
+  using err1 = Ti<ii>; // expected-error {{too few template arguments for class template 'ii'}}
+                       // expected-note@-1 {{different template parameters}}
+  using err2 = Ti<iiPi>; // expected-error {{too few template arguments for class template 'iiPi'}}
+                         // expected-note@-1 {{different template parameters}}
+  using err3 = Ti<t0>; // expected-error@#Ti {{template argument for template type parameter must be a type}}
+                       // expected-note@-1 {{different template parameters}}
+  using err4 = Ti<it>; // expected-error {{too few template arguments for class template 'it'}}
+                       // expected-note@-1 {{different template parameters}}
 }
 
 // These are accepted by the backwards-compatibility "parameter pack in
@@ -42,9 +48,12 @@ namespace IntParam {
 namespace IntPackParam {
   using ok = TPi<Pi>;
   using ok_compat = Pt<TPi<i>, TPi<iDi>, TPi<ii>, TPi<iiPi>>;
-  using err1 = TPi<t0>; // expected-error {{different template parameters}}
-  using err2 = TPi<iDt>; // expected-error {{different template parameters}}
-  using err3 = TPi<it>; // expected-error {{different template parameters}}
+  using err1 = TPi<t0>; // expected-error@#TPi {{template argument for template type parameter must be a type}}
+                        // expected-note@-1 {{different template parameters}}
+  using err2 = TPi<iDt>; // expected-error@#TPi {{template argument for template type parameter must be a type}}
+                         // expected-note@-1 {{different template parameters}}
+  using err3 = TPi<it>; // expected-error@#TPi {{template argument for template type parameter must be a type}}
+                        // expected-note@-1 {{different template parameters}}
 }
 
 namespace IntAndPackParam {
@@ -55,42 +64,50 @@ namespace IntAndPackParam {
 
 namespace DependentType {
   using ok = Pt<tT0<int, i>, tT0<int, iDi>>;
-  using err1 = tT0<int, ii>; // expected-error {{different template parameters}}
+  using err1 = tT0<int, ii>; // expected-error {{too few template arguments for class template 'ii'}}
+                             // expected-note@-1 {{different template parameters}}
   using err2 = tT0<short, i>; // FIXME: should this be OK?
   using err2a = tT0<long long, i>; // FIXME: should this be OK (if long long is larger than int)?
-  using err2b = tT0<void*, i>; // expected-error {{different template parameters}}
-  using err3 = tT0<short, t0>; // expected-error {{different template parameters}}
+  using err2b = tT0<void*, i>; // expected-error@#tT0 {{value of type 'void *' is not implicitly convertible to 'int'}}
+                               // expected-note@-1 {{different template parameters}}
+  using err3 = tT0<short, t0>; // expected-error@#tT0 {{template argument for template type parameter must be a type}}
+                               // expected-note@-1 {{different template parameters}}
 
   using ok2 = Tt0<t0>;
-  using err4 = Tt0<it>; // expected-error {{different template parameters}}
+  using err4 = Tt0<it>; // expected-error@#Tt0 {{template argument for non-type template parameter must be an expression}}
+                        // expected-note@-1 {{different template parameters}}
 }
 
 namespace Auto {
-  template<template<int> typename T> struct TInt {};
-  template<template<int*> typename T> struct TIntPtr {};
+  template<template<int> typename T> struct TInt {}; // #TInt
+  template<template<int*> typename T> struct TIntPtr {}; // #TIntPtr
   template<template<auto> typename T> struct TAuto {};
   template<template<auto*> typename T> struct TAutoPtr {};
   template<template<decltype(auto)> typename T> struct TDecltypeAuto {};
   template<auto> struct Auto;
-  template<auto*> struct AutoPtr;
+  template<auto*> struct AutoPtr; // #AutoPtr
   template<decltype(auto)> struct DecltypeAuto;
   template<int> struct Int;
   template<int*> struct IntPtr;
 
   TInt<Auto> ia;
-  TInt<AutoPtr> iap; // FIXME: ill-formed (?)
+  TInt<AutoPtr> iap; // expected-error@#TInt {{non-type template parameter '' with type 'auto *' has incompatible initializer of type 'int'}}
+                     // expected-note@-1 {{different template parameters}}
   TInt<DecltypeAuto> ida;
   TInt<Int> ii;
-  TInt<IntPtr> iip; // expected-error {{different template parameters}}
+  TInt<IntPtr> iip; // expected-error@#TInt {{conversion from 'int' to 'int *' is not allowed in a converted constant expression}}
+                    // expected-note@-1 {{different template parameters}}
 
   TIntPtr<Auto> ipa;
   TIntPtr<AutoPtr> ipap;
   TIntPtr<DecltypeAuto> ipda;
-  TIntPtr<Int> ipi; // expected-error {{different template parameters}}
+  TIntPtr<Int> ipi; // expected-error@#TIntPtr {{value of type 'int *' is not implicitly convertible to 'int'}}
+                    // expected-note@-1 {{different template parameters}}
   TIntPtr<IntPtr> ipip;
 
   TAuto<Auto> aa;
-  TAuto<AutoPtr> aap; // FIXME: ill-formed (?)
+  TAuto<AutoPtr> aap; // expected-error@#AutoPtr {{could not match 'auto *' against 'auto'}}
+                      // expected-note@-1 {{different template parameters}}
   TAuto<Int> ai; // FIXME: ill-formed (?)
   TAuto<IntPtr> aip; // FIXME: ill-formed (?)
 
@@ -111,7 +128,8 @@ namespace Auto {
   // parameters (such as 'user-defined-type &') that are not valid 'auto'
   // parameters.
   TDecltypeAuto<Auto> daa;
-  TDecltypeAuto<AutoPtr> daap; // FIXME: should probably be ill-formed
+  TDecltypeAuto<AutoPtr> daap; // expected-error@#AutoPtr {{could not match 'auto *' against 'decltype(auto)'}}
+                               // expected-note@-1 {{different template parameters}}
 
   int n;
   template<auto A, decltype(A) B = &n> struct SubstFailure;
@@ -128,7 +146,7 @@ namespace GH62529 {
 } // namespace GH62529
 
 namespace GH101394 {
-  struct X {};
+  struct X {}; // #X
   struct Y {
     constexpr Y(const X &) {}
   };
@@ -139,8 +157,12 @@ namespace GH101394 {
     template struct A<B>;
   } // namespace t1
   namespace t2 {
-    template<template<Y> class> struct A {};
-    template<X> struct B;
-    template struct A<B>; // expected-error {{different template parameters}}
+    template<template<Y> class> struct A {}; // #A
+    template<X> struct B; // #B
+    template struct A<B>;
+    // expected-error@#A {{no viable conversion from 'const Y' to 'X'}}
+    // expected-note@-2  {{different template parameters}}
+    // expected-note@#X 2{{not viable}}
+    // expected-note@#B  {{passing argument to parameter here}}
   } // namespace t2
 } // namespace GH101394
diff --git a/clang/test/Templight/templight-empty-entries-fix.cpp b/clang/test/Templight/templight-empty-entries-fix.cpp
index e17be9012e59c..d13b748068efe 100644
--- a/clang/test/Templight/templight-empty-entries-fix.cpp
+++ b/clang/test/Templight/templight-empty-entries-fix.cpp
@@ -314,6 +314,18 @@ void foo() {
 // CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
 // CHECK: {{^poi:[ ]+''$}}
 // CHECK-LABEL: {{^---$}}
+// CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
+// CHECK: {{^kind:[ ]+PartialOrderingTTP$}}
+// CHECK: {{^event:[ ]+Begin$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:5'$}}
+// CHECK-LABEL: {{^---$}}
+// CHECK: {{^name:[ ]+unnamed template template parameter 0 of d$}}
+// CHECK: {{^kind:[ ]+PartialOrderingTTP$}}
+// CHECK: {{^event:[ ]+End$}}
+// CHECK: {{^orig:[ ]+'.*templight-empty-entries-fix.cpp:289:35'$}}
+// CHECK: {{^poi:[ ]+'.*templight-empty-entries-fix.cpp:295:5'$}}
+// CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+d$}}
 // CHECK: {{^kind:[ ]+ExplicitTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
diff --git a/clang/test/Templight/templight-prior-template-arg.cpp b/clang/test/Templight/templight-prior-template-arg.cpp
index e9b1dd47bb603..14bcb6a4d48f6 100644
--- a/clang/test/Templight/templight-prior-template-arg.cpp
+++ b/clang/test/Templight/templight-prior-template-arg.cpp
@@ -10,63 +10,76 @@ class B {};
 // CHECK: {{^kind:[ ]+PriorTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:5:40'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:72:1'$}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:1'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'B::Outer'$}}
 // CHECK: {{^kind:[ ]+PriorTemplateArgumentSubstitution$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:5:40'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:72:1'$}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:1'$}}
+//
+// CHECK-LABEL: {{^---$}}
+// CHECK: {{^name:[ ]+'B::Outer'$}}
+// CHECK: {{^kind:[ ]+PartialOrderingTTP$}}
+// CHECK: {{^event:[ ]+Begin$}}
+// CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:5:40'}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:3'$}}
+// CHECK-LABEL: {{^---$}}
+// CHECK: {{^name:[ ]+'B::Outer'$}}
+// CHECK: {{^kind:[ ]+PartialOrderingTTP$}}
+// CHECK: {{^event:[ ]+End$}}
+// CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:5:40'}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:3'$}}
 //
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'B<A>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:6:7'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:72:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:6'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'B<A>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:6:7'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:72:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:6'$}}
 //
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'B<A>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:6:7'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:72:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:6'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'B<A>'$}}
 // CHECK: {{^kind:[ ]+TemplateInstantiation$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:6:7'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:72:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:6'$}}
 //
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'B<A>'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:6:7'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:72:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:6'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'B<A>'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:6:7'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:72:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:6'$}}
 //
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'B<A>'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+Begin$}}
 // CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:6:7'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:72:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:6'$}}
 // CHECK-LABEL: {{^---$}}
 // CHECK: {{^name:[ ]+'B<A>'$}}
 // CHECK: {{^kind:[ ]+Memoization$}}
 // CHECK: {{^event:[ ]+End$}}
 // CHECK: {{^orig:[ ]+'.*templight-prior-template-arg.cpp:6:7'}}
-// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:72:6'$}}
+// CHECK: {{^poi:[ ]+'.*templight-prior-template-arg.cpp:85:6'$}}
 B<A> b;
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index d188f794bad20..e6c323775c999 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2839,6 +2839,11 @@ void OpenACCClauseEnqueue::VisitNumGangsClause(const OpenACCNumGangsClause &C) {
     Visitor.AddStmt(IE);
 }
 
+void OpenACCClauseEnqueue::VisitTileClause(const OpenACCTileClause &C) {
+  for (Expr *IE : C.getSizeExprs())
+    Visitor.AddStmt(IE);
+}
+
 void OpenACCClauseEnqueue::VisitPrivateClause(const OpenACCPrivateClause &C) {
   VisitVarList(C);
 }
diff --git a/clang/tools/libclang/CXCursor.cpp b/clang/tools/libclang/CXCursor.cpp
index 4e068f272a153..562228cc334f3 100644
--- a/clang/tools/libclang/CXCursor.cpp
+++ b/clang/tools/libclang/CXCursor.cpp
@@ -337,6 +337,7 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
   case Stmt::SYCLUniqueStableNameExprClass:
   case Stmt::EmbedExprClass:
   case Stmt::HLSLOutArgExprClass:
+  case Stmt::OpenACCAsteriskSizeExprClass:
     K = CXCursor_UnexposedExpr;
     break;
 
diff --git a/clang/tools/libclang/CXIndexDataConsumer.cpp b/clang/tools/libclang/CXIndexDataConsumer.cpp
index 8d364ed8876a1..ced94e13baf12 100644
--- a/clang/tools/libclang/CXIndexDataConsumer.cpp
+++ b/clang/tools/libclang/CXIndexDataConsumer.cpp
@@ -952,18 +952,12 @@ void CXIndexDataConsumer::addContainerInMap(const DeclContext *DC,
   if (!DC)
     return;
 
-  ContainerMapTy::iterator I = ContainerMap.find(DC);
-  if (I == ContainerMap.end()) {
-    if (container)
-      ContainerMap[DC] = container;
-    return;
-  }
   // Allow changing the container of a previously seen DeclContext so we
   // can handle invalid user code, like a function re-definition.
   if (container)
-    I->second = container;
+    ContainerMap[DC] = container;
   else
-    ContainerMap.erase(I);
+    ContainerMap.erase(DC);
 }
 
 CXIdxClientEntity CXIndexDataConsumer::getClientEntity(const Decl *D) const {
diff --git a/clang/unittests/AST/ByteCode/Descriptor.cpp b/clang/unittests/AST/ByteCode/Descriptor.cpp
index 9e09c7f15ac9a..b3517d8731c85 100644
--- a/clang/unittests/AST/ByteCode/Descriptor.cpp
+++ b/clang/unittests/AST/ByteCode/Descriptor.cpp
@@ -251,7 +251,7 @@ TEST(Descriptor, Primitives) {
     ASSERT_TRUE(NE1.isLive());
     ASSERT_EQ(NE1.getIndex(), 0);
     ASSERT_TRUE(NE1.isInitialized());
-    ASSERT_FALSE(NE1.isArrayElement());
+    ASSERT_TRUE(NE1.isArrayElement());
     ASSERT_TRUE(NE1.isField());
     ASSERT_FALSE(NE1.inArray());
     ASSERT_FALSE(NE1.isArrayRoot());
@@ -279,7 +279,7 @@ TEST(Descriptor, Primitives) {
     ASSERT_TRUE(NE2.isLive());
     ASSERT_EQ(NE2.getIndex(), 0);
     ASSERT_TRUE(NE2.isInitialized());
-    ASSERT_FALSE(NE2.isArrayElement());
+    ASSERT_TRUE(NE2.isArrayElement());
     ASSERT_TRUE(NE2.isField());
     ASSERT_FALSE(NE2.inArray());
     ASSERT_FALSE(NE2.isArrayRoot());
@@ -344,7 +344,7 @@ TEST(Descriptor, Primitives) {
     ASSERT_NE(NE1, PF4);
     ASSERT_NE(NE1, E1);
     ASSERT_TRUE(NE1.isLive());
-    ASSERT_FALSE(NE1.isArrayElement());
+    ASSERT_TRUE(NE1.isArrayElement());
     ASSERT_TRUE(NE1.isArrayRoot());
     ASSERT_FALSE(NE1.getFieldDesc()->isCompositeArray());
     ASSERT_TRUE(NE1.getFieldDesc()->isPrimitiveArray());
@@ -375,7 +375,7 @@ TEST(Descriptor, Primitives) {
     ASSERT_NE(NE3, PF4);
     ASSERT_NE(NE3, E1);
     ASSERT_TRUE(NE3.isLive());
-    ASSERT_FALSE(NE3.isArrayElement());
+    ASSERT_TRUE(NE3.isArrayElement());
     ASSERT_TRUE(NE3.isArrayRoot());
     ASSERT_FALSE(NE3.getFieldDesc()->isCompositeArray());
     ASSERT_TRUE(NE3.getFieldDesc()->isPrimitiveArray());
diff --git a/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp b/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp
index 4227a6bfdeba8..877bfce8b2709 100644
--- a/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp
@@ -2873,6 +2873,40 @@ TEST_P(UncheckedOptionalAccessTest, OptionalValueStruct) {
   )");
 }
 
+// FIXME: A case that we should handle but currently don't.
+// When there is a field of type reference to non-optional, we may
+// stop recursively creating storage locations.
+// E.g., the field `second` below in `pair` should eventually lead to
+// the optional `x` in `A`.
+TEST_P(UncheckedOptionalAccessTest, NestedOptionalThroughNonOptionalRefField) {
+  ExpectDiagnosticsFor(R"(
+    #include "unchecked_optional_access_test.h"
+
+    struct A {
+      $ns::$optional<int> x;
+    };
+
+    struct pair {
+      int first;
+      const A &second;
+    };
+
+    struct B {
+      $ns::$optional<pair>& nonConstGetRef();
+    };
+
+    void target(B b) {
+      const auto& maybe_pair = b.nonConstGetRef();
+      if (!maybe_pair.has_value())
+        return;
+
+      if(!maybe_pair->second.x.has_value())
+        return;
+      maybe_pair->second.x.value();  // [[unsafe]]
+    }
+  )");
+}
+
 TEST_P(UncheckedOptionalAccessTest, OptionalValueInitialization) {
   ExpectDiagnosticsFor(
       R"(
diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp
index 1884d41a5f23f..a89adfa3f4fdd 100644
--- a/clang/unittests/Format/TokenAnnotatorTest.cpp
+++ b/clang/unittests/Format/TokenAnnotatorTest.cpp
@@ -308,6 +308,11 @@ TEST_F(TokenAnnotatorTest, UnderstandsUsesOfStarAndAmp) {
   EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
   EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
 
+  Tokens = annotate("if (Foo *&foo{a})");
+  ASSERT_EQ(Tokens.size(), 11u) << Tokens;
+  EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
+  EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
+
   FormatStyle Style = getLLVMStyle();
   Style.TypeNames.push_back("MYI");
   Tokens = annotate("if (MYI *p{nullptr})", Style);
@@ -362,6 +367,14 @@ TEST_F(TokenAnnotatorTest, UnderstandsUsesOfStarAndAmp) {
                     "}");
   ASSERT_EQ(Tokens.size(), 20u) << Tokens;
   EXPECT_TOKEN(Tokens[14], tok::star, TT_PointerOrReference);
+
+  Tokens = annotate("Thingy kConfig = {\n"
+                    "    1,\n"
+                    "    (uint16_t)(kScale * height_pixels),\n"
+                    "};");
+  ASSERT_EQ(Tokens.size(), 18u) << Tokens;
+  EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_CastRParen);
+  EXPECT_TOKEN(Tokens[11], tok::star, TT_BinaryOperator);
 }
 
 TEST_F(TokenAnnotatorTest, UnderstandsUsesOfPlusAndMinus) {
@@ -809,6 +822,12 @@ TEST_F(TokenAnnotatorTest, UnderstandsCasts) {
   ASSERT_EQ(Tokens.size(), 13u) << Tokens;
   EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_CastRParen);
 
+  Tokens = annotate("return (Foo (*)(void *, Bar, ...))&foo;");
+  ASSERT_EQ(Tokens.size(), 19u) << Tokens;
+  EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionTypeLParen);
+  EXPECT_TOKEN(Tokens[14], tok::r_paren, TT_CastRParen);
+  EXPECT_TOKEN(Tokens[15], tok::amp, TT_UnaryOperator);
+
   auto Style = getLLVMStyle();
   Style.TypeNames.push_back("Foo");
   Tokens = annotate("#define FOO(bar) foo((Foo)&bar)", Style);
@@ -3505,6 +3524,30 @@ TEST_F(TokenAnnotatorTest, SplitPenalty) {
   EXPECT_SPLIT_PENALTY(Tokens[7], 23u);
 }
 
+TEST_F(TokenAnnotatorTest, TemplateName) {
+  constexpr StringRef Code{"return Foo < A || B > (C ^ D);"};
+
+  auto Tokens = annotate(Code);
+  ASSERT_EQ(Tokens.size(), 14u) << Tokens;
+  EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);
+  EXPECT_TOKEN(Tokens[6], tok::greater, TT_BinaryOperator);
+
+  auto Style = getLLVMStyle();
+  Style.TemplateNames.push_back("Foo");
+
+  Tokens = annotate(Code, Style);
+  EXPECT_TOKEN(Tokens[1], tok::identifier, TT_TemplateName);
+  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
+  EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);
+}
+
+TEST_F(TokenAnnotatorTest, TemplateInstantiation) {
+  auto Tokens = annotate("return FixedInt<N | M>();");
+  ASSERT_EQ(Tokens.size(), 11u) << Tokens;
+  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
+  EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);
+}
+
 } // namespace
 } // namespace format
 } // namespace clang
diff --git a/clang/unittests/Interpreter/CMakeLists.txt b/clang/unittests/Interpreter/CMakeLists.txt
index ec6f81ea19b96..1ed1216c772e8 100644
--- a/clang/unittests/Interpreter/CMakeLists.txt
+++ b/clang/unittests/Interpreter/CMakeLists.txt
@@ -16,13 +16,15 @@ add_clang_unittest(ClangReplInterpreterTests
 
   EXPORT_SYMBOLS
   )
-target_link_libraries(ClangReplInterpreterTests PUBLIC
+  
+target_link_libraries(ClangReplInterpreterTests PUBLIC LLVMTestingSupport)
+
+clang_target_link_libraries(ClangReplInterpreterTests PRIVATE
   clangAST
   clangBasic
   clangInterpreter
   clangFrontend
   clangSema
-  LLVMTestingSupport
   )
 
 # Exceptions on Windows are not yet supported.
diff --git a/clang/utils/TableGen/ASTTableGen.h b/clang/utils/TableGen/ASTTableGen.h
index 827fce9e213cb..02b97636cf5f2 100644
--- a/clang/utils/TableGen/ASTTableGen.h
+++ b/clang/utils/TableGen/ASTTableGen.h
@@ -320,7 +320,7 @@ class PropertyType : public WrappedRecord {
   }
 
   std::vector<const llvm::Record *> getBufferElementTypes() const {
-    return get()->getValueAsListOfConstDefs(BufferElementTypesFieldName);
+    return get()->getValueAsListOfDefs(BufferElementTypesFieldName);
   }
 
   static llvm::StringRef getTableGenNodeClassName() {
diff --git a/clang/utils/TableGen/ClangAttrEmitter.cpp b/clang/utils/TableGen/ClangAttrEmitter.cpp
index e5d92b343b3dd..28b7ec8f822cf 100644
--- a/clang/utils/TableGen/ClangAttrEmitter.cpp
+++ b/clang/utils/TableGen/ClangAttrEmitter.cpp
@@ -1529,9 +1529,8 @@ createArgument(const Record &Arg, StringRef Attr,
 
   if (!Ptr) {
     // Search in reverse order so that the most-derived type is handled first.
-    ArrayRef<std::pair<Record*, SMRange>> Bases = Search->getSuperClasses();
-    for (const auto &Base : reverse(Bases)) {
-      if ((Ptr = createArgument(Arg, Attr, Base.first)))
+    for (const auto &[Base, _] : reverse(Search->getSuperClasses())) {
+      if ((Ptr = createArgument(Arg, Attr, Base)))
         break;
     }
   }
@@ -1746,8 +1745,7 @@ getSpellingListIndex(const std::vector<FlattenedSpelling> &SpellingList,
 }
 
 static void writeAttrAccessorDefinition(const Record &R, raw_ostream &OS) {
-  std::vector<const Record *> Accessors =
-      R.getValueAsListOfConstDefs("Accessors");
+  std::vector<const Record *> Accessors = R.getValueAsListOfDefs("Accessors");
   if (Accessors.empty())
     return;
 
@@ -1964,7 +1962,7 @@ struct AttributeSubjectMatchRule {
 
   std::vector<const Record *> getSubjects() const {
     return (Constraint ? Constraint : MetaSubject)
-        ->getValueAsListOfConstDefs("Subjects");
+        ->getValueAsListOfDefs("Subjects");
   }
 
   std::vector<const Record *> getLangOpts() const {
@@ -1972,11 +1970,11 @@ struct AttributeSubjectMatchRule {
       // Lookup the options in the sub-rule first, in case the sub-rule
       // overrides the rules options.
       std::vector<const Record *> Opts =
-          Constraint->getValueAsListOfConstDefs("LangOpts");
+          Constraint->getValueAsListOfDefs("LangOpts");
       if (!Opts.empty())
         return Opts;
     }
-    return MetaSubject->getValueAsListOfConstDefs("LangOpts");
+    return MetaSubject->getValueAsListOfDefs("LangOpts");
   }
 
   // Abstract rules are used only for sub-rules
@@ -2105,7 +2103,7 @@ PragmaClangAttributeSupport::PragmaClangAttributeSupport(
                                        const Record *Constraint) {
     Rules.emplace_back(MetaSubject, Constraint);
     for (const Record *Subject :
-         SubjectContainer->getValueAsListOfConstDefs("Subjects")) {
+         SubjectContainer->getValueAsListOfDefs("Subjects")) {
       bool Inserted =
           SubjectsToRules
               .try_emplace(Subject, RuleOrAggregateRuleSet::getRule(
@@ -2503,7 +2501,7 @@ static void emitClangAttrTypeArgList(const RecordKeeper &Records,
   std::map<std::string, FSIVecTy> FSIMap;
   for (const auto *Attr : Records.getAllDerivedDefinitions("Attr")) {
     // Determine whether the first argument is a type.
-    std::vector<const Record *> Args = Attr->getValueAsListOfConstDefs("Args");
+    std::vector<const Record *> Args = Attr->getValueAsListOfDefs("Args");
     if (Args.empty())
       continue;
 
@@ -2581,7 +2579,7 @@ static void emitClangAttrVariadicIdentifierArgList(const RecordKeeper &Records,
   std::map<std::string, FSIVecTy> FSIMap;
   for (const auto *A : Records.getAllDerivedDefinitions("Attr")) {
     // Determine whether the first argument is a variadic identifier.
-    std::vector<const Record *> Args = A->getValueAsListOfConstDefs("Args");
+    std::vector<const Record *> Args = A->getValueAsListOfDefs("Args");
     if (Args.empty() || !isVariadicIdentifierArgument(Args[0]))
       continue;
     generateFlattenedSpellingInfo(*A, FSIMap);
@@ -2614,7 +2612,7 @@ emitClangAttrUnevaluatedStringLiteralList(const RecordKeeper &Records,
   std::map<std::string, FSIVecTy> FSIMap;
   for (const auto *Attr : Records.getAllDerivedDefinitions("Attr")) {
     // Determine whether there are any string arguments.
-    uint32_t ArgMask = MakeMask(Attr->getValueAsListOfConstDefs("Args"));
+    uint32_t ArgMask = MakeMask(Attr->getValueAsListOfDefs("Args"));
     if (!ArgMask)
       continue;
     generateFlattenedSpellingInfo(*Attr, FSIMap, ArgMask);
@@ -2630,7 +2628,7 @@ static void emitClangAttrIdentifierArgList(const RecordKeeper &Records,
   std::map<std::string, FSIVecTy> FSIMap;
   for (const auto *Attr : Records.getAllDerivedDefinitions("Attr")) {
     // Determine whether the first argument is an identifier.
-    std::vector<const Record *> Args = Attr->getValueAsListOfConstDefs("Args");
+    std::vector<const Record *> Args = Attr->getValueAsListOfDefs("Args");
     if (Args.empty() || !isIdentifierArgument(Args[0]))
       continue;
     generateFlattenedSpellingInfo(*Attr, FSIMap);
@@ -2648,7 +2646,7 @@ static void emitClangAttrStrictIdentifierArgList(const RecordKeeper &Records,
     if (!Attr->getValueAsBit("StrictEnumParameters"))
       continue;
     // Check that there is really an identifier argument.
-    std::vector<const Record *> Args = Attr->getValueAsListOfConstDefs("Args");
+    std::vector<const Record *> Args = Attr->getValueAsListOfDefs("Args");
     if (none_of(Args, [&](const Record *R) { return isIdentifierArgument(R); }))
       continue;
     generateFlattenedSpellingInfo(*Attr, FSIMap);
@@ -2670,7 +2668,7 @@ static void emitClangAttrThisIsaIdentifierArgList(const RecordKeeper &Records,
   std::map<std::string, FSIVecTy> FSIMap;
   for (const auto *A : Records.getAllDerivedDefinitions("Attr")) {
     // Determine whether the first argument is a variadic identifier.
-    std::vector<const Record *> Args = A->getValueAsListOfConstDefs("Args");
+    std::vector<const Record *> Args = A->getValueAsListOfDefs("Args");
     if (Args.empty() || !keywordThisIsaIdentifierInArgument(Args[0]))
       continue;
     generateFlattenedSpellingInfo(*A, FSIMap);
@@ -2745,12 +2743,11 @@ static void emitAttributes(const RecordKeeper &Records, raw_ostream &OS,
     if (!R.getValueAsBit("ASTNode"))
       continue;
 
-    ArrayRef<std::pair<Record *, SMRange>> Supers = R.getSuperClasses();
+    ArrayRef<std::pair<const Record *, SMRange>> Supers = R.getSuperClasses();
     assert(!Supers.empty() && "Forgot to specify a superclass for the attr");
     std::string SuperName;
     bool Inheritable = false;
-    for (const auto &Super : reverse(Supers)) {
-      const Record *R = Super.first;
+    for (const auto &[R, _] : reverse(Supers)) {
       if (R->getName() != "TargetSpecificAttr" &&
           R->getName() != "DeclOrTypeAttr" && SuperName.empty())
         SuperName = std::string(R->getName());
@@ -2763,8 +2760,7 @@ static void emitAttributes(const RecordKeeper &Records, raw_ostream &OS,
     else
       OS << "\n// " << R.getName() << "Attr implementation\n\n";
 
-    std::vector<const Record *> ArgRecords =
-        R.getValueAsListOfConstDefs("Args");
+    std::vector<const Record *> ArgRecords = R.getValueAsListOfDefs("Args");
     std::vector<std::unique_ptr<Argument>> Args;
     Args.reserve(ArgRecords.size());
 
@@ -3436,7 +3432,7 @@ namespace {
     }
 
   private:
-    AttrClass *findClassByRecord(Record *R) const {
+    AttrClass *findClassByRecord(const Record *R) const {
       for (auto &Class : Classes) {
         if (Class->TheRecord == R)
           return Class.get();
@@ -3539,7 +3535,7 @@ void EmitClangAttrPCHRead(const RecordKeeper &Records, raw_ostream &OS) {
           std::make_unique<VariadicExprArgument>("DelayedArgs", R.getName());
       DelayedArgs->writePCHReadDecls(OS);
     }
-    ArgRecords = R.getValueAsListOfConstDefs("Args");
+    ArgRecords = R.getValueAsListOfDefs("Args");
     Args.clear();
     for (const auto *Arg : ArgRecords) {
       Args.emplace_back(createArgument(*Arg, R.getName()));
@@ -3578,7 +3574,7 @@ void EmitClangAttrPCHWrite(const RecordKeeper &Records, raw_ostream &OS) {
     if (!R.getValueAsBit("ASTNode"))
       continue;
     OS << "  case attr::" << R.getName() << ": {\n";
-    std::vector<const Record *> Args = R.getValueAsListOfConstDefs("Args");
+    std::vector<const Record *> Args = R.getValueAsListOfDefs("Args");
     if (R.isSubClassOf(InhClass) || !Args.empty())
       OS << "    const auto *SA = cast<" << R.getName()
          << "Attr>(A);\n";
@@ -3769,7 +3765,7 @@ void EmitClangRegularKeywordAttributeInfo(const RecordKeeper &Records,
     for (const auto &S : GetFlattenedSpellings(*R)) {
       if (!isRegularKeywordAttribute(S))
         continue;
-      std::vector<const Record *> Args = R->getValueAsListOfConstDefs("Args");
+      std::vector<const Record *> Args = R->getValueAsListOfDefs("Args");
       bool HasArgs = any_of(
           Args, [](const Record *Arg) { return !Arg->getValueAsBit("Fake"); });
 
@@ -3999,8 +3995,7 @@ void EmitClangAttrTemplateInstantiateHelper(ArrayRef<const Record *> Attrs,
       continue;
     }
 
-    std::vector<const Record *> ArgRecords =
-        R.getValueAsListOfConstDefs("Args");
+    std::vector<const Record *> ArgRecords = R.getValueAsListOfDefs("Args");
     std::vector<std::unique_ptr<Argument>> Args;
     Args.reserve(ArgRecords.size());
 
@@ -4205,7 +4200,7 @@ static void GenerateAppertainsTo(const Record &Attr, raw_ostream &OS) {
 
   const Record *SubjectObj = Attr.getValueAsDef("Subjects");
   std::vector<const Record *> Subjects =
-      SubjectObj->getValueAsListOfConstDefs("Subjects");
+      SubjectObj->getValueAsListOfDefs("Subjects");
 
   // If the list of subjects is empty, it is assumed that the attribute
   // appertains to everything.
@@ -4337,7 +4332,7 @@ static void GenerateMutualExclusionsChecks(const Record &Attr,
   for (const Record *Exclusion :
        Records.getAllDerivedDefinitions("MutualExclusions")) {
     std::vector<const Record *> MutuallyExclusiveAttrs =
-        Exclusion->getValueAsListOfConstDefs("Exclusions");
+        Exclusion->getValueAsListOfDefs("Exclusions");
     auto IsCurAttr = [Attr](const Record *R) {
       return R->getName() == Attr.getName();
     };
@@ -4483,8 +4478,7 @@ static void GenerateLangOptRequirements(const Record &R,
                                         raw_ostream &OS) {
   // If the attribute has an empty or unset list of language requirements,
   // use the default handler.
-  std::vector<const Record *> LangOpts =
-      R.getValueAsListOfConstDefs("LangOpts");
+  std::vector<const Record *> LangOpts = R.getValueAsListOfDefs("LangOpts");
   if (LangOpts.empty())
     return;
 
@@ -4629,7 +4623,7 @@ static bool isParamExpr(const Record *Arg) {
 void GenerateIsParamExpr(const Record &Attr, raw_ostream &OS) {
   OS << "bool isParamExpr(size_t N) const override {\n";
   OS << "  return ";
-  auto Args = Attr.getValueAsListOfConstDefs("Args");
+  auto Args = Attr.getValueAsListOfDefs("Args");
   for (size_t I = 0; I < Args.size(); ++I)
     if (isParamExpr(Args[I]))
       OS << "(N == " << I << ") || ";
@@ -4743,8 +4737,8 @@ void EmitClangAttrParsedAttrImpl(const RecordKeeper &Records, raw_ostream &OS) {
       if (Arg->getValueAsBitOrUnset("Fake", UnusedUnset))
         continue;
       ArgNames.push_back(Arg->getValueAsString("Name").str());
-      for (const auto &Class : Arg->getSuperClasses()) {
-        if (Class.first->getName().starts_with("Variadic")) {
+      for (const auto &[Class, _] : Arg->getSuperClasses()) {
+        if (Class->getName().starts_with("Variadic")) {
           ArgNames.back().append("...");
           break;
         }
@@ -4792,7 +4786,7 @@ void EmitClangAttrParsedAttrImpl(const RecordKeeper &Records, raw_ostream &OS) {
     GenerateLangOptRequirements(Attr, OS);
     GenerateTargetRequirements(Attr, Dupes, OS);
     GenerateSpellingTargetRequirements(
-        Attr, Attr.getValueAsListOfConstDefs("TargetSpecificSpellings"), OS);
+        Attr, Attr.getValueAsListOfDefs("TargetSpecificSpellings"), OS);
     GenerateSpellingIndexToSemanticSpelling(Attr, OS);
     PragmaAttributeSupport.generateStrictConformsTo(*I->second, OS);
     GenerateHandleDeclAttribute(Attr, OS);
@@ -4959,7 +4953,7 @@ void EmitClangAttrTextNodeDump(const RecordKeeper &Records, raw_ostream &OS) {
     if (Spellings.size() > 1 && !SpellingNamesAreCommon(Spellings))
       SS << "    OS << \" \" << A->getSpelling();\n";
 
-    std::vector<const Record *> Args = R.getValueAsListOfConstDefs("Args");
+    std::vector<const Record *> Args = R.getValueAsListOfDefs("Args");
     for (const auto *Arg : Args)
       createArgument(*Arg, R.getName())->writeDump(SS);
 
@@ -4989,7 +4983,7 @@ void EmitClangAttrNodeTraverse(const RecordKeeper &Records, raw_ostream &OS) {
     std::string FunctionContent;
     raw_string_ostream SS(FunctionContent);
 
-    std::vector<const Record *> Args = R.getValueAsListOfConstDefs("Args");
+    std::vector<const Record *> Args = R.getValueAsListOfDefs("Args");
     for (const auto *Arg : Args)
       createArgument(*Arg, R.getName())->writeDumpChildren(SS);
     if (Attr->getValueAsBit("AcceptsExprPack"))
@@ -5033,8 +5027,7 @@ void EmitClangAttrDocTable(const RecordKeeper &Records, raw_ostream &OS) {
   for (const auto *A : Records.getAllDerivedDefinitions("Attr")) {
     if (!A->getValueAsBit("ASTNode"))
       continue;
-    std::vector<const Record *> Docs =
-        A->getValueAsListOfConstDefs("Documentation");
+    std::vector<const Record *> Docs = A->getValueAsListOfDefs("Documentation");
     assert(!Docs.empty());
     // Only look at the first documentation if there are several.
     // (Currently there's only one such attr, revisit if this becomes common).
@@ -5254,7 +5247,7 @@ void EmitClangAttrDocs(const RecordKeeper &Records, raw_ostream &OS) {
   for (const auto *A : Records.getAllDerivedDefinitions("Attr")) {
     const Record &Attr = *A;
     std::vector<const Record *> Docs =
-        Attr.getValueAsListOfConstDefs("Documentation");
+        Attr.getValueAsListOfDefs("Documentation");
     for (const auto *D : Docs) {
       const Record &Doc = *D;
       const Record *Category = Doc.getValueAsDef("Category");
diff --git a/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp b/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
index d9bb0630aff5f..325d63de1563d 100644
--- a/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
+++ b/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
@@ -48,7 +48,7 @@ class DiagGroupParentMap {
         Records.getAllDerivedDefinitions("DiagGroup");
     for (unsigned i = 0, e = DiagGroups.size(); i != e; ++i) {
       std::vector<const Record *> SubGroups =
-          DiagGroups[i]->getValueAsListOfConstDefs("SubGroups");
+          DiagGroups[i]->getValueAsListOfDefs("SubGroups");
       for (unsigned j = 0, e = SubGroups.size(); j != e; ++j)
         Mapping[SubGroups[j]].push_back(DiagGroups[i]);
     }
diff --git a/clang/utils/TableGen/ClangOpcodesEmitter.cpp b/clang/utils/TableGen/ClangOpcodesEmitter.cpp
index 94c361c7d544f..a18220fe23c41 100644
--- a/clang/utils/TableGen/ClangOpcodesEmitter.cpp
+++ b/clang/utils/TableGen/ClangOpcodesEmitter.cpp
@@ -69,7 +69,7 @@ void Enumerate(const Record *R, StringRef N,
 
     if (const auto *TypeClass = dyn_cast<DefInit>(Types->getElement(I))) {
       for (const auto *Type :
-           TypeClass->getDef()->getValueAsListOfConstDefs("Types")) {
+           TypeClass->getDef()->getValueAsListOfDefs("Types")) {
         TypePath.push_back(Type);
         Rec(I + 1, ID + Type->getName());
         TypePath.pop_back();
@@ -117,7 +117,7 @@ void ClangOpcodesEmitter::EmitInterp(raw_ostream &OS, StringRef N,
             [this, R, &OS, &N](ArrayRef<const Record *> TS, const Twine &ID) {
               bool CanReturn = R->getValueAsBit("CanReturn");
               bool ChangesPC = R->getValueAsBit("ChangesPC");
-              const auto &Args = R->getValueAsListOfConstDefs("Args");
+              const auto &Args = R->getValueAsListOfDefs("Args");
 
               OS << "case OP_" << ID << ": {\n";
 
@@ -176,7 +176,7 @@ void ClangOpcodesEmitter::EmitDisasm(raw_ostream &OS, StringRef N,
     OS << "  PrintName(\"" << ID << "\");\n";
     OS << "  OS << \"\\t\"";
 
-    for (const auto *Arg : R->getValueAsListOfConstDefs("Args")) {
+    for (const auto *Arg : R->getValueAsListOfDefs("Args")) {
       OS << " << ReadArg<" << Arg->getValueAsString("Name") << ">(P, PC)";
       OS << " << \" \"";
     }
@@ -194,7 +194,7 @@ void ClangOpcodesEmitter::EmitEmitter(raw_ostream &OS, StringRef N,
 
   OS << "#ifdef GET_LINK_IMPL\n";
   Enumerate(R, N, [R, &OS](ArrayRef<const Record *>, const Twine &ID) {
-    const auto &Args = R->getValueAsListOfConstDefs("Args");
+    const auto &Args = R->getValueAsListOfDefs("Args");
 
     // Emit the list of arguments.
     OS << "bool ByteCodeEmitter::emit" << ID << "(";
@@ -227,7 +227,7 @@ void ClangOpcodesEmitter::EmitEmitter(raw_ostream &OS, StringRef N,
 void ClangOpcodesEmitter::EmitProto(raw_ostream &OS, StringRef N,
                                     const Record *R) {
   OS << "#if defined(GET_EVAL_PROTO) || defined(GET_LINK_PROTO)\n";
-  auto Args = R->getValueAsListOfConstDefs("Args");
+  auto Args = R->getValueAsListOfDefs("Args");
   Enumerate(R, N, [&OS, &Args](ArrayRef<const Record *> TS, const Twine &ID) {
     OS << "bool emit" << ID << "(";
     for (size_t I = 0, N = Args.size(); I < N; ++I) {
@@ -268,7 +268,7 @@ void ClangOpcodesEmitter::EmitGroup(raw_ostream &OS, StringRef N,
     return;
 
   const auto *Types = R->getValueAsListInit("Types");
-  const auto &Args = R->getValueAsListOfConstDefs("Args");
+  const auto &Args = R->getValueAsListOfDefs("Args");
 
   Twine EmitFuncName = "emit" + N;
 
@@ -333,7 +333,7 @@ void ClangOpcodesEmitter::EmitGroup(raw_ostream &OS, StringRef N,
     // Print a switch statement selecting T.
     if (auto *TypeClass = dyn_cast<DefInit>(Types->getElement(I))) {
       OS << "  switch (T" << I << ") {\n";
-      auto Cases = TypeClass->getDef()->getValueAsListOfConstDefs("Types");
+      auto Cases = TypeClass->getDef()->getValueAsListOfDefs("Types");
       for (auto *Case : Cases) {
         OS << "  case PT_" << Case->getName() << ":\n";
         TS.push_back(Case);
@@ -364,7 +364,7 @@ void ClangOpcodesEmitter::EmitEval(raw_ostream &OS, StringRef N,
   OS << "#ifdef GET_EVAL_IMPL\n";
   Enumerate(R, N,
             [this, R, &N, &OS](ArrayRef<const Record *> TS, const Twine &ID) {
-              auto Args = R->getValueAsListOfConstDefs("Args");
+              auto Args = R->getValueAsListOfDefs("Args");
 
               OS << "bool EvalEmitter::emit" << ID << "(";
               for (size_t I = 0, N = Args.size(); I < N; ++I) {
diff --git a/clang/utils/TableGen/ClangOpenCLBuiltinEmitter.cpp b/clang/utils/TableGen/ClangOpenCLBuiltinEmitter.cpp
index 6607086f0b117..386a965445ce8 100644
--- a/clang/utils/TableGen/ClangOpenCLBuiltinEmitter.cpp
+++ b/clang/utils/TableGen/ClangOpenCLBuiltinEmitter.cpp
@@ -480,9 +480,8 @@ static void VerifySignature(ArrayRef<const Record *> Signature,
       }
 
       // Check number of data types.
-      unsigned NTypes = T->getValueAsDef("TypeList")
-                            ->getValueAsListOfConstDefs("List")
-                            .size();
+      unsigned NTypes =
+          T->getValueAsDef("TypeList")->getValueAsListOfDefs("List").size();
       if (NTypes != GenTypeTypes && NTypes != 1) {
         if (GenTypeTypes > 1) {
           // We already saw a gentype with a different number of types.
@@ -512,7 +511,7 @@ void BuiltinNameEmitter::GetOverloads() {
     StringRef BName = B->getValueAsString("Name");
     FctOverloadMap.try_emplace(BName);
 
-    auto Signature = B->getValueAsListOfConstDefs("Signature");
+    auto Signature = B->getValueAsListOfDefs("Signature");
     // Reuse signatures to avoid unnecessary duplicates.
     auto it =
         find_if(SignaturesList,
@@ -636,8 +635,8 @@ void BuiltinNameEmitter::EmitBuiltinTable() {
           Overload.first->getValueAsDef("MaxVersion")->getValueAsInt("ID");
 
       OS << "  { " << Overload.second << ", "
-         << Overload.first->getValueAsListOfConstDefs("Signature").size()
-         << ", " << (Overload.first->getValueAsBit("IsPure")) << ", "
+         << Overload.first->getValueAsListOfDefs("Signature").size() << ", "
+         << (Overload.first->getValueAsBit("IsPure")) << ", "
          << (Overload.first->getValueAsBit("IsConst")) << ", "
          << (Overload.first->getValueAsBit("IsConv")) << ", "
          << FunctionExtensionIndex[ExtName] << ", "
@@ -852,7 +851,7 @@ static void OCL2Qual(Sema &S, const OpenCLTypeStruct &Ty,
     // the plain scalar types for now; other type information such as vector
     // size and type qualifiers will be added after the switch statement.
     std::vector<const Record *> BaseTypes =
-        GenType->getValueAsDef("TypeList")->getValueAsListOfConstDefs("List");
+        GenType->getValueAsDef("TypeList")->getValueAsListOfDefs("List");
 
     // Collect all QualTypes for a single vector size into TypeList.
     OS << "      SmallVector<QualType, " << BaseTypes.size() << "> TypeList;\n";
@@ -1028,8 +1027,7 @@ void OpenCLBuiltinFileEmitterBase::getTypeLists(
     std::vector<int64_t> &VectorList) const {
   bool isGenType = Type->isSubClassOf("GenericType");
   if (isGenType) {
-    TypeList =
-        Type->getValueAsDef("TypeList")->getValueAsListOfConstDefs("List");
+    TypeList = Type->getValueAsDef("TypeList")->getValueAsListOfDefs("List");
     VectorList =
         Type->getValueAsDef("VectorList")->getValueAsListOfInts("List");
     return;
@@ -1215,7 +1213,7 @@ void OpenCLBuiltinTestEmitter::emit() {
     StringRef Name = B->getValueAsString("Name");
 
     SmallVector<SmallVector<std::string, 2>, 4> FTypes;
-    expandTypesInSignature(B->getValueAsListOfConstDefs("Signature"), FTypes);
+    expandTypesInSignature(B->getValueAsListOfDefs("Signature"), FTypes);
 
     OS << "// Test " << Name << "\n";
 
@@ -1284,7 +1282,7 @@ void OpenCLBuiltinHeaderEmitter::emit() {
     std::string OptionalVersionEndif = emitVersionGuard(B);
 
     SmallVector<SmallVector<std::string, 2>, 4> FTypes;
-    expandTypesInSignature(B->getValueAsListOfConstDefs("Signature"), FTypes);
+    expandTypesInSignature(B->getValueAsListOfDefs("Signature"), FTypes);
 
     for (const auto &Signature : FTypes) {
       StringRef OptionalTypeExtEndif = emitTypeExtensionGuards(Signature);
diff --git a/clang/utils/TableGen/ClangOptionDocEmitter.cpp b/clang/utils/TableGen/ClangOptionDocEmitter.cpp
index d8a467fd41a2e..b67c5d1d1146c 100644
--- a/clang/utils/TableGen/ClangOptionDocEmitter.cpp
+++ b/clang/utils/TableGen/ClangOptionDocEmitter.cpp
@@ -181,7 +181,7 @@ const unsigned UnlimitedArgs = unsigned(-1);
 
 // Get the number of arguments expected for an option, or -1 if any number of
 // arguments are accepted.
-unsigned getNumArgsForKind(Record *OptionKind, const Record *Option) {
+unsigned getNumArgsForKind(const Record *OptionKind, const Record *Option) {
   return StringSwitch<unsigned>(OptionKind->getName())
     .Cases("KIND_JOINED", "KIND_JOINED_OR_SEPARATE", "KIND_SEPARATE", 1)
     .Cases("KIND_REMAINING_ARGS", "KIND_REMAINING_ARGS_JOINED",
@@ -282,11 +282,12 @@ void emitOptionName(StringRef Prefix, const Record *Option, raw_ostream &OS) {
     }
   }
 
-  emitOptionWithArgs(Prefix, Option, std::vector<StringRef>(Args.begin(), Args.end()), OS);
+  emitOptionWithArgs(Prefix, Option,
+                     std::vector<StringRef>(Args.begin(), Args.end()), OS);
 
   auto AliasArgs = Option->getValueAsListOfStrings("AliasArgs");
   if (!AliasArgs.empty()) {
-    Record *Alias = Option->getValueAsDef("Alias");
+    const Record *Alias = Option->getValueAsDef("Alias");
     OS << " (equivalent to ";
     emitOptionWithArgs(
         Alias->getValueAsListOfStrings("Prefixes").front(), Alias,
@@ -363,9 +364,8 @@ void emitOption(const DocumentedOption &Option, const Record *DocInfo,
 
   // Prefer a program specific help string.
   // This is a list of (visibilities, string) pairs.
-  std::vector<Record *> VisibilitiesHelp =
-      R->getValueAsListOfDefs("HelpTextsForVariants");
-  for (Record *VisibilityHelp : VisibilitiesHelp) {
+  for (const Record *VisibilityHelp :
+       R->getValueAsListOfDefs("HelpTextsForVariants")) {
     // This is a list of visibilities.
     ArrayRef<Init *> Visibilities =
         VisibilityHelp->getValueAsListInit("Visibilities")->getValues();
diff --git a/clang/utils/TableGen/ClangSACheckersEmitter.cpp b/clang/utils/TableGen/ClangSACheckersEmitter.cpp
index 998c5ed2079d7..bebdcac321261 100644
--- a/clang/utils/TableGen/ClangSACheckersEmitter.cpp
+++ b/clang/utils/TableGen/ClangSACheckersEmitter.cpp
@@ -229,9 +229,8 @@ void clang::EmitClangSACheckers(const RecordKeeper &Records, raw_ostream &OS) {
     if (Package->isValueUnset("PackageOptions"))
       continue;
 
-    std::vector<Record *> PackageOptions = Package
-                                       ->getValueAsListOfDefs("PackageOptions");
-    for (Record *PackageOpt : PackageOptions) {
+    for (const Record *PackageOpt :
+         Package->getValueAsListOfDefs("PackageOptions")) {
       OS << "PACKAGE_OPTION(";
       printOption(OS, getPackageFullName(Package), *PackageOpt);
       OS << ")\n";
@@ -325,13 +324,11 @@ void clang::EmitClangSACheckers(const RecordKeeper &Records, raw_ostream &OS) {
   OS << "\n"
         "#ifdef GET_CHECKER_OPTIONS\n";
   for (const Record *Checker : checkers) {
-
     if (Checker->isValueUnset("CheckerOptions"))
       continue;
 
-    std::vector<Record *> CheckerOptions = Checker
-                                       ->getValueAsListOfDefs("CheckerOptions");
-    for (Record *CheckerOpt : CheckerOptions) {
+    for (const Record *CheckerOpt :
+         Checker->getValueAsListOfDefs("CheckerOptions")) {
       OS << "CHECKER_OPTION(";
       printOption(OS, getCheckerFullName(Checker), *CheckerOpt);
       OS << ")\n";
diff --git a/clang/utils/TableGen/MveEmitter.cpp b/clang/utils/TableGen/MveEmitter.cpp
index 57e6353e60a14..915e914d6b928 100644
--- a/clang/utils/TableGen/MveEmitter.cpp
+++ b/clang/utils/TableGen/MveEmitter.cpp
@@ -994,9 +994,10 @@ class EmitterBase {
   const VectorType *getVectorType(const ScalarType *ST, unsigned Lanes) {
     std::tuple<ScalarTypeKind, unsigned, unsigned> key(ST->kind(),
                                                        ST->sizeInBits(), Lanes);
-    if (VectorTypes.find(key) == VectorTypes.end())
-      VectorTypes[key] = std::make_unique<VectorType>(ST, Lanes);
-    return VectorTypes[key].get();
+    auto [It, Inserted] = VectorTypes.try_emplace(key);
+    if (Inserted)
+      It->second = std::make_unique<VectorType>(ST, Lanes);
+    return It->second.get();
   }
   const VectorType *getVectorType(const ScalarType *ST) {
     return getVectorType(ST, 128 / ST->sizeInBits());
@@ -1004,22 +1005,25 @@ class EmitterBase {
   const MultiVectorType *getMultiVectorType(unsigned Registers,
                                             const VectorType *VT) {
     std::pair<std::string, unsigned> key(VT->cNameBase(), Registers);
-    if (MultiVectorTypes.find(key) == MultiVectorTypes.end())
-      MultiVectorTypes[key] = std::make_unique<MultiVectorType>(Registers, VT);
-    return MultiVectorTypes[key].get();
+    auto [It, Inserted] = MultiVectorTypes.try_emplace(key);
+    if (Inserted)
+      It->second = std::make_unique<MultiVectorType>(Registers, VT);
+    return It->second.get();
   }
   const PredicateType *getPredicateType(unsigned Lanes) {
     unsigned key = Lanes;
-    if (PredicateTypes.find(key) == PredicateTypes.end())
-      PredicateTypes[key] = std::make_unique<PredicateType>(Lanes);
-    return PredicateTypes[key].get();
+    auto [It, Inserted] = PredicateTypes.try_emplace(key);
+    if (Inserted)
+      It->second = std::make_unique<PredicateType>(Lanes);
+    return It->second.get();
   }
   const PointerType *getPointerType(const Type *T, bool Const) {
     PointerType PT(T, Const);
     std::string key = PT.cName();
-    if (PointerTypes.find(key) == PointerTypes.end())
-      PointerTypes[key] = std::make_unique<PointerType>(PT);
-    return PointerTypes[key].get();
+    auto [It, Inserted] = PointerTypes.try_emplace(key);
+    if (Inserted)
+      It->second = std::make_unique<PointerType>(PT);
+    return It->second.get();
   }
 
   // Methods to construct a type from various pieces of Tablegen. These are
diff --git a/clang/utils/TableGen/NeonEmitter.cpp b/clang/utils/TableGen/NeonEmitter.cpp
index 202573e2ac740..d4b42360e7fd3 100644
--- a/clang/utils/TableGen/NeonEmitter.cpp
+++ b/clang/utils/TableGen/NeonEmitter.cpp
@@ -410,8 +410,7 @@ class Intrinsic {
     }
 
     int ArgIdx, Kind, TypeArgIdx;
-    std::vector<Record *> ImmCheckList = R->getValueAsListOfDefs("ImmChecks");
-    for (const auto *I : ImmCheckList) {
+    for (const Record *I : R->getValueAsListOfDefs("ImmChecks")) {
       unsigned EltSizeInBits = 0, VecSizeInBits = 0;
 
       ArgIdx = I->getValueAsInt("ImmArgIdx");
diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp
index 2f9747e7de3de..8e1dcb45f289f 100644
--- a/clang/utils/TableGen/SveEmitter.cpp
+++ b/clang/utils/TableGen/SveEmitter.cpp
@@ -1161,11 +1161,9 @@ void SVEEmitter::createIntrinsic(
   uint64_t Merge = R->getValueAsInt("Merge");
   StringRef MergeSuffix = R->getValueAsString("MergeSuffix");
   uint64_t MemEltType = R->getValueAsInt("MemEltType");
-  std::vector<Record*> FlagsList = R->getValueAsListOfDefs("Flags");
-  std::vector<Record*> ImmCheckList = R->getValueAsListOfDefs("ImmChecks");
 
   int64_t Flags = 0;
-  for (auto FlagRec : FlagsList)
+  for (const Record *FlagRec : R->getValueAsListOfDefs("Flags"))
     Flags |= FlagRec->getValueAsInt("Value");
 
   // Create a dummy TypeSpec for non-overloaded builtins.
@@ -1195,10 +1193,10 @@ void SVEEmitter::createIntrinsic(
   for (auto TS : TypeSpecs) {
     // Collate a list of range/option checks for the immediates.
     SmallVector<ImmCheck, 2> ImmChecks;
-    for (auto *R : ImmCheckList) {
-      int64_t ArgIdx = R->getValueAsInt("ImmArgIdx");
-      int64_t EltSizeArgIdx = R->getValueAsInt("TypeContextArgIdx");
-      int64_t Kind = R->getValueAsDef("Kind")->getValueAsInt("Value");
+    for (const Record *ImmR : R->getValueAsListOfDefs("ImmChecks")) {
+      int64_t ArgIdx = ImmR->getValueAsInt("ImmArgIdx");
+      int64_t EltSizeArgIdx = ImmR->getValueAsInt("TypeContextArgIdx");
+      int64_t Kind = ImmR->getValueAsDef("Kind")->getValueAsInt("Value");
       assert(ArgIdx >= 0 && Kind >= 0 &&
              "ImmArgIdx and Kind must be nonnegative");
 
diff --git a/compiler-rt/cmake/caches/hexagon-linux-builtins.cmake b/compiler-rt/cmake/caches/hexagon-linux-builtins.cmake
new file mode 100644
index 0000000000000..d9c9ff2a4655e
--- /dev/null
+++ b/compiler-rt/cmake/caches/hexagon-linux-builtins.cmake
@@ -0,0 +1,15 @@
+set(CMAKE_ASM_FLAGS "-G0 -mlong-calls -fno-pic" CACHE STRING "")
+
+set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR OFF CACHE BOOL "")
+set(LLVM_TARGET_TRIPLE hexagon-unknown-linux-musl CACHE STRING "")
+set(COMPILER_RT_DEFAULT_TARGET_TRIPLE hexagon-unknown-linux-musl CACHE STRING "")
+set(COMPILER_RT_BUILD_BUILTINS ON CACHE BOOL "")
+set(COMPILER_RT_BUILD_SANITIZERS OFF CACHE BOOL "")
+set(COMPILER_RT_BUILD_XRAY OFF CACHE BOOL "")
+set(COMPILER_RT_BUILD_LIBFUZZER OFF CACHE BOOL "")
+set(COMPILER_RT_BUILD_PROFILE OFF CACHE BOOL "")
+set(COMPILER_RT_BUILD_MEMPROF OFF CACHE BOOL "")
+set(COMPILER_RT_BUILD_ORC OFF CACHE BOOL "")
+set(COMPILER_RT_BUILD_GWP_ASAN OFF CACHE BOOL "")
+set(COMPILER_RT_BUILTINS_ENABLE_PIC OFF CACHE BOOL "")
+set(COMPILER_RT_SUPPORTED_ARCH hexagon CACHE STRING "")
diff --git a/compiler-rt/cmake/caches/hexagon-linux-clangrt.cmake b/compiler-rt/cmake/caches/hexagon-linux-clangrt.cmake
new file mode 100644
index 0000000000000..baf149306f91b
--- /dev/null
+++ b/compiler-rt/cmake/caches/hexagon-linux-clangrt.cmake
@@ -0,0 +1,14 @@
+
+set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR OFF CACHE BOOL "")
+set(LLVM_DEFAULT_TARGET_TRIPLE hexagon-unknown-linux-musl CACHE STRING "")
+
+set(COMPILER_RT_USE_LLVM_UNWINDER ON CACHE BOOL "")
+
+# Some build failures here, including the inline asm in
+# `compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h`, so
+# we can just disable these for now:
+set(COMPILER_RT_BUILD_BUILTINS OFF CACHE BOOL "")
+set(COMPILER_RT_BUILD_SANITIZERS OFF CACHE BOOL "")
+set(COMPILER_RT_BUILD_XRAY OFF CACHE BOOL "")
+set(COMPILER_RT_BUILD_MEMPROF OFF CACHE BOOL "")
+
diff --git a/compiler-rt/lib/asan/asan_win_common_runtime_thunk.cpp b/compiler-rt/lib/asan/asan_win_common_runtime_thunk.cpp
index 274d07a19bb7c..056e49336d326 100644
--- a/compiler-rt/lib/asan/asan_win_common_runtime_thunk.cpp
+++ b/compiler-rt/lib/asan/asan_win_common_runtime_thunk.cpp
@@ -85,11 +85,11 @@ static void WINAPI asan_thread_init(void *mod, unsigned long reason,
 // initializer is needed as a backup.
 extern "C" __declspec(allocate(".CRT$XIB")) int (*__asan_thunk_init)() =
     asan_thunk_init;
-WIN_FORCE_LINK(__asan_thunk_init);
+WIN_FORCE_LINK(__asan_thunk_init)
 
 extern "C" __declspec(allocate(".CRT$XLAB")) void(WINAPI *__asan_tls_init)(
     void *, unsigned long, void *) = asan_thread_init;
-WIN_FORCE_LINK(__asan_tls_init);
+WIN_FORCE_LINK(__asan_tls_init)
 
 ////////////////////////////////////////////////////////////////////////////////
 // ASan SEH handling.
@@ -103,7 +103,7 @@ static int SetSEHFilter() { return __asan_set_seh_filter(); }
 // __asan_intercept_seh gets optimized out, so we have to use an extra function.
 extern "C" __declspec(allocate(".CRT$XCAB")) int (*__asan_seh_interceptor)() =
     SetSEHFilter;
-WIN_FORCE_LINK(__asan_seh_interceptor);
+WIN_FORCE_LINK(__asan_seh_interceptor)
 }
 
 WIN_FORCE_LINK(__asan_dso_reg_hook)
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
index 139c80b4f4a53..d3c41221d5a94 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -10350,24 +10350,6 @@ INTERCEPTOR(SSIZE_T, pwritev2, int fd, __sanitizer_iovec *iov, int iovcnt,
 #define INIT_PWRITEV2
 #endif
 
-#if SANITIZER_INTERCEPT_FREADLINK
-INTERCEPTOR(SSIZE_T, freadlink, int fd, char *buf, SIZE_T bufsiz) {
-  void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, freadlink, fd, buf, bufsiz);
-  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
-  SSIZE_T res = REAL(freadlink)(fd, buf, bufsiz);
-  if (res > 0)
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, res);
-  if (res >= 0 && fd > 0)
-    COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
-  return res;
-}
-
-#  define INIT_FREADLINK COMMON_INTERCEPT_FUNCTION(freadlink)
-#else
-#  define INIT_FREADLINK
-#endif
-
 #include "sanitizer_common_interceptors_netbsd_compat.inc"
 
 namespace __sanitizer {
@@ -10689,7 +10671,6 @@ static void InitializeCommonInterceptors() {
   INIT_CPUSET_GETAFFINITY;
   INIT_PREADV2;
   INIT_PWRITEV2;
-  INIT_FREADLINK;
 
   INIT___PRINTF_CHK;
 }
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
index 272e4a02e6a7f..b1dc1ec204bc8 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
@@ -607,13 +607,6 @@
 // FIXME: also available from musl 1.2.5
 #define SANITIZER_INTERCEPT_PREADV2 (SI_LINUX && __GLIBC_PREREQ(2, 26))
 #define SANITIZER_INTERCEPT_PWRITEV2 (SI_LINUX && __GLIBC_PREREQ(2, 26))
-#if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && \
-    __MAC_OS_X_VERSION_MIN_REQUIRED >= 130000
-#  define SI_MAC_OS_DEPLOYMENT_MIN_13_00 1
-#else
-#  define SI_MAC_OS_DEPLOYMENT_MIN_13_00 0
-#endif
-#define SANITIZER_INTERCEPT_FREADLINK (SI_MAC && SI_MAC_OS_DEPLOYMENT_MIN_13_00)
 
 // This macro gives a way for downstream users to override the above
 // interceptor macros irrespective of the platform they are on. They have
diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h
index a3876479ec815..97188a5ac235c 100644
--- a/compiler-rt/lib/scudo/standalone/primary64.h
+++ b/compiler-rt/lib/scudo/standalone/primary64.h
@@ -532,6 +532,11 @@ template <typename Config> class SizeClassAllocator64 {
     uptr BytesInFreeListAtLastCheckpoint;
     uptr RangesReleased;
     uptr LastReleasedBytes;
+    // The minimum size of pushed blocks to trigger page release.
+    uptr TryReleaseThreshold;
+    // The number of bytes not triggering `releaseToOSMaybe()` because of
+    // the length of release interval.
+    uptr PendingPushedBytesDelta;
     u64 LastReleaseAtNs;
   };
 
@@ -560,8 +565,6 @@ template <typename Config> class SizeClassAllocator64 {
     u32 RandState GUARDED_BY(MMLock) = 0;
     BlocksInfo FreeListInfo GUARDED_BY(FLLock);
     PagesInfo MemMapInfo GUARDED_BY(MMLock);
-    // The minimum size of pushed blocks to trigger page release.
-    uptr TryReleaseThreshold GUARDED_BY(MMLock) = 0;
     ReleaseToOsInfo ReleaseInfo GUARDED_BY(MMLock) = {};
     bool Exhausted GUARDED_BY(MMLock) = false;
     bool isPopulatingFreeList GUARDED_BY(FLLock) = false;
@@ -610,9 +613,8 @@ template <typename Config> class SizeClassAllocator64 {
     return BlockSize < PageSize / 16U;
   }
 
-  ALWAYS_INLINE static bool isLargeBlock(uptr BlockSize) {
-    const uptr PageSize = getPageSizeCached();
-    return BlockSize > PageSize;
+  ALWAYS_INLINE uptr getMinReleaseAttemptSize(uptr BlockSize) {
+    return roundUp(BlockSize, getPageSizeCached());
   }
 
   ALWAYS_INLINE void initRegion(RegionInfo *Region, uptr ClassId,
@@ -631,12 +633,16 @@ template <typename Config> class SizeClassAllocator64 {
           (getRandomModN(&Region->RandState, 16) + 1) * PageSize;
     }
 
+    const uptr BlockSize = getSizeByClassId(ClassId);
     // Releasing small blocks is expensive, set a higher threshold to avoid
     // frequent page releases.
-    if (isSmallBlock(getSizeByClassId(ClassId)))
-      Region->TryReleaseThreshold = PageSize * SmallerBlockReleasePageDelta;
-    else
-      Region->TryReleaseThreshold = PageSize;
+    if (isSmallBlock(BlockSize)) {
+      Region->ReleaseInfo.TryReleaseThreshold =
+          PageSize * SmallerBlockReleasePageDelta;
+    } else {
+      Region->ReleaseInfo.TryReleaseThreshold =
+          getMinReleaseAttemptSize(BlockSize);
+    }
   }
 
   void pushBatchClassBlocks(RegionInfo *Region, CompactPtrT *Array, u32 Size)
@@ -1245,6 +1251,7 @@ template <typename Config> class SizeClassAllocator64 {
     uptr BytesInFreeList;
     const uptr AllocatedUserEnd =
         Region->MemMapInfo.AllocatedUser + Region->RegionBeg;
+    uptr RegionPushedBytesDelta = 0;
     SinglyLinkedList<BatchGroupT> GroupsToRelease;
 
     {
@@ -1267,6 +1274,12 @@ template <typename Config> class SizeClassAllocator64 {
         return 0;
       }
 
+      // Given that we will unlock the freelist for block operations, cache the
+      // value here so that when we are adapting the `TryReleaseThreshold`
+      // later, we are using the right metric.
+      RegionPushedBytesDelta =
+          BytesInFreeList - Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint;
+
       // ==================================================================== //
       // 2. Determine which groups can release the pages. Use a heuristic to
       //    gather groups that are candidates for doing a release.
@@ -1310,12 +1323,45 @@ template <typename Config> class SizeClassAllocator64 {
     auto SkipRegion = [](UNUSED uptr RegionIndex) { return false; };
     releaseFreeMemoryToOS(Context, Recorder, SkipRegion);
     if (Recorder.getReleasedRangesCount() > 0) {
+      // This is the case that we didn't hit the release threshold but it has
+      // been past a certain period of time. Thus we try to release some pages
+      // and if it does release some additional pages, it's hint that we are
+      // able to lower the threshold. Currently, this case happens when the
+      // `RegionPushedBytesDelta` is over half of the `TryReleaseThreshold`. As
+      // a result, we shrink the threshold to half accordingly.
+      // TODO(chiahungduan): Apply the same adjustment strategy to small blocks.
+      if (!isSmallBlock(BlockSize)) {
+        if (RegionPushedBytesDelta < Region->ReleaseInfo.TryReleaseThreshold &&
+            Recorder.getReleasedBytes() >
+                Region->ReleaseInfo.LastReleasedBytes +
+                    getMinReleaseAttemptSize(BlockSize)) {
+          Region->ReleaseInfo.TryReleaseThreshold =
+              Max(Region->ReleaseInfo.TryReleaseThreshold / 2,
+                  getMinReleaseAttemptSize(BlockSize));
+        }
+      }
+
       Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList;
       Region->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount();
       Region->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes();
     }
     Region->ReleaseInfo.LastReleaseAtNs = getMonotonicTimeFast();
 
+    if (Region->ReleaseInfo.PendingPushedBytesDelta > 0) {
+      // Instead of increasing the threshold by the amount of
+      // `PendingPushedBytesDelta`, we only increase half of the amount so that
+      // it won't be a leap (which may lead to higher memory pressure) because
+      // of certain memory usage bursts which don't happen frequently.
+      Region->ReleaseInfo.TryReleaseThreshold +=
+          Region->ReleaseInfo.PendingPushedBytesDelta / 2;
+      // This is another guard of avoiding the growth of threshold indefinitely.
+      // Note that we may consider to make this configurable if we have a better
+      // way to model this.
+      Region->ReleaseInfo.TryReleaseThreshold = Min<uptr>(
+          Region->ReleaseInfo.TryReleaseThreshold, (1UL << GroupSizeLog) / 2);
+      Region->ReleaseInfo.PendingPushedBytesDelta = 0;
+    }
+
     // ====================================================================== //
     // 5. Merge the `GroupsToRelease` back to the freelist.
     // ====================================================================== //
@@ -1329,8 +1375,6 @@ template <typename Config> class SizeClassAllocator64 {
       REQUIRES(Region->MMLock, Region->FLLock) {
     DCHECK_GE(Region->FreeListInfo.PoppedBlocks,
               Region->FreeListInfo.PushedBlocks);
-    const uptr PageSize = getPageSizeCached();
-
     // Always update `BytesInFreeListAtLastCheckpoint` with the smallest value
     // so that we won't underestimate the releasable pages. For example, the
     // following is the region usage,
@@ -1354,34 +1398,44 @@ template <typename Config> class SizeClassAllocator64 {
 
     const uptr RegionPushedBytesDelta =
         BytesInFreeList - Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint;
-    if (RegionPushedBytesDelta < PageSize)
-      return false;
-
-    // Releasing smaller blocks is expensive, so we want to make sure that a
-    // significant amount of bytes are free, and that there has been a good
-    // amount of batches pushed to the freelist before attempting to release.
-    if (isSmallBlock(BlockSize) && ReleaseType == ReleaseToOS::Normal)
-      if (RegionPushedBytesDelta < Region->TryReleaseThreshold)
-        return false;
 
     if (ReleaseType == ReleaseToOS::Normal) {
-      const s32 IntervalMs = atomic_load_relaxed(&ReleaseToOsIntervalMs);
+      if (RegionPushedBytesDelta < Region->ReleaseInfo.TryReleaseThreshold / 2)
+        return false;
+
+      const s64 IntervalMs = atomic_load_relaxed(&ReleaseToOsIntervalMs);
       if (IntervalMs < 0)
         return false;
 
-      // The constant 8 here is selected from profiling some apps and the number
-      // of unreleased pages in the large size classes is around 16 pages or
-      // more. Choose half of it as a heuristic and which also avoids page
-      // release every time for every pushBlocks() attempt by large blocks.
-      const bool ByPassReleaseInterval =
-          isLargeBlock(BlockSize) && RegionPushedBytesDelta > 8 * PageSize;
-      if (!ByPassReleaseInterval) {
-        if (Region->ReleaseInfo.LastReleaseAtNs +
-                static_cast<u64>(IntervalMs) * 1000000 >
-            getMonotonicTimeFast()) {
-          // Memory was returned recently.
+      const u64 IntervalNs = static_cast<u64>(IntervalMs) * 1000000;
+      const u64 CurTimeNs = getMonotonicTimeFast();
+      const u64 DiffSinceLastReleaseNs =
+          CurTimeNs - Region->ReleaseInfo.LastReleaseAtNs;
+
+      // At here, `RegionPushedBytesDelta` is more than half of
+      // `TryReleaseThreshold`. If the last release happened 2 release interval
+      // before, we will still try to see if there's any chance to release some
+      // memory even it doesn't exceed the threshold.
+      if (RegionPushedBytesDelta < Region->ReleaseInfo.TryReleaseThreshold) {
+        // We want the threshold to have a shorter response time to the variant
+        // memory usage patterns. According to data collected during experiments
+        // (which were done with 1, 2, 4, 8 intervals), `2` strikes the better
+        // balance between the memory usage and number of page release attempts.
+        if (DiffSinceLastReleaseNs < 2 * IntervalNs)
           return false;
-        }
+      } else if (DiffSinceLastReleaseNs < IntervalNs) {
+        // In this case, we are over the threshold but we just did some page
+        // release in the same release interval. This is a hint that we may want
+        // a higher threshold so that we can release more memory at once.
+        // `TryReleaseThreshold` will be adjusted according to how many bytes
+        // are not released, i.e., the `PendingPushedBytesdelta` here.
+        // TODO(chiahungduan): Apply the same adjustment strategy to small
+        // blocks.
+        if (!isSmallBlock(BlockSize))
+          Region->ReleaseInfo.PendingPushedBytesDelta = RegionPushedBytesDelta;
+
+        // Memory was returned recently.
+        return false;
       }
     } // if (ReleaseType == ReleaseToOS::Normal)
 
@@ -1397,10 +1451,10 @@ template <typename Config> class SizeClassAllocator64 {
     SinglyLinkedList<BatchGroupT> GroupsToRelease;
 
     // We are examining each group and will take the minimum distance to the
-    // release threshold as the next Region::TryReleaseThreshold(). Note that if
-    // the size of free blocks has reached the release threshold, the distance
-    // to the next release will be PageSize * SmallerBlockReleasePageDelta. See
-    // the comment on `SmallerBlockReleasePageDelta` for more details.
+    // release threshold as the next `TryReleaseThreshold`. Note that if the
+    // size of free blocks has reached the release threshold, the distance to
+    // the next release will be PageSize * SmallerBlockReleasePageDelta. See the
+    // comment on `SmallerBlockReleasePageDelta` for more details.
     uptr MinDistToThreshold = GroupSize;
 
     for (BatchGroupT *BG = Region->FreeListInfo.BlockList.front(),
@@ -1548,7 +1602,7 @@ template <typename Config> class SizeClassAllocator64 {
       // back to normal.
       if (MinDistToThreshold == GroupSize)
         MinDistToThreshold = PageSize * SmallerBlockReleasePageDelta;
-      Region->TryReleaseThreshold = MinDistToThreshold;
+      Region->ReleaseInfo.TryReleaseThreshold = MinDistToThreshold;
     }
 
     return GroupsToRelease;
diff --git a/compiler-rt/lib/scudo/standalone/tests/memtag_test.cpp b/compiler-rt/lib/scudo/standalone/tests/memtag_test.cpp
index 1fae651865359..09093e11452dd 100644
--- a/compiler-rt/lib/scudo/standalone/tests/memtag_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/memtag_test.cpp
@@ -19,7 +19,7 @@ namespace scudo {
 
 TEST(MemtagBasicDeathTest, Unsupported) {
   if (archSupportsMemoryTagging())
-    TEST_SKIP("Memory tagging is not supported");
+    TEST_SKIP("Memory tagging is not unsupported");
   // Skip when running with HWASan.
   if (&__hwasan_init != 0)
     TEST_SKIP("Incompatible with HWASan");
diff --git a/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp b/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp
index 369449303f3e0..26dc0f0d29a6d 100644
--- a/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp
+++ b/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp
@@ -6,6 +6,9 @@
 // FIXME: https://code.google.com/p/address-sanitizer/issues/detail?id=186
 // XFAIL: target={{.*windows-msvc.*}}
 
+// Fails on some Darwin bots, probably iOS.
+// XFAIL: ios
+
 #include <stdio.h>
 
 extern int y;
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Darwin/freadlink.c b/compiler-rt/test/sanitizer_common/TestCases/Darwin/freadlink.c
deleted file mode 100644
index 53658cdb66aa3..0000000000000
--- a/compiler-rt/test/sanitizer_common/TestCases/Darwin/freadlink.c
+++ /dev/null
@@ -1,29 +0,0 @@
-// RUN: %clang -O0 %s -o %t && %run %t
-
-#include <assert.h>
-#include <fcntl.h>
-#include <limits.h>
-#include <stdio.h>
-#include <string.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-int main(int argc, char **argv) {
-  char symlink_path[PATH_MAX];
-  snprintf(symlink_path, sizeof(symlink_path), "%s_%d.symlink", argv[0],
-           getpid());
-  remove(symlink_path);
-  int res = symlink(argv[0], symlink_path);
-  assert(!res);
-
-  int fd;
-  char readlink_path[PATH_MAX];
-  fd = open(symlink_path, O_RDONLY);
-  ssize_t res2 = freadlink(fd, readlink_path, sizeof(readlink_path));
-  assert(res2 >= 0);
-  readlink_path[res2] = '\0';
-  assert(!strcmp(readlink_path, argv[0]));
-  close(fd);
-
-  return 0;
-}
diff --git a/flang/docs/Extensions.md b/flang/docs/Extensions.md
index ed1ef49f8b77a..3ffd2949e45bf 100644
--- a/flang/docs/Extensions.md
+++ b/flang/docs/Extensions.md
@@ -780,6 +780,12 @@ character j
 print *, [(j,j=1,10)]
 ```
 
+* The Fortran standard doesn't mention integer overflow explicitly. In many cases,
+  however, integer overflow makes programs non-conforming.
+  F18 follows other widely-used Fortran compilers. Specifically, f18 assumes
+  integer overflow never occurs in address calculations and increment of
+  do-variable unless the option `-fwrapv` is enabled.
+
 ## De Facto Standard Features
 
 * `EXTENDS_TYPE_OF()` returns `.TRUE.` if both of its arguments have the
diff --git a/flang/docs/Intrinsics.md b/flang/docs/Intrinsics.md
index d6f48a7fd87d7..0196be10d8655 100644
--- a/flang/docs/Intrinsics.md
+++ b/flang/docs/Intrinsics.md
@@ -765,7 +765,7 @@ This phase currently supports all the intrinsic procedures listed above but the
 | Coarray intrinsic functions | COSHAPE |
 | Object characteristic inquiry functions | ALLOCATED, ASSOCIATED, EXTENDS_TYPE_OF, IS_CONTIGUOUS, PRESENT, RANK, SAME_TYPE, STORAGE_SIZE |
 | Type inquiry intrinsic functions | BIT_SIZE, DIGITS, EPSILON, HUGE, KIND, MAXEXPONENT, MINEXPONENT, NEW_LINE, PRECISION, RADIX, RANGE, TINY|
-| Non-standard intrinsic functions | AND, OR, XOR, SHIFT, ZEXT, IZEXT, COSD, SIND, TAND, ACOSD, ASIND, ATAND, ATAN2D, COMPL, EQV, NEQV, INT8, JINT, JNINT, KNINT, QCMPLX, DREAL, DFLOAT, QEXT, QFLOAT, QREAL, DNUM, NUM, JNUM, KNUM, QNUM, RNUM, RAN, RANF, ILEN, SIZEOF, MCLOCK, SECNDS, COTAN, IBCHNG, ISHA, ISHC, ISHL, IXOR, IARG, IARGC, NARGS, GETPID, NUMARG, BADDRESS, IADDR, CACHESIZE, EOF, FP_CLASS, INT_PTR_KIND, ISNAN, MALLOC, FREE |
+| Non-standard intrinsic functions | AND, OR, XOR, SHIFT, ZEXT, IZEXT, COSD, SIND, TAND, ACOSD, ASIND, ATAND, ATAN2D, COMPL, EQV, NEQV, INT8, JINT, JNINT, KNINT, QCMPLX, DREAL, DFLOAT, QEXT, QFLOAT, QREAL, DNUM, NUM, JNUM, KNUM, QNUM, RNUM, RAN, RANF, ILEN, SIZEOF, MCLOCK, SECNDS, COTAN, IBCHNG, ISHA, ISHC, ISHL, IXOR, IARG, IARGC, NARGS, GETPID, NUMARG, BADDRESS, IADDR, CACHESIZE, EOF, FP_CLASS, INT_PTR_KIND, ISNAN, MALLOC, FREE, GETUID, GETGID |
 | Intrinsic subroutines |MVBITS (elemental), CPU_TIME, DATE_AND_TIME, EVENT_QUERY, EXECUTE_COMMAND_LINE, GET_COMMAND, GET_COMMAND_ARGUMENT, GET_ENVIRONMENT_VARIABLE, MOVE_ALLOC, RANDOM_INIT, RANDOM_NUMBER, RANDOM_SEED, SIGNAL, SLEEP, SYSTEM, SYSTEM_CLOCK |
 | Atomic intrinsic subroutines | ATOMIC_ADD |
 | Collective intrinsic subroutines | CO_REDUCE |
diff --git a/flang/include/flang/Common/Fortran-features.h b/flang/include/flang/Common/Fortran-features.h
index 86c6e02b0f2ff..f813cbae40a57 100644
--- a/flang/include/flang/Common/Fortran-features.h
+++ b/flang/include/flang/Common/Fortran-features.h
@@ -12,6 +12,7 @@
 #include "flang/Common/Fortran.h"
 #include "flang/Common/enum-set.h"
 #include "flang/Common/idioms.h"
+#include <optional>
 #include <vector>
 
 namespace Fortran::common {
@@ -48,7 +49,7 @@ ENUM_CLASS(LanguageFeature, BackslashEscapes, OldDebugLines,
     ImpliedDoIndexScope, DistinctCommonSizes, OddIndexVariableRestrictions,
     IndistinguishableSpecifics, SubroutineAndFunctionSpecifics,
     EmptySequenceType, NonSequenceCrayPointee, BranchIntoConstruct,
-    BadBranchTarget, ConvertedArgument, HollerithPolymorphic, ListDirectedSize,
+    BadBranchTarget, HollerithPolymorphic, ListDirectedSize,
     NonBindCInteroperability, CudaManaged, CudaUnified,
     PolymorphicActualAllocatableOrPointerToMonomorphicDummy, RelaxedPureDummy,
     UndefinableAsynchronousOrVolatileActual, AutomaticInMainProgram, PrintCptr,
@@ -76,80 +77,12 @@ ENUM_CLASS(UsageWarning, Portability, PointerToUndefinable,
 using LanguageFeatures = EnumSet<LanguageFeature, LanguageFeature_enumSize>;
 using UsageWarnings = EnumSet<UsageWarning, UsageWarning_enumSize>;
 
+std::optional<LanguageFeature> FindLanguageFeature(const char *);
+std::optional<UsageWarning> FindUsageWarning(const char *);
+
 class LanguageFeatureControl {
 public:
-  LanguageFeatureControl() {
-    // These features must be explicitly enabled by command line options.
-    disable_.set(LanguageFeature::OldDebugLines);
-    disable_.set(LanguageFeature::OpenACC);
-    disable_.set(LanguageFeature::OpenMP);
-    disable_.set(LanguageFeature::CUDA); // !@cuf
-    disable_.set(LanguageFeature::CudaManaged);
-    disable_.set(LanguageFeature::CudaUnified);
-    disable_.set(LanguageFeature::ImplicitNoneTypeNever);
-    disable_.set(LanguageFeature::ImplicitNoneTypeAlways);
-    disable_.set(LanguageFeature::DefaultSave);
-    disable_.set(LanguageFeature::SaveMainProgram);
-    // These features, if enabled, conflict with valid standard usage,
-    // so there are disabled here by default.
-    disable_.set(LanguageFeature::BackslashEscapes);
-    disable_.set(LanguageFeature::LogicalAbbreviations);
-    disable_.set(LanguageFeature::XOROperator);
-    disable_.set(LanguageFeature::OldStyleParameter);
-    // These warnings are enabled by default, but only because they used
-    // to be unconditional.  TODO: prune this list
-    warnLanguage_.set(LanguageFeature::ExponentMatchingKindParam);
-    warnLanguage_.set(LanguageFeature::RedundantAttribute);
-    warnLanguage_.set(LanguageFeature::SubroutineAndFunctionSpecifics);
-    warnLanguage_.set(LanguageFeature::EmptySequenceType);
-    warnLanguage_.set(LanguageFeature::NonSequenceCrayPointee);
-    warnLanguage_.set(LanguageFeature::BranchIntoConstruct);
-    warnLanguage_.set(LanguageFeature::BadBranchTarget);
-    warnLanguage_.set(LanguageFeature::ConvertedArgument);
-    warnLanguage_.set(LanguageFeature::HollerithPolymorphic);
-    warnLanguage_.set(LanguageFeature::ListDirectedSize);
-    warnUsage_.set(UsageWarning::ShortArrayActual);
-    warnUsage_.set(UsageWarning::FoldingException);
-    warnUsage_.set(UsageWarning::FoldingAvoidsRuntimeCrash);
-    warnUsage_.set(UsageWarning::FoldingValueChecks);
-    warnUsage_.set(UsageWarning::FoldingFailure);
-    warnUsage_.set(UsageWarning::FoldingLimit);
-    warnUsage_.set(UsageWarning::Interoperability);
-    warnUsage_.set(UsageWarning::Bounds);
-    warnUsage_.set(UsageWarning::Preprocessing);
-    warnUsage_.set(UsageWarning::Scanning);
-    warnUsage_.set(UsageWarning::OpenAccUsage);
-    warnUsage_.set(UsageWarning::ProcPointerCompatibility);
-    warnUsage_.set(UsageWarning::VoidMold);
-    warnUsage_.set(UsageWarning::KnownBadImplicitInterface);
-    warnUsage_.set(UsageWarning::EmptyCase);
-    warnUsage_.set(UsageWarning::CaseOverflow);
-    warnUsage_.set(UsageWarning::CUDAUsage);
-    warnUsage_.set(UsageWarning::IgnoreTKRUsage);
-    warnUsage_.set(UsageWarning::ExternalInterfaceMismatch);
-    warnUsage_.set(UsageWarning::DefinedOperatorArgs);
-    warnUsage_.set(UsageWarning::Final);
-    warnUsage_.set(UsageWarning::ZeroDoStep);
-    warnUsage_.set(UsageWarning::UnusedForallIndex);
-    warnUsage_.set(UsageWarning::OpenMPUsage);
-    warnUsage_.set(UsageWarning::ModuleFile);
-    warnUsage_.set(UsageWarning::DataLength);
-    warnUsage_.set(UsageWarning::IgnoredDirective);
-    warnUsage_.set(UsageWarning::HomonymousSpecific);
-    warnUsage_.set(UsageWarning::HomonymousResult);
-    warnUsage_.set(UsageWarning::IgnoredIntrinsicFunctionType);
-    warnUsage_.set(UsageWarning::PreviousScalarUse);
-    warnUsage_.set(UsageWarning::RedeclaredInaccessibleComponent);
-    warnUsage_.set(UsageWarning::ImplicitShared);
-    warnUsage_.set(UsageWarning::IndexVarRedefinition);
-    warnUsage_.set(UsageWarning::IncompatibleImplicitInterfaces);
-    warnUsage_.set(UsageWarning::BadTypeForTarget);
-    warnUsage_.set(UsageWarning::VectorSubscriptFinalization);
-    warnUsage_.set(UsageWarning::UndefinedFunctionResult);
-    warnUsage_.set(UsageWarning::UselessIomsg);
-    // New warnings, on by default
-    warnLanguage_.set(LanguageFeature::SavedLocalInSpecExpr);
-  }
+  LanguageFeatureControl();
   LanguageFeatureControl(const LanguageFeatureControl &) = default;
 
   void Enable(LanguageFeature f, bool yes = true) { disable_.set(f, !yes); }
diff --git a/flang/include/flang/Evaluate/target.h b/flang/include/flang/Evaluate/target.h
index d076fcbf08307..b347c549e012d 100644
--- a/flang/include/flang/Evaluate/target.h
+++ b/flang/include/flang/Evaluate/target.h
@@ -102,6 +102,11 @@ class TargetCharacteristics {
   bool isPPC() const { return isPPC_; }
   void set_isPPC(bool isPPC = false);
 
+  bool isOSWindows() const { return isOSWindows_; }
+  void set_isOSWindows(bool isOSWindows = false) {
+    isOSWindows_ = isOSWindows;
+  };
+
   IeeeFeatures &ieeeFeatures() { return ieeeFeatures_; }
   const IeeeFeatures &ieeeFeatures() const { return ieeeFeatures_; }
 
@@ -111,6 +116,7 @@ class TargetCharacteristics {
   std::uint8_t align_[common::TypeCategory_enumSize][maxKind]{};
   bool isBigEndian_{false};
   bool isPPC_{false};
+  bool isOSWindows_{false};
   bool areSubnormalsFlushedToZero_{false};
   Rounding roundingMode_{defaultRounding};
   std::size_t procedurePointerByteSize_{8};
diff --git a/flang/include/flang/Lower/LoweringOptions.def b/flang/include/flang/Lower/LoweringOptions.def
index 7594a57a26291..d3f17c3f939c1 100644
--- a/flang/include/flang/Lower/LoweringOptions.def
+++ b/flang/include/flang/Lower/LoweringOptions.def
@@ -34,8 +34,14 @@ ENUM_LOWERINGOPT(NoPPCNativeVecElemOrder, unsigned, 1, 0)
 /// On by default.
 ENUM_LOWERINGOPT(Underscoring, unsigned, 1, 1)
 
+/// If true, assume the behavior of integer overflow is defined
+/// (i.e. wraps around as two's complement). On by default.
+/// TODO: make the default off
+ENUM_LOWERINGOPT(IntegerWrapAround, unsigned, 1, 1)
+
 /// If true, add nsw flags to loop variable increments.
 /// Off by default.
+/// TODO: integrate this option with the above
 ENUM_LOWERINGOPT(NSWOnLoopVarInc, unsigned, 1, 0)
 
 #undef LOWERINGOPT
diff --git a/flang/include/flang/Optimizer/Builder/Complex.h b/flang/include/flang/Optimizer/Builder/Complex.h
index 3286e41ee039e..cd0a876a4cef0 100644
--- a/flang/include/flang/Optimizer/Builder/Complex.h
+++ b/flang/include/flang/Optimizer/Builder/Complex.h
@@ -32,13 +32,12 @@ class Complex {
   mlir::Type getComplexPartType(mlir::Value cplx) const;
   mlir::Type getComplexPartType(mlir::Type complexType) const;
 
-  /// Complex operation creation. They create MLIR operations.
-  mlir::Value createComplex(fir::KindTy kind, mlir::Value real,
-                            mlir::Value imag);
-
   /// Create a complex value.
   mlir::Value createComplex(mlir::Type complexType, mlir::Value real,
                             mlir::Value imag);
+  /// Create a complex value given the real and imag parts real type (which
+  /// must be the same).
+  mlir::Value createComplex(mlir::Value real, mlir::Value imag);
 
   /// Returns the Real/Imag part of \p cplx
   mlir::Value extractComplexPart(mlir::Value cplx, bool isImagPart) {
diff --git a/flang/include/flang/Optimizer/Builder/FIRBuilder.h b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
index 180e2c8ab33ea..09f7b892f1ecb 100644
--- a/flang/include/flang/Optimizer/Builder/FIRBuilder.h
+++ b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
@@ -85,13 +85,16 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
   // The listener self-reference has to be updated in case of copy-construction.
   FirOpBuilder(const FirOpBuilder &other)
       : OpBuilder(other), OpBuilder::Listener(), kindMap{other.kindMap},
-        fastMathFlags{other.fastMathFlags}, symbolTable{other.symbolTable} {
+        fastMathFlags{other.fastMathFlags},
+        integerOverflowFlags{other.integerOverflowFlags},
+        symbolTable{other.symbolTable} {
     setListener(this);
   }
 
   FirOpBuilder(FirOpBuilder &&other)
       : OpBuilder(other), OpBuilder::Listener(),
         kindMap{std::move(other.kindMap)}, fastMathFlags{other.fastMathFlags},
+        integerOverflowFlags{other.integerOverflowFlags},
         symbolTable{other.symbolTable} {
     setListener(this);
   }
@@ -521,6 +524,18 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
     return fmfString;
   }
 
+  /// Set default IntegerOverflowFlags value for all operations
+  /// supporting mlir::arith::IntegerOverflowFlagsAttr that will be created
+  /// by this builder.
+  void setIntegerOverflowFlags(mlir::arith::IntegerOverflowFlags flags) {
+    integerOverflowFlags = flags;
+  }
+
+  /// Get current IntegerOverflowFlags value.
+  mlir::arith::IntegerOverflowFlags getIntegerOverflowFlags() const {
+    return integerOverflowFlags;
+  }
+
   /// Dump the current function. (debug)
   LLVM_DUMP_METHOD void dumpFunc();
 
@@ -547,6 +562,10 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
   /// mlir::arith::FastMathAttr.
   mlir::arith::FastMathFlags fastMathFlags{};
 
+  /// IntegerOverflowFlags that need to be set for operations that support
+  /// mlir::arith::IntegerOverflowFlagsAttr.
+  mlir::arith::IntegerOverflowFlags integerOverflowFlags{};
+
   /// fir::GlobalOp and func::FuncOp symbol table to speed-up
   /// lookups.
   mlir::SymbolTable *symbolTable = nullptr;
diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index 1cfd6b83353b4..868a8b4e28742 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -257,6 +257,10 @@ struct IntrinsicLibrary {
                         llvm::ArrayRef<mlir::Value> args);
   void genGetCommandArgument(mlir::ArrayRef<fir::ExtendedValue> args);
   void genGetEnvironmentVariable(llvm::ArrayRef<fir::ExtendedValue>);
+  mlir::Value genGetGID(mlir::Type resultType,
+                        llvm::ArrayRef<mlir::Value> args);
+  mlir::Value genGetUID(mlir::Type resultType,
+                        llvm::ArrayRef<mlir::Value> args);
   fir::ExtendedValue genIall(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genIand(mlir::Type, llvm::ArrayRef<mlir::Value>);
   fir::ExtendedValue genIany(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
@@ -664,7 +668,7 @@ static inline mlir::Type getTypeHelper(mlir::MLIRContext *context,
     r = builder.getRealType(kind);
     break;
   case ParamTypeId::Complex:
-    r = fir::ComplexType::get(context, kind);
+    r = mlir::ComplexType::get(builder.getRealType(kind));
     break;
   }
 
diff --git a/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h b/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h
index f62071a49e3bf..02b9b68da0db4 100644
--- a/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h
+++ b/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h
@@ -48,6 +48,10 @@ void genEtime(fir::FirOpBuilder &builder, mlir::Location loc,
               mlir::Value values, mlir::Value time);
 
 void genFree(fir::FirOpBuilder &builder, mlir::Location loc, mlir::Value ptr);
+
+mlir::Value genGetUID(fir::FirOpBuilder &, mlir::Location);
+mlir::Value genGetGID(fir::FirOpBuilder &, mlir::Location);
+
 mlir::Value genMalloc(fir::FirOpBuilder &builder, mlir::Location loc,
                       mlir::Value size);
 
diff --git a/flang/include/flang/Optimizer/Builder/Runtime/RTBuilder.h b/flang/include/flang/Optimizer/Builder/Runtime/RTBuilder.h
index a103861f1510b..66e11b5585d52 100644
--- a/flang/include/flang/Optimizer/Builder/Runtime/RTBuilder.h
+++ b/flang/include/flang/Optimizer/Builder/Runtime/RTBuilder.h
@@ -400,17 +400,19 @@ constexpr TypeBuilderFunc getModel<bool &>() {
     return fir::ReferenceType::get(f(context));
   };
 }
-template <>
-constexpr TypeBuilderFunc getModel<std::complex<float>>() {
-  return [](mlir::MLIRContext *context) -> mlir::Type {
-    return mlir::ComplexType::get(mlir::FloatType::getF32(context));
-  };
-}
+
+// getModel<std::complex<T>> are not implemented on purpose.
+// Prefer passing/returning the complex by reference in the runtime to
+// avoid ABI issues.
+// C++ std::complex is not an intrinsic type, and while it is storage
+// compatible with C/Fortran complex type, it follows the struct value passing
+// ABI rule, which may differ from how C complex are passed on some platforms.
+
 template <>
 constexpr TypeBuilderFunc getModel<std::complex<float> &>() {
   return [](mlir::MLIRContext *context) -> mlir::Type {
-    TypeBuilderFunc f{getModel<std::complex<float>>()};
-    return fir::ReferenceType::get(f(context));
+    mlir::Type floatTy = getModel<float>()(context);
+    return fir::ReferenceType::get(mlir::ComplexType::get(floatTy));
   };
 }
 template <>
@@ -422,16 +424,10 @@ constexpr TypeBuilderFunc getModel<const std::complex<float> *>() {
   return getModel<std::complex<float> *>();
 }
 template <>
-constexpr TypeBuilderFunc getModel<std::complex<double>>() {
-  return [](mlir::MLIRContext *context) -> mlir::Type {
-    return mlir::ComplexType::get(mlir::FloatType::getF64(context));
-  };
-}
-template <>
 constexpr TypeBuilderFunc getModel<std::complex<double> &>() {
   return [](mlir::MLIRContext *context) -> mlir::Type {
-    TypeBuilderFunc f{getModel<std::complex<double>>()};
-    return fir::ReferenceType::get(f(context));
+    mlir::Type floatTy = getModel<double>()(context);
+    return fir::ReferenceType::get(mlir::ComplexType::get(floatTy));
   };
 }
 template <>
@@ -445,13 +441,15 @@ constexpr TypeBuilderFunc getModel<const std::complex<double> *>() {
 template <>
 constexpr TypeBuilderFunc getModel<c_float_complex_t>() {
   return [](mlir::MLIRContext *context) -> mlir::Type {
-    return fir::ComplexType::get(context, sizeof(float));
+    mlir::Type floatTy = getModel<float>()(context);
+    return mlir::ComplexType::get(floatTy);
   };
 }
 template <>
 constexpr TypeBuilderFunc getModel<c_double_complex_t>() {
   return [](mlir::MLIRContext *context) -> mlir::Type {
-    return fir::ComplexType::get(context, sizeof(double));
+    mlir::Type floatTy = getModel<double>()(context);
+    return mlir::ComplexType::get(floatTy);
   };
 }
 template <>
@@ -521,10 +519,45 @@ REDUCTION_VALUE_OPERATION_MODEL(double)
 REDUCTION_REF_OPERATION_MODEL(long double)
 REDUCTION_VALUE_OPERATION_MODEL(long double)
 
-REDUCTION_REF_OPERATION_MODEL(std::complex<float>)
-REDUCTION_VALUE_OPERATION_MODEL(std::complex<float>)
-REDUCTION_REF_OPERATION_MODEL(std::complex<double>)
-REDUCTION_VALUE_OPERATION_MODEL(std::complex<double>)
+// FIXME: the runtime is not using the correct ABIs when calling complex
+// callbacks. lowering either need to create wrappers or just have an inline
+// implementation for it. https://github.com/llvm/llvm-project/issues/110674
+template <>
+constexpr TypeBuilderFunc
+getModel<Fortran::runtime::ValueReductionOperation<std::complex<float>>>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    mlir::Type cplx = mlir::ComplexType::get(getModel<float>()(context));
+    auto refTy = fir::ReferenceType::get(cplx);
+    return mlir::FunctionType::get(context, {cplx, cplx}, refTy);
+  };
+}
+template <>
+constexpr TypeBuilderFunc
+getModel<Fortran::runtime::ValueReductionOperation<std::complex<double>>>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    mlir::Type cplx = mlir::ComplexType::get(getModel<double>()(context));
+    auto refTy = fir::ReferenceType::get(cplx);
+    return mlir::FunctionType::get(context, {cplx, cplx}, refTy);
+  };
+}
+template <>
+constexpr TypeBuilderFunc
+getModel<Fortran::runtime::ReferenceReductionOperation<std::complex<float>>>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    mlir::Type cplx = mlir::ComplexType::get(getModel<float>()(context));
+    auto refTy = fir::ReferenceType::get(cplx);
+    return mlir::FunctionType::get(context, {refTy, refTy}, refTy);
+  };
+}
+template <>
+constexpr TypeBuilderFunc getModel<
+    Fortran::runtime::ReferenceReductionOperation<std::complex<double>>>() {
+  return [](mlir::MLIRContext *context) -> mlir::Type {
+    mlir::Type cplx = mlir::ComplexType::get(getModel<double>()(context));
+    auto refTy = fir::ReferenceType::get(cplx);
+    return mlir::FunctionType::get(context, {refTy, refTy}, refTy);
+  };
+}
 
 REDUCTION_CHAR_OPERATION_MODEL(char)
 REDUCTION_CHAR_OPERATION_MODEL(char16_t)
diff --git a/flang/include/flang/Optimizer/CodeGen/TypeConverter.h b/flang/include/flang/Optimizer/CodeGen/TypeConverter.h
index 7215abf3a8f81..7c317ddeea1fa 100644
--- a/flang/include/flang/Optimizer/CodeGen/TypeConverter.h
+++ b/flang/include/flang/Optimizer/CodeGen/TypeConverter.h
@@ -89,25 +89,10 @@ class LLVMTypeConverter : public mlir::LLVMTypeConverter {
   // fir.char<k,n>  -->  llvm.array<n x "ix">
   mlir::Type convertCharType(fir::CharacterType charTy) const;
 
-  // Use the target specifics to figure out how to map complex to LLVM IR. The
-  // use of complex values in function signatures is handled before conversion
-  // to LLVM IR dialect here.
-  //
-  // fir.complex<T> | std.complex<T>    --> llvm<"{t,t}">
-  template <typename C>
-  mlir::Type convertComplexType(C cmplx) const {
-    auto eleTy = cmplx.getElementType();
-    return convertType(specifics->complexMemoryType(eleTy));
-  }
-
   template <typename A> mlir::Type convertPointerLike(A &ty) const {
     return mlir::LLVM::LLVMPointerType::get(ty.getContext());
   }
 
-  // convert a front-end kind value to either a std or LLVM IR dialect type
-  // fir.real<n>  -->  llvm.anyfloat  where anyfloat is a kind mapping
-  mlir::Type convertRealType(fir::KindTy kind) const;
-
   // fir.array<c ... :any>  -->  llvm<"[...[c x any]]">
   mlir::Type convertSequenceType(SequenceType seq) const;
 
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 68847cfa2cbc2..d467f0ab4e1c1 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -2516,7 +2516,8 @@ def fir_DispatchOp : fir_Op<"dispatch", []> {
     StrAttr:$method,
     fir_ClassType:$object,
     Variadic<AnyType>:$args,
-    OptionalAttr<I32Attr>:$pass_arg_pos
+    OptionalAttr<I32Attr>:$pass_arg_pos,
+    OptionalAttr<fir_FortranProcedureFlagsAttr>:$procedure_attrs
   );
 
   let results = (outs Variadic<AnyType>:$results);
@@ -2525,7 +2526,8 @@ def fir_DispatchOp : fir_Op<"dispatch", []> {
 
   let assemblyFormat = [{
     $method `(` $object `:` qualified(type($object)) `)`
-    ( `(` $args^ `:` type($args) `)` )? (`->` type($results)^)? attr-dict
+    ( `(` $args^ `:` type($args) `)` )? (`->` type($results)^)?
+    (`proc_attrs` $procedure_attrs^)? attr-dict
   }];
 
   let extraClassDeclaration = [{
@@ -2609,38 +2611,15 @@ class fir_UnaryArithmeticOp<string mnemonic, list<Trait> traits = []> :
   let assemblyFormat = "operands attr-dict `:` type($result)";
 }
 
-def fir_ConstcOp : fir_Op<"constc", [NoMemoryEffect]> {
-  let summary = "create a complex constant";
-
-  let description = [{
-    A complex constant. Similar to the standard dialect complex type, but this
-    extension allows constants with APFloat values that are not supported in
-    the standard dialect.
-  }];
-
-  let results = (outs fir_ComplexType);
-
-  let hasCustomAssemblyFormat = 1;
-  let hasVerifier = 1;
-
-  let extraClassDeclaration = [{
-    static constexpr llvm::StringRef getRealAttrName() { return "real"; }
-    static constexpr llvm::StringRef getImagAttrName() { return "imaginary"; }
-
-    mlir::Attribute getReal() { return (*this)->getAttr(getRealAttrName()); }
-    mlir::Attribute getImaginary() { return (*this)->getAttr(getImagAttrName()); }
-  }];
-}
-
 class ComplexUnaryArithmeticOp<string mnemonic, list<Trait> traits = []> :
       fir_UnaryArithmeticOp<mnemonic, traits>,
-      Arguments<(ins fir_ComplexType:$operand)>;
+      Arguments<(ins AnyFirComplex:$operand)>;
 
 def fir_NegcOp : ComplexUnaryArithmeticOp<"negc">;
 
 class ComplexArithmeticOp<string mnemonic, list<Trait> traits = []> :
       fir_ArithmeticOp<mnemonic, traits>,
-      Arguments<(ins fir_ComplexType:$lhs, fir_ComplexType:$rhs,
+      Arguments<(ins AnyFirComplex:$lhs, AnyFirComplex:$rhs,
           DefaultValuedAttr<Arith_FastMathAttr,
                             "::mlir::arith::FastMathFlags::none">:$fastmath)>;
 
@@ -2664,8 +2643,8 @@ def fir_CmpcOp : fir_Op<"cmpc",
   }];
 
   let arguments = (ins
-      fir_ComplexType:$lhs,
-      fir_ComplexType:$rhs,
+      AnyFirComplex:$lhs,
+      AnyFirComplex:$rhs,
       DefaultValuedAttr<Arith_FastMathAttr, "::mlir::arith::FastMathFlags::none">:$fastmath);
 
   let results = (outs AnyLogicalLike);
@@ -2756,8 +2735,8 @@ def fir_ConvertOp : fir_SimpleOneResultOp<"convert", [NoMemoryEffect]> {
 }
 
 def FortranTypeAttr : Attr<And<[CPred<"mlir::isa<mlir::TypeAttr>($_self)">,
-    Or<[CPred<"mlir::isa<fir::CharacterType, fir::ComplexType, "
-              "fir::IntegerType, fir::LogicalType, fir::RealType, "
+    Or<[CPred<"mlir::isa<fir::CharacterType, fir::IntegerType,"
+              "fir::LogicalType, mlir::FloatType, mlir::ComplexType,"
               "fir::RecordType>(mlir::cast<mlir::TypeAttr>($_self).getValue())"
     >]>]>, "Fortran surface type"> {
   let storageType = [{ ::mlir::TypeAttr }];
diff --git a/flang/include/flang/Optimizer/Dialect/FIRType.h b/flang/include/flang/Optimizer/Dialect/FIRType.h
index 3e505b81e1190..ac638d98980d1 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRType.h
+++ b/flang/include/flang/Optimizer/Dialect/FIRType.h
@@ -159,10 +159,8 @@ mlir::Type dyn_cast_ptrEleTy(mlir::Type t);
 /// `t` is not a memory reference or box type, then returns a null `Type`.
 mlir::Type dyn_cast_ptrOrBoxEleTy(mlir::Type t);
 
-/// Is `t` a FIR Real or MLIR Float type?
-inline bool isa_real(mlir::Type t) {
-  return mlir::isa<fir::RealType, mlir::FloatType>(t);
-}
+/// Is `t` a real type?
+inline bool isa_real(mlir::Type t) { return mlir::isa<mlir::FloatType>(t); }
 
 /// Is `t` an integral type?
 inline bool isa_integer(mlir::Type t) {
@@ -182,9 +180,11 @@ void printFirType(FIROpsDialect *, mlir::Type ty, mlir::DialectAsmPrinter &p);
 /// Index, or FIR Int). Aborts execution if condition is false.
 void verifyIntegralType(mlir::Type type);
 
-/// Is `t` a FIR or MLIR Complex type?
+/// Is `t` a floating point complex type?
 inline bool isa_complex(mlir::Type t) {
-  return mlir::isa<fir::ComplexType, mlir::ComplexType>(t);
+  return mlir::isa<mlir::ComplexType>(t) &&
+         mlir::isa<mlir::FloatType>(
+             mlir::cast<mlir::ComplexType>(t).getElementType());
 }
 
 /// Is `t` a CHARACTER type? Does not check the length.
diff --git a/flang/include/flang/Optimizer/Dialect/FIRTypes.td b/flang/include/flang/Optimizer/Dialect/FIRTypes.td
index ae984de63db42..7ac8e0822ecc8 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRTypes.td
+++ b/flang/include/flang/Optimizer/Dialect/FIRTypes.td
@@ -158,26 +158,6 @@ def fir_ClassType : FIR_Type<"Class", "class", [], "BaseBoxType"> {
   let assemblyFormat = "`<` $eleTy `>`";
 }
 
-def fir_ComplexType : FIR_Type<"Complex", "complex"> {
-  let summary = "Complex type";
-
-  let description = [{
-    Model of a Fortran COMPLEX intrinsic type, including the KIND type
-    parameter. COMPLEX is a floating point type with a real and imaginary
-    member.
-  }];
-
-  let parameters = (ins "KindTy":$fKind);
-  let hasCustomAssemblyFormat = 1;
-
-  let extraClassDeclaration = [{
-    using KindTy = unsigned;
-
-    mlir::Type getElementType() const;
-    mlir::Type getEleType(const fir::KindMapping &kindMap) const;
-  }];
-}
-
 def fir_FieldType : FIR_Type<"Field", "field"> {
   let summary = "A field (in a RecordType) argument's type";
 
@@ -313,26 +293,6 @@ def fir_PointerType : FIR_Type<"Pointer", "ptr"> {
   }];
 }
 
-def fir_RealType : FIR_Type<"Real", "real"> {
-  let summary = "FIR real type";
-
-  let description = [{
-    Model of a Fortran REAL (and DOUBLE PRECISION) intrinsic type, including the
-    KIND type parameter.
-  }];
-
-  let parameters = (ins "KindTy":$fKind);
-  let hasCustomAssemblyFormat = 1;
-
-  let extraClassDeclaration = [{
-    using KindTy = unsigned;
-    // Get MLIR float type with same semantics.
-    mlir::Type getFloatType(const fir::KindMapping &kindMap) const;
-  }];
-
-  let genVerifyDecl = 1;
-}
-
 def fir_RecordType : FIR_Type<"Record", "type"> {
   let summary = "FIR derived type";
 
@@ -597,16 +557,17 @@ def AnyIntegerLike : TypeConstraint<Or<[SignlessIntegerLike.predicate,
     AnySignedInteger.predicate, fir_IntegerType.predicate]>, "any integer">;
 def AnyLogicalLike : TypeConstraint<Or<[BoolLike.predicate,
     fir_LogicalType.predicate]>, "any logical">;
-def AnyRealLike : TypeConstraint<Or<[FloatLike.predicate,
-    fir_RealType.predicate]>, "any real">;
+def AnyRealLike : TypeConstraint<FloatLike.predicate, "any real">;
 def AnyIntegerType : Type<AnyIntegerLike.predicate, "any integer">;
 
+def AnyFirComplexLike :  TypeConstraint<CPred<"::fir::isa_complex($_self)">,
+  "any floating point complex type">;
+def AnyFirComplex : Type<AnyFirComplexLike.predicate,
+  "any floating point complex type">;
+
 // Composable types
-// Note that we include both fir_ComplexType and AnyComplex, so we can use both
-// the FIR ComplexType and the MLIR ComplexType (the former is used to represent
-// Fortran complex and the latter for C++ std::complex).
 def AnyCompositeLike : TypeConstraint<Or<[fir_RecordType.predicate,
-    fir_SequenceType.predicate, fir_ComplexType.predicate, AnyComplex.predicate,
+    fir_SequenceType.predicate, AnyFirComplexLike.predicate,
     fir_VectorType.predicate, IsTupleTypePred, fir_CharacterType.predicate]>,
     "any composite">;
 
diff --git a/flang/include/flang/Optimizer/HLFIR/HLFIROpBase.td b/flang/include/flang/Optimizer/HLFIR/HLFIROpBase.td
index 492e3337137cf..d967a407a7588 100644
--- a/flang/include/flang/Optimizer/HLFIR/HLFIROpBase.td
+++ b/flang/include/flang/Optimizer/HLFIR/HLFIROpBase.td
@@ -93,7 +93,7 @@ def AnyFortranVariable : Type<IsFortranVariablePred, "any HLFIR variable type">;
 
 def AnyFortranValue : TypeConstraint<Or<[AnyLogicalLike.predicate,
     AnyIntegerLike.predicate, AnyRealLike.predicate,
-    fir_ComplexType.predicate, AnyComplex.predicate,
+    AnyFirComplexLike.predicate,
     hlfir_ExprType.predicate]>, "any Fortran value type">;
 
 
diff --git a/flang/include/flang/Optimizer/Support/Utils.h b/flang/include/flang/Optimizer/Support/Utils.h
index 06cf9c0be157c..2e25ef5f19bbe 100644
--- a/flang/include/flang/Optimizer/Support/Utils.h
+++ b/flang/include/flang/Optimizer/Support/Utils.h
@@ -84,25 +84,33 @@ inline std::string mlirTypeToString(mlir::Type type) {
   return result;
 }
 
-inline std::string mlirTypeToIntrinsicFortran(fir::FirOpBuilder &builder,
-                                              mlir::Type type,
-                                              mlir::Location loc,
-                                              const llvm::Twine &name) {
+inline std::optional<int> mlirFloatTypeToKind(mlir::Type type) {
   if (type.isF16())
-    return "REAL(KIND=2)";
+    return 2;
   else if (type.isBF16())
-    return "REAL(KIND=3)";
-  else if (type.isTF32())
-    return "REAL(KIND=unknown)";
+    return 3;
   else if (type.isF32())
-    return "REAL(KIND=4)";
+    return 4;
   else if (type.isF64())
-    return "REAL(KIND=8)";
+    return 8;
   else if (type.isF80())
-    return "REAL(KIND=10)";
+    return 10;
   else if (type.isF128())
-    return "REAL(KIND=16)";
-  else if (type.isInteger(8))
+    return 16;
+  return std::nullopt;
+}
+
+inline std::string mlirTypeToIntrinsicFortran(fir::FirOpBuilder &builder,
+                                              mlir::Type type,
+                                              mlir::Location loc,
+                                              const llvm::Twine &name) {
+  if (auto floatTy = mlir::dyn_cast<mlir::FloatType>(type)) {
+    if (std::optional<int> kind = mlirFloatTypeToKind(type))
+      return "REAL(KIND="s + std::to_string(*kind) + ")";
+  } else if (auto cplxTy = mlir::dyn_cast<mlir::ComplexType>(type)) {
+    if (std::optional<int> kind = mlirFloatTypeToKind(cplxTy.getElementType()))
+      return "COMPLEX(KIND+"s + std::to_string(*kind) + ")";
+  } else if (type.isInteger(8))
     return "INTEGER(KIND=1)";
   else if (type.isInteger(16))
     return "INTEGER(KIND=2)";
@@ -112,18 +120,6 @@ inline std::string mlirTypeToIntrinsicFortran(fir::FirOpBuilder &builder,
     return "INTEGER(KIND=8)";
   else if (type.isInteger(128))
     return "INTEGER(KIND=16)";
-  else if (type == fir::ComplexType::get(builder.getContext(), 2))
-    return "COMPLEX(KIND=2)";
-  else if (type == fir::ComplexType::get(builder.getContext(), 3))
-    return "COMPLEX(KIND=3)";
-  else if (type == fir::ComplexType::get(builder.getContext(), 4))
-    return "COMPLEX(KIND=4)";
-  else if (type == fir::ComplexType::get(builder.getContext(), 8))
-    return "COMPLEX(KIND=8)";
-  else if (type == fir::ComplexType::get(builder.getContext(), 10))
-    return "COMPLEX(KIND=10)";
-  else if (type == fir::ComplexType::get(builder.getContext(), 16))
-    return "COMPLEX(KIND=16)";
   else if (type == fir::LogicalType::get(builder.getContext(), 1))
     return "LOGICAL(KIND=1)";
   else if (type == fir::LogicalType::get(builder.getContext(), 2))
@@ -132,9 +128,9 @@ inline std::string mlirTypeToIntrinsicFortran(fir::FirOpBuilder &builder,
     return "LOGICAL(KIND=4)";
   else if (type == fir::LogicalType::get(builder.getContext(), 8))
     return "LOGICAL(KIND=8)";
-  else
-    fir::emitFatalError(loc, "unsupported type in " + name + ": " +
-                                 fir::mlirTypeToString(type));
+
+  fir::emitFatalError(loc, "unsupported type in " + name + ": " +
+                               fir::mlirTypeToString(type));
 }
 
 inline void intrinsicTypeTODO(fir::FirOpBuilder &builder, mlir::Type type,
@@ -159,19 +155,13 @@ inline void intrinsicTypeTODO2(fir::FirOpBuilder &builder, mlir::Type type1,
 
 inline std::pair<Fortran::common::TypeCategory, KindMapping::KindTy>
 mlirTypeToCategoryKind(mlir::Location loc, mlir::Type type) {
-  if (type.isF16())
-    return {Fortran::common::TypeCategory::Real, 2};
-  else if (type.isBF16())
-    return {Fortran::common::TypeCategory::Real, 3};
-  else if (type.isF32())
-    return {Fortran::common::TypeCategory::Real, 4};
-  else if (type.isF64())
-    return {Fortran::common::TypeCategory::Real, 8};
-  else if (type.isF80())
-    return {Fortran::common::TypeCategory::Real, 10};
-  else if (type.isF128())
-    return {Fortran::common::TypeCategory::Real, 16};
-  else if (type.isInteger(8))
+  if (auto floatTy = mlir::dyn_cast<mlir::FloatType>(type)) {
+    if (std::optional<int> kind = mlirFloatTypeToKind(type))
+      return {Fortran::common::TypeCategory::Real, *kind};
+  } else if (auto cplxTy = mlir::dyn_cast<mlir::ComplexType>(type)) {
+    if (std::optional<int> kind = mlirFloatTypeToKind(cplxTy.getElementType()))
+      return {Fortran::common::TypeCategory::Complex, *kind};
+  } else if (type.isInteger(8))
     return {Fortran::common::TypeCategory::Integer, 1};
   else if (type.isInteger(16))
     return {Fortran::common::TypeCategory::Integer, 2};
@@ -181,17 +171,13 @@ mlirTypeToCategoryKind(mlir::Location loc, mlir::Type type) {
     return {Fortran::common::TypeCategory::Integer, 8};
   else if (type.isInteger(128))
     return {Fortran::common::TypeCategory::Integer, 16};
-  else if (auto complexType = mlir::dyn_cast<fir::ComplexType>(type))
-    return {Fortran::common::TypeCategory::Complex, complexType.getFKind()};
   else if (auto logicalType = mlir::dyn_cast<fir::LogicalType>(type))
     return {Fortran::common::TypeCategory::Logical, logicalType.getFKind()};
   else if (auto charType = mlir::dyn_cast<fir::CharacterType>(type))
     return {Fortran::common::TypeCategory::Character, charType.getFKind()};
   else if (mlir::isa<fir::RecordType>(type))
     return {Fortran::common::TypeCategory::Derived, 0};
-  else
-    fir::emitFatalError(loc,
-                        "unsupported type: " + fir::mlirTypeToString(type));
+  fir::emitFatalError(loc, "unsupported type: " + fir::mlirTypeToString(type));
 }
 
 /// Find the fir.type_info that was created for this \p recordType in \p module,
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 0b4ebcbaa24c4..bf00e6b43d066 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -522,6 +522,8 @@ class ParseTreeDumper {
   NODE(parser, OmpEndSectionsDirective)
   NODE(parser, OmpIfClause)
   NODE_ENUM(OmpIfClause, DirectiveNameModifier)
+  NODE_ENUM(OmpLastprivateClause, LastprivateModifier)
+  NODE(parser, OmpLastprivateClause)
   NODE(parser, OmpLinearClause)
   NODE(OmpLinearClause, WithModifier)
   NODE(OmpLinearClause, WithoutModifier)
diff --git a/flang/include/flang/Parser/message.h b/flang/include/flang/Parser/message.h
index 668559aeec947..bc38f571ca3df 100644
--- a/flang/include/flang/Parser/message.h
+++ b/flang/include/flang/Parser/message.h
@@ -15,6 +15,7 @@
 #include "char-block.h"
 #include "char-set.h"
 #include "provenance.h"
+#include "flang/Common/Fortran-features.h"
 #include "flang/Common/idioms.h"
 #include "flang/Common/reference-counted.h"
 #include "flang/Common/restorer.h"
@@ -202,6 +203,26 @@ class Message : public common::ReferenceCounted<Message> {
   Message(ProvenanceRange pr, const MessageExpectedText &t)
       : location_{pr}, text_{t} {}
 
+  Message(common::LanguageFeature feature, ProvenanceRange pr,
+      const MessageFixedText &t)
+      : location_{pr}, text_{t}, languageFeature_{feature} {}
+  Message(common::LanguageFeature feature, ProvenanceRange pr,
+      const MessageFormattedText &s)
+      : location_{pr}, text_{s}, languageFeature_{feature} {}
+  Message(common::LanguageFeature feature, ProvenanceRange pr,
+      MessageFormattedText &&s)
+      : location_{pr}, text_{std::move(s)}, languageFeature_{feature} {}
+
+  Message(common::UsageWarning warning, ProvenanceRange pr,
+      const MessageFixedText &t)
+      : location_{pr}, text_{t}, usageWarning_{warning} {}
+  Message(common::UsageWarning warning, ProvenanceRange pr,
+      const MessageFormattedText &s)
+      : location_{pr}, text_{s}, usageWarning_{warning} {}
+  Message(common::UsageWarning warning, ProvenanceRange pr,
+      MessageFormattedText &&s)
+      : location_{pr}, text_{std::move(s)}, usageWarning_{warning} {}
+
   Message(CharBlock csr, const MessageFixedText &t)
       : location_{csr}, text_{t} {}
   Message(CharBlock csr, const MessageFormattedText &s)
@@ -211,10 +232,41 @@ class Message : public common::ReferenceCounted<Message> {
   Message(CharBlock csr, const MessageExpectedText &t)
       : location_{csr}, text_{t} {}
 
+  Message(
+      common::LanguageFeature feature, CharBlock csr, const MessageFixedText &t)
+      : location_{csr}, text_{t}, languageFeature_{feature} {}
+  Message(common::LanguageFeature feature, CharBlock csr,
+      const MessageFormattedText &s)
+      : location_{csr}, text_{s}, languageFeature_{feature} {}
+  Message(
+      common::LanguageFeature feature, CharBlock csr, MessageFormattedText &&s)
+      : location_{csr}, text_{std::move(s)}, languageFeature_{feature} {}
+
+  Message(
+      common::UsageWarning warning, CharBlock csr, const MessageFixedText &t)
+      : location_{csr}, text_{t}, usageWarning_{warning} {}
+  Message(common::UsageWarning warning, CharBlock csr,
+      const MessageFormattedText &s)
+      : location_{csr}, text_{s}, usageWarning_{warning} {}
+  Message(common::UsageWarning warning, CharBlock csr, MessageFormattedText &&s)
+      : location_{csr}, text_{std::move(s)}, usageWarning_{warning} {}
+
   template <typename RANGE, typename A, typename... As>
   Message(RANGE r, const MessageFixedText &t, A &&x, As &&...xs)
       : location_{r}, text_{MessageFormattedText{
                           t, std::forward<A>(x), std::forward<As>(xs)...}} {}
+  template <typename RANGE, typename A, typename... As>
+  Message(common::LanguageFeature feature, RANGE r, const MessageFixedText &t,
+      A &&x, As &&...xs)
+      : location_{r}, text_{MessageFormattedText{
+                          t, std::forward<A>(x), std::forward<As>(xs)...}},
+        languageFeature_{feature} {}
+  template <typename RANGE, typename A, typename... As>
+  Message(common::UsageWarning warning, RANGE r, const MessageFixedText &t,
+      A &&x, As &&...xs)
+      : location_{r}, text_{MessageFormattedText{
+                          t, std::forward<A>(x), std::forward<As>(xs)...}},
+        usageWarning_{warning} {}
 
   Reference attachment() const { return attachment_; }
 
@@ -232,6 +284,10 @@ class Message : public common::ReferenceCounted<Message> {
   bool IsFatal() const;
   Severity severity() const;
   Message &set_severity(Severity);
+  std::optional<common::LanguageFeature> languageFeature() const;
+  Message &set_languageFeature(common::LanguageFeature);
+  std::optional<common::UsageWarning> usageWarning() const;
+  Message &set_usageWarning(common::UsageWarning);
   std::string ToString() const;
   std::optional<ProvenanceRange> GetProvenanceRange(
       const AllCookedSources &) const;
@@ -256,6 +312,8 @@ class Message : public common::ReferenceCounted<Message> {
       text_;
   bool attachmentIsContext_{false};
   Reference attachment_;
+  std::optional<common::LanguageFeature> languageFeature_;
+  std::optional<common::UsageWarning> usageWarning_;
 };
 
 class Messages {
@@ -275,6 +333,16 @@ class Messages {
     return messages_.emplace_back(std::forward<A>(args)...);
   }
 
+  template <typename... A>
+  Message &Say(common::LanguageFeature feature, A &&...args) {
+    return Say(std::forward<A>(args)...).set_languageFeature(feature);
+  }
+
+  template <typename... A>
+  Message &Say(common::UsageWarning warning, A &&...args) {
+    return Say(std::forward<A>(args)...).set_usageWarning(warning);
+  }
+
   void Annex(Messages &&that) {
     messages_.splice(messages_.end(), that.messages_);
   }
@@ -330,6 +398,10 @@ class ContextualMessages {
     return common::ScopedSet(messages_, nullptr);
   }
 
+  template <typename... A> Message *Say(A &&...args) {
+    return Say(at_, std::forward<A>(args)...);
+  }
+
   template <typename... A> Message *Say(CharBlock at, A &&...args) {
     if (messages_ != nullptr) {
       auto &msg{messages_->Say(at, std::forward<A>(args)...)};
@@ -347,8 +419,22 @@ class ContextualMessages {
     return Say(at.value_or(at_), std::forward<A>(args)...);
   }
 
-  template <typename... A> Message *Say(A &&...args) {
-    return Say(at_, std::forward<A>(args)...);
+  template <typename... A>
+  Message *Say(common::LanguageFeature feature, A &&...args) {
+    Message *msg{Say(std::forward<A>(args)...)};
+    if (msg) {
+      msg->set_languageFeature(feature);
+    }
+    return msg;
+  }
+
+  template <typename... A>
+  Message *Say(common::UsageWarning warning, A &&...args) {
+    Message *msg{Say(std::forward<A>(args)...)};
+    if (msg) {
+      msg->set_usageWarning(warning);
+    }
+    return msg;
   }
 
   Message *Say(Message &&msg) {
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 548fcc81984b2..7057ac2267aa1 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3648,6 +3648,15 @@ struct OmpAtomicDefaultMemOrderClause {
       OmpAtomicDefaultMemOrderClause, common::OmpAtomicDefaultMemOrderType);
 };
 
+// OMP 5.0 2.19.4.5 lastprivate-clause ->
+//                    LASTPRIVATE ([lastprivate-modifier :] list)
+//                  lastprivate-modifier -> CONDITIONAL
+struct OmpLastprivateClause {
+  TUPLE_CLASS_BOILERPLATE(OmpLastprivateClause);
+  ENUM_CLASS(LastprivateModifier, Conditional);
+  std::tuple<std::optional<LastprivateModifier>, OmpObjectList> t;
+};
+
 // OpenMP Clauses
 struct OmpClause {
   UNION_CLASS_BOILERPLATE(OmpClause);
diff --git a/flang/include/flang/Runtime/extensions.h b/flang/include/flang/Runtime/extensions.h
index 8b7607be7e999..a855c694e0090 100644
--- a/flang/include/flang/Runtime/extensions.h
+++ b/flang/include/flang/Runtime/extensions.h
@@ -20,6 +20,14 @@
 #include <cstddef>
 #include <cstdint>
 
+#ifdef _WIN32
+// UID and GID don't exist on Windows, these exist to avoid errors.
+typedef std::uint32_t uid_t;
+typedef std::uint32_t gid_t;
+#else
+#include "sys/types.h" //pid_t
+#endif
+
 extern "C" {
 
 // CALL FLUSH(n) antedates the Fortran 2003 FLUSH statement.
@@ -37,6 +45,12 @@ std::int32_t FORTRAN_PROCEDURE_NAME(iargc)();
 void FORTRAN_PROCEDURE_NAME(getarg)(
     std::int32_t &n, char *arg, std::int64_t length);
 
+// Calls getgid()
+gid_t RTNAME(GetGID)();
+
+// Calls getuid()
+uid_t RTNAME(GetUID)();
+
 // GNU extension subroutine GETLOG(C).
 void FORTRAN_PROCEDURE_NAME(getlog)(char *name, std::int64_t length);
 
diff --git a/flang/include/flang/Semantics/expression.h b/flang/include/flang/Semantics/expression.h
index c90c8c4b3cc70..a90801db7338c 100644
--- a/flang/include/flang/Semantics/expression.h
+++ b/flang/include/flang/Semantics/expression.h
@@ -123,6 +123,16 @@ class ExpressionAnalyzer {
   template <typename... A> parser::Message *Say(A &&...args) {
     return GetContextualMessages().Say(std::forward<A>(args)...);
   }
+  template <typename FeatureOrUsageWarning, typename... A>
+  parser::Message *Warn(
+      FeatureOrUsageWarning warning, parser::CharBlock at, A &&...args) {
+    return context_.Warn(warning, at, std::forward<A>(args)...);
+  }
+  template <typename FeatureOrUsageWarning, typename... A>
+  parser::Message *Warn(FeatureOrUsageWarning warning, A &&...args) {
+    return Warn(
+        warning, GetContextualMessages().at(), std::forward<A>(args)...);
+  }
 
   template <typename T, typename... A>
   parser::Message *SayAt(const T &parsed, A &&...args) {
diff --git a/flang/include/flang/Semantics/semantics.h b/flang/include/flang/Semantics/semantics.h
index 2a326074b3dc6..606afbe288c38 100644
--- a/flang/include/flang/Semantics/semantics.h
+++ b/flang/include/flang/Semantics/semantics.h
@@ -188,6 +188,24 @@ class SemanticsContext {
     return message;
   }
 
+  template <typename FeatureOrUsageWarning, typename... A>
+  parser::Message *Warn(
+      FeatureOrUsageWarning warning, parser::CharBlock at, A &&...args) {
+    if (languageFeatures_.ShouldWarn(warning) && !IsInModuleFile(at)) {
+      parser::Message &msg{
+          messages_.Say(warning, at, std::forward<A>(args)...)};
+      return &msg;
+    } else {
+      return nullptr;
+    }
+  }
+
+  template <typename FeatureOrUsageWarning, typename... A>
+  parser::Message *Warn(FeatureOrUsageWarning warning, A &&...args) {
+    CHECK(location_);
+    return Warn(warning, *location_, std::forward<A>(args)...);
+  }
+
   const Scope &FindScope(parser::CharBlock) const;
   Scope &FindScope(parser::CharBlock);
   void UpdateScopeIndex(Scope &, parser::CharBlock);
@@ -270,7 +288,7 @@ class SemanticsContext {
       std::multimap<parser::CharBlock, Scope &, ScopeIndexComparator>;
   ScopeIndex::iterator SearchScopeIndex(parser::CharBlock);
 
-  void CheckIndexVarRedefine(
+  parser::Message *CheckIndexVarRedefine(
       const parser::CharBlock &, const Symbol &, parser::MessageFixedText &&);
   void CheckError(const Symbol &);
 
diff --git a/flang/include/flang/Semantics/symbol.h b/flang/include/flang/Semantics/symbol.h
index b4db6689a9427..cf0350735b5b9 100644
--- a/flang/include/flang/Semantics/symbol.h
+++ b/flang/include/flang/Semantics/symbol.h
@@ -755,7 +755,7 @@ class Symbol {
       OmpDeclarativeAllocateDirective, OmpExecutableAllocateDirective,
       OmpDeclareSimd, OmpDeclareTarget, OmpThreadprivate, OmpDeclareReduction,
       OmpFlushed, OmpCriticalLock, OmpIfSpecified, OmpNone, OmpPreDetermined,
-      OmpImplicit, OmpFromStmtFunction);
+      OmpImplicit);
   using Flags = common::EnumSet<Flag, Flag_enumSize>;
 
   const Scope &owner() const { return *owner_; }
diff --git a/flang/include/flang/Tools/TargetSetup.h b/flang/include/flang/Tools/TargetSetup.h
index c8d32e8e87cf1..f52b5ddaa8d49 100644
--- a/flang/include/flang/Tools/TargetSetup.h
+++ b/flang/include/flang/Tools/TargetSetup.h
@@ -59,6 +59,9 @@ namespace Fortran::tools {
   if (targetTriple.isPPC())
     targetCharacteristics.set_isPPC(true);
 
+  if (targetTriple.isOSWindows())
+    targetCharacteristics.set_isOSWindows(true);
+
   // TODO: use target machine data layout to set-up the target characteristics
   // type size and alignment info.
 }
diff --git a/flang/lib/Common/Fortran-features.cpp b/flang/lib/Common/Fortran-features.cpp
index 25a948818e656..59f570e6ab6e9 100644
--- a/flang/lib/Common/Fortran-features.cpp
+++ b/flang/lib/Common/Fortran-features.cpp
@@ -12,6 +12,131 @@
 
 namespace Fortran::common {
 
+LanguageFeatureControl::LanguageFeatureControl() {
+  // These features must be explicitly enabled by command line options.
+  disable_.set(LanguageFeature::OldDebugLines);
+  disable_.set(LanguageFeature::OpenACC);
+  disable_.set(LanguageFeature::OpenMP);
+  disable_.set(LanguageFeature::CUDA); // !@cuf
+  disable_.set(LanguageFeature::CudaManaged);
+  disable_.set(LanguageFeature::CudaUnified);
+  disable_.set(LanguageFeature::ImplicitNoneTypeNever);
+  disable_.set(LanguageFeature::ImplicitNoneTypeAlways);
+  disable_.set(LanguageFeature::DefaultSave);
+  disable_.set(LanguageFeature::SaveMainProgram);
+  // These features, if enabled, conflict with valid standard usage,
+  // so there are disabled here by default.
+  disable_.set(LanguageFeature::BackslashEscapes);
+  disable_.set(LanguageFeature::LogicalAbbreviations);
+  disable_.set(LanguageFeature::XOROperator);
+  disable_.set(LanguageFeature::OldStyleParameter);
+  // These warnings are enabled by default, but only because they used
+  // to be unconditional.  TODO: prune this list
+  warnLanguage_.set(LanguageFeature::ExponentMatchingKindParam);
+  warnLanguage_.set(LanguageFeature::RedundantAttribute);
+  warnLanguage_.set(LanguageFeature::SubroutineAndFunctionSpecifics);
+  warnLanguage_.set(LanguageFeature::EmptySequenceType);
+  warnLanguage_.set(LanguageFeature::NonSequenceCrayPointee);
+  warnLanguage_.set(LanguageFeature::BranchIntoConstruct);
+  warnLanguage_.set(LanguageFeature::BadBranchTarget);
+  warnLanguage_.set(LanguageFeature::HollerithPolymorphic);
+  warnLanguage_.set(LanguageFeature::ListDirectedSize);
+  warnUsage_.set(UsageWarning::ShortArrayActual);
+  warnUsage_.set(UsageWarning::FoldingException);
+  warnUsage_.set(UsageWarning::FoldingAvoidsRuntimeCrash);
+  warnUsage_.set(UsageWarning::FoldingValueChecks);
+  warnUsage_.set(UsageWarning::FoldingFailure);
+  warnUsage_.set(UsageWarning::FoldingLimit);
+  warnUsage_.set(UsageWarning::Interoperability);
+  warnUsage_.set(UsageWarning::Bounds);
+  warnUsage_.set(UsageWarning::Preprocessing);
+  warnUsage_.set(UsageWarning::Scanning);
+  warnUsage_.set(UsageWarning::OpenAccUsage);
+  warnUsage_.set(UsageWarning::ProcPointerCompatibility);
+  warnUsage_.set(UsageWarning::VoidMold);
+  warnUsage_.set(UsageWarning::KnownBadImplicitInterface);
+  warnUsage_.set(UsageWarning::EmptyCase);
+  warnUsage_.set(UsageWarning::CaseOverflow);
+  warnUsage_.set(UsageWarning::CUDAUsage);
+  warnUsage_.set(UsageWarning::IgnoreTKRUsage);
+  warnUsage_.set(UsageWarning::ExternalInterfaceMismatch);
+  warnUsage_.set(UsageWarning::DefinedOperatorArgs);
+  warnUsage_.set(UsageWarning::Final);
+  warnUsage_.set(UsageWarning::ZeroDoStep);
+  warnUsage_.set(UsageWarning::UnusedForallIndex);
+  warnUsage_.set(UsageWarning::OpenMPUsage);
+  warnUsage_.set(UsageWarning::ModuleFile);
+  warnUsage_.set(UsageWarning::DataLength);
+  warnUsage_.set(UsageWarning::IgnoredDirective);
+  warnUsage_.set(UsageWarning::HomonymousSpecific);
+  warnUsage_.set(UsageWarning::HomonymousResult);
+  warnUsage_.set(UsageWarning::IgnoredIntrinsicFunctionType);
+  warnUsage_.set(UsageWarning::PreviousScalarUse);
+  warnUsage_.set(UsageWarning::RedeclaredInaccessibleComponent);
+  warnUsage_.set(UsageWarning::ImplicitShared);
+  warnUsage_.set(UsageWarning::IndexVarRedefinition);
+  warnUsage_.set(UsageWarning::IncompatibleImplicitInterfaces);
+  warnUsage_.set(UsageWarning::BadTypeForTarget);
+  warnUsage_.set(UsageWarning::VectorSubscriptFinalization);
+  warnUsage_.set(UsageWarning::UndefinedFunctionResult);
+  warnUsage_.set(UsageWarning::UselessIomsg);
+  // New warnings, on by default
+  warnLanguage_.set(LanguageFeature::SavedLocalInSpecExpr);
+}
+
+// Ignore case and any inserted punctuation (like '-'/'_')
+static std::optional<char> GetWarningChar(char ch) {
+  if (ch >= 'a' && ch <= 'z') {
+    return ch;
+  } else if (ch >= 'A' && ch <= 'Z') {
+    return ch - 'A' + 'a';
+  } else if (ch >= '0' && ch <= '9') {
+    return ch;
+  } else {
+    return std::nullopt;
+  }
+}
+
+static bool WarningNameMatch(const char *a, const char *b) {
+  while (true) {
+    auto ach{GetWarningChar(*a)};
+    while (!ach && *a) {
+      ach = GetWarningChar(*++a);
+    }
+    auto bch{GetWarningChar(*b)};
+    while (!bch && *b) {
+      bch = GetWarningChar(*++b);
+    }
+    if (!ach && !bch) {
+      return true;
+    } else if (!ach || !bch || *ach != *bch) {
+      return false;
+    }
+    ++a, ++b;
+  }
+}
+
+template <typename ENUM, std::size_t N>
+std::optional<ENUM> ScanEnum(const char *name) {
+  if (name) {
+    for (std::size_t j{0}; j < N; ++j) {
+      auto feature{static_cast<ENUM>(j)};
+      if (WarningNameMatch(name, EnumToString(feature).data())) {
+        return feature;
+      }
+    }
+  }
+  return std::nullopt;
+}
+
+std::optional<LanguageFeature> FindLanguageFeature(const char *name) {
+  return ScanEnum<LanguageFeature, LanguageFeature_enumSize>(name);
+}
+
+std::optional<UsageWarning> FindUsageWarning(const char *name) {
+  return ScanEnum<UsageWarning, UsageWarning_enumSize>(name);
+}
+
 std::vector<const char *> LanguageFeatureControl::GetNames(
     LogicalOperator opr) const {
   std::vector<const char *> result;
diff --git a/flang/lib/Evaluate/check-expression.cpp b/flang/lib/Evaluate/check-expression.cpp
index a1ede7d7553bf..38794a2d8aacc 100644
--- a/flang/lib/Evaluate/check-expression.cpp
+++ b/flang/lib/Evaluate/check-expression.cpp
@@ -412,6 +412,7 @@ std::optional<Expr<SomeType>> NonPointerInitializationExpr(const Symbol &symbol,
           symbol.owner().context().ShouldWarn(
               common::LanguageFeature::LogicalIntegerAssignment)) {
         context.messages().Say(
+            common::LanguageFeature::LogicalIntegerAssignment,
             "nonstandard usage: initialization of %s with %s"_port_en_US,
             symTS->type().AsFortran(), x.GetType().value().AsFortran());
       }
@@ -565,7 +566,7 @@ class CheckSpecificationExprHelper
       if (!scope_.IsModuleFile() &&
           context_.languageFeatures().ShouldWarn(
               common::LanguageFeature::SavedLocalInSpecExpr)) {
-        context_.messages().Say(
+        context_.messages().Say(common::LanguageFeature::SavedLocalInSpecExpr,
             "specification expression refers to local object '%s' (initialized and saved)"_port_en_US,
             ultimate.name().ToString());
       }
@@ -1102,44 +1103,53 @@ class StmtFunctionChecker
 public:
   using Result = std::optional<parser::Message>;
   using Base = AnyTraverse<StmtFunctionChecker, Result>;
+
+  static constexpr auto feature{
+      common::LanguageFeature::StatementFunctionExtensions};
+
   StmtFunctionChecker(const Symbol &sf, FoldingContext &context)
       : Base{*this}, sf_{sf}, context_{context} {
-    if (!context_.languageFeatures().IsEnabled(
-            common::LanguageFeature::StatementFunctionExtensions)) {
+    if (!context_.languageFeatures().IsEnabled(feature)) {
       severity_ = parser::Severity::Error;
-    } else if (context_.languageFeatures().ShouldWarn(
-                   common::LanguageFeature::StatementFunctionExtensions)) {
+    } else if (context_.languageFeatures().ShouldWarn(feature)) {
       severity_ = parser::Severity::Portability;
     }
   }
   using Base::operator();
 
-  template <typename T> Result operator()(const ArrayConstructor<T> &) const {
+  Result Return(parser::Message &&msg) const {
     if (severity_) {
-      auto msg{
-          "Statement function '%s' should not contain an array constructor"_port_en_US};
       msg.set_severity(*severity_);
-      return parser::Message{sf_.name(), std::move(msg), sf_.name()};
+      if (*severity_ != parser::Severity::Error) {
+        msg.set_languageFeature(feature);
+      }
+    }
+    return std::move(msg);
+  }
+
+  template <typename T> Result operator()(const ArrayConstructor<T> &) const {
+    if (severity_) {
+      return Return(parser::Message{sf_.name(),
+          "Statement function '%s' should not contain an array constructor"_port_en_US,
+          sf_.name()});
     } else {
       return std::nullopt;
     }
   }
   Result operator()(const StructureConstructor &) const {
     if (severity_) {
-      auto msg{
-          "Statement function '%s' should not contain a structure constructor"_port_en_US};
-      msg.set_severity(*severity_);
-      return parser::Message{sf_.name(), std::move(msg), sf_.name()};
+      return Return(parser::Message{sf_.name(),
+          "Statement function '%s' should not contain a structure constructor"_port_en_US,
+          sf_.name()});
     } else {
       return std::nullopt;
     }
   }
   Result operator()(const TypeParamInquiry &) const {
     if (severity_) {
-      auto msg{
-          "Statement function '%s' should not contain a type parameter inquiry"_port_en_US};
-      msg.set_severity(*severity_);
-      return parser::Message{sf_.name(), std::move(msg), sf_.name()};
+      return Return(parser::Message{sf_.name(),
+          "Statement function '%s' should not contain a type parameter inquiry"_port_en_US,
+          sf_.name()});
     } else {
       return std::nullopt;
     }
@@ -1161,21 +1171,18 @@ class StmtFunctionChecker
               proc, context_, /*emitError=*/true)}) {
         if (!chars->CanBeCalledViaImplicitInterface()) {
           if (severity_) {
-            auto msg{
-                "Statement function '%s' should not reference function '%s' that requires an explicit interface"_port_en_US};
-            msg.set_severity(*severity_);
-            return parser::Message{
-                sf_.name(), std::move(msg), sf_.name(), symbol->name()};
+            return Return(parser::Message{sf_.name(),
+                "Statement function '%s' should not reference function '%s' that requires an explicit interface"_port_en_US,
+                sf_.name(), symbol->name()});
           }
         }
       }
     }
     if (proc.Rank() > 0) {
       if (severity_) {
-        auto msg{
-            "Statement function '%s' should not reference a function that returns an array"_port_en_US};
-        msg.set_severity(*severity_);
-        return parser::Message{sf_.name(), std::move(msg), sf_.name()};
+        return Return(parser::Message{sf_.name(),
+            "Statement function '%s' should not reference a function that returns an array"_port_en_US,
+            sf_.name()});
       }
     }
     return std::nullopt;
@@ -1187,10 +1194,9 @@ class StmtFunctionChecker
       }
       if (expr->Rank() > 0 && !UnwrapWholeSymbolOrComponentDataRef(*expr)) {
         if (severity_) {
-          auto msg{
-              "Statement function '%s' should not pass an array argument that is not a whole array"_port_en_US};
-          msg.set_severity(*severity_);
-          return parser::Message{sf_.name(), std::move(msg), sf_.name()};
+          return Return(parser::Message{sf_.name(),
+              "Statement function '%s' should not pass an array argument that is not a whole array"_port_en_US,
+              sf_.name()});
         }
       }
     }
diff --git a/flang/lib/Evaluate/common.cpp b/flang/lib/Evaluate/common.cpp
index c633bff57b1ec..6a960d46166e9 100644
--- a/flang/lib/Evaluate/common.cpp
+++ b/flang/lib/Evaluate/common.cpp
@@ -15,23 +15,25 @@ namespace Fortran::evaluate {
 
 void RealFlagWarnings(
     FoldingContext &context, const RealFlags &flags, const char *operation) {
-  if (context.languageFeatures().ShouldWarn(
-          common::UsageWarning::FoldingException)) {
+  static constexpr auto warning{common::UsageWarning::FoldingException};
+  if (context.languageFeatures().ShouldWarn(warning)) {
     if (flags.test(RealFlag::Overflow)) {
-      context.messages().Say("overflow on %s"_warn_en_US, operation);
+      context.messages().Say(warning, "overflow on %s"_warn_en_US, operation);
     }
     if (flags.test(RealFlag::DivideByZero)) {
       if (std::strcmp(operation, "division") == 0) {
-        context.messages().Say("division by zero"_warn_en_US);
+        context.messages().Say(warning, "division by zero"_warn_en_US);
       } else {
-        context.messages().Say("division by zero on %s"_warn_en_US, operation);
+        context.messages().Say(
+            warning, "division by zero on %s"_warn_en_US, operation);
       }
     }
     if (flags.test(RealFlag::InvalidArgument)) {
-      context.messages().Say("invalid argument on %s"_warn_en_US, operation);
+      context.messages().Say(
+          warning, "invalid argument on %s"_warn_en_US, operation);
     }
     if (flags.test(RealFlag::Underflow)) {
-      context.messages().Say("underflow on %s"_warn_en_US, operation);
+      context.messages().Say(warning, "underflow on %s"_warn_en_US, operation);
     }
   }
 }
diff --git a/flang/lib/Evaluate/fold-character.cpp b/flang/lib/Evaluate/fold-character.cpp
index 5bdfa539eb0e0..76ac497e16646 100644
--- a/flang/lib/Evaluate/fold-character.cpp
+++ b/flang/lib/Evaluate/fold-character.cpp
@@ -60,7 +60,7 @@ Expr<Type<TypeCategory::Character, KIND>> FoldIntrinsicFunction(
           if (i.IsNegative() || i.BGE(Scalar<IntT>{0}.IBSET(8 * KIND))) {
             if (context.languageFeatures().ShouldWarn(
                     common::UsageWarning::FoldingValueChecks)) {
-              context.messages().Say(
+              context.messages().Say(common::UsageWarning::FoldingValueChecks,
                   "%s(I=%jd) is out of range for CHARACTER(KIND=%d)"_warn_en_US,
                   parser::ToUpperCaseLetters(name),
                   static_cast<std::intmax_t>(i.ToInt64()), KIND);
@@ -108,7 +108,7 @@ Expr<Type<TypeCategory::Character, KIND>> FoldIntrinsicFunction(
           (1 << 20)) { // sanity limit of 1MiB
         if (context.languageFeatures().ShouldWarn(
                 common::UsageWarning::FoldingLimit)) {
-          context.messages().Say(
+          context.messages().Say(common::UsageWarning::FoldingLimit,
               "Result of REPEAT() is too large to compute at compilation time (%g characters)"_port_en_US,
               static_cast<double>(n) * str.size());
         }
diff --git a/flang/lib/Evaluate/fold-complex.cpp b/flang/lib/Evaluate/fold-complex.cpp
index d44cc9c69dd68..3eb8e1f3f1fc2 100644
--- a/flang/lib/Evaluate/fold-complex.cpp
+++ b/flang/lib/Evaluate/fold-complex.cpp
@@ -31,7 +31,7 @@ Expr<Type<TypeCategory::Complex, KIND>> FoldIntrinsicFunction(
           context, std::move(funcRef), *callable);
     } else if (context.languageFeatures().ShouldWarn(
                    common::UsageWarning::FoldingFailure)) {
-      context.messages().Say(
+      context.messages().Say(common::UsageWarning::FoldingFailure,
           "%s(complex(kind=%d)) cannot be folded on host"_warn_en_US, name,
           KIND);
     }
diff --git a/flang/lib/Evaluate/fold-implementation.h b/flang/lib/Evaluate/fold-implementation.h
index 89477dfb36435..b9c75448b7542 100644
--- a/flang/lib/Evaluate/fold-implementation.h
+++ b/flang/lib/Evaluate/fold-implementation.h
@@ -1735,7 +1735,7 @@ Expr<TO> FoldOperation(
               if (converted.overflow &&
                   msvcWorkaround.context.languageFeatures().ShouldWarn(
                       common::UsageWarning::FoldingException)) {
-                ctx.messages().Say(
+                ctx.messages().Say(common::UsageWarning::FoldingException,
                     "conversion of %s_%d to INTEGER(%d) overflowed; result is %s"_warn_en_US,
                     value->SignedDecimal(), Operand::kind, TO::kind,
                     converted.value.SignedDecimal());
@@ -1746,7 +1746,7 @@ Expr<TO> FoldOperation(
               if (msvcWorkaround.context.languageFeatures().ShouldWarn(
                       common::UsageWarning::FoldingException)) {
                 if (converted.flags.test(RealFlag::InvalidArgument)) {
-                  ctx.messages().Say(
+                  ctx.messages().Say(common::UsageWarning::FoldingException,
                       "REAL(%d) to INTEGER(%d) conversion: invalid argument"_warn_en_US,
                       Operand::kind, TO::kind);
                 } else if (converted.flags.test(RealFlag::Overflow)) {
@@ -1865,7 +1865,7 @@ Expr<T> FoldOperation(FoldingContext &context, Negate<T> &&x) {
       if (negated.overflow &&
           context.languageFeatures().ShouldWarn(
               common::UsageWarning::FoldingException)) {
-        context.messages().Say(
+        context.messages().Say(common::UsageWarning::FoldingException,
             "INTEGER(%d) negation overflowed"_warn_en_US, T::kind);
       }
       return Expr<T>{Constant<T>{std::move(negated.value)}};
@@ -1907,7 +1907,7 @@ Expr<T> FoldOperation(FoldingContext &context, Add<T> &&x) {
       if (sum.overflow &&
           context.languageFeatures().ShouldWarn(
               common::UsageWarning::FoldingException)) {
-        context.messages().Say(
+        context.messages().Say(common::UsageWarning::FoldingException,
             "INTEGER(%d) addition overflowed"_warn_en_US, T::kind);
       }
       return Expr<T>{Constant<T>{sum.value}};
@@ -1935,7 +1935,7 @@ Expr<T> FoldOperation(FoldingContext &context, Subtract<T> &&x) {
       if (difference.overflow &&
           context.languageFeatures().ShouldWarn(
               common::UsageWarning::FoldingException)) {
-        context.messages().Say(
+        context.messages().Say(common::UsageWarning::FoldingException,
             "INTEGER(%d) subtraction overflowed"_warn_en_US, T::kind);
       }
       return Expr<T>{Constant<T>{difference.value}};
@@ -1963,7 +1963,7 @@ Expr<T> FoldOperation(FoldingContext &context, Multiply<T> &&x) {
       if (product.SignedMultiplicationOverflowed() &&
           context.languageFeatures().ShouldWarn(
               common::UsageWarning::FoldingException)) {
-        context.messages().Say(
+        context.messages().Say(common::UsageWarning::FoldingException,
             "INTEGER(%d) multiplication overflowed"_warn_en_US, T::kind);
       }
       return Expr<T>{Constant<T>{product.lower}};
@@ -2009,7 +2009,7 @@ Expr<T> FoldOperation(FoldingContext &context, Divide<T> &&x) {
       if (quotAndRem.divisionByZero) {
         if (context.languageFeatures().ShouldWarn(
                 common::UsageWarning::FoldingException)) {
-          context.messages().Say(
+          context.messages().Say(common::UsageWarning::FoldingException,
               "INTEGER(%d) division by zero"_warn_en_US, T::kind);
         }
         return Expr<T>{std::move(x)};
@@ -2017,7 +2017,7 @@ Expr<T> FoldOperation(FoldingContext &context, Divide<T> &&x) {
       if (quotAndRem.overflow &&
           context.languageFeatures().ShouldWarn(
               common::UsageWarning::FoldingException)) {
-        context.messages().Say(
+        context.messages().Say(common::UsageWarning::FoldingException,
             "INTEGER(%d) division overflowed"_warn_en_US, T::kind);
       }
       return Expr<T>{Constant<T>{quotAndRem.quotient}};
@@ -2060,13 +2060,13 @@ Expr<T> FoldOperation(FoldingContext &context, Power<T> &&x) {
       if (context.languageFeatures().ShouldWarn(
               common::UsageWarning::FoldingException)) {
         if (power.divisionByZero) {
-          context.messages().Say(
+          context.messages().Say(common::UsageWarning::FoldingException,
               "INTEGER(%d) zero to negative power"_warn_en_US, T::kind);
         } else if (power.overflow) {
-          context.messages().Say(
+          context.messages().Say(common::UsageWarning::FoldingException,
               "INTEGER(%d) power overflowed"_warn_en_US, T::kind);
         } else if (power.zeroToZero) {
-          context.messages().Say(
+          context.messages().Say(common::UsageWarning::FoldingException,
               "INTEGER(%d) 0**0 is not defined"_warn_en_US, T::kind);
         }
       }
@@ -2077,7 +2077,7 @@ Expr<T> FoldOperation(FoldingContext &context, Power<T> &&x) {
             Constant<T>{(*callable)(context, folded->first, folded->second)}};
       } else if (context.languageFeatures().ShouldWarn(
                      common::UsageWarning::FoldingFailure)) {
-        context.messages().Say(
+        context.messages().Say(common::UsageWarning::FoldingFailure,
             "Power for %s cannot be folded on host"_warn_en_US,
             T{}.AsFortran());
       }
@@ -2163,7 +2163,7 @@ Expr<Type<TypeCategory::Real, KIND>> ToReal(
           if (original != converted &&
               context.languageFeatures().ShouldWarn(
                   common::UsageWarning::FoldingValueChecks)) { // C1601
-            context.messages().Say(
+            context.messages().Say(common::UsageWarning::FoldingValueChecks,
                 "Nonzero bits truncated from BOZ literal constant in REAL intrinsic"_warn_en_US);
           }
         } else if constexpr (IsNumericCategoryExpr<From>()) {
diff --git a/flang/lib/Evaluate/fold-integer.cpp b/flang/lib/Evaluate/fold-integer.cpp
index 821fa4e5dadfd..594a614a5f2e7 100644
--- a/flang/lib/Evaluate/fold-integer.cpp
+++ b/flang/lib/Evaluate/fold-integer.cpp
@@ -300,7 +300,7 @@ static Expr<T> FoldCount(FoldingContext &context, FunctionRef<T> &&ref) {
     if (accumulator.overflow() &&
         context.languageFeatures().ShouldWarn(
             common::UsageWarning::FoldingException)) {
-      context.messages().Say(
+      context.messages().Say(common::UsageWarning::FoldingException,
           "Result of intrinsic function COUNT overflows its result type"_warn_en_US);
     }
     return Expr<T>{std::move(result)};
@@ -562,7 +562,7 @@ Expr<Type<TypeCategory::Integer, KIND>> FoldIntrinsicFunction(
     if (result.ToInt64() != n &&
         context.languageFeatures().ShouldWarn(
             common::UsageWarning::FoldingException)) {
-      context.messages().Say(
+      context.messages().Say(common::UsageWarning::FoldingException,
           "Result of intrinsic function '%s' (%jd) overflows its result type"_warn_en_US,
           name, std::intmax_t{n});
     }
@@ -575,7 +575,7 @@ Expr<Type<TypeCategory::Integer, KIND>> FoldIntrinsicFunction(
           if (j.overflow &&
               context.languageFeatures().ShouldWarn(
                   common::UsageWarning::FoldingException)) {
-            context.messages().Say(
+            context.messages().Say(common::UsageWarning::FoldingException,
                 "abs(integer(kind=%d)) folding overflowed"_warn_en_US, KIND);
           }
           return j.value;
@@ -598,6 +598,7 @@ Expr<Type<TypeCategory::Integer, KIND>> FoldIntrinsicFunction(
                       context.languageFeatures().ShouldWarn(
                           common::UsageWarning::FoldingException)) {
                     context.messages().Say(
+                        common::UsageWarning::FoldingException,
                         "%s intrinsic folding overflow"_warn_en_US, name);
                   }
                   return y.value;
@@ -646,7 +647,8 @@ Expr<Type<TypeCategory::Integer, KIND>> FoldIntrinsicFunction(
           if (result.overflow &&
               context.languageFeatures().ShouldWarn(
                   common::UsageWarning::FoldingException)) {
-            context.messages().Say("DIM intrinsic folding overflow"_warn_en_US);
+            context.messages().Say(common::UsageWarning::FoldingException,
+                "DIM intrinsic folding overflow"_warn_en_US);
           }
           return result.value;
         }));
@@ -708,7 +710,7 @@ Expr<Type<TypeCategory::Integer, KIND>> FoldIntrinsicFunction(
           context.languageFeatures().ShouldWarn(
               common::UsageWarning::Portability)) {
         // Do not die, this was not checked before
-        context.messages().Say(
+        context.messages().Say(common::UsageWarning::Portability,
             "Character in intrinsic function %s should have length one"_port_en_US,
             name);
       } else {
@@ -1127,7 +1129,8 @@ Expr<Type<TypeCategory::Integer, KIND>> FoldIntrinsicFunction(
           pConst->IsZero() &&
           context.languageFeatures().ShouldWarn(
               common::UsageWarning::FoldingAvoidsRuntimeCrash)) {
-        context.messages().Say("MOD: P argument is zero"_warn_en_US);
+        context.messages().Say(common::UsageWarning::FoldingAvoidsRuntimeCrash,
+            "MOD: P argument is zero"_warn_en_US);
         badPConst = true;
       }
     }
@@ -1139,9 +1142,13 @@ Expr<Type<TypeCategory::Integer, KIND>> FoldIntrinsicFunction(
               if (context.languageFeatures().ShouldWarn(
                       common::UsageWarning::FoldingAvoidsRuntimeCrash)) {
                 if (!badPConst && quotRem.divisionByZero) {
-                  context.messages().Say("mod() by zero"_warn_en_US);
+                  context.messages().Say(
+                      common::UsageWarning::FoldingAvoidsRuntimeCrash,
+                      "mod() by zero"_warn_en_US);
                 } else if (quotRem.overflow) {
-                  context.messages().Say("mod() folding overflowed"_warn_en_US);
+                  context.messages().Say(
+                      common::UsageWarning::FoldingAvoidsRuntimeCrash,
+                      "mod() folding overflowed"_warn_en_US);
                 }
               }
               return quotRem.remainder;
@@ -1154,7 +1161,8 @@ Expr<Type<TypeCategory::Integer, KIND>> FoldIntrinsicFunction(
           pConst->IsZero() &&
           context.languageFeatures().ShouldWarn(
               common::UsageWarning::FoldingAvoidsRuntimeCrash)) {
-        context.messages().Say("MODULO: P argument is zero"_warn_en_US);
+        context.messages().Say(common::UsageWarning::FoldingAvoidsRuntimeCrash,
+            "MODULO: P argument is zero"_warn_en_US);
         badPConst = true;
       }
     }
@@ -1166,7 +1174,8 @@ Expr<Type<TypeCategory::Integer, KIND>> FoldIntrinsicFunction(
           if (!badPConst && result.overflow &&
               context.languageFeatures().ShouldWarn(
                   common::UsageWarning::FoldingException)) {
-            context.messages().Say("modulo() folding overflowed"_warn_en_US);
+            context.messages().Say(common::UsageWarning::FoldingException,
+                "modulo() folding overflowed"_warn_en_US);
           }
           return result.value;
         }));
@@ -1303,7 +1312,7 @@ Expr<Type<TypeCategory::Integer, KIND>> FoldIntrinsicFunction(
           if (result.overflow &&
               context.languageFeatures().ShouldWarn(
                   common::UsageWarning::FoldingException)) {
-            context.messages().Say(
+            context.messages().Say(common::UsageWarning::FoldingException,
                 "sign(integer(kind=%d)) folding overflowed"_warn_en_US, KIND);
           }
           return result.value;
@@ -1363,7 +1372,8 @@ Expr<Type<TypeCategory::Integer, KIND>> FoldIntrinsicFunction(
       if (intBytes != realBytes &&
           context.languageFeatures().ShouldWarn(
               common::UsageWarning::FoldingValueChecks)) {
-        context.messages().Say(*context.moduleFileName(),
+        context.messages().Say(common::UsageWarning::FoldingValueChecks,
+            *context.moduleFileName(),
             "NUMERIC_STORAGE_SIZE from ISO_FORTRAN_ENV is not well-defined when default INTEGER and REAL are not consistent due to compiler options"_warn_en_US);
       }
       return Expr<T>{8 * std::min(intBytes, realBytes)};
diff --git a/flang/lib/Evaluate/fold-logical.cpp b/flang/lib/Evaluate/fold-logical.cpp
index ee6655f838717..f5bbe7e429335 100644
--- a/flang/lib/Evaluate/fold-logical.cpp
+++ b/flang/lib/Evaluate/fold-logical.cpp
@@ -534,7 +534,8 @@ static Expr<Type<TypeCategory::Logical, KIND>> RewriteOutOfRange(
                   context.languageFeatures().ShouldWarn(
                       common::UsageWarning::OptionalMustBePresent)) {
                 if (auto source{args[2]->sourceLocation()}) {
-                  context.messages().Say(*source,
+                  context.messages().Say(
+                      common::UsageWarning::OptionalMustBePresent, *source,
                       "ROUND= argument to OUT_OF_RANGE() is an optional dummy argument that must be present at execution"_warn_en_US);
                 }
               }
diff --git a/flang/lib/Evaluate/fold-matmul.h b/flang/lib/Evaluate/fold-matmul.h
index a799cfb80a59d..be9c547d45286 100644
--- a/flang/lib/Evaluate/fold-matmul.h
+++ b/flang/lib/Evaluate/fold-matmul.h
@@ -95,7 +95,7 @@ static Expr<T> FoldMatmul(FoldingContext &context, FunctionRef<T> &&funcRef) {
   if (overflow &&
       context.languageFeatures().ShouldWarn(
           common::UsageWarning::FoldingException)) {
-    context.messages().Say(
+    context.messages().Say(common::UsageWarning::FoldingException,
         "MATMUL of %s data overflowed during computation"_warn_en_US,
         T::AsFortran());
   }
diff --git a/flang/lib/Evaluate/fold-real.cpp b/flang/lib/Evaluate/fold-real.cpp
index fd8eca28a679c..ed7749e880301 100644
--- a/flang/lib/Evaluate/fold-real.cpp
+++ b/flang/lib/Evaluate/fold-real.cpp
@@ -37,7 +37,7 @@ static Expr<T> FoldTransformationalBessel(
           std::move(results), ConstantSubscripts{std::max(n2 - n1 + 1, 0)}}};
     } else if (context.languageFeatures().ShouldWarn(
                    common::UsageWarning::FoldingFailure)) {
-      context.messages().Say(
+      context.messages().Say(common::UsageWarning::FoldingFailure,
           "%s(integer(kind=4), real(kind=%d)) cannot be folded on host"_warn_en_US,
           name, T::kind);
     }
@@ -134,7 +134,7 @@ static Expr<Type<TypeCategory::Real, KIND>> FoldNorm2(FoldingContext &context,
     if (norm2Accumulator.overflow() &&
         context.languageFeatures().ShouldWarn(
             common::UsageWarning::FoldingException)) {
-      context.messages().Say(
+      context.messages().Say(common::UsageWarning::FoldingException,
           "NORM2() of REAL(%d) data overflowed"_warn_en_US, KIND);
     }
     return Expr<T>{std::move(result)};
@@ -167,7 +167,7 @@ Expr<Type<TypeCategory::Real, KIND>> FoldIntrinsicFunction(
           context, std::move(funcRef), *callable);
     } else if (context.languageFeatures().ShouldWarn(
                    common::UsageWarning::FoldingFailure)) {
-      context.messages().Say(
+      context.messages().Say(common::UsageWarning::FoldingFailure,
           "%s(real(kind=%d)) cannot be folded on host"_warn_en_US, name, KIND);
     }
   } else if (name == "amax0" || name == "amin0" || name == "amin1" ||
@@ -181,7 +181,7 @@ Expr<Type<TypeCategory::Real, KIND>> FoldIntrinsicFunction(
           context, std::move(funcRef), *callable);
     } else if (context.languageFeatures().ShouldWarn(
                    common::UsageWarning::FoldingFailure)) {
-      context.messages().Say(
+      context.messages().Say(common::UsageWarning::FoldingFailure,
           "%s(real(kind=%d), real(kind%d)) cannot be folded on host"_warn_en_US,
           name, KIND, KIND);
     }
@@ -193,7 +193,7 @@ Expr<Type<TypeCategory::Real, KIND>> FoldIntrinsicFunction(
             context, std::move(funcRef), *callable);
       } else if (context.languageFeatures().ShouldWarn(
                      common::UsageWarning::FoldingFailure)) {
-        context.messages().Say(
+        context.messages().Say(common::UsageWarning::FoldingFailure,
             "%s(integer(kind=4), real(kind=%d)) cannot be folded on host"_warn_en_US,
             name, KIND);
       }
@@ -213,7 +213,7 @@ Expr<Type<TypeCategory::Real, KIND>> FoldIntrinsicFunction(
             if (y.flags.test(RealFlag::Overflow) &&
                 context.languageFeatures().ShouldWarn(
                     common::UsageWarning::FoldingException)) {
-              context.messages().Say(
+              context.messages().Say(common::UsageWarning::FoldingException,
                   "complex ABS intrinsic folding overflow"_warn_en_US, name);
             }
             return y.value;
@@ -237,7 +237,7 @@ Expr<Type<TypeCategory::Real, KIND>> FoldIntrinsicFunction(
               if (y.flags.test(RealFlag::Overflow) &&
                   context.languageFeatures().ShouldWarn(
                       common::UsageWarning::FoldingException)) {
-                context.messages().Say(
+                context.messages().Say(common::UsageWarning::FoldingException,
                     "%s intrinsic folding overflow"_warn_en_US, name);
               }
               return y.value;
@@ -250,7 +250,8 @@ Expr<Type<TypeCategory::Real, KIND>> FoldIntrinsicFunction(
           if (result.flags.test(RealFlag::Overflow) &&
               context.languageFeatures().ShouldWarn(
                   common::UsageWarning::FoldingException)) {
-            context.messages().Say("DIM intrinsic folding overflow"_warn_en_US);
+            context.messages().Say(common::UsageWarning::FoldingException,
+                "DIM intrinsic folding overflow"_warn_en_US);
           }
           return result.value;
         }));
@@ -284,7 +285,7 @@ Expr<Type<TypeCategory::Real, KIND>> FoldIntrinsicFunction(
               if (result.flags.test(RealFlag::Overflow) &&
                   context.languageFeatures().ShouldWarn(
                       common::UsageWarning::FoldingException)) {
-                context.messages().Say(
+                context.messages().Say(common::UsageWarning::FoldingException,
                     "HYPOT intrinsic folding overflow"_warn_en_US);
               }
               return result.value;
@@ -310,7 +311,8 @@ Expr<Type<TypeCategory::Real, KIND>> FoldIntrinsicFunction(
           pConst->IsZero() &&
           context.languageFeatures().ShouldWarn(
               common::UsageWarning::FoldingAvoidsRuntimeCrash)) {
-        context.messages().Say("MOD: P argument is zero"_warn_en_US);
+        context.messages().Say(common::UsageWarning::FoldingAvoidsRuntimeCrash,
+            "MOD: P argument is zero"_warn_en_US);
         badPConst = true;
       }
     }
@@ -322,6 +324,7 @@ Expr<Type<TypeCategory::Real, KIND>> FoldIntrinsicFunction(
               context.languageFeatures().ShouldWarn(
                   common::UsageWarning::FoldingAvoidsRuntimeCrash)) {
             context.messages().Say(
+                common::UsageWarning::FoldingAvoidsRuntimeCrash,
                 "second argument to MOD must not be zero"_warn_en_US);
           }
           return result.value;
@@ -335,7 +338,8 @@ Expr<Type<TypeCategory::Real, KIND>> FoldIntrinsicFunction(
           pConst->IsZero() &&
           context.languageFeatures().ShouldWarn(
               common::UsageWarning::FoldingAvoidsRuntimeCrash)) {
-        context.messages().Say("MODULO: P argument is zero"_warn_en_US);
+        context.messages().Say(common::UsageWarning::FoldingAvoidsRuntimeCrash,
+            "MODULO: P argument is zero"_warn_en_US);
         badPConst = true;
       }
     }
@@ -347,6 +351,7 @@ Expr<Type<TypeCategory::Real, KIND>> FoldIntrinsicFunction(
               context.languageFeatures().ShouldWarn(
                   common::UsageWarning::FoldingAvoidsRuntimeCrash)) {
             context.messages().Say(
+                common::UsageWarning::FoldingAvoidsRuntimeCrash,
                 "second argument to MODULO must not be zero"_warn_en_US);
           }
           return result.value;
@@ -362,7 +367,8 @@ Expr<Type<TypeCategory::Real, KIND>> FoldIntrinsicFunction(
                 (sConst->IsZero() || sConst->IsNotANumber()) &&
                 context.languageFeatures().ShouldWarn(
                     common::UsageWarning::FoldingValueChecks)) {
-              context.messages().Say("NEAREST: S argument is %s"_warn_en_US,
+              context.messages().Say(common::UsageWarning::FoldingValueChecks,
+                  "NEAREST: S argument is %s"_warn_en_US,
                   sConst->IsZero() ? "zero" : "NaN");
               badSConst = true;
             }
@@ -373,6 +379,7 @@ Expr<Type<TypeCategory::Real, KIND>> FoldIntrinsicFunction(
                       context.languageFeatures().ShouldWarn(
                           common::UsageWarning::FoldingValueChecks)) {
                     context.messages().Say(
+                        common::UsageWarning::FoldingValueChecks,
                         "NEAREST: S argument is %s"_warn_en_US,
                         s.IsZero() ? "zero" : "NaN");
                   }
@@ -381,6 +388,7 @@ Expr<Type<TypeCategory::Real, KIND>> FoldIntrinsicFunction(
                           common::UsageWarning::FoldingException)) {
                     if (result.flags.test(RealFlag::InvalidArgument)) {
                       context.messages().Say(
+                          common::UsageWarning::FoldingException,
                           "NEAREST intrinsic folding: bad argument"_warn_en_US);
                     }
                   }
@@ -423,6 +431,7 @@ Expr<Type<TypeCategory::Real, KIND>> FoldIntrinsicFunction(
                           context.languageFeatures().ShouldWarn(
                               common::UsageWarning::FoldingException)) {
                         context.messages().Say(
+                            common::UsageWarning::FoldingException,
                             "SCALE intrinsic folding overflow"_warn_en_US);
                       }
                       return result.value;
@@ -475,6 +484,7 @@ Expr<Type<TypeCategory::Real, KIND>> FoldIntrinsicFunction(
                     if (context.languageFeatures().ShouldWarn(
                             common::UsageWarning::FoldingValueChecks)) {
                       context.messages().Say(
+                          common::UsageWarning::FoldingValueChecks,
                           "IEEE_NEXT_AFTER intrinsic folding: arguments are unordered"_warn_en_US);
                     }
                     return x.NotANumber();
@@ -500,7 +510,7 @@ Expr<Type<TypeCategory::Real, KIND>> FoldIntrinsicFunction(
           if (context.languageFeatures().ShouldWarn(
                   common::UsageWarning::FoldingException)) {
             if (result.flags.test(RealFlag::InvalidArgument)) {
-              context.messages().Say(
+              context.messages().Say(common::UsageWarning::FoldingException,
                   "%s intrinsic folding: argument is NaN"_warn_en_US, iName);
             }
           }
diff --git a/flang/lib/Evaluate/fold-reduction.h b/flang/lib/Evaluate/fold-reduction.h
index fbdae8f4eee01..8ca0794ab0fc7 100644
--- a/flang/lib/Evaluate/fold-reduction.h
+++ b/flang/lib/Evaluate/fold-reduction.h
@@ -108,7 +108,7 @@ static Expr<T> FoldDotProduct(
     if (overflow &&
         context.languageFeatures().ShouldWarn(
             common::UsageWarning::FoldingException)) {
-      context.messages().Say(
+      context.messages().Say(common::UsageWarning::FoldingException,
           "DOT_PRODUCT of %s data overflowed during computation"_warn_en_US,
           T::AsFortran());
     }
@@ -326,7 +326,7 @@ static Expr<T> FoldProduct(
     if (accumulator.overflow() &&
         context.languageFeatures().ShouldWarn(
             common::UsageWarning::FoldingException)) {
-      context.messages().Say(
+      context.messages().Say(common::UsageWarning::FoldingException,
           "PRODUCT() of %s data overflowed"_warn_en_US, T::AsFortran());
     }
     return result;
@@ -394,7 +394,7 @@ static Expr<T> FoldSum(FoldingContext &context, FunctionRef<T> &&ref) {
     if (accumulator.overflow() &&
         context.languageFeatures().ShouldWarn(
             common::UsageWarning::FoldingException)) {
-      context.messages().Say(
+      context.messages().Say(common::UsageWarning::FoldingException,
           "SUM() of %s data overflowed"_warn_en_US, T::AsFortran());
     }
     return result;
diff --git a/flang/lib/Evaluate/host.cpp b/flang/lib/Evaluate/host.cpp
index 31bc438385803..187bb2f09806b 100644
--- a/flang/lib/Evaluate/host.cpp
+++ b/flang/lib/Evaluate/host.cpp
@@ -102,7 +102,7 @@ void HostFloatingPointEnvironment::SetUpHostFloatingPointEnvironment(
     fesetround(FE_TONEAREST);
     if (context.languageFeatures().ShouldWarn(
             common::UsageWarning::FoldingFailure)) {
-      context.messages().Say(
+      context.messages().Say(common::UsageWarning::FoldingFailure,
           "TiesAwayFromZero rounding mode is not available when folding "
           "constants"
           " with host runtime; using TiesToEven instead"_warn_en_US);
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index a89e9732228cb..1f48fc21662eb 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -523,7 +523,9 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
         {{"c", DefaultChar, Rank::scalar, Optionality::required,
             common::Intent::Out}},
         TypePattern{IntType, KindCode::greaterOrEqualToKind, 4}},
+    {"getgid", {}, DefaultInt},
     {"getpid", {}, DefaultInt},
+    {"getuid", {}, DefaultInt},
     {"huge",
         {{"x", SameIntOrReal, Rank::anyOrAssumedRank, Optionality::required,
             common::Intent::In, {ArgFlag::canBeMoldNull}}},
@@ -2373,10 +2375,10 @@ std::optional<SpecificCall> IntrinsicInterface::Match(
             if (context.languageFeatures().ShouldWarn(
                     common::UsageWarning::OptionalMustBePresent)) {
               if (rank == Rank::scalarIfDim || arrayRank.value_or(-1) == 1) {
-                messages.Say(
+                messages.Say(common::UsageWarning::OptionalMustBePresent,
                     "The actual argument for DIM= is optional, pointer, or allocatable, and it is assumed to be present and equal to 1 at execution time"_warn_en_US);
               } else {
-                messages.Say(
+                messages.Say(common::UsageWarning::OptionalMustBePresent,
                     "The actual argument for DIM= is optional, pointer, or allocatable, and may not be absent during execution; parenthesize to silence this warning"_warn_en_US);
               }
             }
@@ -2849,11 +2851,11 @@ IntrinsicProcTable::Implementation::HandleC_F_Pointer(
           if (context.languageFeatures().ShouldWarn(
                   common::UsageWarning::Interoperability)) {
             if (type->IsUnlimitedPolymorphic()) {
-              context.messages().Say(at,
+              context.messages().Say(common::UsageWarning::Interoperability, at,
                   "FPTR= argument to C_F_POINTER() should not be unlimited polymorphic"_warn_en_US);
             } else if (!type->GetDerivedTypeSpec().typeSymbol().attrs().test(
                            semantics::Attr::BIND_C)) {
-              context.messages().Say(at,
+              context.messages().Say(common::UsageWarning::Interoperability, at,
                   "FPTR= argument to C_F_POINTER() should not have a derived type that is not BIND(C)"_warn_en_US);
             }
           }
@@ -2862,7 +2864,7 @@ IntrinsicProcTable::Implementation::HandleC_F_Pointer(
                         .value_or(true) &&
             context.languageFeatures().ShouldWarn(
                 common::UsageWarning::Interoperability)) {
-          context.messages().Say(at,
+          context.messages().Say(common::UsageWarning::Interoperability, at,
               "FPTR= argument to C_F_POINTER() should not have the non-interoperable intrinsic type %s"_warn_en_US,
               type->AsFortran());
         }
@@ -2964,7 +2966,8 @@ std::optional<SpecificCall> IntrinsicProcTable::Implementation::HandleC_Loc(
           !IsInteroperableIntrinsicType(typeAndShape->type()).value_or(true) &&
           context.languageFeatures().ShouldWarn(
               common::UsageWarning::Interoperability)) {
-        context.messages().Say(arguments[0]->sourceLocation(),
+        context.messages().Say(common::UsageWarning::Interoperability,
+            arguments[0]->sourceLocation(),
             "C_LOC() argument has non-interoperable intrinsic type, kind, or length"_warn_en_US);
       }
 
@@ -3309,6 +3312,8 @@ std::optional<SpecificCall> IntrinsicProcTable::Implementation::Probe(
                         common::LanguageFeature::
                             UseGenericIntrinsicWhenSpecificDoesntMatch)) {
                   context.messages().Say(
+                      common::LanguageFeature::
+                          UseGenericIntrinsicWhenSpecificDoesntMatch,
                       "Argument types do not match specific intrinsic '%s' requirements; using '%s' generic instead and converting the result to %s if needed"_port_en_US,
                       call.name, genericName, newType.AsFortran());
                 }
diff --git a/flang/lib/Evaluate/variable.cpp b/flang/lib/Evaluate/variable.cpp
index b074ae6d811ac..707a2065ca30a 100644
--- a/flang/lib/Evaluate/variable.cpp
+++ b/flang/lib/Evaluate/variable.cpp
@@ -215,7 +215,7 @@ std::optional<Expr<SomeCharacter>> Substring::Fold(FoldingContext &context) {
   if (!result) { // error cases
     if (*lbi < 1) {
       if (context.languageFeatures().ShouldWarn(common::UsageWarning::Bounds)) {
-        context.messages().Say(
+        context.messages().Say(common::UsageWarning::Bounds,
             "Lower bound (%jd) on substring is less than one"_warn_en_US,
             static_cast<std::intmax_t>(*lbi));
       }
@@ -224,7 +224,7 @@ std::optional<Expr<SomeCharacter>> Substring::Fold(FoldingContext &context) {
     }
     if (length && *ubi > *length) {
       if (context.languageFeatures().ShouldWarn(common::UsageWarning::Bounds)) {
-        context.messages().Say(
+        context.messages().Say(common::UsageWarning::Bounds,
             "Upper bound (%jd) on substring is greater than character length (%jd)"_warn_en_US,
             static_cast<std::intmax_t>(*ubi),
             static_cast<std::intmax_t>(*length));
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index ebcb761396966..0894a5903635e 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -5323,10 +5323,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
           LLVM_ATTRIBUTE_UNUSED auto getBitWidth = [this](mlir::Type ty) {
             // 15.6.2.6.3: differering result types should be integer, real,
             // complex or logical
-            if (auto cmplx = mlir::dyn_cast_or_null<fir::ComplexType>(ty)) {
-              fir::KindTy kind = cmplx.getFKind();
-              return 2 * builder->getKindMap().getRealBitsize(kind);
-            }
+            if (auto cmplx = mlir::dyn_cast_or_null<mlir::ComplexType>(ty))
+              return 2 * cmplx.getElementType().getIntOrFloatBitWidth();
             if (auto logical = mlir::dyn_cast_or_null<fir::LogicalType>(ty)) {
               fir::KindTy kind = logical.getFKind();
               return builder->getKindMap().getLogicalBitsize(kind);
diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index ee5eb225f0d7e..9f5b58590fb79 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -523,6 +523,8 @@ std::pair<fir::ExtendedValue, bool> Fortran::lower::genCallOpAndResult(
 
   mlir::Value callResult;
   unsigned callNumResults;
+  fir::FortranProcedureFlagsEnumAttr procAttrs =
+      caller.getProcedureAttrs(builder.getContext());
 
   if (!caller.getCallDescription().chevrons().empty()) {
     // A call to a CUDA kernel with the chevron syntax.
@@ -610,7 +612,7 @@ std::pair<fir::ExtendedValue, bool> Fortran::lower::genCallOpAndResult(
       dispatch = builder.create<fir::DispatchOp>(
           loc, funcType.getResults(), builder.getStringAttr(procName),
           caller.getInputs()[*passArg], operands,
-          builder.getI32IntegerAttr(*passArg));
+          builder.getI32IntegerAttr(*passArg), procAttrs);
     } else {
       // NOPASS
       const Fortran::evaluate::Component *component =
@@ -625,15 +627,13 @@ std::pair<fir::ExtendedValue, bool> Fortran::lower::genCallOpAndResult(
         passObject = builder.create<fir::LoadOp>(loc, passObject);
       dispatch = builder.create<fir::DispatchOp>(
           loc, funcType.getResults(), builder.getStringAttr(procName),
-          passObject, operands, nullptr);
+          passObject, operands, nullptr, procAttrs);
     }
     callNumResults = dispatch.getNumResults();
     if (callNumResults != 0)
       callResult = dispatch.getResult(0);
   } else {
     // Standard procedure call with fir.call.
-    fir::FortranProcedureFlagsEnumAttr procAttrs =
-        caller.getProcedureAttrs(builder.getContext());
     auto call = builder.create<fir::CallOp>(
         loc, funcType.getResults(), funcSymbolAttr, operands, procAttrs);
 
@@ -2570,9 +2570,26 @@ genIntrinsicRef(const Fortran::evaluate::SpecificIntrinsic *intrinsic,
           hlfir::Entity{*var}, /*isPresent=*/std::nullopt});
       continue;
     }
+    // arguments of bitwise comparison functions may not have nsw flag
+    // even if -fno-wrapv is enabled
+    mlir::arith::IntegerOverflowFlags iofBackup{};
+    auto isBitwiseComparison = [](const std::string intrinsicName) -> bool {
+      if (intrinsicName == "bge" || intrinsicName == "bgt" ||
+          intrinsicName == "ble" || intrinsicName == "blt")
+        return true;
+      return false;
+    };
+    if (isBitwiseComparison(callContext.getProcedureName())) {
+      iofBackup = callContext.getBuilder().getIntegerOverflowFlags();
+      callContext.getBuilder().setIntegerOverflowFlags(
+          mlir::arith::IntegerOverflowFlags::none);
+    }
     auto loweredActual = Fortran::lower::convertExprToHLFIR(
         loc, callContext.converter, *expr, callContext.symMap,
         callContext.stmtCtx);
+    if (isBitwiseComparison(callContext.getProcedureName()))
+      callContext.getBuilder().setIntegerOverflowFlags(iofBackup);
+
     std::optional<mlir::Value> isPresent;
     if (argLowering) {
       fir::ArgLoweringRule argRules =
diff --git a/flang/lib/Lower/ConvertConstant.cpp b/flang/lib/Lower/ConvertConstant.cpp
index 3361817ee27ee..748be508235f1 100644
--- a/flang/lib/Lower/ConvertConstant.cpp
+++ b/flang/lib/Lower/ConvertConstant.cpp
@@ -152,9 +152,6 @@ class DenseGlobalBuilder {
                       : TC;
     attributeElementType = Fortran::lower::getFIRType(
         builder.getContext(), attrTc, KIND, std::nullopt);
-    if (auto firCTy = mlir::dyn_cast<fir::ComplexType>(attributeElementType))
-      attributeElementType =
-          mlir::ComplexType::get(firCTy.getEleType(builder.getKindMap()));
     for (auto element : constant.values())
       attributes.push_back(
           convertToAttribute<TC, KIND>(builder, element, attributeElementType));
@@ -264,14 +261,11 @@ static mlir::Value genScalarLit(
       return genRealConstant<KIND>(builder, loc, floatVal);
     }
   } else if constexpr (TC == Fortran::common::TypeCategory::Complex) {
-    mlir::Value realPart =
-        genScalarLit<Fortran::common::TypeCategory::Real, KIND>(builder, loc,
-                                                                value.REAL());
-    mlir::Value imagPart =
-        genScalarLit<Fortran::common::TypeCategory::Real, KIND>(builder, loc,
-                                                                value.AIMAG());
-    return fir::factory::Complex{builder, loc}.createComplex(KIND, realPart,
-                                                             imagPart);
+    mlir::Value real = genScalarLit<Fortran::common::TypeCategory::Real, KIND>(
+        builder, loc, value.REAL());
+    mlir::Value imag = genScalarLit<Fortran::common::TypeCategory::Real, KIND>(
+        builder, loc, value.AIMAG());
+    return fir::factory::Complex{builder, loc}.createComplex(real, imag);
   } else /*constexpr*/ {
     llvm_unreachable("unhandled constant");
   }
diff --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp
index 62a7615e1af13..87e2114e41305 100644
--- a/flang/lib/Lower/ConvertExpr.cpp
+++ b/flang/lib/Lower/ConvertExpr.cpp
@@ -1127,7 +1127,7 @@ class ScalarExprLowering {
   ExtValue genval(const Fortran::evaluate::ComplexConstructor<KIND> &op) {
     mlir::Value realPartValue = genunbox(op.left());
     return fir::factory::Complex{builder, getLoc()}.createComplex(
-        KIND, realPartValue, genunbox(op.right()));
+        realPartValue, genunbox(op.right()));
   }
 
   template <int KIND>
@@ -5242,7 +5242,7 @@ class ArrayExprLowering {
     return [=](IterSpace iters) -> ExtValue {
       mlir::Value lhs = fir::getBase(lf(iters));
       mlir::Value rhs = fir::getBase(rf(iters));
-      return fir::factory::Complex{builder, loc}.createComplex(KIND, lhs, rhs);
+      return fir::factory::Complex{builder, loc}.createComplex(lhs, rhs);
     };
   }
 
@@ -5710,10 +5710,6 @@ class ArrayExprLowering {
             fir::applyPathToType(seqTy.getEleTy(), components.suffixComponents);
         if (!eleTy)
           fir::emitFatalError(loc, "slicing path is ill-formed");
-        if (auto realTy = mlir::dyn_cast<fir::RealType>(eleTy))
-          eleTy = Fortran::lower::convertReal(realTy.getContext(),
-                                              realTy.getFKind());
-
         // create the type of the projected array.
         arrTy = fir::SequenceType::get(seqTy.getShape(), eleTy);
         LLVM_DEBUG(llvm::dbgs()
diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index 1933f38f735b5..93b78fd3357fa 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -1218,7 +1218,7 @@ struct BinaryOp<Fortran::evaluate::ComplexConstructor<KIND>> {
                                          fir::FirOpBuilder &builder, const Op &,
                                          hlfir::Entity lhs, hlfir::Entity rhs) {
     mlir::Value res =
-        fir::factory::Complex{builder, loc}.createComplex(KIND, lhs, rhs);
+        fir::factory::Complex{builder, loc}.createComplex(lhs, rhs);
     return hlfir::EntityWithAttributes{res};
   }
 };
@@ -1584,9 +1584,14 @@ class HlfirBuilder {
       auto rightVal = hlfir::loadTrivialScalar(l, b, rightElement);
       return binaryOp.gen(l, b, op.derived(), leftVal, rightVal);
     };
+    auto iofBackup = builder.getIntegerOverflowFlags();
+    // nsw is never added to operations on vector subscripts
+    // even if -fno-wrapv is enabled.
+    builder.setIntegerOverflowFlags(mlir::arith::IntegerOverflowFlags::none);
     mlir::Value elemental = hlfir::genElementalOp(loc, builder, elementType,
                                                   shape, typeParams, genKernel,
                                                   /*isUnordered=*/true);
+    builder.setIntegerOverflowFlags(iofBackup);
     fir::FirOpBuilder *bldr = &builder;
     getStmtCtx().attachCleanup(
         [=]() { bldr->create<hlfir::DestroyOp>(loc, elemental); });
@@ -1899,10 +1904,17 @@ class HlfirBuilder {
 template <typename T>
 hlfir::Entity
 HlfirDesignatorBuilder::genSubscript(const Fortran::evaluate::Expr<T> &expr) {
+  fir::FirOpBuilder &builder = getBuilder();
+  mlir::arith::IntegerOverflowFlags iofBackup{};
+  if (!getConverter().getLoweringOptions().getIntegerWrapAround()) {
+    iofBackup = builder.getIntegerOverflowFlags();
+    builder.setIntegerOverflowFlags(mlir::arith::IntegerOverflowFlags::nsw);
+  }
   auto loweredExpr =
       HlfirBuilder(getLoc(), getConverter(), getSymMap(), getStmtCtx())
           .gen(expr);
-  fir::FirOpBuilder &builder = getBuilder();
+  if (!getConverter().getLoweringOptions().getIntegerWrapAround())
+    builder.setIntegerOverflowFlags(iofBackup);
   // Skip constant conversions that litters designators and makes generated
   // IR harder to read: directly use index constants for constant subscripts.
   mlir::Type idxTy = builder.getIndexType();
diff --git a/flang/lib/Lower/ConvertType.cpp b/flang/lib/Lower/ConvertType.cpp
index a47fc99ea9f45..8664477b50078 100644
--- a/flang/lib/Lower/ConvertType.cpp
+++ b/flang/lib/Lower/ConvertType.cpp
@@ -96,10 +96,7 @@ static mlir::Type genCharacterType(
 }
 
 static mlir::Type genComplexType(mlir::MLIRContext *context, int KIND) {
-  if (Fortran::evaluate::IsValidKindOfIntrinsicType(
-          Fortran::common::TypeCategory::Complex, KIND))
-    return fir::ComplexType::get(context, KIND);
-  return {};
+  return mlir::ComplexType::get(genRealType(context, KIND));
 }
 
 static mlir::Type
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index f76d44f5479d3..8b03d60e47ca6 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -46,10 +46,10 @@
 #include "llvm/Support/Debug.h"
 #include <optional>
 
-static llvm::cl::opt<bool> allowAssumedRank(
-    "allow-assumed-rank",
-    llvm::cl::desc("Enable assumed rank lowering - experimental"),
-    llvm::cl::init(false));
+static llvm::cl::opt<bool>
+    allowAssumedRank("allow-assumed-rank",
+                     llvm::cl::desc("Enable assumed rank lowering"),
+                     llvm::cl::init(true));
 
 #define DEBUG_TYPE "flang-lower-variable"
 
@@ -519,7 +519,7 @@ static fir::GlobalOp defineGlobal(Fortran::lower::AbstractConverter &converter,
   if (mlir::isa<fir::SequenceType>(symTy) &&
       !Fortran::semantics::IsAllocatableOrPointer(sym)) {
     mlir::Type eleTy = mlir::cast<fir::SequenceType>(symTy).getEleTy();
-    if (mlir::isa<mlir::IntegerType, mlir::FloatType, fir::ComplexType,
+    if (mlir::isa<mlir::IntegerType, mlir::FloatType, mlir::ComplexType,
                   fir::LogicalType>(eleTy)) {
       const auto *details =
           sym.detailsIf<Fortran::semantics::ObjectEntityDetails>();
diff --git a/flang/lib/Lower/DirectivesCommon.h b/flang/lib/Lower/DirectivesCommon.h
index a32f0b287e049..da192ded4aa97 100644
--- a/flang/lib/Lower/DirectivesCommon.h
+++ b/flang/lib/Lower/DirectivesCommon.h
@@ -129,8 +129,7 @@ static void processOmpAtomicTODO(mlir::Type elementType,
     // Based on assertion for supported element types in OMPIRBuilder.cpp
     // createAtomicRead
     mlir::Type unwrappedEleTy = fir::unwrapRefType(elementType);
-    bool supportedAtomicType =
-        (!fir::isa_complex(unwrappedEleTy) && fir::isa_trivial(unwrappedEleTy));
+    bool supportedAtomicType = fir::isa_trivial(unwrappedEleTy);
     if (!supportedAtomicType)
       TODO(loc, "Unsupported atomic type");
   }
diff --git a/flang/lib/Lower/IO.cpp b/flang/lib/Lower/IO.cpp
index 9e98b230b676f..1894b0cfd1bec 100644
--- a/flang/lib/Lower/IO.cpp
+++ b/flang/lib/Lower/IO.cpp
@@ -684,9 +684,9 @@ static mlir::func::FuncOp getOutputFunc(mlir::Location loc,
       return getIORuntimeFunc<mkIOKey(OutputReal64)>(loc, builder);
   }
   auto kindMap = fir::getKindMapping(builder.getModule());
-  if (auto ty = mlir::dyn_cast<fir::ComplexType>(type)) {
+  if (auto ty = mlir::dyn_cast<mlir::ComplexType>(type)) {
     // COMPLEX(KIND=k) corresponds to a pair of REAL(KIND=k).
-    auto width = kindMap.getRealBitsize(ty.getFKind());
+    auto width = mlir::cast<mlir::FloatType>(ty.getElementType()).getWidth();
     if (width == 32)
       return getIORuntimeFunc<mkIOKey(OutputComplex32)>(loc, builder);
     else if (width == 64)
@@ -788,8 +788,8 @@ static mlir::func::FuncOp getInputFunc(mlir::Location loc,
       return getIORuntimeFunc<mkIOKey(InputReal64)>(loc, builder);
   }
   auto kindMap = fir::getKindMapping(builder.getModule());
-  if (auto ty = mlir::dyn_cast<fir::ComplexType>(type)) {
-    auto width = kindMap.getRealBitsize(ty.getFKind());
+  if (auto ty = mlir::dyn_cast<mlir::ComplexType>(type)) {
+    auto width = mlir::cast<mlir::FloatType>(ty.getElementType()).getWidth();
     if (width == 32)
       return getIORuntimeFunc<mkIOKey(InputComplex32)>(loc, builder);
     else if (width == 64)
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index c97398fc43f92..878dccc4ecbc4 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -998,7 +998,7 @@ static mlir::Value getReductionInitValue(fir::FirOpBuilder &builder,
         builder.getIntegerAttr(ty, getReductionInitValue<llvm::APInt>(op, ty)));
   if (op == mlir::acc::ReductionOperator::AccMin ||
       op == mlir::acc::ReductionOperator::AccMax) {
-    if (mlir::isa<fir::ComplexType>(ty))
+    if (mlir::isa<mlir::ComplexType>(ty))
       llvm::report_fatal_error(
           "min/max reduction not supported for complex type");
     if (auto floatTy = mlir::dyn_cast_or_null<mlir::FloatType>(ty))
@@ -1010,14 +1010,13 @@ static mlir::Value getReductionInitValue(fir::FirOpBuilder &builder,
     return builder.create<mlir::arith::ConstantOp>(
         loc, ty,
         builder.getFloatAttr(ty, getReductionInitValue<int64_t>(op, ty)));
-  } else if (auto cmplxTy = mlir::dyn_cast_or_null<fir::ComplexType>(ty)) {
-    mlir::Type floatTy =
-        Fortran::lower::convertReal(builder.getContext(), cmplxTy.getFKind());
+  } else if (auto cmplxTy = mlir::dyn_cast_or_null<mlir::ComplexType>(ty)) {
+    mlir::Type floatTy = cmplxTy.getElementType();
     mlir::Value realInit = builder.createRealConstant(
         loc, floatTy, getReductionInitValue<int64_t>(op, cmplxTy));
     mlir::Value imagInit = builder.createRealConstant(loc, floatTy, 0.0);
-    return fir::factory::Complex{builder, loc}.createComplex(
-        cmplxTy.getFKind(), realInit, imagInit);
+    return fir::factory::Complex{builder, loc}.createComplex(cmplxTy, realInit,
+                                                             imagInit);
   }
 
   if (auto seqTy = mlir::dyn_cast<fir::SequenceType>(ty))
@@ -1136,7 +1135,7 @@ static mlir::Value genScalarCombiner(fir::FirOpBuilder &builder,
       return builder.create<mlir::arith::AddIOp>(loc, value1, value2);
     if (mlir::isa<mlir::FloatType>(ty))
       return builder.create<mlir::arith::AddFOp>(loc, value1, value2);
-    if (auto cmplxTy = mlir::dyn_cast_or_null<fir::ComplexType>(ty))
+    if (auto cmplxTy = mlir::dyn_cast_or_null<mlir::ComplexType>(ty))
       return builder.create<fir::AddcOp>(loc, value1, value2);
     TODO(loc, "reduction add type");
   }
@@ -1146,7 +1145,7 @@ static mlir::Value genScalarCombiner(fir::FirOpBuilder &builder,
       return builder.create<mlir::arith::MulIOp>(loc, value1, value2);
     if (mlir::isa<mlir::FloatType>(ty))
       return builder.create<mlir::arith::MulFOp>(loc, value1, value2);
-    if (mlir::isa<fir::ComplexType>(ty))
+    if (mlir::isa<mlir::ComplexType>(ty))
       return builder.create<fir::MulcOp>(loc, value1, value2);
     TODO(loc, "reduction mul type");
   }
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index e9ef8579100e9..a4d2524bccf5c 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -684,7 +684,7 @@ void TypeInfo::typeScan(mlir::Type ty) {
     typeScan(pty.getEleTy());
   } else {
     // The scan ends when reaching any built-in or record type.
-    assert(ty.isIntOrIndexOrFloat() || mlir::isa<fir::ComplexType>(ty) ||
+    assert(ty.isIntOrIndexOrFloat() || mlir::isa<mlir::ComplexType>(ty) ||
            mlir::isa<fir::LogicalType>(ty) || mlir::isa<fir::RecordType>(ty));
   }
 }
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index efac7757ca585..c93767eb8a550 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -783,9 +783,22 @@ IsDevicePtr make(const parser::OmpClause::IsDevicePtr &inp,
 
 Lastprivate make(const parser::OmpClause::Lastprivate &inp,
                  semantics::SemanticsContext &semaCtx) {
-  // inp.v -> parser::OmpObjectList
-  return Lastprivate{{/*LastprivateModifier=*/std::nullopt,
-                      /*List=*/makeObjects(inp.v, semaCtx)}};
+  // inp.v -> parser::OmpLastprivateClause
+  using wrapped = parser::OmpLastprivateClause;
+
+  CLAUSET_ENUM_CONVERT( //
+      convert, parser::OmpLastprivateClause::LastprivateModifier,
+      Lastprivate::LastprivateModifier,
+      // clang-format off
+      MS(Conditional, Conditional)
+      // clang-format on
+  );
+
+  auto &t0 = std::get<std::optional<wrapped::LastprivateModifier>>(inp.v.t);
+  auto &t1 = std::get<parser::OmpObjectList>(inp.v.t);
+
+  return Lastprivate{{/*LastprivateModifier=*/maybeApply(convert, t0),
+                      /*List=*/makeObjects(t1, semaCtx)}};
 }
 
 Linear make(const parser::OmpClause::Linear &inp,
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index 23a171c657638..2369dcd32d5e5 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -151,6 +151,8 @@ void DataSharingProcessor::collectSymbolsForPrivatization() {
                                  explicitlyPrivatizedSymbols);
     } else if (const auto &lastPrivateClause =
                    std::get_if<omp::clause::Lastprivate>(&clause.u)) {
+      lastprivateModifierNotSupported(*lastPrivateClause,
+                                      converter.getCurrentLocation());
       const ObjectList &objects = std::get<ObjectList>(lastPrivateClause->t);
       collectOmpObjectListSymbol(objects, explicitlyPrivatizedSymbols);
     }
@@ -188,12 +190,12 @@ void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) {
   for (const omp::Clause &clause : clauses) {
     if (clause.id != llvm::omp::OMPC_lastprivate)
       continue;
-    // TODO: Add lastprivate support for simd construct
-    if (mlir::isa<mlir::omp::WsloopOp>(op)) {
+    if (mlir::isa<mlir::omp::WsloopOp>(op) ||
+        mlir::isa<mlir::omp::SimdOp>(op)) {
       // Update the original variable just before exiting the worksharing
       // loop. Conversion as follows:
       //
-      // omp.wsloop {             omp.wsloop {
+      // omp.wsloop / omp.simd {             omp.wsloop / omp.simd {
       //   omp.loop_nest {          omp.loop_nest {
       //     ...                      ...
       //     store          ===>      store
@@ -350,15 +352,6 @@ void DataSharingProcessor::collectSymbols(
                              /*collectSymbols=*/true,
                              /*collectHostAssociatedSymbols=*/true);
 
-  // Add implicitly referenced symbols from statement functions.
-  if (curScope) {
-    for (const auto &sym : curScope->GetSymbols()) {
-      if (sym->test(semantics::Symbol::Flag::OmpFromStmtFunction) &&
-          sym->test(flag))
-        allSymbols.insert(&*sym);
-    }
-  }
-
   llvm::SetVector<const semantics::Symbol *> symbolsInNestedRegions;
   collectSymbolsInNestedRegions(eval, flag, symbolsInNestedRegions);
 
@@ -374,7 +367,8 @@ void DataSharingProcessor::collectSymbols(
     return !semantics::IsProcedure(sym) &&
            !sym.GetUltimate().has<semantics::DerivedTypeDetails>() &&
            !sym.GetUltimate().has<semantics::NamelistDetails>() &&
-           !semantics::IsImpliedDoIndex(sym.GetUltimate());
+           !semantics::IsImpliedDoIndex(sym.GetUltimate()) &&
+           !semantics::IsStmtFunction(sym);
   };
 
   auto shouldCollectSymbol = [&](const semantics::Symbol *sym) {
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 17ebf93edcce1..8195f4a897a90 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -590,9 +590,11 @@ static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
   mlir::Operation *marker = insertMarker(firOpBuilder);
 
   // If it is an unstructured region, create empty blocks for all evaluations.
-  if (info.eval.lowerAsUnstructured())
+  if (lower::omp::isLastItemInQueue(item, queue) &&
+      info.eval.lowerAsUnstructured()) {
     lower::createEmptyRegionBlocks<mlir::omp::TerminatorOp, mlir::omp::YieldOp>(
         firOpBuilder, info.eval.getNestedEvaluations());
+  }
 
   // Start with privatization, so that the lowering of the nested
   // code will use the right symbols.
@@ -966,7 +968,8 @@ static void genBodyOfTargetOp(
 
   // Create blocks for unstructured regions. This has to be done since
   // blocks are initially allocated with the function as the parent region.
-  if (eval.lowerAsUnstructured()) {
+  if (lower::omp::isLastItemInQueue(item, queue) &&
+      eval.lowerAsUnstructured()) {
     lower::createEmptyRegionBlocks<mlir::omp::TerminatorOp, mlir::omp::YieldOp>(
         firOpBuilder, eval.getNestedEvaluations());
   }
@@ -1563,7 +1566,9 @@ genSectionsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
 
   for (const Clause &clause : item->clauses) {
     if (clause.id == llvm::omp::Clause::OMPC_lastprivate) {
-      lastprivates.push_back(&std::get<clause::Lastprivate>(clause.u));
+      auto &lastp = std::get<clause::Lastprivate>(clause.u);
+      lastprivateModifierNotSupported(lastp, converter.getCurrentLocation());
+      lastprivates.push_back(&lastp);
     } else {
       switch (clause.id) {
       case llvm::omp::Clause::OMPC_firstprivate:
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
index c87182abe3d18..9da15ba303a47 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
@@ -178,9 +178,8 @@ ReductionProcessor::getReductionInitValue(mlir::Location loc, mlir::Type type,
   case ReductionIdentifier::OR:
   case ReductionIdentifier::EQV:
   case ReductionIdentifier::NEQV:
-    if (auto cplxTy = mlir::dyn_cast<fir::ComplexType>(type)) {
-      mlir::Type realTy =
-          lower::convertReal(builder.getContext(), cplxTy.getFKind());
+    if (auto cplxTy = mlir::dyn_cast<mlir::ComplexType>(type)) {
+      mlir::Type realTy = cplxTy.getElementType();
       mlir::Value initRe = builder.createRealConstant(
           loc, realTy, getOperationIdentity(redId, loc));
       mlir::Value initIm = builder.createRealConstant(loc, realTy, 0);
diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index 8073b24a1d5b4..47bc12e1b8a03 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -17,6 +17,7 @@
 #include <flang/Lower/ConvertType.h>
 #include <flang/Lower/PFTBuilder.h>
 #include <flang/Optimizer/Builder/FIRBuilder.h>
+#include <flang/Optimizer/Builder/Todo.h>
 #include <flang/Parser/parse-tree.h>
 #include <flang/Parser/tools.h>
 #include <flang/Semantics/tools.h>
@@ -356,6 +357,18 @@ semantics::Symbol *getOmpObjectSymbol(const parser::OmpObject &ompObject) {
   return sym;
 }
 
+void lastprivateModifierNotSupported(const omp::clause::Lastprivate &lastp,
+                                     mlir::Location loc) {
+  using Lastprivate = omp::clause::Lastprivate;
+  auto &maybeMod =
+      std::get<std::optional<Lastprivate::LastprivateModifier>>(lastp.t);
+  if (maybeMod) {
+    assert(*maybeMod == Lastprivate::LastprivateModifier::Conditional &&
+           "Unexpected lastprivate modifier");
+    TODO(loc, "lastprivate clause with CONDITIONAL modifier");
+  }
+}
+
 } // namespace omp
 } // namespace lower
 } // namespace Fortran
diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h
index 0b4fe9044bfa7..658d062e67b27 100644
--- a/flang/lib/Lower/OpenMP/Utils.h
+++ b/flang/lib/Lower/OpenMP/Utils.h
@@ -100,6 +100,9 @@ void genObjectList(const ObjectList &objects,
                    lower::AbstractConverter &converter,
                    llvm::SmallVectorImpl<mlir::Value> &operands);
 
+void lastprivateModifierNotSupported(const omp::clause::Lastprivate &lastp,
+                                     mlir::Location loc);
+
 } // namespace omp
 } // namespace lower
 } // namespace Fortran
diff --git a/flang/lib/Optimizer/Builder/Complex.cpp b/flang/lib/Optimizer/Builder/Complex.cpp
index cbcd4f850014e..69f97dd654ce0 100644
--- a/flang/lib/Optimizer/Builder/Complex.cpp
+++ b/flang/lib/Optimizer/Builder/Complex.cpp
@@ -14,24 +14,23 @@
 
 mlir::Type
 fir::factory::Complex::getComplexPartType(mlir::Type complexType) const {
-  return builder.getRealType(
-      mlir::cast<fir::ComplexType>(complexType).getFKind());
+  return mlir::cast<mlir::ComplexType>(complexType).getElementType();
 }
 
 mlir::Type fir::factory::Complex::getComplexPartType(mlir::Value cplx) const {
   return getComplexPartType(cplx.getType());
 }
 
-mlir::Value fir::factory::Complex::createComplex(fir::KindTy kind,
-                                                 mlir::Value real,
-                                                 mlir::Value imag) {
-  auto complexTy = fir::ComplexType::get(builder.getContext(), kind);
-  return createComplex(complexTy, real, imag);
-}
-
 mlir::Value fir::factory::Complex::createComplex(mlir::Type cplxTy,
                                                  mlir::Value real,
                                                  mlir::Value imag) {
   mlir::Value und = builder.create<fir::UndefOp>(loc, cplxTy);
   return insert<Part::Imag>(insert<Part::Real>(und, real), imag);
 }
+
+mlir::Value fir::factory::Complex::createComplex(mlir::Value real,
+                                                 mlir::Value imag) {
+  assert(real.getType() == imag.getType() && "part types must match");
+  mlir::Type cplxTy = mlir::ComplexType::get(real.getType());
+  return createComplex(cplxTy, real, imag);
+}
diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index 539235f01f5f7..9ad37c8df434a 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -150,8 +150,6 @@ mlir::Value
 fir::FirOpBuilder::createRealConstant(mlir::Location loc, mlir::Type fltTy,
                                       llvm::APFloat::integerPart val) {
   auto apf = [&]() -> llvm::APFloat {
-    if (auto ty = mlir::dyn_cast<fir::RealType>(fltTy))
-      return llvm::APFloat(kindMap.getFloatSemantics(ty.getFKind()), val);
     if (fltTy.isF16())
       return llvm::APFloat(llvm::APFloat::IEEEhalf(), val);
     if (fltTy.isBF16())
@@ -417,10 +415,7 @@ mlir::Value fir::FirOpBuilder::convertWithSemantics(
     // imaginary part is zero
     auto eleTy = helper.getComplexPartType(toTy);
     auto cast = createConvert(loc, eleTy, val);
-    llvm::APFloat zero{kindMap.getFloatSemantics(
-                           mlir::cast<fir::ComplexType>(toTy).getFKind()),
-                       0};
-    auto imag = createRealConstant(loc, eleTy, zero);
+    auto imag = createRealZeroConstant(loc, eleTy);
     return helper.createComplex(toTy, cast, imag);
   }
   if (fir::isa_complex(fromTy) &&
@@ -768,14 +763,23 @@ mlir::Value fir::FirOpBuilder::genAbsentOp(mlir::Location loc,
 
 void fir::FirOpBuilder::setCommonAttributes(mlir::Operation *op) const {
   auto fmi = mlir::dyn_cast<mlir::arith::ArithFastMathInterface>(*op);
-  if (!fmi)
-    return;
-  // TODO: use fmi.setFastMathFlagsAttr() after D137114 is merged.
-  //       For now set the attribute by the name.
-  llvm::StringRef arithFMFAttrName = fmi.getFastMathAttrName();
-  if (fastMathFlags != mlir::arith::FastMathFlags::none)
-    op->setAttr(arithFMFAttrName, mlir::arith::FastMathFlagsAttr::get(
-                                      op->getContext(), fastMathFlags));
+  if (fmi) {
+    // TODO: use fmi.setFastMathFlagsAttr() after D137114 is merged.
+    //       For now set the attribute by the name.
+    llvm::StringRef arithFMFAttrName = fmi.getFastMathAttrName();
+    if (fastMathFlags != mlir::arith::FastMathFlags::none)
+      op->setAttr(arithFMFAttrName, mlir::arith::FastMathFlagsAttr::get(
+                                        op->getContext(), fastMathFlags));
+  }
+  auto iofi =
+      mlir::dyn_cast<mlir::arith::ArithIntegerOverflowFlagsInterface>(*op);
+  if (iofi) {
+    llvm::StringRef arithIOFAttrName = iofi.getIntegerOverflowAttrName();
+    if (integerOverflowFlags != mlir::arith::IntegerOverflowFlags::none)
+      op->setAttr(arithIOFAttrName,
+                  mlir::arith::IntegerOverflowFlagsAttr::get(
+                      op->getContext(), integerOverflowFlags));
+  }
 }
 
 void fir::FirOpBuilder::setFastMathFlags(
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index 6e9fa0a433cb1..92ef3ccbbfe21 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -292,7 +292,9 @@ static constexpr IntrinsicHandler handlers[]{
      &I::genGetCwd,
      {{{"c", asBox}, {"status", asAddr, handleDynamicOptional}}},
      /*isElemental=*/false},
+    {"getgid", &I::genGetGID},
     {"getpid", &I::genGetPID},
+    {"getuid", &I::genGetUID},
     {"iachar", &I::genIchar},
     {"iall",
      &I::genIall,
@@ -836,24 +838,7 @@ mlir::Value genLibSplitComplexArgsCall(fir::FirOpBuilder &builder,
 
   auto getSplitComplexArgsType = [&builder, &args]() -> mlir::FunctionType {
     mlir::Type ctype = args[0].getType();
-    auto fKind = mlir::cast<fir::ComplexType>(ctype).getFKind();
-    mlir::Type ftype;
-
-    if (fKind == 2)
-      ftype = builder.getF16Type();
-    else if (fKind == 3)
-      ftype = builder.getBF16Type();
-    else if (fKind == 4)
-      ftype = builder.getF32Type();
-    else if (fKind == 8)
-      ftype = builder.getF64Type();
-    else if (fKind == 10)
-      ftype = builder.getF80Type();
-    else if (fKind == 16)
-      ftype = builder.getF128Type();
-    else
-      assert(0 && "Unsupported Complex Type");
-
+    auto ftype = mlir::cast<mlir::ComplexType>(ctype).getElementType();
     return builder.getFunctionType({ftype, ftype, ftype, ftype}, {ctype});
   };
 
@@ -944,25 +929,16 @@ mlir::Value genComplexMathOp(fir::FirOpBuilder &builder, mlir::Location loc,
   LLVM_DEBUG(llvm::dbgs() << "Generating '" << mathLibFuncName
                           << "' operation with type ";
              mathLibFuncType.dump(); llvm::dbgs() << "\n");
-  auto type = mlir::cast<fir::ComplexType>(mathLibFuncType.getInput(0));
-  auto kind = mlir::cast<fir::RealType>(type.getElementType()).getFKind();
-  auto realTy = builder.getRealType(kind);
-  auto mComplexTy = mlir::ComplexType::get(realTy);
-
-  llvm::SmallVector<mlir::Value, 2> cargs;
-  for (const mlir::Value &arg : args) {
-    // Convert the fir.complex to a mlir::complex
-    cargs.push_back(builder.createConvert(loc, mComplexTy, arg));
-  }
-
   // Builder expects an extra return type to be provided if different to
   // the argument types for an operation
   if constexpr (T::template hasTrait<
                     mlir::OpTrait::SameOperandsAndResultType>()) {
-    result = builder.create<T>(loc, cargs);
+    result = builder.create<T>(loc, args);
     result = builder.createConvert(loc, mathLibFuncType.getResult(0), result);
   } else {
-    result = builder.create<T>(loc, realTy, cargs);
+    auto complexTy = mlir::cast<mlir::ComplexType>(mathLibFuncType.getInput(0));
+    auto realTy = complexTy.getElementType();
+    result = builder.create<T>(loc, realTy, args);
     result = builder.createConvert(loc, mathLibFuncType.getResult(0), result);
   }
 
@@ -1456,17 +1432,12 @@ class FunctionDistance {
     }
   }
 
-  // Floating point can be mlir::FloatType or fir::real
+  // Floating point can be mlir Float or Complex Type.
   static unsigned getFloatingPointWidth(mlir::Type t) {
     if (auto f{mlir::dyn_cast<mlir::FloatType>(t)})
       return f.getWidth();
-    // FIXME: Get width another way for fir.real/complex
-    // - use fir/KindMapping.h and llvm::Type
-    // - or use evaluate/type.h
-    if (auto r{mlir::dyn_cast<fir::RealType>(t)})
-      return r.getFKind() * 4;
-    if (auto cplx{mlir::dyn_cast<fir::ComplexType>(t)})
-      return cplx.getFKind() * 4;
+    if (auto cplx{mlir::dyn_cast<mlir::ComplexType>(t)})
+      return mlir::cast<mlir::FloatType>(cplx.getElementType()).getWidth();
     llvm_unreachable("not a floating-point type");
   }
 
@@ -1487,13 +1458,10 @@ class FunctionDistance {
                  : Conversion::Extend;
     }
 
-    if (auto fromCplxTy{mlir::dyn_cast<fir::ComplexType>(from)}) {
-      if (auto toCplxTy{mlir::dyn_cast<fir::ComplexType>(to)}) {
-        return getFloatingPointWidth(fromCplxTy) >
-                       getFloatingPointWidth(toCplxTy)
-                   ? Conversion::Narrow
-                   : Conversion::Extend;
-      }
+    if (fir::isa_complex(from) && fir::isa_complex(to)) {
+      return getFloatingPointWidth(from) > getFloatingPointWidth(to)
+                 ? Conversion::Narrow
+                 : Conversion::Extend;
     }
     // Notes:
     // - No conversion between character types, specialization of runtime
@@ -2007,11 +1975,9 @@ static std::string typeToString(mlir::Type t) {
   if (auto i{mlir::dyn_cast<mlir::IntegerType>(t)}) {
     return "i" + std::to_string(i.getWidth());
   }
-  if (auto cplx{mlir::dyn_cast<fir::ComplexType>(t)}) {
-    return "z" + std::to_string(cplx.getFKind());
-  }
-  if (auto real{mlir::dyn_cast<fir::RealType>(t)}) {
-    return "r" + std::to_string(real.getFKind());
+  if (auto cplx{mlir::dyn_cast<mlir::ComplexType>(t)}) {
+    auto eleTy = mlir::cast<mlir::FloatType>(cplx.getElementType());
+    return "z" + std::to_string(eleTy.getWidth());
   }
   if (auto f{mlir::dyn_cast<mlir::FloatType>(t)}) {
     return "f" + std::to_string(f.getWidth());
@@ -3664,6 +3630,14 @@ void IntrinsicLibrary::genGetCommand(llvm::ArrayRef<fir::ExtendedValue> args) {
   }
 }
 
+// GETGID
+mlir::Value IntrinsicLibrary::genGetGID(mlir::Type resultType,
+                                        llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 0 && "getgid takes no input");
+  return builder.createConvert(loc, resultType,
+                               fir::runtime::genGetGID(builder, loc));
+}
+
 // GETPID
 mlir::Value IntrinsicLibrary::genGetPID(mlir::Type resultType,
                                         llvm::ArrayRef<mlir::Value> args) {
@@ -3672,6 +3646,14 @@ mlir::Value IntrinsicLibrary::genGetPID(mlir::Type resultType,
                                fir::runtime::genGetPID(builder, loc));
 }
 
+// GETUID
+mlir::Value IntrinsicLibrary::genGetUID(mlir::Type resultType,
+                                        llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 0 && "getgid takes no input");
+  return builder.createConvert(loc, resultType,
+                               fir::runtime::genGetUID(builder, loc));
+}
+
 // GET_COMMAND_ARGUMENT
 void IntrinsicLibrary::genGetCommandArgument(
     llvm::ArrayRef<fir::ExtendedValue> args) {
diff --git a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
index cf2483d36c027..ded9579f2c1df 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
@@ -129,6 +129,22 @@ void fir::runtime::genFree(fir::FirOpBuilder &builder, mlir::Location loc,
                               builder.createConvert(loc, intPtrTy, ptr));
 }
 
+mlir::Value fir::runtime::genGetGID(fir::FirOpBuilder &builder,
+                                    mlir::Location loc) {
+  auto runtimeFunc =
+      fir::runtime::getRuntimeFunc<mkRTKey(GetGID)>(loc, builder);
+
+  return builder.create<fir::CallOp>(loc, runtimeFunc).getResult(0);
+}
+
+mlir::Value fir::runtime::genGetUID(fir::FirOpBuilder &builder,
+                                    mlir::Location loc) {
+  auto runtimeFunc =
+      fir::runtime::getRuntimeFunc<mkRTKey(GetUID)>(loc, builder);
+
+  return builder.create<fir::CallOp>(loc, runtimeFunc).getResult(0);
+}
+
 mlir::Value fir::runtime::genMalloc(fir::FirOpBuilder &builder,
                                     mlir::Location loc, mlir::Value size) {
   auto runtimeFunc =
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index efc28e9708e19..1cb869bfeb95a 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -589,9 +589,7 @@ struct CallOpConversion : public fir::FIROpConversion<fir::CallOp> {
 } // namespace
 
 static mlir::Type getComplexEleTy(mlir::Type complex) {
-  if (auto cc = mlir::dyn_cast<mlir::ComplexType>(complex))
-    return cc.getElementType();
-  return mlir::cast<fir::ComplexType>(complex).getElementType();
+  return mlir::cast<mlir::ComplexType>(complex).getElementType();
 }
 
 namespace {
@@ -637,33 +635,6 @@ struct CmpcOpConversion : public fir::FIROpConversion<fir::CmpcOp> {
   }
 };
 
-/// Lower complex constants
-struct ConstcOpConversion : public fir::FIROpConversion<fir::ConstcOp> {
-  using FIROpConversion::FIROpConversion;
-
-  llvm::LogicalResult
-  matchAndRewrite(fir::ConstcOp conc, OpAdaptor,
-                  mlir::ConversionPatternRewriter &rewriter) const override {
-    mlir::Location loc = conc.getLoc();
-    mlir::Type ty = convertType(conc.getType());
-    mlir::Type ety = convertType(getComplexEleTy(conc.getType()));
-    auto realPart = rewriter.create<mlir::LLVM::ConstantOp>(
-        loc, ety, getValue(conc.getReal()));
-    auto imPart = rewriter.create<mlir::LLVM::ConstantOp>(
-        loc, ety, getValue(conc.getImaginary()));
-    auto undef = rewriter.create<mlir::LLVM::UndefOp>(loc, ty);
-    auto setReal =
-        rewriter.create<mlir::LLVM::InsertValueOp>(loc, undef, realPart, 0);
-    rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(conc, setReal,
-                                                           imPart, 1);
-    return mlir::success();
-  }
-
-  inline llvm::APFloat getValue(mlir::Attribute attr) const {
-    return mlir::cast<fir::RealAttr>(attr).getValue();
-  }
-};
-
 /// convert value of from-type to value of to-type
 struct ConvertOpConversion : public fir::FIROpConversion<fir::ConvertOp> {
   using FIROpConversion::FIROpConversion;
@@ -3861,12 +3832,12 @@ void fir::populateFIRToLLVMConversionPatterns(
       BoxIsAllocOpConversion, BoxIsArrayOpConversion, BoxIsPtrOpConversion,
       BoxOffsetOpConversion, BoxProcHostOpConversion, BoxRankOpConversion,
       BoxTypeCodeOpConversion, BoxTypeDescOpConversion, CallOpConversion,
-      CmpcOpConversion, ConstcOpConversion, ConvertOpConversion,
-      CoordinateOpConversion, DTEntryOpConversion, DeclareOpConversion,
-      DivcOpConversion, EmboxOpConversion, EmboxCharOpConversion,
-      EmboxProcOpConversion, ExtractValueOpConversion, FieldIndexOpConversion,
-      FirEndOpConversion, FreeMemOpConversion, GlobalLenOpConversion,
-      GlobalOpConversion, InsertOnRangeOpConversion, IsPresentOpConversion,
+      CmpcOpConversion, ConvertOpConversion, CoordinateOpConversion,
+      DTEntryOpConversion, DeclareOpConversion, DivcOpConversion,
+      EmboxOpConversion, EmboxCharOpConversion, EmboxProcOpConversion,
+      ExtractValueOpConversion, FieldIndexOpConversion, FirEndOpConversion,
+      FreeMemOpConversion, GlobalLenOpConversion, GlobalOpConversion,
+      InsertOnRangeOpConversion, IsPresentOpConversion,
       LenParamIndexOpConversion, LoadOpConversion, MulcOpConversion,
       NegcOpConversion, NoReassocOpConversion, SelectCaseOpConversion,
       SelectOpConversion, SelectRankOpConversion, SelectTypeOpConversion,
diff --git a/flang/lib/Optimizer/CodeGen/Target.cpp b/flang/lib/Optimizer/CodeGen/Target.cpp
index 7bc730bff76fe..a12b59413f445 100644
--- a/flang/lib/Optimizer/CodeGen/Target.cpp
+++ b/flang/lib/Optimizer/CodeGen/Target.cpp
@@ -41,8 +41,6 @@ llvm::StringRef Attributes::getIntExtensionAttrName() const {
 static const llvm::fltSemantics &floatToSemantics(const KindMapping &kindMap,
                                                   mlir::Type type) {
   assert(isa_real(type));
-  if (auto ty = mlir::dyn_cast<fir::RealType>(type))
-    return kindMap.getFloatSemantics(ty.getFKind());
   return mlir::cast<mlir::FloatType>(type).getFloatSemantics();
 }
 
@@ -356,7 +354,7 @@ struct TargetX86_64 : public GenericTarget<TargetX86_64> {
           else
             current = ArgClass::Integer;
         })
-        .template Case<mlir::FloatType, fir::RealType>([&](mlir::Type floatTy) {
+        .template Case<mlir::FloatType>([&](mlir::Type floatTy) {
           const auto *sem = &floatToSemantics(kindMap, floatTy);
           if (sem == &llvm::APFloat::x87DoubleExtended()) {
             Lo = ArgClass::X87;
@@ -368,7 +366,7 @@ struct TargetX86_64 : public GenericTarget<TargetX86_64> {
             current = ArgClass::SSE;
           }
         })
-        .template Case<fir::ComplexType>([&](fir::ComplexType cmplx) {
+        .template Case<mlir::ComplexType>([&](mlir::ComplexType cmplx) {
           const auto *sem = &floatToSemantics(kindMap, cmplx.getElementType());
           if (sem == &llvm::APFloat::x87DoubleExtended()) {
             current = ArgClass::ComplexX87;
@@ -540,9 +538,16 @@ struct TargetX86_64 : public GenericTarget<TargetX86_64> {
     if (typeList.size() != 1)
       return {};
     mlir::Type fieldType = typeList[0].second;
-    if (mlir::isa<mlir::FloatType, mlir::IntegerType, fir::RealType,
-                  fir::CharacterType, fir::LogicalType>(fieldType))
+    if (mlir::isa<mlir::FloatType, mlir::IntegerType, fir::LogicalType>(
+            fieldType))
       return fieldType;
+    if (mlir::isa<fir::CharacterType>(fieldType)) {
+      // Only CHARACTER(1) are expected in BIND(C) contexts, which is the only
+      // contexts where derived type may be passed in registers.
+      assert(mlir::cast<fir::CharacterType>(fieldType).getLen() == 1 &&
+             "fir.type value arg character components must have length 1");
+      return fieldType;
+    }
     // Complex field that needs to be split, or array.
     return {};
   }
diff --git a/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp b/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp
index f6cb26ff9613f..fd56fd6bf50f4 100644
--- a/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp
+++ b/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp
@@ -351,11 +351,6 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
     if (fnTy.getResults().size() == 1) {
       mlir::Type ty = fnTy.getResult(0);
       llvm::TypeSwitch<mlir::Type>(ty)
-          .template Case<fir::ComplexType>([&](fir::ComplexType cmplx) {
-            wrap = rewriteCallComplexResultType(loc, cmplx, newResTys,
-                                                newInTyAndAttrs, newOpers,
-                                                savedStackPtr);
-          })
           .template Case<mlir::ComplexType>([&](mlir::ComplexType cmplx) {
             wrap = rewriteCallComplexResultType(loc, cmplx, newResTys,
                                                 newInTyAndAttrs, newOpers,
@@ -414,10 +409,6 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
               }
             }
           })
-          .template Case<fir::ComplexType>([&](fir::ComplexType cmplx) {
-            rewriteCallComplexInputType(loc, cmplx, oper, newInTyAndAttrs,
-                                        newOpers, savedStackPtr);
-          })
           .template Case<mlir::ComplexType>([&](mlir::ComplexType cmplx) {
             rewriteCallComplexInputType(loc, cmplx, oper, newInTyAndAttrs,
                                         newOpers, savedStackPtr);
@@ -507,7 +498,8 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
       fir::DispatchOp dispatchOp = rewriter->create<A>(
           loc, newResTys, rewriter->getStringAttr(callOp.getMethod()),
           callOp.getOperands()[0], newOpers,
-          rewriter->getI32IntegerAttr(*callOp.getPassArgPos() + passArgShift));
+          rewriter->getI32IntegerAttr(*callOp.getPassArgPos() + passArgShift),
+          callOp.getProcedureAttrsAttr());
       if (wrap)
         newCallResults.push_back((*wrap)(dispatchOp.getOperation()));
       else
@@ -537,10 +529,10 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
     }
   }
 
-  // Result type fixup for fir::ComplexType and mlir::ComplexType
-  template <typename A, typename B>
+  // Result type fixup for ComplexType.
+  template <typename Ty>
   void lowerComplexSignatureRes(
-      mlir::Location loc, A cmplx, B &newResTys,
+      mlir::Location loc, mlir::ComplexType cmplx, Ty &newResTys,
       fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs) {
     if (noComplexConversion) {
       newResTys.push_back(cmplx);
@@ -556,10 +548,9 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
     }
   }
 
-  // Argument type fixup for fir::ComplexType and mlir::ComplexType
-  template <typename A>
+  // Argument type fixup for ComplexType.
   void lowerComplexSignatureArg(
-      mlir::Location loc, A cmplx,
+      mlir::Location loc, mlir::ComplexType cmplx,
       fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs) {
     if (noComplexConversion) {
       newInTyAndAttrs.push_back(fir::CodeGenSpecifics::getTypeAndAttr(cmplx));
@@ -601,9 +592,6 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
     auto loc = addrOp.getLoc();
     for (mlir::Type ty : addrTy.getResults()) {
       llvm::TypeSwitch<mlir::Type>(ty)
-          .Case<fir::ComplexType>([&](fir::ComplexType ty) {
-            lowerComplexSignatureRes(loc, ty, newResTys, newInTyAndAttrs);
-          })
           .Case<mlir::ComplexType>([&](mlir::ComplexType ty) {
             lowerComplexSignatureRes(loc, ty, newResTys, newInTyAndAttrs);
           })
@@ -627,9 +615,6 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
               }
             }
           })
-          .Case<fir::ComplexType>([&](fir::ComplexType ty) {
-            lowerComplexSignatureArg(loc, ty, newInTyAndAttrs);
-          })
           .Case<mlir::ComplexType>([&](mlir::ComplexType ty) {
             lowerComplexSignatureArg(loc, ty, newInTyAndAttrs);
           })
@@ -765,12 +750,6 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
     // Convert return value(s)
     for (auto ty : funcTy.getResults())
       llvm::TypeSwitch<mlir::Type>(ty)
-          .Case<fir::ComplexType>([&](fir::ComplexType cmplx) {
-            if (noComplexConversion)
-              newResTys.push_back(cmplx);
-            else
-              doComplexReturn(func, cmplx, newResTys, newInTyAndAttrs, fixups);
-          })
           .Case<mlir::ComplexType>([&](mlir::ComplexType cmplx) {
             if (noComplexConversion)
               newResTys.push_back(cmplx);
@@ -834,9 +813,6 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
               }
             }
           })
-          .Case<fir::ComplexType>([&](fir::ComplexType cmplx) {
-            doComplexArg(func, cmplx, newInTyAndAttrs, fixups);
-          })
           .Case<mlir::ComplexType>([&](mlir::ComplexType cmplx) {
             doComplexArg(func, cmplx, newInTyAndAttrs, fixups);
           })
@@ -1089,10 +1065,11 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
   /// Convert a complex return value. This can involve converting the return
   /// value to a "hidden" first argument or packing the complex into a wide
   /// GPR.
-  template <typename A, typename B, typename C>
-  void doComplexReturn(mlir::func::FuncOp func, A cmplx, B &newResTys,
+  template <typename Ty, typename FIXUPS>
+  void doComplexReturn(mlir::func::FuncOp func, mlir::ComplexType cmplx,
+                       Ty &newResTys,
                        fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs,
-                       C &fixups) {
+                       FIXUPS &fixups) {
     if (noComplexConversion) {
       newResTys.push_back(cmplx);
       return;
@@ -1193,10 +1170,10 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
   /// Convert a complex argument value. This can involve storing the value to
   /// a temporary memory location or factoring the value into two distinct
   /// arguments.
-  template <typename A, typename B>
-  void doComplexArg(mlir::func::FuncOp func, A cmplx,
+  template <typename FIXUPS>
+  void doComplexArg(mlir::func::FuncOp func, mlir::ComplexType cmplx,
                     fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs,
-                    B &fixups) {
+                    FIXUPS &fixups) {
     if (noComplexConversion) {
       newInTyAndAttrs.push_back(fir::CodeGenSpecifics::getTypeAndAttr(cmplx));
       return;
diff --git a/flang/lib/Optimizer/CodeGen/TypeConverter.cpp b/flang/lib/Optimizer/CodeGen/TypeConverter.cpp
index 1043494b6fb48..c23203efcd3df 100644
--- a/flang/lib/Optimizer/CodeGen/TypeConverter.cpp
+++ b/flang/lib/Optimizer/CodeGen/TypeConverter.cpp
@@ -57,8 +57,6 @@ LLVMTypeConverter::LLVMTypeConverter(mlir::ModuleOp module, bool applyTBAA,
       [&](fir::ClassType classTy) { return convertBoxType(classTy); });
   addConversion(
       [&](fir::CharacterType charTy) { return convertCharType(charTy); });
-  addConversion(
-      [&](fir::ComplexType cmplx) { return convertComplexType(cmplx); });
   addConversion([&](fir::FieldType field) {
     // Convert to i32 because of LLVM GEP indexing restriction.
     return mlir::IntegerType::get(field.getContext(), 32);
@@ -86,8 +84,6 @@ LLVMTypeConverter::LLVMTypeConverter(mlir::ModuleOp module, bool applyTBAA,
       [&](fir::RecordType derived, llvm::SmallVectorImpl<mlir::Type> &results) {
         return convertRecordType(derived, results);
       });
-  addConversion(
-      [&](fir::RealType real) { return convertRealType(real.getFKind()); });
   addConversion(
       [&](fir::ReferenceType ref) { return convertPointerLike(ref); });
   addConversion([&](fir::SequenceType sequence) {
@@ -277,13 +273,6 @@ mlir::Type LLVMTypeConverter::convertCharType(fir::CharacterType charTy) const {
   return mlir::LLVM::LLVMArrayType::get(iTy, charTy.getLen());
 }
 
-// convert a front-end kind value to either a std or LLVM IR dialect type
-// fir.real<n>  -->  llvm.anyfloat  where anyfloat is a kind mapping
-mlir::Type LLVMTypeConverter::convertRealType(fir::KindTy kind) const {
-  return fir::fromRealTypeID(&getContext(), kindMapping.getRealTypeID(kind),
-                             kind);
-}
-
 // fir.array<c ... :any>  -->  llvm<"[...[c x any]]">
 mlir::Type LLVMTypeConverter::convertSequenceType(SequenceType seq) const {
   auto baseTy = convertType(seq.getEleTy());
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index 13a30a358a866..8fdc06f6fce3f 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -1304,40 +1304,6 @@ mlir::ParseResult fir::CmpcOp::parse(mlir::OpAsmParser &parser,
   return parseCmpOp<fir::CmpcOp>(parser, result);
 }
 
-//===----------------------------------------------------------------------===//
-// ConstcOp
-//===----------------------------------------------------------------------===//
-
-mlir::ParseResult fir::ConstcOp::parse(mlir::OpAsmParser &parser,
-                                       mlir::OperationState &result) {
-  fir::RealAttr realp;
-  fir::RealAttr imagp;
-  mlir::Type type;
-  if (parser.parseLParen() ||
-      parser.parseAttribute(realp, fir::ConstcOp::getRealAttrName(),
-                            result.attributes) ||
-      parser.parseComma() ||
-      parser.parseAttribute(imagp, fir::ConstcOp::getImagAttrName(),
-                            result.attributes) ||
-      parser.parseRParen() || parser.parseColonType(type) ||
-      parser.addTypesToList(type, result.types))
-    return mlir::failure();
-  return mlir::success();
-}
-
-void fir::ConstcOp::print(mlir::OpAsmPrinter &p) {
-  p << '(';
-  p << getOperation()->getAttr(fir::ConstcOp::getRealAttrName()) << ", ";
-  p << getOperation()->getAttr(fir::ConstcOp::getImagAttrName()) << ") : ";
-  p.printType(getType());
-}
-
-llvm::LogicalResult fir::ConstcOp::verify() {
-  if (!mlir::isa<fir::ComplexType>(getType()))
-    return emitOpError("must be a !fir.complex type");
-  return mlir::success();
-}
-
 //===----------------------------------------------------------------------===//
 // ConvertOp
 //===----------------------------------------------------------------------===//
@@ -1381,7 +1347,7 @@ bool fir::ConvertOp::isIntegerCompatible(mlir::Type ty) {
 }
 
 bool fir::ConvertOp::isFloatCompatible(mlir::Type ty) {
-  return mlir::isa<mlir::FloatType, fir::RealType>(ty);
+  return mlir::isa<mlir::FloatType>(ty);
 }
 
 bool fir::ConvertOp::isPointerCompatible(mlir::Type ty) {
@@ -1567,8 +1533,6 @@ llvm::LogicalResult fir::CoordinateOp::verify() {
       } else if (auto t = mlir::dyn_cast<fir::RecordType>(eleTy)) {
         // FIXME: This is the same as the tuple case.
         return mlir::success();
-      } else if (auto t = mlir::dyn_cast<fir::ComplexType>(eleTy)) {
-        eleTy = t.getElementType();
       } else if (auto t = mlir::dyn_cast<mlir::ComplexType>(eleTy)) {
         eleTy = t.getElementType();
       } else if (auto t = mlir::dyn_cast<fir::CharacterType>(eleTy)) {
@@ -2122,7 +2086,7 @@ struct UndoComplexPattern : public mlir::RewritePattern {
   matchAndRewrite(mlir::Operation *op,
                   mlir::PatternRewriter &rewriter) const override {
     auto insval = mlir::dyn_cast_or_null<fir::InsertValueOp>(op);
-    if (!insval || !mlir::isa<fir::ComplexType>(insval.getType()))
+    if (!insval || !mlir::isa<mlir::ComplexType>(insval.getType()))
       return mlir::failure();
     auto insval2 = mlir::dyn_cast_or_null<fir::InsertValueOp>(
         insval.getAdt().getDefiningOp());
@@ -4423,14 +4387,6 @@ mlir::Type fir::applyPathToType(mlir::Type eleTy, mlir::ValueRange path) {
                       return ty.getType(fir::toInt(off));
                   return mlir::Type{};
                 })
-                .Case<fir::ComplexType>([&](fir::ComplexType ty) {
-                  auto x = *i;
-                  if (auto *op = (*i++).getDefiningOp())
-                    if (fir::isa_integer(x.getType()))
-                      return ty.getEleType(fir::getKindMapping(
-                          op->getParentOfType<mlir::ModuleOp>()));
-                  return mlir::Type{};
-                })
                 .Case<mlir::ComplexType>([&](mlir::ComplexType ty) {
                   if (fir::isa_integer((*i++).getType()))
                     return ty.getElementType();
diff --git a/flang/lib/Optimizer/Dialect/FIRType.cpp b/flang/lib/Optimizer/Dialect/FIRType.cpp
index 7a516298e5ef4..70c0fd66d18ae 100644
--- a/flang/lib/Optimizer/Dialect/FIRType.cpp
+++ b/flang/lib/Optimizer/Dialect/FIRType.cpp
@@ -441,7 +441,6 @@ unsigned getBoxRank(mlir::Type boxTy) {
 
 /// Return the ISO_C_BINDING intrinsic module value of type \p ty.
 int getTypeCode(mlir::Type ty, const fir::KindMapping &kindMap) {
-  unsigned width = 0;
   if (mlir::IntegerType intTy = mlir::dyn_cast<mlir::IntegerType>(ty)) {
     switch (intTy.getWidth()) {
     case 8:
@@ -485,21 +484,12 @@ int getTypeCode(mlir::Type ty, const fir::KindMapping &kindMap) {
     }
     llvm_unreachable("unsupported real type");
   }
-  if (fir::isa_complex(ty)) {
-    if (mlir::ComplexType complexTy = mlir::dyn_cast<mlir::ComplexType>(ty)) {
-      mlir::FloatType floatTy =
-          mlir::cast<mlir::FloatType>(complexTy.getElementType());
-      if (floatTy.isBF16())
-        return CFI_type_bfloat_Complex;
-      width = floatTy.getWidth();
-    } else if (fir::ComplexType complexTy =
-                   mlir::dyn_cast<fir::ComplexType>(ty)) {
-      auto FKind = complexTy.getFKind();
-      if (FKind == 3)
-        return CFI_type_bfloat_Complex;
-      width = kindMap.getRealBitsize(FKind);
-    }
-    switch (width) {
+  if (mlir::ComplexType complexTy = mlir::dyn_cast<mlir::ComplexType>(ty)) {
+    mlir::FloatType floatTy =
+        mlir::cast<mlir::FloatType>(complexTy.getElementType());
+    if (floatTy.isBF16())
+      return CFI_type_bfloat_Complex;
+    switch (floatTy.getWidth()) {
     case 16:
       return CFI_type_half_float_Complex;
     case 32:
@@ -545,14 +535,10 @@ std::string getTypeAsString(mlir::Type ty, const fir::KindMapping &kindMap,
         name << 'i' << ty.getIntOrFloatBitWidth();
       } else if (mlir::isa<mlir::FloatType>(ty)) {
         name << 'f' << ty.getIntOrFloatBitWidth();
-      } else if (fir::isa_complex(ty)) {
+      } else if (auto cplxTy = mlir::dyn_cast_or_null<mlir::ComplexType>(ty)) {
         name << 'z';
-        if (auto cplxTy = mlir::dyn_cast_or_null<mlir::ComplexType>(ty)) {
-          auto floatTy = mlir::cast<mlir::FloatType>(cplxTy.getElementType());
-          name << floatTy.getWidth();
-        } else if (auto cplxTy = mlir::dyn_cast_or_null<fir::ComplexType>(ty)) {
-          name << kindMap.getRealBitsize(cplxTy.getFKind());
-        }
+        auto floatTy = mlir::cast<mlir::FloatType>(cplxTy.getElementType());
+        name << floatTy.getWidth();
       } else if (auto logTy = mlir::dyn_cast_or_null<fir::LogicalType>(ty)) {
         name << 'l' << kindMap.getLogicalBitsize(logTy.getFKind());
       } else {
@@ -778,34 +764,11 @@ fir::ClassType::verify(llvm::function_ref<mlir::InFlightDiagnostic()> emitError,
   if (mlir::isa<fir::RecordType, fir::SequenceType, fir::HeapType,
                 fir::PointerType, mlir::NoneType, mlir::IntegerType,
                 mlir::FloatType, fir::CharacterType, fir::LogicalType,
-                fir::ComplexType, mlir::ComplexType>(eleTy))
+                mlir::ComplexType>(eleTy))
     return mlir::success();
   return emitError() << "invalid element type\n";
 }
 
-//===----------------------------------------------------------------------===//
-// ComplexType
-//===----------------------------------------------------------------------===//
-
-mlir::Type fir::ComplexType::parse(mlir::AsmParser &parser) {
-  return parseKindSingleton<fir::ComplexType>(parser);
-}
-
-void fir::ComplexType::print(mlir::AsmPrinter &printer) const {
-  printer << "<" << getFKind() << '>';
-}
-
-mlir::Type fir::ComplexType::getElementType() const {
-  return fir::RealType::get(getContext(), getFKind());
-}
-
-// Return the MLIR float type of the complex element type.
-mlir::Type fir::ComplexType::getEleType(const fir::KindMapping &kindMap) const {
-  auto fkind = getFKind();
-  auto realTypeID = kindMap.getRealTypeID(fkind);
-  return fir::fromRealTypeID(getContext(), realTypeID, fkind);
-}
-
 //===----------------------------------------------------------------------===//
 // HeapType
 //===----------------------------------------------------------------------===//
@@ -875,32 +838,6 @@ llvm::LogicalResult fir::PointerType::verify(
   return mlir::success();
 }
 
-//===----------------------------------------------------------------------===//
-// RealType
-//===----------------------------------------------------------------------===//
-
-// `real` `<` kind `>`
-mlir::Type fir::RealType::parse(mlir::AsmParser &parser) {
-  return parseKindSingleton<fir::RealType>(parser);
-}
-
-void fir::RealType::print(mlir::AsmPrinter &printer) const {
-  printer << "<" << getFKind() << '>';
-}
-
-llvm::LogicalResult
-fir::RealType::verify(llvm::function_ref<mlir::InFlightDiagnostic()> emitError,
-                      KindTy fKind) {
-  // TODO
-  return mlir::success();
-}
-
-mlir::Type fir::RealType::getFloatType(const fir::KindMapping &kindMap) const {
-  auto fkind = getFKind();
-  auto realTypeID = kindMap.getRealTypeID(fkind);
-  return fir::fromRealTypeID(getContext(), realTypeID, fkind);
-}
-
 //===----------------------------------------------------------------------===//
 // RecordType
 //===----------------------------------------------------------------------===//
@@ -1367,10 +1304,10 @@ bool fir::BaseBoxType::isAssumedRank() const {
 
 void FIROpsDialect::registerTypes() {
   addTypes<BoxType, BoxCharType, BoxProcType, CharacterType, ClassType,
-           fir::ComplexType, FieldType, HeapType, fir::IntegerType, LenType,
-           LogicalType, LLVMPointerType, PointerType, RealType, RecordType,
-           ReferenceType, SequenceType, ShapeType, ShapeShiftType, ShiftType,
-           SliceType, TypeDescType, fir::VectorType, fir::DummyScopeType>();
+           FieldType, HeapType, fir::IntegerType, LenType, LogicalType,
+           LLVMPointerType, PointerType, RecordType, ReferenceType,
+           SequenceType, ShapeType, ShapeShiftType, ShiftType, SliceType,
+           TypeDescType, fir::VectorType, fir::DummyScopeType>();
   fir::ReferenceType::attachInterface<
       OpenMPPointerLikeModel<fir::ReferenceType>>(*getContext());
   fir::ReferenceType::attachInterface<
@@ -1401,19 +1338,6 @@ fir::getTypeSizeAndAlignment(mlir::Location loc, mlir::Type ty,
     unsigned short alignment = dl.getTypeABIAlignment(ty);
     return std::pair{size, alignment};
   }
-  if (auto firCmplx = mlir::dyn_cast<fir::ComplexType>(ty)) {
-    auto result =
-        getTypeSizeAndAlignment(loc, firCmplx.getEleType(kindMap), dl, kindMap);
-    if (!result)
-      return result;
-    auto [floatSize, floatAlign] = *result;
-    return std::pair{llvm::alignTo(floatSize, floatAlign) + floatSize,
-                     floatAlign};
-  }
-  if (auto real = mlir::dyn_cast<fir::RealType>(ty))
-    return getTypeSizeAndAlignment(loc, real.getFloatType(kindMap), dl,
-                                   kindMap);
-
   if (auto seqTy = mlir::dyn_cast<fir::SequenceType>(ty)) {
     auto result = getTypeSizeAndAlignment(loc, seqTy.getEleTy(), dl, kindMap);
     if (!result)
diff --git a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
index ae62affaed6b0..ed301c74c9ede 100644
--- a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
+++ b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
@@ -360,14 +360,11 @@ llvm::LogicalResult hlfir::DesignateOp::verify() {
       return emitOpError("substring must contain 2 indices when provided");
   }
   if (getComplexPart()) {
-    if (!fir::isa_complex(outputElementType))
+    if (auto cplx = mlir::dyn_cast<mlir::ComplexType>(outputElementType))
+      outputElementType = cplx.getElementType();
+    else
       return emitOpError("memref or component must have complex type if "
                          "complex_part is provided");
-    if (auto firCplx = mlir::dyn_cast<fir::ComplexType>(outputElementType))
-      outputElementType = firCplx.getElementType();
-    else
-      outputElementType =
-          mlir::cast<mlir::ComplexType>(outputElementType).getElementType();
   }
   mlir::Type resultBaseType =
       getFortranElementOrSequenceType(getResult().getType());
@@ -383,9 +380,7 @@ llvm::LogicalResult hlfir::DesignateOp::verify() {
   // length may differ because of substrings.
   if (resultElementType != outputElementType &&
       !(mlir::isa<fir::CharacterType>(resultElementType) &&
-        mlir::isa<fir::CharacterType>(outputElementType)) &&
-      !(mlir::isa<mlir::FloatType>(resultElementType) &&
-        mlir::isa<fir::RealType>(outputElementType)))
+        mlir::isa<fir::CharacterType>(outputElementType)))
     return emitOpError(
                "result element type is not consistent with operands, expected ")
            << outputElementType;
diff --git a/flang/lib/Optimizer/Transforms/AbstractResult.cpp b/flang/lib/Optimizer/Transforms/AbstractResult.cpp
index ff37310224e85..7299ff80121e1 100644
--- a/flang/lib/Optimizer/Transforms/AbstractResult.cpp
+++ b/flang/lib/Optimizer/Transforms/AbstractResult.cpp
@@ -154,17 +154,14 @@ class CallConversion : public mlir::OpRewritePattern<Op> {
         newOperands.emplace_back(arg);
       unsigned passArgShift = newOperands.size();
       newOperands.append(op.getOperands().begin() + 1, op.getOperands().end());
-
-      fir::DispatchOp newDispatchOp;
+      mlir::IntegerAttr passArgPos;
       if (op.getPassArgPos())
-        newOp = rewriter.create<fir::DispatchOp>(
-            loc, newResultTypes, rewriter.getStringAttr(op.getMethod()),
-            op.getOperands()[0], newOperands,
-            rewriter.getI32IntegerAttr(*op.getPassArgPos() + passArgShift));
-      else
-        newOp = rewriter.create<fir::DispatchOp>(
-            loc, newResultTypes, rewriter.getStringAttr(op.getMethod()),
-            op.getOperands()[0], newOperands, nullptr);
+        passArgPos =
+            rewriter.getI32IntegerAttr(*op.getPassArgPos() + passArgShift);
+      newOp = rewriter.create<fir::DispatchOp>(
+          loc, newResultTypes, rewriter.getStringAttr(op.getMethod()),
+          op.getOperands()[0], newOperands, passArgPos,
+          op.getProcedureAttrsAttr());
     }
 
     if (isResultBuiltinCPtr) {
diff --git a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
index a921d91d16fe4..927e275b36089 100644
--- a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
+++ b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
@@ -18,6 +18,7 @@
 #include "flang/Optimizer/CodeGen/CGOps.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
+#include "flang/Optimizer/Dialect/FIROpsSupport.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Optimizer/Dialect/Support/FIRContext.h"
 #include "flang/Optimizer/Support/InternalNames.h"
@@ -265,6 +266,14 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
   auto result = fir::NameUniquer::deconstruct(funcName);
   funcName = mlir::StringAttr::get(context, result.second.name);
 
+  // try to use a better function name than _QQmain for the program statement
+  if (funcName == fir::NameUniquer::doProgramEntry()) {
+    mlir::StringAttr bindcName =
+        funcOp->getAttrOfType<mlir::StringAttr>(fir::getSymbolAttrName());
+    if (bindcName)
+      funcName = bindcName;
+  }
+
   llvm::SmallVector<mlir::LLVM::DITypeAttr> types;
   for (auto resTy : funcOp.getResultTypes()) {
     auto tyAttr =
diff --git a/flang/lib/Optimizer/Transforms/CufOpConversion.cpp b/flang/lib/Optimizer/Transforms/CufOpConversion.cpp
index a80ca7a125abd..5870b0985532d 100644
--- a/flang/lib/Optimizer/Transforms/CufOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CufOpConversion.cpp
@@ -196,9 +196,9 @@ static int computeWidth(mlir::Location loc, mlir::Type type,
   } else if (auto t{mlir::dyn_cast<fir::LogicalType>(eleTy)}) {
     int kind = t.getFKind();
     width = kindMap.getLogicalBitsize(kind) / 8;
-  } else if (auto t{mlir::dyn_cast<fir::ComplexType>(eleTy)}) {
-    int kind = t.getFKind();
-    int elemSize = kindMap.getRealBitsize(kind) / 8;
+  } else if (auto t{mlir::dyn_cast<mlir::ComplexType>(eleTy)}) {
+    int elemSize =
+        mlir::cast<mlir::FloatType>(t.getElementType()).getWidth() / 8;
     width = 2 * elemSize;
   } else {
     llvm::report_fatal_error("unsupported type");
diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index 29e61d505bf6a..71e534b4f2e2a 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -78,6 +78,35 @@ static mlir::LLVM::DITypeAttr genPlaceholderType(mlir::MLIRContext *context) {
                       /*bitSize=*/32, llvm::dwarf::DW_ATE_signed);
 }
 
+// Helper function to create DILocalVariableAttr and DbgValueOp when information
+// about the size or dimension of a variable etc lives in an mlir::Value.
+mlir::LLVM::DILocalVariableAttr DebugTypeGenerator::generateArtificialVariable(
+    mlir::MLIRContext *context, mlir::Value val,
+    mlir::LLVM::DIFileAttr fileAttr, mlir::LLVM::DIScopeAttr scope,
+    fir::cg::XDeclareOp declOp) {
+  // There can be multiple artificial variable for a single declOp. To help
+  // distinguish them, we pad the name with a counter. The counter is the
+  // position of 'val' in the operands of declOp.
+  auto varID = std::distance(
+      declOp.getOperands().begin(),
+      std::find(declOp.getOperands().begin(), declOp.getOperands().end(), val));
+  mlir::OpBuilder builder(context);
+  auto name = mlir::StringAttr::get(context, "." + declOp.getUniqName().str() +
+                                                 std::to_string(varID));
+  builder.setInsertionPoint(declOp);
+  mlir::Type type = val.getType();
+  if (!mlir::isa<mlir::IntegerType>(type) || !type.isSignlessInteger()) {
+    type = builder.getIntegerType(64);
+    val = builder.create<fir::ConvertOp>(declOp.getLoc(), type, val);
+  }
+  mlir::LLVM::DITypeAttr Ty = convertType(type, fileAttr, scope, declOp);
+  auto lvAttr = mlir::LLVM::DILocalVariableAttr::get(
+      context, scope, name, fileAttr, /*line=*/0, /*argNo=*/0,
+      /*alignInBits=*/0, Ty, mlir::LLVM::DIFlags::Artificial);
+  builder.create<mlir::LLVM::DbgValueOp>(declOp.getLoc(), val, lvAttr, nullptr);
+  return lvAttr;
+}
+
 mlir::LLVM::DITypeAttr DebugTypeGenerator::convertBoxedSequenceType(
     fir::SequenceType seqTy, mlir::LLVM::DIFileAttr fileAttr,
     mlir::LLVM::DIScopeAttr scope, fir::cg::XDeclareOp declOp,
@@ -122,11 +151,12 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertBoxedSequenceType(
     mlir::Attribute lowerAttr = nullptr;
     // If declaration has a lower bound, use it.
     if (declOp && declOp.getShift().size() > index) {
-      // TODO: Handle case where lower bound is a variable (instead of a
-      // constant as handled here)
       if (std::optional<std::int64_t> optint =
               getIntIfConstant(declOp.getShift()[index]))
         lowerAttr = mlir::IntegerAttr::get(intTy, llvm::APInt(64, *optint));
+      else
+        lowerAttr = generateArtificialVariable(
+            context, declOp.getShift()[index], fileAttr, scope, declOp);
     }
     // FIXME: If `indexSize` happens to be bigger than address size on the
     // system then we may have to change 'DW_OP_deref' here.
@@ -320,31 +350,43 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertSequenceType(
   unsigned index = 0;
   auto intTy = mlir::IntegerType::get(context, 64);
   for (fir::SequenceType::Extent dim : seqTy.getShape()) {
-    int64_t shift = 1;
+    mlir::Attribute lowerAttr = nullptr;
+    mlir::Attribute countAttr = nullptr;
+    // If declOp is present, we use the shift in it to get the lower bound of
+    // the array. If it is constant, that is used. If it is not constant, we
+    // create a variable that represents its location and use that as lower
+    // bound. As an optimization, we don't create a lower bound when shift is a
+    // constant 1 as that is the default.
     if (declOp && declOp.getShift().size() > index) {
       if (std::optional<std::int64_t> optint =
-              getIntIfConstant(declOp.getShift()[index]))
-        shift = *optint;
+              getIntIfConstant(declOp.getShift()[index])) {
+        if (*optint != 1)
+          lowerAttr = mlir::IntegerAttr::get(intTy, llvm::APInt(64, *optint));
+      } else
+        lowerAttr = generateArtificialVariable(
+            context, declOp.getShift()[index], fileAttr, scope, declOp);
     }
+
     if (dim == seqTy.getUnknownExtent()) {
-      mlir::IntegerAttr lowerAttr = nullptr;
-      if (declOp && declOp.getShift().size() > index)
-        lowerAttr = mlir::IntegerAttr::get(intTy, llvm::APInt(64, shift));
-      // FIXME: This path is taken for assumed size arrays but also for arrays
-      // with non constant extent. For the latter case, the DISubrangeAttr
-      // should point to a variable which will have the extent at runtime.
-      auto subrangeTy = mlir::LLVM::DISubrangeAttr::get(
-          context, /*count=*/nullptr, lowerAttr, /*upperBound*/ nullptr,
-          /*stride*/ nullptr);
-      elements.push_back(subrangeTy);
-    } else {
-      auto countAttr = mlir::IntegerAttr::get(intTy, llvm::APInt(64, dim));
-      auto lowerAttr = mlir::IntegerAttr::get(intTy, llvm::APInt(64, shift));
-      auto subrangeTy = mlir::LLVM::DISubrangeAttr::get(
-          context, countAttr, lowerAttr, /*upperBound=*/nullptr,
-          /*stride=*/nullptr);
-      elements.push_back(subrangeTy);
-    }
+      // This path is taken for both assumed size array or when the size of the
+      // array is variable. In the case of variable size, we create a variable
+      // to use as countAttr. Note that fir has a constant size of -1 for
+      // assumed size array. So !optint check makes sure we don't generate
+      // variable in that case.
+      if (declOp && declOp.getShape().size() > index) {
+        std::optional<std::int64_t> optint =
+            getIntIfConstant(declOp.getShape()[index]);
+        if (!optint)
+          countAttr = generateArtificialVariable(
+              context, declOp.getShape()[index], fileAttr, scope, declOp);
+      }
+    } else
+      countAttr = mlir::IntegerAttr::get(intTy, llvm::APInt(64, dim));
+
+    auto subrangeTy = mlir::LLVM::DISubrangeAttr::get(
+        context, countAttr, lowerAttr, /*upperBound=*/nullptr,
+        /*stride=*/nullptr);
+    elements.push_back(subrangeTy);
     ++index;
   }
   // Apart from arrays, the `DICompositeTypeAttr` is used for other things like
@@ -400,23 +442,8 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertCharacterType(
     // variable that will contain that length. This variable is used as
     // 'stringLength' in DIStringTypeAttr.
     if (declOp && !declOp.getTypeparams().empty()) {
-      auto name =
-          mlir::StringAttr::get(context, "." + declOp.getUniqName().str());
-      mlir::OpBuilder builder(context);
-      builder.setInsertionPoint(declOp);
-      mlir::Value sizeVal = declOp.getTypeparams()[0];
-      mlir::Type type = sizeVal.getType();
-      if (!mlir::isa<mlir::IntegerType>(type) || !type.isSignlessInteger()) {
-        type = builder.getIntegerType(64);
-        sizeVal =
-            builder.create<fir::ConvertOp>(declOp.getLoc(), type, sizeVal);
-      }
-      mlir::LLVM::DITypeAttr Ty = convertType(type, fileAttr, scope, declOp);
-      auto lvAttr = mlir::LLVM::DILocalVariableAttr::get(
-          context, scope, name, fileAttr, /*line=*/0, /*argNo=*/0,
-          /*alignInBits=*/0, Ty, mlir::LLVM::DIFlags::Artificial);
-      builder.create<mlir::LLVM::DbgValueOp>(declOp.getLoc(), sizeVal, lvAttr,
-                                             nullptr);
+      mlir::LLVM::DILocalVariableAttr lvAttr = generateArtificialVariable(
+          context, declOp.getTypeparams()[0], fileAttr, scope, declOp);
       varAttr = mlir::cast<mlir::LLVM::DIVariableAttr>(lvAttr);
     }
   }
@@ -467,25 +494,14 @@ DebugTypeGenerator::convertType(mlir::Type Ty, mlir::LLVM::DIFileAttr fileAttr,
   } else if (mlir::isa<mlir::FloatType>(Ty)) {
     return genBasicType(context, mlir::StringAttr::get(context, "real"),
                         Ty.getIntOrFloatBitWidth(), llvm::dwarf::DW_ATE_float);
-  } else if (auto realTy = mlir::dyn_cast_or_null<fir::RealType>(Ty)) {
-    return genBasicType(context, mlir::StringAttr::get(context, "real"),
-                        kindMapping.getRealBitsize(realTy.getFKind()),
-                        llvm::dwarf::DW_ATE_float);
   } else if (auto logTy = mlir::dyn_cast_or_null<fir::LogicalType>(Ty)) {
     return genBasicType(context,
                         mlir::StringAttr::get(context, logTy.getMnemonic()),
                         kindMapping.getLogicalBitsize(logTy.getFKind()),
                         llvm::dwarf::DW_ATE_boolean);
-  } else if (fir::isa_complex(Ty)) {
-    unsigned bitWidth;
-    if (auto cplxTy = mlir::dyn_cast_or_null<mlir::ComplexType>(Ty)) {
-      auto floatTy = mlir::cast<mlir::FloatType>(cplxTy.getElementType());
-      bitWidth = floatTy.getWidth();
-    } else if (auto cplxTy = mlir::dyn_cast_or_null<fir::ComplexType>(Ty)) {
-      bitWidth = kindMapping.getRealBitsize(cplxTy.getFKind());
-    } else {
-      llvm_unreachable("Unhandled complex type");
-    }
+  } else if (auto cplxTy = mlir::dyn_cast_or_null<mlir::ComplexType>(Ty)) {
+    auto floatTy = mlir::cast<mlir::FloatType>(cplxTy.getElementType());
+    unsigned bitWidth = floatTy.getWidth();
     return genBasicType(context, mlir::StringAttr::get(context, "complex"),
                         bitWidth * 2, llvm::dwarf::DW_ATE_complex_float);
   } else if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(Ty)) {
diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
index b8a068e5ba148..ff600e751eb0b 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
@@ -64,6 +64,11 @@ class DebugTypeGenerator {
                                                 fir::cg::XDeclareOp declOp,
                                                 bool genAllocated,
                                                 bool genAssociated);
+  mlir::LLVM::DILocalVariableAttr
+  generateArtificialVariable(mlir::MLIRContext *context, mlir::Value Val,
+                             mlir::LLVM::DIFileAttr fileAttr,
+                             mlir::LLVM::DIScopeAttr scope,
+                             fir::cg::XDeclareOp declOp);
 
   mlir::ModuleOp module;
   mlir::SymbolTable *symbolTable;
diff --git a/flang/lib/Optimizer/Transforms/LoopVersioning.cpp b/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
index 51dc48f0fcb12..adc39861840ab 100644
--- a/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
+++ b/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
@@ -265,7 +265,7 @@ void LoopVersioningPass::runOnOperation() {
         mlir::Type elementType = fir::unwrapSeqOrBoxedSeqType(arg.getType());
         if (mlir::isa<mlir::FloatType>(elementType) ||
             mlir::isa<mlir::IntegerType>(elementType) ||
-            mlir::isa<fir::ComplexType>(elementType)) {
+            mlir::isa<mlir::ComplexType>(elementType)) {
           auto [eleSize, eleAlign] = fir::getTypeSizeAndAlignmentOrCrash(
               arg.getLoc(), elementType, *dl, kindMap);
           typeSize = llvm::alignTo(eleSize, eleAlign);
diff --git a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
index 105f275de8b94..070889a284f48 100644
--- a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
@@ -205,10 +205,8 @@ struct DispatchOpConv : public OpConversionPattern<fir::DispatchOp> {
     // Make the call.
     llvm::SmallVector<mlir::Value> args{funcPtr};
     args.append(dispatch.getArgs().begin(), dispatch.getArgs().end());
-    // FIXME: add procedure_attrs to fir.dispatch and propagate to fir.call.
-    rewriter.replaceOpWithNewOp<fir::CallOp>(
-        dispatch, resTypes, nullptr, args,
-        /*procedure_attrs=*/fir::FortranProcedureFlagsEnumAttr{});
+    rewriter.replaceOpWithNewOp<fir::CallOp>(dispatch, resTypes, nullptr, args,
+                                             dispatch.getProcedureAttrsAttr());
     return mlir::success();
   }
 
diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index a56337d65e5f3..69e4814bf246c 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -185,6 +185,24 @@ Message &Message::set_severity(Severity severity) {
   return *this;
 }
 
+std::optional<common::LanguageFeature> Message::languageFeature() const {
+  return languageFeature_;
+}
+
+Message &Message::set_languageFeature(common::LanguageFeature feature) {
+  languageFeature_ = feature;
+  return *this;
+}
+
+std::optional<common::UsageWarning> Message::usageWarning() const {
+  return usageWarning_;
+}
+
+Message &Message::set_usageWarning(common::UsageWarning warning) {
+  usageWarning_ = warning;
+  return *this;
+}
+
 std::string Message::ToString() const {
   return common::visit(
       common::visitors{
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 52789d6e5f0f6..cc2930cbd7ded 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -232,6 +232,12 @@ TYPE_PARSER(construct<OmpOrderClause>(
 TYPE_PARSER(
     construct<OmpObject>(designator) || construct<OmpObject>("/" >> name / "/"))
 
+// OMP 5.0 2.19.4.5 LASTPRIVATE ([lastprivate-modifier :] list)
+TYPE_PARSER(construct<OmpLastprivateClause>(
+    maybe("CONDITIONAL" >>
+        pure(OmpLastprivateClause::LastprivateModifier::Conditional) / ":"),
+    Parser<OmpObjectList>{}))
+
 TYPE_PARSER(
     "ACQUIRE" >> construct<OmpClause>(construct<OmpClause::Acquire>()) ||
     "ACQ_REL" >> construct<OmpClause>(construct<OmpClause::AcqRel>()) ||
@@ -289,7 +295,7 @@ TYPE_PARSER(
     "IS_DEVICE_PTR" >> construct<OmpClause>(construct<OmpClause::IsDevicePtr>(
                            parenthesized(Parser<OmpObjectList>{}))) ||
     "LASTPRIVATE" >> construct<OmpClause>(construct<OmpClause::Lastprivate>(
-                         parenthesized(Parser<OmpObjectList>{}))) ||
+                         parenthesized(Parser<OmpLastprivateClause>{}))) ||
     "LINEAR" >> construct<OmpClause>(construct<OmpClause::Linear>(
                     parenthesized(Parser<OmpLinearClause>{}))) ||
     "LINK" >> construct<OmpClause>(construct<OmpClause::Link>(
diff --git a/flang/lib/Parser/preprocessor.cpp b/flang/lib/Parser/preprocessor.cpp
index cb3725bc4ea67..f9b8716941b3b 100644
--- a/flang/lib/Parser/preprocessor.cpp
+++ b/flang/lib/Parser/preprocessor.cpp
@@ -610,7 +610,8 @@ void Preprocessor::Directive(const TokenSequence &dir, Prescanner &prescanner) {
       if (dir.IsAnythingLeft(++j)) {
         if (prescanner.features().ShouldWarn(
                 common::UsageWarning::Portability)) {
-          prescanner.Say(dir.GetIntervalProvenanceRange(j, tokens - j),
+          prescanner.Say(common::UsageWarning::Portability,
+              dir.GetIntervalProvenanceRange(j, tokens - j),
               "#undef: excess tokens at end of directive"_port_en_US);
         }
       } else {
@@ -627,7 +628,8 @@ void Preprocessor::Directive(const TokenSequence &dir, Prescanner &prescanner) {
       if (dir.IsAnythingLeft(++j)) {
         if (prescanner.features().ShouldWarn(
                 common::UsageWarning::Portability)) {
-          prescanner.Say(dir.GetIntervalProvenanceRange(j, tokens - j),
+          prescanner.Say(common::UsageWarning::Portability,
+              dir.GetIntervalProvenanceRange(j, tokens - j),
               "#%s: excess tokens at end of directive"_port_en_US, dirName);
         }
       }
@@ -649,7 +651,8 @@ void Preprocessor::Directive(const TokenSequence &dir, Prescanner &prescanner) {
   } else if (dirName == "else") {
     if (dir.IsAnythingLeft(j)) {
       if (prescanner.features().ShouldWarn(common::UsageWarning::Portability)) {
-        prescanner.Say(dir.GetIntervalProvenanceRange(j, tokens - j),
+        prescanner.Say(common::UsageWarning::Portability,
+            dir.GetIntervalProvenanceRange(j, tokens - j),
             "#else: excess tokens at end of directive"_port_en_US);
       }
     } else if (ifStack_.empty()) {
@@ -678,7 +681,8 @@ void Preprocessor::Directive(const TokenSequence &dir, Prescanner &prescanner) {
   } else if (dirName == "endif") {
     if (dir.IsAnythingLeft(j)) {
       if (prescanner.features().ShouldWarn(common::UsageWarning::Portability)) {
-        prescanner.Say(dir.GetIntervalProvenanceRange(j, tokens - j),
+        prescanner.Say(common::UsageWarning::Portability,
+            dir.GetIntervalProvenanceRange(j, tokens - j),
             "#endif: excess tokens at end of directive"_port_en_US);
       }
     } else if (ifStack_.empty()) {
@@ -729,7 +733,8 @@ void Preprocessor::Directive(const TokenSequence &dir, Prescanner &prescanner) {
       if (k >= pathTokens) {
         if (prescanner.features().ShouldWarn(
                 common::UsageWarning::Portability)) {
-          prescanner.Say(dir.GetIntervalProvenanceRange(j, tokens - j),
+          prescanner.Say(common::UsageWarning::Portability,
+              dir.GetIntervalProvenanceRange(j, tokens - j),
               "#include: expected '>' at end of included file"_port_en_US);
         }
       }
@@ -758,7 +763,8 @@ void Preprocessor::Directive(const TokenSequence &dir, Prescanner &prescanner) {
     k = path.SkipBlanks(k + 1);
     if (k < pathTokens && path.TokenAt(k).ToString() != "!") {
       if (prescanner.features().ShouldWarn(common::UsageWarning::Portability)) {
-        prescanner.Say(dir.GetIntervalProvenanceRange(j, tokens - j),
+        prescanner.Say(common::UsageWarning::Portability,
+            dir.GetIntervalProvenanceRange(j, tokens - j),
             "#include: extra stuff ignored after file name"_port_en_US);
       }
     }
diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index eabfcc244001a..47260c0680463 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -243,7 +243,8 @@ void Prescanner::Statement() {
   }
   if (continuationLines_ > 255) {
     if (features_.ShouldWarn(common::LanguageFeature::MiscSourceExtensions)) {
-      Say(GetProvenance(statementStart),
+      Say(common::LanguageFeature::MiscSourceExtensions,
+          GetProvenance(statementStart),
           "%d continuation lines is more than the Fortran standard allows"_port_en_US,
           continuationLines_);
     }
@@ -265,7 +266,8 @@ void Prescanner::Statement() {
     case LineClassification::Kind::DefinitionDirective:
     case LineClassification::Kind::PreprocessorDirective:
       if (features_.ShouldWarn(common::UsageWarning::Preprocessing)) {
-        Say(preprocessed->GetProvenanceRange(),
+        Say(common::UsageWarning::Preprocessing,
+            preprocessed->GetProvenanceRange(),
             "Preprocessed line resembles a preprocessor directive"_warn_en_US);
       }
       CheckAndEmitLine(preprocessed->ToLowerCase(), newlineProvenance);
@@ -400,7 +402,7 @@ void Prescanner::LabelField(TokenSequence &token) {
       // CookedSource::Marshal().
       cooked_.MarkPossibleFixedFormContinuation();
     } else if (features_.ShouldWarn(common::UsageWarning::Scanning)) {
-      Say(GetProvenance(start + *badColumn - 1),
+      Say(common::UsageWarning::Scanning, GetProvenance(start + *badColumn - 1),
           *badColumn == 6
               ? "Statement should not begin with a continuation line"_warn_en_US
               : "Character in fixed-form label field must be a digit"_warn_en_US);
@@ -424,7 +426,7 @@ void Prescanner::LabelField(TokenSequence &token) {
   SkipToNextSignificantCharacter();
   if (IsDecimalDigit(*at_)) {
     if (features_.ShouldWarn(common::LanguageFeature::MiscSourceExtensions)) {
-      Say(GetCurrentProvenance(),
+      Say(common::LanguageFeature::MiscSourceExtensions, GetCurrentProvenance(),
           "Label digit is not in fixed-form label field"_port_en_US);
     }
   }
@@ -627,7 +629,7 @@ bool Prescanner::NextToken(TokenSequence &tokens) {
       // Recognize and skip over classic C style /*comments*/ when
       // outside a character literal.
       if (features_.ShouldWarn(LanguageFeature::ClassicCComments)) {
-        Say(GetCurrentProvenance(),
+        Say(LanguageFeature::ClassicCComments, GetCurrentProvenance(),
             "nonstandard usage: C-style comment"_port_en_US);
       }
       SkipCComments();
@@ -795,7 +797,8 @@ bool Prescanner::NextToken(TokenSequence &tokens) {
       if (IsDecimalDigit(*at_)) {
         if (features_.ShouldWarn(
                 common::LanguageFeature::MiscSourceExtensions)) {
-          Say(GetProvenanceRange(at_, at_ + 1),
+          Say(common::LanguageFeature::MiscSourceExtensions,
+              GetProvenanceRange(at_, at_ + 1),
               "Label should be in the label field"_port_en_US);
         }
       }
@@ -923,7 +926,7 @@ void Prescanner::Hollerith(
     if (PadOutCharacterLiteral(tokens)) {
     } else if (*at_ == '\n') {
       if (features_.ShouldWarn(common::UsageWarning::Scanning)) {
-        Say(GetProvenanceRange(start, at_),
+        Say(common::UsageWarning::Scanning, GetProvenanceRange(start, at_),
             "Possible truncated Hollerith literal"_warn_en_US);
       }
       break;
@@ -1087,7 +1090,7 @@ void Prescanner::FortranInclude(const char *firstQuote) {
     for (; *p != '\n' && *p != '!'; ++p) {
     }
     if (features_.ShouldWarn(common::UsageWarning::Scanning)) {
-      Say(GetProvenanceRange(garbage, p),
+      Say(common::UsageWarning::Scanning, GetProvenanceRange(garbage, p),
           "excess characters after path name"_warn_en_US);
     }
   }
@@ -1228,7 +1231,8 @@ const char *Prescanner::FixedFormContinuationLine(bool mightNeedSpace) {
       // Extension: '&' as continuation marker
       if (features_.ShouldWarn(
               LanguageFeature::FixedFormContinuationWithColumn1Ampersand)) {
-        Say(GetProvenance(nextLine_), "nonstandard usage"_port_en_US);
+        Say(LanguageFeature::FixedFormContinuationWithColumn1Ampersand,
+            GetProvenance(nextLine_), "nonstandard usage"_port_en_US);
       }
       return nextLine_ + 1;
     }
@@ -1294,7 +1298,8 @@ const char *Prescanner::FreeFormContinuationLine(bool ampersand) {
         //   'b'
         if (features_.ShouldWarn(
                 common::LanguageFeature::MiscSourceExtensions)) {
-          Say(GetProvenanceRange(p, p + 1),
+          Say(common::LanguageFeature::MiscSourceExtensions,
+              GetProvenanceRange(p, p + 1),
               "Character literal continuation line should have been preceded by '&'"_port_en_US);
         }
       } else if (p > nextLine_) {
@@ -1339,7 +1344,8 @@ bool Prescanner::FreeFormContinuation() {
     } else if (ampersand && isPossibleMacroCall_ && (*p == ',' || *p == ')')) {
       return false; // allow & at end of a macro argument
     } else if (features_.ShouldWarn(LanguageFeature::CruftAfterAmpersand)) {
-      Say(GetProvenance(p), "missing ! before comment after &"_warn_en_US);
+      Say(LanguageFeature::CruftAfterAmpersand, GetProvenance(p),
+          "missing ! before comment after &"_warn_en_US);
     }
   }
   do {
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index 2511a5dda9d09..ab01d82537c03 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2067,6 +2067,12 @@ class UnparseVisitor {
                   },
         x.u);
   }
+  void Unparse(const OmpLastprivateClause &x) {
+    Walk(
+        std::get<std::optional<OmpLastprivateClause::LastprivateModifier>>(x.t),
+        ":");
+    Walk(std::get<OmpObjectList>(x.t));
+  }
   void Unparse(const OmpMapType::Always &) { Word("ALWAYS,"); }
   void Unparse(const OmpMapClause &x) {
     Walk(std::get<std::optional<OmpMapType>>(x.t), ":");
@@ -2764,6 +2770,8 @@ class UnparseVisitor {
   WALK_NESTED_ENUM(OmpDefaultClause, Type) // OMP DEFAULT
   WALK_NESTED_ENUM(OmpDefaultmapClause, ImplicitBehavior) // OMP DEFAULTMAP
   WALK_NESTED_ENUM(OmpDefaultmapClause, VariableCategory) // OMP DEFAULTMAP
+  WALK_NESTED_ENUM(
+      OmpLastprivateClause, LastprivateModifier) // OMP lastprivate-modifier
   WALK_NESTED_ENUM(OmpScheduleModifierType, ModType) // OMP schedule-modifier
   WALK_NESTED_ENUM(OmpLinearModifier, Type) // OMP linear-modifier
   WALK_NESTED_ENUM(OmpDependenceType, Type) // OMP dependence-type
diff --git a/flang/lib/Semantics/check-acc-structure.cpp b/flang/lib/Semantics/check-acc-structure.cpp
index c0f834c845a82..623e31341a7c2 100644
--- a/flang/lib/Semantics/check-acc-structure.cpp
+++ b/flang/lib/Semantics/check-acc-structure.cpp
@@ -409,16 +409,12 @@ void AccStructureChecker::CheckMultipleOccurrenceInDeclare(
               if (const auto *name = getDesignatorNameIfDataRef(designator)) {
                 if (declareSymbols.contains(&name->symbol->GetUltimate())) {
                   if (declareSymbols[&name->symbol->GetUltimate()] == clause) {
-                    if (context_.languageFeatures().ShouldWarn(
-                            common::UsageWarning::OpenAccUsage)) {
-                      context_.Say(GetContext().clauseSource,
-                          "'%s' in the %s clause is already present in the "
-                          "same "
-                          "clause in this module"_warn_en_US,
-                          name->symbol->name(),
-                          parser::ToUpperCaseLetters(
-                              llvm::acc::getOpenACCClauseName(clause).str()));
-                    }
+                    context_.Warn(common::UsageWarning::OpenAccUsage,
+                        GetContext().clauseSource,
+                        "'%s' in the %s clause is already present in the same clause in this module"_warn_en_US,
+                        name->symbol->name(),
+                        parser::ToUpperCaseLetters(
+                            llvm::acc::getOpenACCClauseName(clause).str()));
                   } else {
                     context_.Say(GetContext().clauseSource,
                         "'%s' in the %s clause is already present in another "
@@ -770,10 +766,8 @@ void AccStructureChecker::Enter(const parser::AccClause::Link &x) {
 }
 
 void AccStructureChecker::Enter(const parser::AccClause::Shortloop &x) {
-  if (CheckAllowed(llvm::acc::Clause::ACCC_shortloop) &&
-      context_.languageFeatures().ShouldWarn(
-          common::UsageWarning::OpenAccUsage)) {
-    context_.Say(GetContext().clauseSource,
+  if (CheckAllowed(llvm::acc::Clause::ACCC_shortloop)) {
+    context_.Warn(common::UsageWarning::OpenAccUsage, GetContext().clauseSource,
         "Non-standard shortloop clause ignored"_warn_en_US);
   }
 }
@@ -793,10 +787,8 @@ void AccStructureChecker::Enter(const parser::AccClause::If &x) {
 }
 
 void AccStructureChecker::Enter(const parser::OpenACCEndConstruct &x) {
-  if (context_.languageFeatures().ShouldWarn(
-          common::UsageWarning::OpenAccUsage)) {
-    context_.Say(x.source, "Misplaced OpenACC end directive"_warn_en_US);
-  }
+  context_.Warn(common::UsageWarning::OpenAccUsage, x.source,
+      "Misplaced OpenACC end directive"_warn_en_US);
 }
 
 void AccStructureChecker::Enter(const parser::Module &) {
diff --git a/flang/lib/Semantics/check-allocate.cpp b/flang/lib/Semantics/check-allocate.cpp
index a5363a6710d31..1e5412324916d 100644
--- a/flang/lib/Semantics/check-allocate.cpp
+++ b/flang/lib/Semantics/check-allocate.cpp
@@ -531,10 +531,8 @@ bool AllocationCheckerHelper::RunChecks(SemanticsContext &context) {
     // Character length distinction is allowed, with a warning
     if (!HaveCompatibleLengths(
             *type_, allocateInfo_.sourceExprType.value())) { // F'2023 C950
-      if (context.ShouldWarn(common::LanguageFeature::AllocateToOtherLength)) {
-        context.Say(name_.source,
-            "Character length of allocatable object in ALLOCATE should be the same as the SOURCE or MOLD"_port_en_US);
-      }
+      context.Warn(common::LanguageFeature::AllocateToOtherLength, name_.source,
+          "Character length of allocatable object in ALLOCATE should be the same as the SOURCE or MOLD"_port_en_US);
       return false;
     }
   }
diff --git a/flang/lib/Semantics/check-call.cpp b/flang/lib/Semantics/check-call.cpp
index 31079174239c2..28a12a5798cb0 100644
--- a/flang/lib/Semantics/check-call.cpp
+++ b/flang/lib/Semantics/check-call.cpp
@@ -169,37 +169,40 @@ static void CheckCharacterActual(evaluate::Expr<evaluate::SomeType> &actual,
                                     actualOffset->offset()) /
                       actualType.type().kind();
                 }
-                if (actualChars < dummyChars &&
-                    (extentErrors ||
-                        context.ShouldWarn(
-                            common::UsageWarning::ShortCharacterActual))) {
-                  auto msg{
-                      "Actual argument has fewer characters remaining in storage sequence (%jd) than %s (%jd)"_warn_en_US};
+                if (actualChars < dummyChars) {
                   if (extentErrors) {
-                    msg.set_severity(parser::Severity::Error);
+                    messages.Say(
+                        "Actual argument has fewer characters remaining in storage sequence (%jd) than %s (%jd)"_err_en_US,
+                        static_cast<std::intmax_t>(actualChars), dummyName,
+                        static_cast<std::intmax_t>(dummyChars));
+                  } else if (context.ShouldWarn(
+                                 common::UsageWarning::ShortCharacterActual)) {
+                    messages.Say(common::UsageWarning::ShortCharacterActual,
+                        "Actual argument has fewer characters remaining in storage sequence (%jd) than %s (%jd)"_warn_en_US,
+                        static_cast<std::intmax_t>(actualChars), dummyName,
+                        static_cast<std::intmax_t>(dummyChars));
                   }
-                  messages.Say(std::move(msg),
-                      static_cast<std::intmax_t>(actualChars), dummyName,
-                      static_cast<std::intmax_t>(dummyChars));
                 }
               }
             } else { // actual.type.Rank() > 0
               if (auto actualSize{evaluate::ToInt64(evaluate::Fold(
                       foldingContext, evaluate::GetSize(actualType.shape())))};
                   actualSize &&
-                  *actualSize * *actualLength < *dummySize * *dummyLength &&
-                  (extentErrors ||
-                      context.ShouldWarn(
-                          common::UsageWarning::ShortCharacterActual))) {
-                auto msg{
-                    "Actual argument array has fewer characters (%jd) than %s array (%jd)"_warn_en_US};
+                  *actualSize * *actualLength < *dummySize * *dummyLength) {
                 if (extentErrors) {
-                  msg.set_severity(parser::Severity::Error);
+                  messages.Say(
+                      "Actual argument array has fewer characters (%jd) than %s array (%jd)"_err_en_US,
+                      static_cast<std::intmax_t>(*actualSize * *actualLength),
+                      dummyName,
+                      static_cast<std::intmax_t>(*dummySize * *dummyLength));
+                } else if (context.ShouldWarn(
+                               common::UsageWarning::ShortCharacterActual)) {
+                  messages.Say(common::UsageWarning::ShortCharacterActual,
+                      "Actual argument array has fewer characters (%jd) than %s array (%jd)"_warn_en_US,
+                      static_cast<std::intmax_t>(*actualSize * *actualLength),
+                      dummyName,
+                      static_cast<std::intmax_t>(*dummySize * *dummyLength));
                 }
-                messages.Say(std::move(msg),
-                    static_cast<std::intmax_t>(*actualSize * *actualLength),
-                    dummyName,
-                    static_cast<std::intmax_t>(*dummySize * *dummyLength));
               }
             }
           }
@@ -217,11 +220,11 @@ static void CheckCharacterActual(evaluate::Expr<evaluate::SomeType> &actual,
             if (context.ShouldWarn(
                     common::UsageWarning::ShortCharacterActual)) {
               if (isVariable) {
-                messages.Say(
+                messages.Say(common::UsageWarning::ShortCharacterActual,
                     "Actual argument variable length '%jd' is less than expected length '%jd'"_warn_en_US,
                     *actualLength, *dummyLength);
               } else {
-                messages.Say(
+                messages.Say(common::UsageWarning::ShortCharacterActual,
                     "Actual argument expression length '%jd' is less than expected length '%jd'"_warn_en_US,
                     *actualLength, *dummyLength);
               }
@@ -260,23 +263,16 @@ static void ConvertIntegerActual(evaluate::Expr<evaluate::SomeType> &actual,
     actual = std::move(*converted);
     if (dummyType.type().kind() < actualType.type().kind()) {
       if (!semanticsContext.IsEnabled(
-              common::LanguageFeature::ActualIntegerConvertedToSmallerKind) ||
-          semanticsContext.ShouldWarn(
               common::LanguageFeature::ActualIntegerConvertedToSmallerKind)) {
-        std::optional<parser::MessageFixedText> msg;
-        if (!semanticsContext.IsEnabled(
-                common::LanguageFeature::ActualIntegerConvertedToSmallerKind)) {
-          msg =
-              "Actual argument scalar expression of type INTEGER(%d) cannot beimplicitly converted to smaller dummy argument type INTEGER(%d)"_err_en_US;
-        } else if (semanticsContext.ShouldWarn(
-                       common::LanguageFeature::ConvertedArgument)) {
-          msg =
-              "Actual argument scalar expression of type INTEGER(%d) was converted to smaller dummy argument type INTEGER(%d)"_port_en_US;
-        }
-        if (msg) {
-          messages.Say(std::move(msg.value()), actualType.type().kind(),
-              dummyType.type().kind());
-        }
+        messages.Say(
+            "Actual argument scalar expression of type INTEGER(%d) cannot beimplicitly converted to smaller dummy argument type INTEGER(%d)"_err_en_US,
+            actualType.type().kind(), dummyType.type().kind());
+      } else if (semanticsContext.ShouldWarn(common::LanguageFeature::
+                         ActualIntegerConvertedToSmallerKind)) {
+        messages.Say(
+            common::LanguageFeature::ActualIntegerConvertedToSmallerKind,
+            "Actual argument scalar expression of type INTEGER(%d) was converted to smaller dummy argument type INTEGER(%d)"_port_en_US,
+            actualType.type().kind(), dummyType.type().kind());
       }
     }
     actualType = dummyType;
@@ -355,7 +351,7 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
         constantChar && constantChar->wasHollerith() &&
         dummy.type.type().IsUnlimitedPolymorphic() &&
         context.ShouldWarn(common::LanguageFeature::HollerithPolymorphic)) {
-      messages.Say(
+      messages.Say(common::LanguageFeature::HollerithPolymorphic,
           "passing Hollerith to unlimited polymorphic as if it were CHARACTER"_port_en_US);
     }
   } else if (dummyRank == 0 && allowActualArgumentConversions) {
@@ -364,7 +360,7 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
             foldingContext, actual, dummy.type.type())}) {
       if (context.ShouldWarn(
               common::LanguageFeature::HollerithOrCharacterAsBOZ)) {
-        messages.Say(
+        messages.Say(common::LanguageFeature::HollerithOrCharacterAsBOZ,
             "passing Hollerith or character literal as if it were BOZ"_port_en_US);
       }
       actual = *converted;
@@ -607,35 +603,38 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
                       *actualSymTypeBytes;
                 }
               }
-              if (actualElements && *actualElements < *dummySize &&
-                  (extentErrors ||
-                      context.ShouldWarn(
-                          common::UsageWarning::ShortArrayActual))) {
-                auto msg{
-                    "Actual argument has fewer elements remaining in storage sequence (%jd) than %s array (%jd)"_warn_en_US};
+              if (actualElements && *actualElements < *dummySize) {
                 if (extentErrors) {
-                  msg.set_severity(parser::Severity::Error);
+                  messages.Say(
+                      "Actual argument has fewer elements remaining in storage sequence (%jd) than %s array (%jd)"_err_en_US,
+                      static_cast<std::intmax_t>(*actualElements), dummyName,
+                      static_cast<std::intmax_t>(*dummySize));
+                } else if (context.ShouldWarn(
+                               common::UsageWarning::ShortArrayActual)) {
+                  messages.Say(common::UsageWarning::ShortArrayActual,
+                      "Actual argument has fewer elements remaining in storage sequence (%jd) than %s array (%jd)"_warn_en_US,
+                      static_cast<std::intmax_t>(*actualElements), dummyName,
+                      static_cast<std::intmax_t>(*dummySize));
                 }
-                messages.Say(std::move(msg),
-                    static_cast<std::intmax_t>(*actualElements), dummyName,
-                    static_cast<std::intmax_t>(*dummySize));
               }
             }
           }
         } else { // actualRank > 0 || actualIsAssumedRank
           if (auto actualSize{evaluate::ToInt64(evaluate::Fold(
                   foldingContext, evaluate::GetSize(actualType.shape())))};
-              actualSize && *actualSize < *dummySize &&
-              (extentErrors ||
-                  context.ShouldWarn(common::UsageWarning::ShortArrayActual))) {
-            auto msg{
-                "Actual argument array has fewer elements (%jd) than %s array (%jd)"_warn_en_US};
+              actualSize && *actualSize < *dummySize) {
             if (extentErrors) {
-              msg.set_severity(parser::Severity::Error);
+              messages.Say(
+                  "Actual argument array has fewer elements (%jd) than %s array (%jd)"_err_en_US,
+                  static_cast<std::intmax_t>(*actualSize), dummyName,
+                  static_cast<std::intmax_t>(*dummySize));
+            } else if (context.ShouldWarn(
+                           common::UsageWarning::ShortArrayActual)) {
+              messages.Say(common::UsageWarning::ShortArrayActual,
+                  "Actual argument array has fewer elements (%jd) than %s array (%jd)"_warn_en_US,
+                  static_cast<std::intmax_t>(*actualSize), dummyName,
+                  static_cast<std::intmax_t>(*dummySize));
             }
-            messages.Say(std::move(msg),
-                static_cast<std::intmax_t>(*actualSize), dummyName,
-                static_cast<std::intmax_t>(*dummySize));
           }
         }
       }
@@ -683,8 +682,11 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
       }
       if (auto whyNot{WhyNotDefinable(messages.at(), *scope, flags, actual)}) {
         if (whyNot->IsFatal()) {
-          if (auto *msg{
-                  messages.Say(std::move(*undefinableMessage), dummyName)}) {
+          if (auto *msg{messages.Say(*undefinableMessage, dummyName)}) {
+            if (!msg->IsFatal()) {
+              msg->set_languageFeature(common::LanguageFeature::
+                      UndefinableAsynchronousOrVolatileActual);
+            }
             msg->Attach(
                 std::move(whyNot->set_severity(parser::Severity::Because)));
           }
@@ -747,7 +749,7 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
         // is treated as an unassociated allocatable.
         if (context.ShouldWarn(
                 common::LanguageFeature::NullActualForAllocatable)) {
-          messages.Say(
+          messages.Say(common::LanguageFeature::NullActualForAllocatable,
               "Allocatable %s is associated with a null pointer"_port_en_US,
               dummyName);
         }
@@ -802,7 +804,7 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
           context.IsEnabled(common::LanguageFeature::RelaxedIntentInChecking)) {
         if (context.ShouldWarn(
                 common::LanguageFeature::RelaxedIntentInChecking)) {
-          messages.Say(
+          messages.Say(common::LanguageFeature::RelaxedIntentInChecking,
               "If a POINTER or ALLOCATABLE dummy or actual argument is unlimited polymorphic, both should be so"_port_en_US);
         }
       } else {
@@ -814,7 +816,7 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
           context.IsEnabled(common::LanguageFeature::RelaxedIntentInChecking)) {
         if (context.ShouldWarn(
                 common::LanguageFeature::RelaxedIntentInChecking)) {
-          messages.Say(
+          messages.Say(common::LanguageFeature::RelaxedIntentInChecking,
               "If a POINTER or ALLOCATABLE dummy or actual argument is polymorphic, both should be so"_port_en_US);
         }
       } else if (actualIsPolymorphic &&
@@ -823,6 +825,8 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
         if (context.ShouldWarn(common::LanguageFeature::
                     PolymorphicActualAllocatableOrPointerToMonomorphicDummy)) {
           messages.Say(
+              common::LanguageFeature::
+                  PolymorphicActualAllocatableOrPointerToMonomorphicDummy,
               "If a POINTER or ALLOCATABLE actual argument is polymorphic, the corresponding dummy argument should also be so"_port_en_US);
         }
       } else {
@@ -838,7 +842,7 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
                 common::LanguageFeature::RelaxedIntentInChecking)) {
           if (context.ShouldWarn(
                   common::LanguageFeature::RelaxedIntentInChecking)) {
-            messages.Say(
+            messages.Say(common::LanguageFeature::RelaxedIntentInChecking,
                 "POINTER or ALLOCATABLE dummy and actual arguments should have the same declared type and kind"_port_en_US);
           }
         } else {
@@ -908,13 +912,13 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
     bool actualIsTemp{!actualIsVariable || HasVectorSubscript(actual) ||
         evaluate::ExtractCoarrayRef(actual)};
     if (actualIsTemp) {
-      messages.Say(
+      messages.Say(common::UsageWarning::NonTargetPassedToTarget,
           "Any pointer associated with TARGET %s during this call will not be associated with the value of '%s' afterwards"_warn_en_US,
           dummyName, actual.AsFortran());
     } else {
       auto actualSymbolVector{GetSymbolVector(actual)};
       if (!evaluate::GetLastTarget(actualSymbolVector)) {
-        messages.Say(
+        messages.Say(common::UsageWarning::NonTargetPassedToTarget,
             "Any pointer associated with TARGET %s during this call must not be used afterwards, as '%s' is not a target"_warn_en_US,
             dummyName, actual.AsFortran());
       }
@@ -1012,15 +1016,16 @@ static void CheckProcedureArg(evaluate::ActualArgument &arg,
           return;
         }
       } else if (argProcSymbol->has<ProcBindingDetails>()) {
-        if (!context.IsEnabled(common::LanguageFeature::BindingAsProcedure) ||
-            context.ShouldWarn(common::LanguageFeature::BindingAsProcedure)) {
-          parser::MessageFixedText msg{
-              "Procedure binding '%s' passed as an actual argument"_port_en_US};
-          if (!context.IsEnabled(common::LanguageFeature::BindingAsProcedure)) {
-            msg.set_severity(parser::Severity::Error);
-          }
-          evaluate::SayWithDeclaration(
-              messages, *argProcSymbol, std::move(msg), argProcSymbol->name());
+        if (!context.IsEnabled(common::LanguageFeature::BindingAsProcedure)) {
+          evaluate::SayWithDeclaration(messages, *argProcSymbol,
+              "Procedure binding '%s' passed as an actual argument"_err_en_US,
+              argProcSymbol->name());
+        } else if (context.ShouldWarn(
+                       common::LanguageFeature::BindingAsProcedure)) {
+          evaluate::SayWithDeclaration(messages, *argProcSymbol,
+              common::LanguageFeature::BindingAsProcedure,
+              "Procedure binding '%s' passed as an actual argument"_port_en_US,
+              argProcSymbol->name());
         }
       }
     }
@@ -1067,13 +1072,13 @@ static void CheckProcedureArg(evaluate::ActualArgument &arg,
                     dummyName);
               } else if (context.ShouldWarn(
                              common::UsageWarning::ImplicitInterfaceActual)) {
-                messages.Say(
+                messages.Say(common::UsageWarning::ImplicitInterfaceActual,
                     "Actual procedure argument has an implicit interface which is not known to be compatible with %s which has an explicit interface"_warn_en_US,
                     dummyName);
               }
             } else if (warning &&
                 context.ShouldWarn(common::UsageWarning::ProcDummyArgShapes)) {
-              messages.Say(
+              messages.Say(common::UsageWarning::ProcDummyArgShapes,
                   "Actual procedure argument has possible interface incompatibility with %s: %s"_warn_en_US,
                   dummyName, std::move(*warning));
             }
@@ -1239,6 +1244,7 @@ static void CheckExplicitInterfaceArg(evaluate::ActualArgument &arg,
                     if (context.ShouldWarn(common::LanguageFeature::
                                 NullActualForAllocatable)) {
                       messages.Say(
+                          common::LanguageFeature::NullActualForAllocatable,
                           "Allocatable %s is associated with NULL()"_port_en_US,
                           dummyName);
                     }
@@ -1430,7 +1436,8 @@ static void CheckAssociated(evaluate::ActualArguments &arguments,
         if (semanticsContext.ShouldWarn(common::UsageWarning::Portability)) {
           if (!evaluate::ExtractDataRef(*pointerExpr) &&
               !evaluate::IsProcedurePointer(*pointerExpr)) {
-            messages.Say(pointerArg->sourceLocation(),
+            messages.Say(common::UsageWarning::Portability,
+                pointerArg->sourceLocation(),
                 "POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer"_port_en_US);
           } else if (scope && !evaluate::UnwrapProcedureRef(*pointerExpr)) {
             if (auto whyNot{WhyNotDefinable(
@@ -1440,7 +1447,8 @@ static void CheckAssociated(evaluate::ActualArguments &arguments,
                         DefinabilityFlag::DoNotNoteDefinition},
                     *pointerExpr)}) {
               if (whyNot->IsFatal()) {
-                if (auto *msg{messages.Say(pointerArg->sourceLocation(),
+                if (auto *msg{messages.Say(common::UsageWarning::Portability,
+                        pointerArg->sourceLocation(),
                         "POINTER= argument of ASSOCIATED() is required by some other compilers to be a valid left-hand side of a pointer assignment statement"_port_en_US)}) {
                   msg->Attach(std::move(
                       whyNot->set_severity(parser::Severity::Because)));
@@ -1474,23 +1482,27 @@ static void CheckAssociated(evaluate::ActualArguments &arguments,
                       CheckProcCompatibility(isCall, pointerProc, &*targetProc,
                           specificIntrinsic, whyNot, warning,
                           /*ignoreImplicitVsExplicit=*/false)};
+                  std::optional<common::UsageWarning> whichWarning;
                   if (!msg && warning &&
                       semanticsContext.ShouldWarn(
                           common::UsageWarning::ProcDummyArgShapes)) {
+                    whichWarning = common::UsageWarning::ProcDummyArgShapes;
                     msg =
                         "Procedures '%s' and '%s' may not be completely compatible: %s"_warn_en_US;
                     whyNot = std::move(*warning);
-                  } else if (msg &&
-                      msg->severity() != parser::Severity::Error &&
-                      !semanticsContext.ShouldWarn(
+                  } else if (msg && !msg->IsFatal() &&
+                      semanticsContext.ShouldWarn(
                           common::UsageWarning::ProcPointerCompatibility)) {
-                    msg.reset();
+                    whichWarning =
+                        common::UsageWarning::ProcPointerCompatibility;
                   }
-                  if (msg) {
-                    msg->set_severity(parser::Severity::Warning);
-                    messages.Say(std::move(*msg),
-                        "pointer '" + pointerExpr->AsFortran() + "'",
-                        targetExpr->AsFortran(), whyNot);
+                  if (msg && (msg->IsFatal() || whichWarning)) {
+                    if (auto *said{messages.Say(std::move(*msg),
+                            "pointer '" + pointerExpr->AsFortran() + "'",
+                            targetExpr->AsFortran(), whyNot)};
+                        said && whichWarning) {
+                      said->set_usageWarning(*whichWarning);
+                    }
                   }
                 }
               } else if (!IsNullProcedurePointer(*targetExpr)) {
@@ -1819,6 +1831,7 @@ static void CheckTransferOperandType(SemanticsContext &context,
   if (type.IsPolymorphic() &&
       context.ShouldWarn(common::UsageWarning::PolymorphicTransferArg)) {
     context.foldingContext().messages().Say(
+        common::UsageWarning::PolymorphicTransferArg,
         "%s of TRANSFER is polymorphic"_warn_en_US, which);
   } else if (!type.IsUnlimitedPolymorphic() &&
       type.category() == TypeCategory::Derived &&
@@ -1827,6 +1840,7 @@ static void CheckTransferOperandType(SemanticsContext &context,
     if (auto bad{std::find_if(directs.begin(), directs.end(), IsDescriptor)};
         bad != directs.end()) {
       evaluate::SayWithDeclaration(context.foldingContext().messages(), *bad,
+          common::UsageWarning::PointerComponentTransferArg,
           "%s of TRANSFER contains allocatable or pointer component %s"_warn_en_US,
           which, bad.BuildResultDesignatorName());
     }
@@ -1856,7 +1870,7 @@ static void CheckTransfer(evaluate::ActualArguments &arguments,
                   "Element size of MOLD= array may not be zero when SOURCE= is not empty"_err_en_US);
             }
           } else if (context.ShouldWarn(common::UsageWarning::VoidMold)) {
-            messages.Say(
+            messages.Say(common::UsageWarning::VoidMold,
                 "Element size of MOLD= array may not be zero unless SOURCE= is empty"_warn_en_US);
           }
         }
@@ -1872,7 +1886,7 @@ static void CheckTransfer(evaluate::ActualArguments &arguments,
         } else if (context.ShouldWarn(
                        common::UsageWarning::TransferSizePresence) &&
             IsAllocatableOrObjectPointer(whole)) {
-          messages.Say(
+          messages.Say(common::UsageWarning::TransferSizePresence,
               "SIZE= argument that is allocatable or pointer must be present at execution; parenthesize to silence this warning"_warn_en_US);
         }
       }
@@ -2042,6 +2056,22 @@ bool CheckPPCIntrinsic(const Symbol &generic, const Symbol &specific,
   return false;
 }
 
+bool CheckWindowsIntrinsic(
+    const Symbol &intrinsic, evaluate::FoldingContext &foldingContext) {
+  parser::ContextualMessages &messages{foldingContext.messages()};
+  // TODO: there are other intrinsics that are unsupported on Windows that
+  // should be added here.
+  if (intrinsic.name() == "getuid") {
+    messages.Say(
+        "User IDs do not exist on Windows. This function will always return 1"_warn_en_US);
+  }
+  if (intrinsic.name() == "getgid") {
+    messages.Say(
+        "Group IDs do not exist on Windows. This function will always return 1"_warn_en_US);
+  }
+  return true;
+}
+
 bool CheckArguments(const characteristics::Procedure &proc,
     evaluate::ActualArguments &actuals, SemanticsContext &context,
     const Scope &scope, bool treatingExternalAsImplicit,
@@ -2078,6 +2108,7 @@ bool CheckArguments(const characteristics::Procedure &proc,
         if (context.ShouldWarn(
                 common::UsageWarning::KnownBadImplicitInterface)) {
           if (auto *msg{messages.Say(
+                  common::UsageWarning::KnownBadImplicitInterface,
                   "If the procedure's interface were explicit, this reference would be in error"_warn_en_US)}) {
             buffer.AttachTo(*msg, parser::Severity::Because);
           }
diff --git a/flang/lib/Semantics/check-call.h b/flang/lib/Semantics/check-call.h
index 8553f3a31efb5..46bc61a601bd3 100644
--- a/flang/lib/Semantics/check-call.h
+++ b/flang/lib/Semantics/check-call.h
@@ -41,6 +41,8 @@ bool CheckArguments(const evaluate::characteristics::Procedure &,
 bool CheckPPCIntrinsic(const Symbol &generic, const Symbol &specific,
     const evaluate::ActualArguments &actuals,
     evaluate::FoldingContext &context);
+bool CheckWindowsIntrinsic(
+    const Symbol &intrinsic, evaluate::FoldingContext &context);
 bool CheckArgumentIsConstantExprInRange(
     const evaluate::ActualArguments &actuals, int index, int lowerBound,
     int upperBound, parser::ContextualMessages &messages);
diff --git a/flang/lib/Semantics/check-case.cpp b/flang/lib/Semantics/check-case.cpp
index d296460127e12..caa8f8b6e70be 100644
--- a/flang/lib/Semantics/check-case.cpp
+++ b/flang/lib/Semantics/check-case.cpp
@@ -49,10 +49,8 @@ template <typename T> class CaseValues {
               for (const auto &range : ranges) {
                 auto pair{ComputeBounds(range)};
                 if (pair.first && pair.second && *pair.first > *pair.second) {
-                  if (context_.ShouldWarn(common::UsageWarning::EmptyCase)) {
-                    context_.Say(stmt.source,
-                        "CASE has lower bound greater than upper bound"_warn_en_US);
-                  }
+                  context_.Warn(common::UsageWarning::EmptyCase, stmt.source,
+                      "CASE has lower bound greater than upper bound"_warn_en_US);
                 } else {
                   if constexpr (T::category == TypeCategory::Logical) { // C1148
                     if ((pair.first || pair.second) &&
@@ -95,11 +93,9 @@ template <typename T> class CaseValues {
               x->v = converted;
               return value;
             } else {
-              if (context_.ShouldWarn(common::UsageWarning::CaseOverflow)) {
-                context_.Say(expr.source,
-                    "CASE value (%s) overflows type (%s) of SELECT CASE expression"_warn_en_US,
-                    folded.AsFortran(), caseExprType_.AsFortran());
-              }
+              context_.Warn(common::UsageWarning::CaseOverflow, expr.source,
+                  "CASE value (%s) overflows type (%s) of SELECT CASE expression"_warn_en_US,
+                  folded.AsFortran(), caseExprType_.AsFortran());
               hasErrors_ = true;
               return std::nullopt;
             }
diff --git a/flang/lib/Semantics/check-cuda.cpp b/flang/lib/Semantics/check-cuda.cpp
index d8bd435bf09f3..eaf1d52a9fc1a 100644
--- a/flang/lib/Semantics/check-cuda.cpp
+++ b/flang/lib/Semantics/check-cuda.cpp
@@ -296,10 +296,8 @@ template <bool IsCUFKernelDo> class DeviceContextChecker {
     return false;
   }
   void WarnOnIoStmt(const parser::CharBlock &source) {
-    if (context_.ShouldWarn(common::UsageWarning::CUDAUsage)) {
-      context_.Say(
-          source, "I/O statement might not be supported on device"_warn_en_US);
-    }
+    context_.Warn(common::UsageWarning::CUDAUsage, source,
+        "I/O statement might not be supported on device"_warn_en_US);
   }
   template <typename A>
   void WarnIfNotInternal(const A &stmt, const parser::CharBlock &source) {
diff --git a/flang/lib/Semantics/check-data.cpp b/flang/lib/Semantics/check-data.cpp
index 9d3e8c5a4ea8f..d6f1351c12d3c 100644
--- a/flang/lib/Semantics/check-data.cpp
+++ b/flang/lib/Semantics/check-data.cpp
@@ -89,9 +89,8 @@ class DataVarChecker : public evaluate::AllTraverse<DataVarChecker, true> {
             "Procedure pointer '%s' may not appear in a DATA statement"_err_en_US,
             symbol.name());
         return false;
-      } else if (context_.ShouldWarn(
-                     common::LanguageFeature::DataStmtExtensions)) {
-        context_.Say(source_,
+      } else {
+        context_.Warn(common::LanguageFeature::DataStmtExtensions, source_,
             "Procedure pointer '%s' in a DATA statement is not standard"_port_en_US,
             symbol.name());
       }
@@ -102,9 +101,8 @@ class DataVarChecker : public evaluate::AllTraverse<DataVarChecker, true> {
             "Blank COMMON object '%s' may not appear in a DATA statement"_err_en_US,
             symbol.name());
         return false;
-      } else if (context_.ShouldWarn(
-                     common::LanguageFeature::DataStmtExtensions)) {
-        context_.Say(source_,
+      } else {
+        context_.Warn(common::LanguageFeature::DataStmtExtensions, source_,
             "Blank COMMON object '%s' in a DATA statement is not standard"_port_en_US,
             symbol.name());
       }
diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index dfd49db74eea7..7778561fb5bd3 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -124,19 +124,23 @@ class CheckHelper {
     return FindModuleFileContaining(context_.FindScope(messages_.at())) !=
         nullptr;
   }
-  template <typename... A> parser::Message *WarnIfNotInModuleFile(A &&...x) {
-    if (InModuleFile()) {
+  template <typename FeatureOrUsageWarning, typename... A>
+  parser::Message *Warn(FeatureOrUsageWarning warning, A &&...x) {
+    if (!context_.ShouldWarn(warning) || InModuleFile()) {
       return nullptr;
     } else {
-      return messages_.Say(std::forward<A>(x)...);
+      return messages_.Say(warning, std::forward<A>(x)...);
     }
   }
-  template <typename... A>
-  parser::Message *WarnIfNotInModuleFile(parser::CharBlock source, A &&...x) {
-    if (FindModuleFileContaining(context_.FindScope(source))) {
+  template <typename FeatureOrUsageWarning, typename... A>
+  parser::Message *Warn(
+      FeatureOrUsageWarning warning, parser::CharBlock source, A &&...x) {
+    if (!context_.ShouldWarn(warning) ||
+        FindModuleFileContaining(context_.FindScope(source))) {
       return nullptr;
+    } else {
+      return messages_.Say(warning, source, std::forward<A>(x)...);
     }
-    return messages_.Say(source, std::forward<A>(x)...);
   }
   bool IsResultOkToDiffer(const FunctionResult &);
   void CheckGlobalName(const Symbol &);
@@ -261,11 +265,9 @@ void CheckHelper::Check(const Symbol &symbol) {
   }
   if (symbol.name().size() > common::maxNameLen &&
       &symbol == &symbol.GetUltimate()) {
-    if (context_.ShouldWarn(common::LanguageFeature::LongNames)) {
-      WarnIfNotInModuleFile(symbol.name(),
-          "%s has length %d, which is greater than the maximum name length %d"_port_en_US,
-          symbol.name(), symbol.name().size(), common::maxNameLen);
-    }
+    Warn(common::LanguageFeature::LongNames, symbol.name(),
+        "%s has length %d, which is greater than the maximum name length %d"_port_en_US,
+        symbol.name(), symbol.name().size(), common::maxNameLen);
   }
   if (context_.HasError(symbol)) {
     return;
@@ -448,10 +450,8 @@ void CheckHelper::Check(const Symbol &symbol) {
       }
     }
     if (IsProcedurePointer(symbol) && IsDummy(symbol)) {
-      if (context_.ShouldWarn(common::UsageWarning::Portability)) {
-        messages_.Say(
-            "A dummy procedure pointer should not have assumed-length CHARACTER(*) result type"_port_en_US);
-      }
+      Warn(common::UsageWarning::Portability,
+          "A dummy procedure pointer should not have assumed-length CHARACTER(*) result type"_port_en_US);
       // The non-dummy case is a hard error that's caught elsewhere.
     }
   }
@@ -481,12 +481,9 @@ void CheckHelper::Check(const Symbol &symbol) {
           symbol.name());
     } else if (symbol.owner().kind() == Scope::Kind::MainProgram) {
       if (context_.IsEnabled(common::LanguageFeature::AutomaticInMainProgram)) {
-        if (context_.ShouldWarn(
-                common::LanguageFeature::AutomaticInMainProgram)) {
-          messages_.Say(
-              "Automatic data object '%s' should not appear in the specification part of a main program"_port_en_US,
-              symbol.name());
-        }
+        Warn(common::LanguageFeature::AutomaticInMainProgram,
+            "Automatic data object '%s' should not appear in the specification part of a main program"_port_en_US,
+            symbol.name());
       } else {
         messages_.Say(
             "Automatic data object '%s' may not appear in the specification part of a main program"_err_en_US,
@@ -602,10 +599,9 @@ void CheckHelper::CheckValue(
     messages_.Say(
         "VALUE attribute may not apply to an assumed-rank array"_err_en_US);
   }
-  if (context_.ShouldWarn(common::UsageWarning::Portability) &&
-      IsAssumedLengthCharacter(symbol)) {
+  if (IsAssumedLengthCharacter(symbol)) {
     // F'2008 feature not widely implemented
-    messages_.Say(
+    Warn(common::UsageWarning::Portability,
         "VALUE attribute on assumed-length CHARACTER may not be portable"_port_en_US);
   }
 }
@@ -779,10 +775,8 @@ void CheckHelper::CheckObjectEntity(
       }
       if (ok && InFunction() && !InModuleFile() && !InElemental()) {
         if (context_.IsEnabled(common::LanguageFeature::RelaxedPureDummy)) {
-          if (context_.ShouldWarn(common::LanguageFeature::RelaxedPureDummy)) {
-            messages_.Say(
-                "non-POINTER dummy argument of pure function should be INTENT(IN) or VALUE"_warn_en_US);
-          }
+          Warn(common::LanguageFeature::RelaxedPureDummy,
+              "non-POINTER dummy argument of pure function should be INTENT(IN) or VALUE"_warn_en_US);
         } else {
           messages_.Say(
               "non-POINTER dummy argument of pure function must be INTENT(IN) or VALUE"_err_en_US);
@@ -806,25 +800,19 @@ void CheckHelper::CheckObjectEntity(
       if (IsPassedViaDescriptor(symbol)) {
         if (IsAllocatableOrObjectPointer(&symbol)) {
           if (inExplicitExternalInterface) {
-            if (context_.ShouldWarn(common::UsageWarning::IgnoreTKRUsage)) {
-              WarnIfNotInModuleFile(
-                  "!DIR$ IGNORE_TKR should not apply to an allocatable or pointer"_warn_en_US);
-            }
+            Warn(common::UsageWarning::IgnoreTKRUsage,
+                "!DIR$ IGNORE_TKR should not apply to an allocatable or pointer"_warn_en_US);
           } else {
             messages_.Say(
                 "!DIR$ IGNORE_TKR may not apply to an allocatable or pointer"_err_en_US);
           }
         } else if (ignoreTKR.test(common::IgnoreTKR::Rank)) {
           if (ignoreTKR.count() == 1 && evaluate::IsAssumedRank(symbol)) {
-            if (context_.ShouldWarn(common::UsageWarning::IgnoreTKRUsage)) {
-              WarnIfNotInModuleFile(
-                  "!DIR$ IGNORE_TKR(R) is not meaningful for an assumed-rank array"_warn_en_US);
-            }
+            Warn(common::UsageWarning::IgnoreTKRUsage,
+                "!DIR$ IGNORE_TKR(R) is not meaningful for an assumed-rank array"_warn_en_US);
           } else if (inExplicitExternalInterface) {
-            if (context_.ShouldWarn(common::UsageWarning::IgnoreTKRUsage)) {
-              WarnIfNotInModuleFile(
-                  "!DIR$ IGNORE_TKR(R) should not apply to a dummy argument passed via descriptor"_warn_en_US);
-            }
+            Warn(common::UsageWarning::IgnoreTKRUsage,
+                "!DIR$ IGNORE_TKR(R) should not apply to a dummy argument passed via descriptor"_warn_en_US);
           } else {
             messages_.Say(
                 "!DIR$ IGNORE_TKR(R) may not apply to a dummy argument passed via descriptor"_err_en_US);
@@ -884,10 +872,8 @@ void CheckHelper::CheckObjectEntity(
     } else if (IsFunctionResult(symbol)) {
       messages_.Say("A function result must not be initialized"_err_en_US);
     } else if (IsInBlankCommon(symbol)) {
-      if (context_.ShouldWarn(common::LanguageFeature::InitBlankCommon)) {
-        WarnIfNotInModuleFile(
-            "A variable in blank COMMON should not be initialized"_port_en_US);
-      }
+      Warn(common::LanguageFeature::InitBlankCommon,
+          "A variable in blank COMMON should not be initialized"_port_en_US);
     }
   }
   if (symbol.owner().kind() == Scope::Kind::BlockData) {
@@ -929,31 +915,25 @@ void CheckHelper::CheckObjectEntity(
   bool inDeviceSubprogram{IsCUDADeviceContext(&symbol.owner())};
   if (inDeviceSubprogram) {
     if (IsSaved(symbol)) {
-      if (context_.ShouldWarn(common::UsageWarning::CUDAUsage)) {
-        WarnIfNotInModuleFile(
-            "'%s' should not have the SAVE attribute or initialization in a device subprogram"_warn_en_US,
-            symbol.name());
-      }
+      Warn(common::UsageWarning::CUDAUsage,
+          "'%s' should not have the SAVE attribute or initialization in a device subprogram"_warn_en_US,
+          symbol.name());
     }
     if (IsPointer(symbol)) {
-      if (context_.ShouldWarn(common::UsageWarning::CUDAUsage)) {
-        WarnIfNotInModuleFile(
-            "Pointer '%s' may not be associated in a device subprogram"_warn_en_US,
-            symbol.name());
-      }
+      Warn(common::UsageWarning::CUDAUsage,
+          "Pointer '%s' may not be associated in a device subprogram"_warn_en_US,
+          symbol.name());
     }
     if (details.isDummy() &&
         details.cudaDataAttr().value_or(common::CUDADataAttr::Device) !=
             common::CUDADataAttr::Device &&
         details.cudaDataAttr().value_or(common::CUDADataAttr::Device) !=
             common::CUDADataAttr::Managed) {
-      if (context_.ShouldWarn(common::UsageWarning::CUDAUsage)) {
-        WarnIfNotInModuleFile(
-            "Dummy argument '%s' may not have ATTRIBUTES(%s) in a device subprogram"_warn_en_US,
-            symbol.name(),
-            parser::ToUpperCaseLetters(
-                common::EnumToString(*details.cudaDataAttr())));
-      }
+      Warn(common::UsageWarning::CUDAUsage,
+          "Dummy argument '%s' may not have ATTRIBUTES(%s) in a device subprogram"_warn_en_US,
+          symbol.name(),
+          parser::ToUpperCaseLetters(
+              common::EnumToString(*details.cudaDataAttr())));
     }
   }
   if (details.cudaDataAttr()) {
@@ -1003,23 +983,17 @@ void CheckHelper::CheckObjectEntity(
       break;
     case common::CUDADataAttr::Pinned:
       if (inDeviceSubprogram) {
-        if (context_.ShouldWarn(common::UsageWarning::CUDAUsage)) {
-          WarnIfNotInModuleFile(
-              "Object '%s' with ATTRIBUTES(PINNED) may not be declared in a device subprogram"_warn_en_US,
-              symbol.name());
-        }
+        Warn(common::UsageWarning::CUDAUsage,
+            "Object '%s' with ATTRIBUTES(PINNED) may not be declared in a device subprogram"_warn_en_US,
+            symbol.name());
       } else if (IsPointer(symbol)) {
-        if (context_.ShouldWarn(common::UsageWarning::CUDAUsage)) {
-          WarnIfNotInModuleFile(
-              "Object '%s' with ATTRIBUTES(PINNED) may not be a pointer"_warn_en_US,
-              symbol.name());
-        }
+        Warn(common::UsageWarning::CUDAUsage,
+            "Object '%s' with ATTRIBUTES(PINNED) may not be a pointer"_warn_en_US,
+            symbol.name());
       } else if (!IsAllocatable(symbol)) {
-        if (context_.ShouldWarn(common::UsageWarning::CUDAUsage)) {
-          WarnIfNotInModuleFile(
-              "Object '%s' with ATTRIBUTES(PINNED) should also be allocatable"_warn_en_US,
-              symbol.name());
-        }
+        Warn(common::UsageWarning::CUDAUsage,
+            "Object '%s' with ATTRIBUTES(PINNED) should also be allocatable"_warn_en_US,
+            symbol.name());
       }
       break;
     case common::CUDADataAttr::Shared:
@@ -1281,10 +1255,8 @@ void CheckHelper::CheckProcEntity(
       // because it is explicitly legal to *pass* the specific intrinsic
       // function SIN as an actual argument.
       if (interface->attrs().test(Attr::INTRINSIC)) {
-        if (context_.ShouldWarn(common::UsageWarning::Portability)) {
-          messages_.Say(
-              "A dummy procedure should not have an ELEMENTAL intrinsic as its interface"_port_en_US);
-        }
+        Warn(common::UsageWarning::Portability,
+            "A dummy procedure should not have an ELEMENTAL intrinsic as its interface"_port_en_US);
       } else {
         messages_.Say("A dummy procedure may not be ELEMENTAL"_err_en_US);
       }
@@ -1302,11 +1274,9 @@ void CheckHelper::CheckProcEntity(
               "to procedure pointer '%s'"_err_en_US,
               interface->name(), symbol.name());
         } else if (IsElementalProcedure(*interface)) {
-          if (context_.ShouldWarn(common::UsageWarning::Portability)) {
-            messages_.Say(
-                "Procedure pointer '%s' should not have an ELEMENTAL intrinsic as its interface"_port_en_US,
-                symbol.name()); // C1517
-          }
+          Warn(common::UsageWarning::Portability,
+              "Procedure pointer '%s' should not have an ELEMENTAL intrinsic as its interface"_port_en_US,
+              symbol.name()); // C1517
         }
       } else if (IsElementalProcedure(*interface)) {
         messages_.Say("Procedure pointer '%s' may not be ELEMENTAL"_err_en_US,
@@ -1470,13 +1440,11 @@ void CheckHelper::CheckSubprogram(
       // 15.6.4 p2 weird requirement
       if (const Symbol *
           host{symbol.owner().parent().FindSymbol(symbol.name())}) {
-        if (context_.ShouldWarn(
-                common::LanguageFeature::StatementFunctionExtensions)) {
-          evaluate::AttachDeclaration(
-              messages_.Say(symbol.name(),
-                  "An implicitly typed statement function should not appear when the same symbol is available in its host scope"_port_en_US),
-              *host);
-        }
+        evaluate::AttachDeclaration(
+            Warn(common::LanguageFeature::StatementFunctionExtensions,
+                symbol.name(),
+                "An implicitly typed statement function should not appear when the same symbol is available in its host scope"_port_en_US),
+            *host);
       }
     }
     if (GetProgramUnitOrBlockConstructContaining(symbol).kind() ==
@@ -1566,9 +1534,8 @@ void CheckHelper::CheckExternal(const Symbol &symbol) {
         parser::Message *msg{nullptr};
         if (!IsProcedure(*global)) {
           if ((symbol.flags().test(Symbol::Flag::Function) ||
-                  symbol.flags().test(Symbol::Flag::Subroutine)) &&
-              context_.ShouldWarn(common::UsageWarning::ExternalNameConflict)) {
-            msg = WarnIfNotInModuleFile(
+                  symbol.flags().test(Symbol::Flag::Subroutine))) {
+            msg = Warn(common::UsageWarning::ExternalNameConflict,
                 "The global entity '%s' corresponding to the local procedure '%s' is not a callable subprogram"_warn_en_US,
                 global->name(), symbol.name());
           }
@@ -1577,20 +1544,16 @@ void CheckHelper::CheckExternal(const Symbol &symbol) {
             if (chars->HasExplicitInterface()) {
               std::string whyNot;
               if (!chars->IsCompatibleWith(*globalChars,
-                      /*ignoreImplicitVsExplicit=*/false, &whyNot) &&
-                  context_.ShouldWarn(
-                      common::UsageWarning::ExternalInterfaceMismatch)) {
-                msg = WarnIfNotInModuleFile(
+                      /*ignoreImplicitVsExplicit=*/false, &whyNot)) {
+                msg = Warn(common::UsageWarning::ExternalInterfaceMismatch,
                     "The global subprogram '%s' is not compatible with its local procedure declaration (%s)"_warn_en_US,
                     global->name(), whyNot);
               }
-            } else if (!globalChars->CanBeCalledViaImplicitInterface() &&
-                context_.ShouldWarn(
-                    common::UsageWarning::ExternalInterfaceMismatch)) {
+            } else if (!globalChars->CanBeCalledViaImplicitInterface()) {
               // TODO: This should be a hard error if the procedure has
               // actually been called (as opposed to just being used as a
               // procedure pointer target or passed as an actual argument).
-              msg = WarnIfNotInModuleFile(
+              msg = Warn(common::UsageWarning::ExternalInterfaceMismatch,
                   "The global subprogram '%s' should not be referenced via the implicit interface '%s'"_warn_en_US,
                   global->name(), symbol.name());
             }
@@ -1611,10 +1574,8 @@ void CheckHelper::CheckExternal(const Symbol &symbol) {
         if (auto previousChars{Characterize(previous)}) {
           std::string whyNot;
           if (!chars->IsCompatibleWith(*previousChars,
-                  /*ignoreImplicitVsExplicit=*/false, &whyNot) &&
-              context_.ShouldWarn(
-                  common::UsageWarning::ExternalInterfaceMismatch)) {
-            if (auto *msg{WarnIfNotInModuleFile(
+                  /*ignoreImplicitVsExplicit=*/false, &whyNot)) {
+            if (auto *msg{Warn(common::UsageWarning::ExternalInterfaceMismatch,
                     "The external interface '%s' is not compatible with an earlier definition (%s)"_warn_en_US,
                     symbol.name(), whyNot)}) {
               evaluate::AttachDeclaration(msg, previous);
@@ -1891,22 +1852,20 @@ void CheckHelper::CheckSpecifics(
       auto intrinsic{context_.intrinsics().IsSpecificIntrinsicFunction(
           specific.name().ToString())};
       if (intrinsic && !intrinsic->isRestrictedSpecific) {
-        if (context_.ShouldWarn(common::LanguageFeature::IntrinsicAsSpecific)) {
-          if (auto *msg{messages_.Say(specific.name(),
-                  "Specific procedure '%s' of generic interface '%s' should not be INTRINSIC"_port_en_US,
-                  specific.name(), generic.name())}) {
-            msg->Attach(
-                generic.name(), "Definition of '%s'"_en_US, generic.name());
-          }
+        if (auto *msg{Warn(common::LanguageFeature::IntrinsicAsSpecific,
+                specific.name(),
+                "Specific procedure '%s' of generic interface '%s' should not be INTRINSIC"_port_en_US,
+                specific.name(), generic.name())}) {
+          msg->Attach(
+              generic.name(), "Definition of '%s'"_en_US, generic.name());
         }
       } else {
-        if (context_.ShouldWarn(common::LanguageFeature::IntrinsicAsSpecific)) {
-          if (auto *msg{messages_.Say(specific.name(),
-                  "Procedure '%s' of generic interface '%s' is INTRINSIC but not an unrestricted specific intrinsic function"_port_en_US,
-                  specific.name(), generic.name())}) {
-            msg->Attach(
-                generic.name(), "Definition of '%s'"_en_US, generic.name());
-          }
+        if (auto *msg{Warn(common::LanguageFeature::IntrinsicAsSpecific,
+                specific.name(),
+                "Procedure '%s' of generic interface '%s' is INTRINSIC but not an unrestricted specific intrinsic function"_port_en_US,
+                specific.name(), generic.name())}) {
+          msg->Attach(
+              generic.name(), "Definition of '%s'"_en_US, generic.name());
         }
         continue;
       }
@@ -2041,23 +2000,27 @@ bool CheckHelper::CheckDefinedOperator(SourceName opName, GenericKind kind,
     msg = "%s function '%s' may not have assumed-length CHARACTER(*)"
           " result"_err_en_US;
   } else if (auto m{CheckNumberOfArgs(kind, proc.dummyArguments.size())}) {
-    msg = std::move(m);
+    if (m->IsFatal()) {
+      msg = *m;
+    } else {
+      evaluate::AttachDeclaration(
+          Warn(common::UsageWarning::DefinedOperatorArgs, specific.name(),
+              std::move(*m), MakeOpName(opName), specific.name()),
+          specific);
+      return true;
+    }
   } else if (!checkDefinedOperatorArgs(opName, specific, proc)) {
     return false; // error was reported
   } else if (ConflictsWithIntrinsicOperator(kind, proc)) {
     msg = "%s function '%s' conflicts with intrinsic operator"_err_en_US;
-  } else {
-    return true; // OK
   }
-  bool isFatal{msg->IsFatal()};
-  if (isFatal || !FindModuleFileContaining(specific.owner())) {
+  if (msg) {
     SayWithDeclaration(
         specific, std::move(*msg), MakeOpName(opName), specific.name());
-  }
-  if (isFatal) {
     context_.SetError(specific);
+    return false;
   }
-  return !isFatal;
+  return true;
 }
 
 // If the number of arguments is wrong for this intrinsic operator, return
@@ -2114,32 +2077,29 @@ bool CheckHelper::CheckDefinedOperatorArg(const SourceName &opName,
   auto &arg{proc.dummyArguments.at(pos)};
   std::optional<parser::MessageFixedText> msg;
   if (arg.IsOptional()) {
-    msg = "In %s function '%s', dummy argument '%s' may not be"
-          " OPTIONAL"_err_en_US;
+    msg =
+        "In %s function '%s', dummy argument '%s' may not be OPTIONAL"_err_en_US;
   } else if (const auto *dataObject{std::get_if<DummyDataObject>(&arg.u)};
              dataObject == nullptr) {
-    msg = "In %s function '%s', dummy argument '%s' must be a"
-          " data object"_err_en_US;
+    msg =
+        "In %s function '%s', dummy argument '%s' must be a data object"_err_en_US;
   } else if (dataObject->intent == common::Intent::Out) {
     msg =
         "In %s function '%s', dummy argument '%s' may not be INTENT(OUT)"_err_en_US;
   } else if (dataObject->intent != common::Intent::In &&
       !dataObject->attrs.test(DummyDataObject::Attr::Value)) {
-    if (context_.ShouldWarn(common::UsageWarning::DefinedOperatorArgs)) {
-      msg =
-          "In %s function '%s', dummy argument '%s' should have INTENT(IN) or VALUE attribute"_warn_en_US;
-    }
+    evaluate::AttachDeclaration(
+        Warn(common::UsageWarning::DefinedOperatorArgs,
+            "In %s function '%s', dummy argument '%s' should have INTENT(IN) or VALUE attribute"_warn_en_US,
+            parser::ToUpperCaseLetters(opName.ToString()), symbol.name(),
+            arg.name),
+        symbol);
+    return true;
   }
   if (msg) {
-    bool isFatal{msg->IsFatal()};
-    if (isFatal || !FindModuleFileContaining(symbol.owner())) {
-      SayWithDeclaration(symbol, std::move(*msg),
-          parser::ToUpperCaseLetters(opName.ToString()), symbol.name(),
-          arg.name);
-    }
-    if (isFatal) {
-      return false;
-    }
+    SayWithDeclaration(symbol, std::move(*msg),
+        parser::ToUpperCaseLetters(opName.ToString()), symbol.name(), arg.name);
+    return false;
   }
   return true;
 }
@@ -2190,10 +2150,8 @@ bool CheckHelper::CheckDefinedAssignmentArg(
               " may not have INTENT(IN)"_err_en_US;
       } else if (dataObject->intent != common::Intent::Out &&
           dataObject->intent != common::Intent::InOut) {
-        if (context_.ShouldWarn(common::UsageWarning::DefinedOperatorArgs)) {
-          msg =
-              "In defined assignment subroutine '%s', first dummy argument '%s' should have INTENT(OUT) or INTENT(INOUT)"_warn_en_US;
-        }
+        msg =
+            "In defined assignment subroutine '%s', first dummy argument '%s' should have INTENT(OUT) or INTENT(INOUT)"_warn_en_US;
       }
     } else if (pos == 1) {
       if (dataObject->intent == common::Intent::Out) {
@@ -2201,10 +2159,8 @@ bool CheckHelper::CheckDefinedAssignmentArg(
               " argument '%s' may not have INTENT(OUT)"_err_en_US;
       } else if (dataObject->intent != common::Intent::In &&
           !dataObject->attrs.test(DummyDataObject::Attr::Value)) {
-        if (context_.ShouldWarn(common::UsageWarning::DefinedOperatorArgs)) {
-          msg =
-              "In defined assignment subroutine '%s', second dummy argument '%s' should have INTENT(IN) or VALUE attribute"_warn_en_US;
-        }
+        msg =
+            "In defined assignment subroutine '%s', second dummy argument '%s' should have INTENT(IN) or VALUE attribute"_warn_en_US;
       } else if (dataObject->attrs.test(DummyDataObject::Attr::Pointer)) {
         msg =
             "In defined assignment subroutine '%s', second dummy argument '%s' must not be a pointer"_err_en_US;
@@ -2220,13 +2176,15 @@ bool CheckHelper::CheckDefinedAssignmentArg(
           " must be a data object"_err_en_US;
   }
   if (msg) {
-    bool isFatal{msg->IsFatal()};
-    if (isFatal || !FindModuleFileContaining(symbol.owner())) {
+    if (msg->IsFatal()) {
       SayWithDeclaration(symbol, std::move(*msg), symbol.name(), arg.name);
-    }
-    if (isFatal) {
       context_.SetError(symbol);
       return false;
+    } else {
+      evaluate::AttachDeclaration(
+          Warn(common::UsageWarning::DefinedOperatorArgs, std::move(*msg),
+              symbol.name(), arg.name),
+          symbol);
     }
   }
   return true;
@@ -2258,13 +2216,12 @@ void CheckHelper::WarnMissingFinal(const Symbol &symbol) {
   while (const auto *derivedDetails{
       derivedSym ? derivedSym->detailsIf<DerivedTypeDetails>() : nullptr}) {
     if (!derivedDetails->finals().empty() &&
-        !derivedDetails->GetFinalForRank(rank) &&
-        context_.ShouldWarn(common::UsageWarning::Final)) {
+        !derivedDetails->GetFinalForRank(rank)) {
       if (auto *msg{derivedSym == initialDerivedSym
-                  ? WarnIfNotInModuleFile(symbol.name(),
+                  ? Warn(common::UsageWarning::Final, symbol.name(),
                         "'%s' of derived type '%s' does not have a FINAL subroutine for its rank (%d)"_warn_en_US,
                         symbol.name(), derivedSym->name(), rank)
-                  : WarnIfNotInModuleFile(symbol.name(),
+                  : Warn(common::UsageWarning::Final, symbol.name(),
                         "'%s' of derived type '%s' extended from '%s' does not have a FINAL subroutine for its rank (%d)"_warn_en_US,
                         symbol.name(), initialDerivedSym->name(),
                         derivedSym->name(), rank)}) {
@@ -2316,16 +2273,17 @@ void CheckHelper::CheckContiguous(const Symbol &symbol) {
   if (evaluate::IsVariable(symbol) &&
       ((IsPointer(symbol) && symbol.Rank() > 0) || IsAssumedShape(symbol) ||
           evaluate::IsAssumedRank(symbol))) {
-  } else if (!context_.IsEnabled(
-                 common::LanguageFeature::RedundantContiguous) ||
-      context_.ShouldWarn(common::LanguageFeature::RedundantContiguous)) {
+  } else {
     parser::MessageFixedText msg{symbol.owner().IsDerivedType()
             ? "CONTIGUOUS component '%s' should be an array with the POINTER attribute"_port_en_US
             : "CONTIGUOUS entity '%s' should be an array pointer, assumed-shape, or assumed-rank"_port_en_US};
     if (!context_.IsEnabled(common::LanguageFeature::RedundantContiguous)) {
       msg.set_severity(parser::Severity::Error);
+      messages_.Say(std::move(msg), symbol.name());
+    } else {
+      Warn(common::LanguageFeature::RedundantContiguous, std::move(msg),
+          symbol.name());
     }
-    messages_.Say(std::move(msg), symbol.name());
   }
 }
 
@@ -2615,26 +2573,24 @@ void CheckHelper::Check(const Scope &scope) {
       auto iter{scope.find(*name)};
       if (iter != scope.end()) {
         const char *kind{nullptr};
-        if (context_.ShouldWarn(common::LanguageFeature::BenignNameClash)) {
-          switch (scope.kind()) {
-          case Scope::Kind::Module:
-            kind = scope.symbol()->get<ModuleDetails>().isSubmodule()
-                ? "submodule"
-                : "module";
-            break;
-          case Scope::Kind::MainProgram:
-            kind = "main program";
-            break;
-          case Scope::Kind::BlockData:
-            kind = "BLOCK DATA subprogram";
-            break;
-          default:;
-          }
-          if (kind) {
-            messages_.Say(iter->second->name(),
-                "Name '%s' declared in a %s should not have the same name as the %s"_port_en_US,
-                *name, kind, kind);
-          }
+        switch (scope.kind()) {
+        case Scope::Kind::Module:
+          kind = scope.symbol()->get<ModuleDetails>().isSubmodule()
+              ? "submodule"
+              : "module";
+          break;
+        case Scope::Kind::MainProgram:
+          kind = "main program";
+          break;
+        case Scope::Kind::BlockData:
+          kind = "BLOCK DATA subprogram";
+          break;
+        default:;
+        }
+        if (kind) {
+          Warn(common::LanguageFeature::BenignNameClash, iter->second->name(),
+              "Name '%s' declared in a %s should not have the same name as the %s"_port_en_US,
+              *name, kind, kind);
         }
       }
     }
@@ -2876,17 +2832,13 @@ void CheckHelper::CheckGlobalName(const Symbol &symbol) {
               !IsExternalProcedureDefinition(other))) {
         // both are procedures/BLOCK DATA, not both definitions
       } else if (symbol.has<ModuleDetails>()) {
-        if (context_.ShouldWarn(common::LanguageFeature::BenignNameClash)) {
-          messages_.Say(symbol.name(),
-              "Module '%s' conflicts with a global name"_port_en_US,
-              pair.first->first);
-        }
+        Warn(common::LanguageFeature::BenignNameClash, symbol.name(),
+            "Module '%s' conflicts with a global name"_port_en_US,
+            pair.first->first);
       } else if (other.has<ModuleDetails>()) {
-        if (context_.ShouldWarn(common::LanguageFeature::BenignNameClash)) {
-          messages_.Say(symbol.name(),
-              "Global name '%s' conflicts with a module"_port_en_US,
-              pair.first->first);
-        }
+        Warn(common::LanguageFeature::BenignNameClash, symbol.name(),
+            "Global name '%s' conflicts with a module"_port_en_US,
+            pair.first->first);
       } else if (auto *msg{messages_.Say(symbol.name(),
                      "Two entities have the same global name '%s'"_err_en_US,
                      pair.first->first)}) {
@@ -3012,13 +2964,13 @@ parser::Messages CheckHelper::WhyNotInteroperableDerivedType(
                         .value_or(false)) {
           if (type->category() == DeclTypeSpec::Logical) {
             if (context_.ShouldWarn(common::UsageWarning::LogicalVsCBool)) {
-              msgs.Say(component.name(),
+              msgs.Say(common::UsageWarning::LogicalVsCBool, component.name(),
                   "A LOGICAL component of an interoperable type should have the interoperable KIND=C_BOOL"_port_en_US);
             }
           } else if (type->category() == DeclTypeSpec::Character && dyType &&
               dyType->kind() == 1) {
             if (context_.ShouldWarn(common::UsageWarning::BindCCharLength)) {
-              msgs.Say(component.name(),
+              msgs.Say(common::UsageWarning::BindCCharLength, component.name(),
                   "A CHARACTER component of an interoperable type should have length 1"_port_en_US);
             }
           } else {
@@ -3036,7 +2988,7 @@ parser::Messages CheckHelper::WhyNotInteroperableDerivedType(
     }
     if (derived->componentNames().empty()) { // F'2023 C1805
       if (context_.ShouldWarn(common::LanguageFeature::EmptyBindCDerivedType)) {
-        msgs.Say(symbol.name(),
+        msgs.Say(common::LanguageFeature::EmptyBindCDerivedType, symbol.name(),
             "A derived type with the BIND attribute should not be empty"_warn_en_US);
       }
     }
@@ -3119,13 +3071,12 @@ parser::Messages CheckHelper::WhyNotInteroperableObject(const Symbol &symbol) {
       // when processing a module file, since the module file might have been
       // compiled with CUDA while the client is not.
     } else if (type->category() == DeclTypeSpec::Logical) {
-      if (context_.ShouldWarn(common::UsageWarning::LogicalVsCBool) &&
-          !InModuleFile()) {
+      if (context_.ShouldWarn(common::UsageWarning::LogicalVsCBool)) {
         if (IsDummy(symbol)) {
-          msgs.Say(symbol.name(),
+          msgs.Say(common::UsageWarning::LogicalVsCBool, symbol.name(),
               "A BIND(C) LOGICAL dummy argument should have the interoperable KIND=C_BOOL"_port_en_US);
         } else {
-          msgs.Say(symbol.name(),
+          msgs.Say(common::UsageWarning::LogicalVsCBool, symbol.name(),
               "A BIND(C) LOGICAL object should have the interoperable KIND=C_BOOL"_port_en_US);
         }
       }
diff --git a/flang/lib/Semantics/check-directive-structure.h b/flang/lib/Semantics/check-directive-structure.h
index b32c10b0f3330..a1aff52f3a684 100644
--- a/flang/lib/Semantics/check-directive-structure.h
+++ b/flang/lib/Semantics/check-directive-structure.h
@@ -463,12 +463,11 @@ void DirectiveStructureChecker<D, C, PC,
   }
   // No clause matched in the actual clauses list
   if (warnInsteadOfError) {
-    if (context_.ShouldWarn(common::UsageWarning::Portability)) {
-      context_.Say(GetContext().directiveSource,
-          "At least one of %s clause should appear on the %s directive"_port_en_US,
-          ClauseSetToString(GetContext().requiredClauses),
-          ContextDirectiveAsFortran());
-    }
+    context_.Warn(common::UsageWarning::Portability,
+        GetContext().directiveSource,
+        "At least one of %s clause should appear on the %s directive"_port_en_US,
+        ClauseSetToString(GetContext().requiredClauses),
+        ContextDirectiveAsFortran());
   } else {
     context_.Say(GetContext().directiveSource,
         "At least one of %s clause must appear on the %s directive"_err_en_US,
@@ -493,13 +492,11 @@ bool DirectiveStructureChecker<D, C, PC, ClauseEnumSize>::CheckAllowed(
       !GetContext().allowedExclusiveClauses.test(clause) &&
       !GetContext().requiredClauses.test(clause)) {
     if (warnInsteadOfError) {
-      if (context_.ShouldWarn(common::UsageWarning::Portability)) {
-        context_.Say(GetContext().clauseSource,
-            "%s clause is not allowed on the %s directive and will be ignored"_port_en_US,
-            parser::ToUpperCaseLetters(getClauseName(clause).str()),
-            parser::ToUpperCaseLetters(
-                GetContext().directiveSource.ToString()));
-      }
+      context_.Warn(common::UsageWarning::Portability,
+          GetContext().clauseSource,
+          "%s clause is not allowed on the %s directive and will be ignored"_port_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clause).str()),
+          parser::ToUpperCaseLetters(GetContext().directiveSource.ToString()));
     } else {
       context_.Say(GetContext().clauseSource,
           "%s clause is not allowed on the %s directive"_err_en_US,
diff --git a/flang/lib/Semantics/check-do-forall.cpp b/flang/lib/Semantics/check-do-forall.cpp
index b9778e96080a6..84e6b6455cc61 100644
--- a/flang/lib/Semantics/check-do-forall.cpp
+++ b/flang/lib/Semantics/check-do-forall.cpp
@@ -495,10 +495,8 @@ class DoContext {
 
   void CheckDoControl(const parser::CharBlock &sourceLocation, bool isReal) {
     if (isReal) {
-      if (context_.ShouldWarn(common::LanguageFeature::RealDoControls)) {
-        context_.Say(
-            sourceLocation, "DO controls should be INTEGER"_port_en_US);
-      }
+      context_.Warn(common::LanguageFeature::RealDoControls, sourceLocation,
+          "DO controls should be INTEGER"_port_en_US);
     } else {
       SayBadDoControl(sourceLocation);
     }
@@ -552,9 +550,9 @@ class DoContext {
     CheckDoExpression(bounds.upper);
     if (bounds.step) {
       CheckDoExpression(*bounds.step);
-      if (IsZero(*bounds.step) &&
-          context_.ShouldWarn(common::UsageWarning::ZeroDoStep)) {
-        context_.Say(bounds.step->thing.value().source,
+      if (IsZero(*bounds.step)) {
+        context_.Warn(common::UsageWarning::ZeroDoStep,
+            bounds.step->thing.value().source,
             "DO step expression should not be zero"_warn_en_US);
       }
     }
@@ -679,10 +677,9 @@ class DoContext {
       if (std::holds_alternative<parser::LocalitySpec::DefaultNone>(ls.u)) {
         if (hasDefaultNone) {
           // F'2023 C1129, you can only have one DEFAULT(NONE)
-          if (context_.ShouldWarn(common::LanguageFeature::BenignRedundancy)) {
-            context_.Say(currentStatementSourcePosition_,
-                "Only one DEFAULT(NONE) may appear"_port_en_US);
-          }
+          context_.Warn(common::LanguageFeature::BenignRedundancy,
+              currentStatementSourcePosition_,
+              "Only one DEFAULT(NONE) may appear"_port_en_US);
           break;
         }
         hasDefaultNone = true;
@@ -890,10 +887,9 @@ class DoContext {
           },
           assignment.u);
       for (const Symbol &index : indexVars) {
-        if (symbols.count(index) == 0 &&
-            context_.ShouldWarn(common::UsageWarning::UnusedForallIndex)) {
-          context_.Say("FORALL index variable '%s' not used on left-hand side"
-                       " of assignment"_warn_en_US,
+        if (symbols.count(index) == 0) {
+          context_.Warn(common::UsageWarning::UnusedForallIndex,
+              "FORALL index variable '%s' not used on left-hand side of assignment"_warn_en_US,
               index.name());
         }
       }
diff --git a/flang/lib/Semantics/check-io.cpp b/flang/lib/Semantics/check-io.cpp
index 46f07842b92cb..eeeda553d8a46 100644
--- a/flang/lib/Semantics/check-io.cpp
+++ b/flang/lib/Semantics/check-io.cpp
@@ -232,8 +232,8 @@ void IoChecker::Enter(const parser::Format &spec) {
               if (!IsVariable(*expr)) {
                 context_.Say(format.source,
                     "Assigned format label must be a scalar variable"_err_en_US);
-              } else if (context_.ShouldWarn(common::LanguageFeature::Assign)) {
-                context_.Say(format.source,
+              } else {
+                context_.Warn(common::LanguageFeature::Assign, format.source,
                     "Assigned format labels are deprecated"_port_en_US);
               }
               return;
@@ -245,11 +245,9 @@ void IoChecker::Enter(const parser::Format &spec) {
                     common::LanguageFeature::NonCharacterFormat)) {
               // Legacy extension: using non-character variables, typically
               // DATA-initialized with Hollerith, as format expressions.
-              if (context_.ShouldWarn(
-                      common::LanguageFeature::NonCharacterFormat)) {
-                context_.Say(format.source,
-                    "Non-character format expression is not standard"_port_en_US);
-              }
+              context_.Warn(common::LanguageFeature::NonCharacterFormat,
+                  format.source,
+                  "Non-character format expression is not standard"_port_en_US);
             } else if (!type ||
                 type->kind() !=
                     context_.defaultKinds().GetDefaultKind(type->category())) {
@@ -936,11 +934,8 @@ void IoChecker::CheckStringValue(IoSpecKind specKind, const std::string &value,
   auto upper{Normalize(value)};
   if (specValues.at(specKind).count(upper) == 0) {
     if (specKind == IoSpecKind::Access && upper == "APPEND") {
-      if (context_.ShouldWarn(common::LanguageFeature::OpenAccessAppend)) {
-        context_.Say(source,
-            "ACCESS='%s' interpreted as POSITION='%s'"_port_en_US, value,
-            upper);
-      }
+      context_.Warn(common::LanguageFeature::OpenAccessAppend, source,
+          "ACCESS='%s' interpreted as POSITION='%s'"_port_en_US, value, upper);
     } else {
       context_.Say(source, "Invalid %s value '%s'"_err_en_US,
           parser::ToUpperCaseLetters(common::EnumToString(specKind)), value);
diff --git a/flang/lib/Semantics/check-namelist.cpp b/flang/lib/Semantics/check-namelist.cpp
index b000e1771755e..c2804c5d874e9 100644
--- a/flang/lib/Semantics/check-namelist.cpp
+++ b/flang/lib/Semantics/check-namelist.cpp
@@ -34,4 +34,19 @@ void NamelistChecker::Leave(const parser::NamelistStmt &nmlStmt) {
   }
 }
 
+void NamelistChecker::Leave(const parser::LocalitySpec::Reduce &x) {
+  for (const parser::Name &name : std::get<std::list<parser::Name>>(x.t)) {
+    Symbol *sym{name.symbol};
+    // This is not disallowed by the standard, but would be difficult to
+    // support. This has to go here not with the other checks for locality specs
+    // in resolve-names.cpp so that it is done after the InNamelist flag is
+    // applied.
+    if (sym && sym->GetUltimate().test(Symbol::Flag::InNamelist)) {
+      context_.Say(name.source,
+          "NAMELIST variable '%s' not allowed in a REDUCE locality-spec"_err_en_US,
+          name.ToString());
+    }
+  }
+}
+
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/check-namelist.h b/flang/lib/Semantics/check-namelist.h
index a23b7d717b022..c51beb2cf25ae 100644
--- a/flang/lib/Semantics/check-namelist.h
+++ b/flang/lib/Semantics/check-namelist.h
@@ -17,6 +17,7 @@ class NamelistChecker : public virtual BaseChecker {
 public:
   NamelistChecker(SemanticsContext &context) : context_{context} {}
   void Leave(const parser::NamelistStmt &);
+  void Leave(const parser::LocalitySpec::Reduce &);
 
 private:
   SemanticsContext &context_;
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 51341b3faf3a4..5ef504aa72326 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -666,11 +666,10 @@ void OmpStructureChecker::CheckTargetNest(const parser::OpenMPConstruct &c) {
           [&](const auto &c) {},
       },
       c.u);
-  if (!eligibleTarget &&
-      context_.ShouldWarn(common::UsageWarning::Portability)) {
-    context_.Say(parser::FindSourceLocation(c),
-        "If %s directive is nested inside TARGET region, the behaviour "
-        "is unspecified"_port_en_US,
+  if (!eligibleTarget) {
+    context_.Warn(common::UsageWarning::Portability,
+        parser::FindSourceLocation(c),
+        "If %s directive is nested inside TARGET region, the behaviour is unspecified"_port_en_US,
         parser::ToUpperCaseLetters(
             getDirectiveName(ineligibleTargetDir).str()));
   }
@@ -1077,12 +1076,10 @@ void OmpStructureChecker::CheckThreadprivateOrDeclareTargetVar(
                         ContextDirectiveAsFortran());
                   else if (GetContext().directive ==
                       llvm::omp::Directive::OMPD_declare_target)
-                    if (context_.ShouldWarn(
-                            common::UsageWarning::OpenMPUsage)) {
-                      context_.Say(name->source,
-                          "The entity with PARAMETER attribute is used in a %s directive"_warn_en_US,
-                          ContextDirectiveAsFortran());
-                    }
+                    context_.Warn(common::UsageWarning::OpenMPUsage,
+                        name->source,
+                        "The entity with PARAMETER attribute is used in a %s directive"_warn_en_US,
+                        ContextDirectiveAsFortran());
                 } else if (FindCommonBlockContaining(*name->symbol)) {
                   context_.Say(name->source,
                       "A variable in a %s directive cannot be an element of a "
@@ -1249,8 +1246,8 @@ void OmpStructureChecker::Leave(const parser::OmpDeclareTargetWithClause &x) {
       context_.Say(x.source,
           "If the DECLARE TARGET directive has a clause, it must contain at least one ENTER clause or LINK clause"_err_en_US);
     }
-    if (toClause && context_.ShouldWarn(common::UsageWarning::OpenMPUsage)) {
-      context_.Say(toClause->source,
+    if (toClause) {
+      context_.Warn(common::UsageWarning::OpenMPUsage, toClause->source,
           "The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead."_warn_en_US);
     }
   }
@@ -3143,9 +3140,8 @@ void OmpStructureChecker::CheckCopyingPolymorphicAllocatable(
       const auto *symbol{it->first};
       const auto source{it->second};
       if (IsPolymorphicAllocatable(*symbol)) {
-        context_.Say(source,
-            "If a polymorphic variable with allocatable attribute '%s' is in "
-            "%s clause, the behavior is unspecified"_port_en_US,
+        context_.Warn(common::UsageWarning::Portability, source,
+            "If a polymorphic variable with allocatable attribute '%s' is in %s clause, the behavior is unspecified"_port_en_US,
             symbol->name(),
             parser::ToUpperCaseLetters(getClauseName(clause).str()));
       }
@@ -3174,11 +3170,13 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Copyprivate &x) {
 void OmpStructureChecker::Enter(const parser::OmpClause::Lastprivate &x) {
   CheckAllowedClause(llvm::omp::Clause::OMPC_lastprivate);
 
-  CheckIsVarPartOfAnotherVar(GetContext().clauseSource, x.v, "LASTPRIVATE");
+  const auto &objectList{std::get<parser::OmpObjectList>(x.v.t)};
+  CheckIsVarPartOfAnotherVar(
+      GetContext().clauseSource, objectList, "LASTPRIVATE");
 
   DirectivesClauseTriple dirClauseTriple;
   SymbolSourceMap currSymbols;
-  GetSymbolsInObjectList(x.v, currSymbols);
+  GetSymbolsInObjectList(objectList, currSymbols);
   CheckDefinableObjects(currSymbols, GetClauseKindForParserClass(x));
   CheckCopyingPolymorphicAllocatable(
       currSymbols, llvm::omp::Clause::OMPC_lastprivate);
@@ -3193,6 +3191,21 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Lastprivate &x) {
 
   CheckPrivateSymbolsInOuterCxt(
       currSymbols, dirClauseTriple, GetClauseKindForParserClass(x));
+
+  using LastprivateModifier = parser::OmpLastprivateClause::LastprivateModifier;
+  const auto &maybeMod{std::get<std::optional<LastprivateModifier>>(x.v.t)};
+  if (maybeMod) {
+    unsigned version{context_.langOptions().OpenMPVersion};
+    unsigned allowedInVersion = 50;
+    if (version < allowedInVersion) {
+      std::string thisVersion{
+          std::to_string(version / 10) + "." + std::to_string(version % 10)};
+      context_.Say(GetContext().clauseSource,
+          "LASTPRIVATE clause with CONDITIONAL modifier is not "
+          "allowed in OpenMP v%s, try -fopenmp-version=%d"_err_en_US,
+          thisVersion, allowedInVersion);
+    }
+  }
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Copyin &x) {
@@ -3246,11 +3259,10 @@ void OmpStructureChecker::Enter(const parser::OmpClause::UseDevicePtr &x) {
       if (const auto *name{parser::Unwrap<parser::Name>(ompObject)}) {
         if (name->symbol) {
           if (!(IsBuiltinCPtr(*(name->symbol)))) {
-            if (context_.ShouldWarn(common::UsageWarning::OpenMPUsage)) {
-              context_.Say(itr->second->source,
-                  "Use of non-C_PTR type '%s' in USE_DEVICE_PTR is deprecated, use USE_DEVICE_ADDR instead"_warn_en_US,
-                  name->ToString());
-            }
+            context_.Warn(common::UsageWarning::OpenMPUsage,
+                itr->second->source,
+                "Use of non-C_PTR type '%s' in USE_DEVICE_PTR is deprecated, use USE_DEVICE_ADDR instead"_warn_en_US,
+                name->ToString());
           } else {
             useDevicePtrNameList.push_back(*name);
           }
@@ -3307,20 +3319,16 @@ void OmpStructureChecker::Enter(const parser::OmpClause::IsDevicePtr &x) {
             "Variable '%s' in IS_DEVICE_PTR clause must be of type C_PTR"_err_en_US,
             source.ToString());
       } else if (!(IsDummy(*symbol))) {
-        if (context_.ShouldWarn(common::UsageWarning::OpenMPUsage)) {
-          context_.Say(itr->second->source,
-              "Variable '%s' in IS_DEVICE_PTR clause must be a dummy argument. "
-              "This semantic check is deprecated from OpenMP 5.2 and later."_warn_en_US,
-              source.ToString());
-        }
+        context_.Warn(common::UsageWarning::OpenMPUsage, itr->second->source,
+            "Variable '%s' in IS_DEVICE_PTR clause must be a dummy argument. "
+            "This semantic check is deprecated from OpenMP 5.2 and later."_warn_en_US,
+            source.ToString());
       } else if (IsAllocatableOrPointer(*symbol) || IsValue(*symbol)) {
-        if (context_.ShouldWarn(common::UsageWarning::OpenMPUsage)) {
-          context_.Say(itr->second->source,
-              "Variable '%s' in IS_DEVICE_PTR clause must be a dummy argument "
-              "that does not have the ALLOCATABLE, POINTER or VALUE attribute. "
-              "This semantic check is deprecated from OpenMP 5.2 and later."_warn_en_US,
-              source.ToString());
-        }
+        context_.Warn(common::UsageWarning::OpenMPUsage, itr->second->source,
+            "Variable '%s' in IS_DEVICE_PTR clause must be a dummy argument "
+            "that does not have the ALLOCATABLE, POINTER or VALUE attribute. "
+            "This semantic check is deprecated from OpenMP 5.2 and later."_warn_en_US,
+            source.ToString());
       }
     }
   }
@@ -3617,18 +3625,17 @@ const parser::OmpObjectList *OmpStructureChecker::GetOmpObjectList(
     const parser::OmpClause &clause) {
 
   // Clauses with OmpObjectList as its data member
-  using MemberObjectListClauses =
-      std::tuple<parser::OmpClause::Copyprivate, parser::OmpClause::Copyin,
-          parser::OmpClause::Firstprivate, parser::OmpClause::From,
-          parser::OmpClause::Lastprivate, parser::OmpClause::Link,
-          parser::OmpClause::Private, parser::OmpClause::Shared,
-          parser::OmpClause::To, parser::OmpClause::Enter,
-          parser::OmpClause::UseDevicePtr, parser::OmpClause::UseDeviceAddr>;
+  using MemberObjectListClauses = std::tuple<parser::OmpClause::Copyprivate,
+      parser::OmpClause::Copyin, parser::OmpClause::Firstprivate,
+      parser::OmpClause::From, parser::OmpClause::Link,
+      parser::OmpClause::Private, parser::OmpClause::Shared,
+      parser::OmpClause::To, parser::OmpClause::Enter,
+      parser::OmpClause::UseDevicePtr, parser::OmpClause::UseDeviceAddr>;
 
   // Clauses with OmpObjectList in the tuple
-  using TupleObjectListClauses =
-      std::tuple<parser::OmpClause::Allocate, parser::OmpClause::Map,
-          parser::OmpClause::Reduction, parser::OmpClause::Aligned>;
+  using TupleObjectListClauses = std::tuple<parser::OmpClause::Allocate,
+      parser::OmpClause::Lastprivate, parser::OmpClause::Map,
+      parser::OmpClause::Reduction, parser::OmpClause::Aligned>;
 
   // TODO:: Generate the tuples using TableGen.
   // Handle other constructs with OmpObjectList such as OpenMPThreadprivate.
diff --git a/flang/lib/Semantics/check-return.cpp b/flang/lib/Semantics/check-return.cpp
index ec2600bac3c68..22729f659ed5b 100644
--- a/flang/lib/Semantics/check-return.cpp
+++ b/flang/lib/Semantics/check-return.cpp
@@ -36,9 +36,9 @@ void ReturnStmtChecker::Leave(const parser::ReturnStmt &returnStmt) {
             IsFunction(*subprogramScope->GetSymbol()))) {
       context_.Say(
           "RETURN with expression is only allowed in SUBROUTINE subprogram"_err_en_US);
-    } else if (subprogramScope->kind() == Scope::Kind::MainProgram &&
-        context_.ShouldWarn(common::LanguageFeature::ProgramReturn)) {
-      context_.Say("RETURN should not appear in a main program"_port_en_US);
+    } else if (subprogramScope->kind() == Scope::Kind::MainProgram) {
+      context_.Warn(common::LanguageFeature::ProgramReturn,
+          "RETURN should not appear in a main program"_port_en_US);
     }
   }
 }
diff --git a/flang/lib/Semantics/compute-offsets.cpp b/flang/lib/Semantics/compute-offsets.cpp
index b5a58ddca0ecd..028633813a91b 100644
--- a/flang/lib/Semantics/compute-offsets.cpp
+++ b/flang/lib/Semantics/compute-offsets.cpp
@@ -167,11 +167,9 @@ void ComputeOffsetsHelper::DoCommonBlock(Symbol &commonBlock) {
     auto errorSite{
         commonBlock.name().empty() ? symbol.name() : commonBlock.name()};
     if (std::size_t padding{DoSymbol(symbol.GetUltimate())}) {
-      if (context_.ShouldWarn(common::UsageWarning::CommonBlockPadding)) {
-        context_.Say(errorSite,
-            "COMMON block /%s/ requires %zd bytes of padding before '%s' for alignment"_port_en_US,
-            commonBlock.name(), padding, symbol.name());
-      }
+      context_.Warn(common::UsageWarning::CommonBlockPadding, errorSite,
+          "COMMON block /%s/ requires %zd bytes of padding before '%s' for alignment"_port_en_US,
+          commonBlock.name(), padding, symbol.name());
     }
     previous.emplace(symbol);
     auto eqIter{equivalenceBlock_.end()};
diff --git a/flang/lib/Semantics/data-to-inits.cpp b/flang/lib/Semantics/data-to-inits.cpp
index 0f8b36de46086..c9b86a930437a 100644
--- a/flang/lib/Semantics/data-to-inits.cpp
+++ b/flang/lib/Semantics/data-to-inits.cpp
@@ -298,12 +298,10 @@ DataInitializationCompiler<DSV>::ConvertElement(
   if (context.IsEnabled(common::LanguageFeature::LogicalIntegerAssignment)) {
     if (MaybeExpr converted{evaluate::DataConstantConversionExtension(
             exprAnalyzer_.GetFoldingContext(), type, expr)}) {
-      if (context.ShouldWarn(
-              common::LanguageFeature::LogicalIntegerAssignment)) {
-        context.Say(
-            "nonstandard usage: initialization of %s with %s"_port_en_US,
-            type.AsFortran(), expr.GetType().value().AsFortran());
-      }
+      context.Warn(common::LanguageFeature::LogicalIntegerAssignment,
+          exprAnalyzer_.GetFoldingContext().messages().at(),
+          "nonstandard usage: initialization of %s with %s"_port_en_US,
+          type.AsFortran(), expr.GetType().value().AsFortran());
       return {std::make_pair(std::move(*converted), false)};
     }
   }
@@ -434,16 +432,11 @@ bool DataInitializationCompiler<DSV>::InitElement(
       // value non-pointer initialization
       if (IsBOZLiteral(*expr) &&
           designatorType->category() != TypeCategory::Integer) { // 8.6.7(11)
-        if (exprAnalyzer_.context().ShouldWarn(
-                common::LanguageFeature::DataStmtExtensions)) {
-          exprAnalyzer_.Say(
-              "BOZ literal should appear in a DATA statement only as a value for an integer object, but '%s' is '%s'"_port_en_US,
-              DescribeElement(), designatorType->AsFortran());
-        }
-      } else if (converted->second &&
-          exprAnalyzer_.context().ShouldWarn(
-              common::LanguageFeature::DataStmtExtensions)) {
-        exprAnalyzer_.context().Say(
+        exprAnalyzer_.Warn(common::LanguageFeature::DataStmtExtensions,
+            "BOZ literal should appear in a DATA statement only as a value for an integer object, but '%s' is '%s'"_port_en_US,
+            DescribeElement(), designatorType->AsFortran());
+      } else if (converted->second) {
+        exprAnalyzer_.Warn(common::LanguageFeature::DataStmtExtensions,
             "DATA statement value initializes '%s' of type '%s' with CHARACTER"_port_en_US,
             DescribeElement(), designatorType->AsFortran());
       }
@@ -462,12 +455,9 @@ bool DataInitializationCompiler<DSV>::InitElement(
       } else if (status == evaluate::InitialImage::OutOfRange) {
         OutOfRangeError();
       } else if (status == evaluate::InitialImage::LengthMismatch) {
-        if (exprAnalyzer_.context().ShouldWarn(
-                common::UsageWarning::DataLength)) {
-          exprAnalyzer_.Say(
-              "DATA statement value '%s' for '%s' has the wrong length"_warn_en_US,
-              folded.AsFortran(), DescribeElement());
-        }
+        exprAnalyzer_.Warn(common::UsageWarning::DataLength,
+            "DATA statement value '%s' for '%s' has the wrong length"_warn_en_US,
+            folded.AsFortran(), DescribeElement());
         return true;
       } else if (status == evaluate::InitialImage::TooManyElems) {
         exprAnalyzer_.Say("DATA statement has too many elements"_err_en_US);
diff --git a/flang/lib/Semantics/definable.cpp b/flang/lib/Semantics/definable.cpp
index 5c41376d2a42b..88f9463e35c78 100644
--- a/flang/lib/Semantics/definable.cpp
+++ b/flang/lib/Semantics/definable.cpp
@@ -349,7 +349,8 @@ std::optional<parser::Message> WhyNotDefinable(parser::CharBlock at,
                 if (!portabilityWarning &&
                     scope.context().languageFeatures().ShouldWarn(
                         common::UsageWarning::VectorSubscriptFinalization)) {
-                  portabilityWarning = parser::Message{at,
+                  portabilityWarning = parser::Message{
+                      common::UsageWarning::VectorSubscriptFinalization, at,
                       "Variable '%s' has a vector subscript and will be finalized by non-elemental subroutine '%s'"_port_en_US,
                       expr.AsFortran(), anyRankMatch->name()};
                 }
diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index 182ea5d441956..c70c8a8aecc2f 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -663,10 +663,8 @@ struct IntTypeVisitor {
         auto unsignedNum{Int::Read(p, 10, false /*unsigned*/)};
         num.value = unsignedNum.value.Negate().value;
         num.overflow = unsignedNum.overflow || num.value > Int{0};
-        if (!num.overflow && num.value.Negate().overflow &&
-            analyzer.context().ShouldWarn(LanguageFeature::BigIntLiterals) &&
-            !analyzer.context().IsInModuleFile(digits)) {
-          analyzer.Say(digits,
+        if (!num.overflow && num.value.Negate().overflow) {
+          analyzer.Warn(LanguageFeature::BigIntLiterals, digits,
               "negated maximum INTEGER(KIND=%d) literal"_port_en_US, T::kind);
         }
       } else {
@@ -677,9 +675,8 @@ struct IntTypeVisitor {
           if (!isDefaultKind ||
               !analyzer.context().IsEnabled(LanguageFeature::BigIntLiterals)) {
             return std::nullopt;
-          } else if (analyzer.context().ShouldWarn(
-                         LanguageFeature::BigIntLiterals)) {
-            analyzer.Say(digits,
+          } else {
+            analyzer.Warn(LanguageFeature::BigIntLiterals, digits,
                 "Integer literal is too large for default INTEGER(KIND=%d); "
                 "assuming INTEGER(KIND=%d)"_port_en_US,
                 kind, T::kind);
@@ -809,16 +806,12 @@ MaybeExpr ExpressionAnalyzer::Analyze(const parser::RealLiteralConstant &x) {
   auto kind{AnalyzeKindParam(x.kind, defaultKind)};
   if (letterKind && expoLetter != 'e') {
     if (kind != *letterKind) {
-      if (context_.ShouldWarn(
-              common::LanguageFeature::ExponentMatchingKindParam)) {
-        Say("Explicit kind parameter on real constant disagrees with exponent letter '%c'"_warn_en_US,
-            expoLetter);
-      }
-    } else if (x.kind &&
-        context_.ShouldWarn(
-            common::LanguageFeature::ExponentMatchingKindParam)) {
-      Say("Explicit kind parameter together with non-'E' exponent letter "
-          "is not standard"_port_en_US);
+      Warn(common::LanguageFeature::ExponentMatchingKindParam,
+          "Explicit kind parameter on real constant disagrees with exponent letter '%c'"_warn_en_US,
+          expoLetter);
+    } else if (x.kind) {
+      Warn(common::LanguageFeature::ExponentMatchingKindParam,
+          "Explicit kind parameter together with non-'E' exponent letter is not standard"_port_en_US);
     }
   }
   auto result{common::SearchTypes(
@@ -1657,11 +1650,8 @@ void ArrayConstructorContext::Push(MaybeExpr &&x) {
   if (!type_) {
     if (auto *boz{std::get_if<BOZLiteralConstant>(&x->u)}) {
       // Treat an array constructor of BOZ as if default integer.
-      if (exprAnalyzer_.context().ShouldWarn(
-              common::LanguageFeature::BOZAsDefaultInteger)) {
-        exprAnalyzer_.Say(
-            "BOZ literal in array constructor without explicit type is assumed to be default INTEGER"_port_en_US);
-      }
+      exprAnalyzer_.Warn(common::LanguageFeature::BOZAsDefaultInteger,
+          "BOZ literal in array constructor without explicit type is assumed to be default INTEGER"_port_en_US);
       x = AsGenericExpr(ConvertToKind<TypeCategory::Integer>(
           exprAnalyzer_.GetDefaultKind(TypeCategory::Integer),
           std::move(*boz)));
@@ -1672,11 +1662,8 @@ void ArrayConstructorContext::Push(MaybeExpr &&x) {
     if (auto *boz{std::get_if<BOZLiteralConstant>(&x->u)}) {
       if (!type_) {
         // Treat an array constructor of BOZ as if default integer.
-        if (exprAnalyzer_.context().ShouldWarn(
-                common::LanguageFeature::BOZAsDefaultInteger)) {
-          exprAnalyzer_.Say(
-              "BOZ literal in array constructor without explicit type is assumed to be default INTEGER"_port_en_US);
-        }
+        exprAnalyzer_.Warn(common::LanguageFeature::BOZAsDefaultInteger,
+            "BOZ literal in array constructor without explicit type is assumed to be default INTEGER"_port_en_US);
         x = AsGenericExpr(ConvertToKind<TypeCategory::Integer>(
             exprAnalyzer_.GetDefaultKind(TypeCategory::Integer),
             std::move(*boz)));
@@ -1740,15 +1727,12 @@ void ArrayConstructorContext::Push(MaybeExpr &&x) {
       auto xLen{xType.LEN()};
       if (auto thisLen{ToInt64(xLen)}) {
         if (constantLength_) {
-          if (exprAnalyzer_.context().ShouldWarn(
-                  common::LanguageFeature::DistinctArrayConstructorLengths) &&
-              *thisLen != *constantLength_) {
-            if (!(messageDisplayedSet_ & 1)) {
-              exprAnalyzer_.Say(
-                  "Character literal in array constructor without explicit "
-                  "type has different length than earlier elements"_port_en_US);
-              messageDisplayedSet_ |= 1;
-            }
+          if (*thisLen != *constantLength_ && !(messageDisplayedSet_ & 1)) {
+            exprAnalyzer_.Warn(
+                common::LanguageFeature::DistinctArrayConstructorLengths,
+                "Character literal in array constructor without explicit "
+                "type has different length than earlier elements"_port_en_US);
+            messageDisplayedSet_ |= 1;
           }
           if (*thisLen > *constantLength_) {
             // Language extension: use the longest literal to determine the
@@ -2091,11 +2075,9 @@ MaybeExpr ExpressionAnalyzer::Analyze(
               valueType->IsEquivalentTo(*parentType)) {
             symbol = &*parent;
             nextAnonymous = ++parent;
-            if (context().ShouldWarn(LanguageFeature::AnonymousParents)) {
-              Say(source,
-                  "Whole parent component '%s' in structure constructor should not be anonymous"_port_en_US,
-                  symbol->name());
-            }
+            Warn(LanguageFeature::AnonymousParents, source,
+                "Whole parent component '%s' in structure constructor should not be anonymous"_port_en_US,
+                symbol->name());
             break;
           }
         }
@@ -2166,14 +2148,13 @@ MaybeExpr ExpressionAnalyzer::Analyze(
               continue;
             }
             if (IsNullObjectPointer(*value)) {
-              if (context().ShouldWarn(common::LanguageFeature::
-                          NullMoldAllocatableComponentValue)) {
-                AttachDeclaration(
-                    Say(expr.source,
-                        "NULL() with arguments is not standard conforming as the value for allocatable component '%s'"_port_en_US,
-                        symbol->name()),
-                    *symbol);
-              }
+              AttachDeclaration(
+                  Warn(common::LanguageFeature::
+                           NullMoldAllocatableComponentValue,
+                      expr.source,
+                      "NULL() with arguments is not standard conforming as the value for allocatable component '%s'"_port_en_US,
+                      symbol->name()),
+                  *symbol);
               // proceed to check type & shape
             } else {
               AttachDeclaration(
@@ -2459,13 +2440,11 @@ auto ExpressionAnalyzer::AnalyzeProcedureComponentRef(
               sym->attrs().test(semantics::Attr::NOPASS)) {
             // F'2023 C1529 seems unnecessary and most compilers don't
             // enforce it.
-            if (context().ShouldWarn(
-                    common::LanguageFeature::NopassScalarBase)) {
-              AttachDeclaration(
-                  Say(sc.component.source,
-                      "Base of NOPASS type-bound procedure reference should be scalar"_port_en_US),
-                  *sym);
-            }
+            AttachDeclaration(
+                Warn(common::LanguageFeature::NopassScalarBase,
+                    sc.component.source,
+                    "Base of NOPASS type-bound procedure reference should be scalar"_port_en_US),
+                *sym);
           } else if (IsProcedurePointer(*sym)) { // C919
             Say(sc.component.source,
                 "Base of procedure component reference must be scalar"_err_en_US);
@@ -2916,6 +2895,9 @@ auto ExpressionAnalyzer::GetCalleeAndArguments(const parser::Name &name,
   } else {
     resolution = symbol;
   }
+  if (resolution && context_.targetCharacteristics().isOSWindows()) {
+    semantics::CheckWindowsIntrinsic(*resolution, GetFoldingContext());
+  }
   if (!resolution || resolution->attrs().test(semantics::Attr::INTRINSIC)) {
     auto name{resolution ? resolution->name() : ultimate.name()};
     if (std::optional<SpecificCall> specificCall{context_.intrinsics().Probe(
@@ -2967,10 +2949,9 @@ void ExpressionAnalyzer::CheckBadExplicitType(
       if (const auto *typeAndShape{result->GetTypeAndShape()}) {
         if (auto declared{
                 typeAndShape->Characterize(intrinsic, GetFoldingContext())}) {
-          if (!declared->type().IsTkCompatibleWith(typeAndShape->type()) &&
-              context_.ShouldWarn(
-                  common::UsageWarning::IgnoredIntrinsicFunctionType)) {
-            if (auto *msg{Say(
+          if (!declared->type().IsTkCompatibleWith(typeAndShape->type())) {
+            if (auto *msg{Warn(
+                    common::UsageWarning::IgnoredIntrinsicFunctionType,
                     "The result type '%s' of the intrinsic function '%s' is not the explicit declared type '%s'"_warn_en_US,
                     typeAndShape->AsFortran(), intrinsic.name(),
                     declared->AsFortran())}) {
@@ -3342,10 +3323,10 @@ std::optional<characteristics::Procedure> ExpressionAnalyzer::CheckCall(
           iter != implicitInterfaces_.end()) {
         std::string whyNot;
         if (!chars->IsCompatibleWith(iter->second.second,
-                /*ignoreImplicitVsExplicit=*/false, &whyNot) &&
-            context_.ShouldWarn(
-                common::UsageWarning::IncompatibleImplicitInterfaces)) {
-          if (auto *msg{Say(callSite,
+                /*ignoreImplicitVsExplicit=*/false, &whyNot)) {
+          if (auto *msg{Warn(
+                  common::UsageWarning::IncompatibleImplicitInterfaces,
+                  callSite,
                   "Reference to the procedure '%s' has an implicit interface that is distinct from another reference: %s"_warn_en_US,
                   name, whyNot)}) {
             msg->Attach(
@@ -3555,10 +3536,8 @@ MaybeExpr ExpressionAnalyzer::Analyze(const parser::Expr::Subtract &x) {
 
 MaybeExpr ExpressionAnalyzer::Analyze(
     const parser::Expr::ComplexConstructor &z) {
-  if (context_.ShouldWarn(common::LanguageFeature::ComplexConstructor)) {
-    context_.Say(
-        "nonstandard usage: generalized COMPLEX constructor"_port_en_US);
-  }
+  Warn(common::LanguageFeature::ComplexConstructor,
+      "nonstandard usage: generalized COMPLEX constructor"_port_en_US);
   return AnalyzeComplex(Analyze(std::get<0>(z.t).value()),
       Analyze(std::get<1>(z.t).value()), "complex constructor");
 }
@@ -4037,11 +4016,9 @@ bool ExpressionAnalyzer::CheckIntrinsicKind(
     return true;
   } else if (foldingContext_.targetCharacteristics().CanSupportType(
                  category, kind)) {
-    if (context_.ShouldWarn(common::UsageWarning::BadTypeForTarget) &&
-        !context_.IsInModuleFile(GetContextualMessages().at())) {
-      Say("%s(KIND=%jd) is not an enabled type for this target"_warn_en_US,
-          ToUpperCase(EnumToString(category)), kind);
-    }
+    Warn(common::UsageWarning::BadTypeForTarget,
+        "%s(KIND=%jd) is not an enabled type for this target"_warn_en_US,
+        ToUpperCase(EnumToString(category)), kind);
     return true;
   } else {
     Say("%s(KIND=%jd) is not a supported type"_err_en_US,
@@ -4067,10 +4044,9 @@ bool ExpressionAnalyzer::CheckIntrinsicSize(
     return true;
   } else if (foldingContext_.targetCharacteristics().CanSupportType(
                  category, kind)) {
-    if (context_.ShouldWarn(common::UsageWarning::BadTypeForTarget)) {
-      Say("%s*%jd is not an enabled type for this target"_warn_en_US,
-          ToUpperCase(EnumToString(category)), size);
-    }
+    Warn(common::UsageWarning::BadTypeForTarget,
+        "%s*%jd is not an enabled type for this target"_warn_en_US,
+        ToUpperCase(EnumToString(category)), size);
     return true;
   } else {
     Say("%s*%jd is not a supported type"_err_en_US,
@@ -4174,13 +4150,13 @@ MaybeExpr ExpressionAnalyzer::MakeFunctionRef(
 
 MaybeExpr ExpressionAnalyzer::AnalyzeComplex(
     MaybeExpr &&re, MaybeExpr &&im, const char *what) {
-  if (context().ShouldWarn(common::LanguageFeature::ComplexConstructor)) {
-    if (re && re->Rank() > 0) {
-      Say("Real part of %s is not scalar"_port_en_US, what);
-    }
-    if (im && im->Rank() > 0) {
-      Say("Imaginary part of %s is not scalar"_port_en_US, what);
-    }
+  if (re && re->Rank() > 0) {
+    Warn(common::LanguageFeature::ComplexConstructor,
+        "Real part of %s is not scalar"_port_en_US, what);
+  }
+  if (im && im->Rank() > 0) {
+    Warn(common::LanguageFeature::ComplexConstructor,
+        "Imaginary part of %s is not scalar"_port_en_US, what);
   }
   if (re && im) {
     ConformabilityCheck(GetContextualMessages(), *re, *im);
@@ -4591,10 +4567,8 @@ bool ArgumentAnalyzer::OkLogicalIntegerAssignment(
   } else {
     return false;
   }
-  if (context_.context().ShouldWarn(
-          common::LanguageFeature::LogicalIntegerAssignment)) {
-    context_.Say(std::move(*msg));
-  }
+  context_.Warn(
+      common::LanguageFeature::LogicalIntegerAssignment, std::move(*msg));
   return true;
 }
 
diff --git a/flang/lib/Semantics/mod-file.cpp b/flang/lib/Semantics/mod-file.cpp
index 52a5a321aba64..075f33cb68488 100644
--- a/flang/lib/Semantics/mod-file.cpp
+++ b/flang/lib/Semantics/mod-file.cpp
@@ -1482,14 +1482,16 @@ Scope *ModFileReader::Read(SourceName name, std::optional<bool> isIntrinsic,
   if (!checkSum) {
     if (context_.ShouldWarn(common::UsageWarning::ModuleFile)) {
       Say(name, ancestorName, "File has invalid checksum: %s"_warn_en_US,
-          sourceFile->path());
+          sourceFile->path())
+          .set_usageWarning(common::UsageWarning::ModuleFile);
     }
     return nullptr;
   } else if (requiredHash && *requiredHash != *checkSum) {
     if (context_.ShouldWarn(common::UsageWarning::ModuleFile)) {
       Say(name, ancestorName,
           "File is not the right module file for %s"_warn_en_US,
-          "'"s + name.ToString() + "': "s + sourceFile->path());
+          "'"s + name.ToString() + "': "s + sourceFile->path())
+          .set_usageWarning(common::UsageWarning::ModuleFile);
     }
     return nullptr;
   }
diff --git a/flang/lib/Semantics/pointer-assignment.cpp b/flang/lib/Semantics/pointer-assignment.cpp
index 2813a0cf968cb..2450ce39215ec 100644
--- a/flang/lib/Semantics/pointer-assignment.cpp
+++ b/flang/lib/Semantics/pointer-assignment.cpp
@@ -77,6 +77,8 @@ class PointerAssignmentChecker {
       const evaluate::SpecificIntrinsic *specific = nullptr);
   bool LhsOkForUnlimitedPoly() const;
   template <typename... A> parser::Message *Say(A &&...);
+  template <typename FeatureOrUsageWarning, typename... A>
+  parser::Message *Warn(FeatureOrUsageWarning, A &&...);
 
   SemanticsContext &context_;
   evaluate::FoldingContext &foldingContext_{context_.foldingContext()};
@@ -215,20 +217,17 @@ bool PointerAssignmentChecker::Check(const SomeExpr &rhs) {
         Say("CONTIGUOUS pointer may not be associated with a discontiguous target"_err_en_US);
         return false;
       }
-    } else if (context_.ShouldWarn(
-                   common::UsageWarning::PointerToPossibleNoncontiguous)) {
-      Say("Target of CONTIGUOUS pointer association is not known to be contiguous"_warn_en_US);
+    } else {
+      Warn(common::UsageWarning::PointerToPossibleNoncontiguous,
+          "Target of CONTIGUOUS pointer association is not known to be contiguous"_warn_en_US);
     }
   }
   // Warn about undefinable data targets
-  if (context_.ShouldWarn(common::UsageWarning::PointerToUndefinable)) {
-    if (auto because{WhyNotDefinable(
-            foldingContext_.messages().at(), scope_, {}, rhs)}) {
-      if (auto *msg{
-              Say("Pointer target is not a definable variable"_warn_en_US)}) {
-        msg->Attach(
-            std::move(because->set_severity(parser::Severity::Because)));
-      }
+  if (auto because{
+          WhyNotDefinable(foldingContext_.messages().at(), scope_, {}, rhs)}) {
+    if (auto *msg{Warn(common::UsageWarning::PointerToUndefinable,
+            "Pointer target is not a definable variable"_warn_en_US)}) {
+      msg->Attach(std::move(because->set_severity(parser::Severity::Because)));
       return false;
     }
   }
@@ -270,10 +269,11 @@ bool PointerAssignmentChecker::Check(const evaluate::FunctionRef<T> &f) {
           " that is a not a pointer"_err_en_US;
   } else if (isContiguous_ &&
       !funcResult->attrs.test(FunctionResult::Attr::Contiguous)) {
-    if (context_.ShouldWarn(
-            common::UsageWarning::PointerToPossibleNoncontiguous)) {
-      msg =
-          "CONTIGUOUS %s is associated with the result of reference to function '%s' that is not known to be contiguous"_warn_en_US;
+    auto restorer{common::ScopedSet(lhs_, symbol)};
+    if (Warn(common::UsageWarning::PointerToPossibleNoncontiguous,
+            "CONTIGUOUS %s is associated with the result of reference to function '%s' that is not known to be contiguous"_warn_en_US,
+            description_, funcName)) {
+      return false;
     }
   } else if (lhsType_) {
     const auto *frTypeAndShape{funcResult->GetTypeAndShape()};
@@ -377,9 +377,9 @@ bool PointerAssignmentChecker::Check(parser::CharBlock rhsName, bool isCall,
     Say(std::move(*msg), description_, rhsName, whyNot);
     return false;
   }
-  if (context_.ShouldWarn(common::UsageWarning::ProcDummyArgShapes) &&
-      warning) {
-    Say("%s and %s may not be completely compatible procedures: %s"_warn_en_US,
+  if (warning) {
+    Warn(common::UsageWarning::ProcDummyArgShapes,
+        "%s and %s may not be completely compatible procedures: %s"_warn_en_US,
         description_, rhsName, std::move(*warning));
   }
   return true;
@@ -396,11 +396,12 @@ bool PointerAssignmentChecker::Check(const evaluate::ProcedureDesignator &d) {
             symbol->name());
         return false;
       }
-    } else if (symbol->has<ProcBindingDetails>() &&
-        context_.ShouldWarn(common::LanguageFeature::BindingAsProcedure)) {
-      evaluate::SayWithDeclaration(foldingContext_.messages(), *symbol,
-          "Procedure binding '%s' used as target of a pointer assignment"_port_en_US,
-          symbol->name());
+    } else if (symbol->has<ProcBindingDetails>()) {
+      evaluate::AttachDeclaration(
+          Warn(common::LanguageFeature::BindingAsProcedure,
+              "Procedure binding '%s' used as target of a pointer assignment"_port_en_US,
+              symbol->name()),
+          *symbol);
     }
   }
   if (auto chars{
@@ -447,6 +448,22 @@ parser::Message *PointerAssignmentChecker::Say(A &&...x) {
   return msg;
 }
 
+template <typename FeatureOrUsageWarning, typename... A>
+parser::Message *PointerAssignmentChecker::Warn(
+    FeatureOrUsageWarning warning, A &&...x) {
+  auto *msg{context_.Warn(
+      warning, foldingContext_.messages().at(), std::forward<A>(x)...)};
+  if (msg) {
+    if (lhs_) {
+      return evaluate::AttachDeclaration(msg, *lhs_);
+    }
+    if (!source_.empty()) {
+      msg->Attach(source_, "Declaration of %s"_en_US, description_);
+    }
+  }
+  return msg;
+}
+
 // Verify that any bounds on the LHS of a pointer assignment are valid.
 // Return true if it is a bound-remapping so we can perform further checks.
 static bool CheckPointerBounds(
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 45fe17c0122c0..3d3630e8f388c 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -455,7 +455,8 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
     return false;
   }
   bool Pre(const parser::OmpClause::Lastprivate &x) {
-    ResolveOmpObjectList(x.v, Symbol::Flag::OmpLastPrivate);
+    const auto &objList{std::get<parser::OmpObjectList>(x.v.t)};
+    ResolveOmpObjectList(objList, Symbol::Flag::OmpLastPrivate);
     return false;
   }
   bool Pre(const parser::OmpClause::Copyin &x) {
@@ -718,7 +719,7 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
   void CheckDataCopyingClause(
       const parser::Name &, const Symbol &, Symbol::Flag);
   void CheckAssocLoopLevel(std::int64_t level, const parser::OmpClause *clause);
-  void CheckObjectInNamelistOrAssociate(
+  void CheckObjectIsPrivatizable(
       const parser::Name &, const Symbol &, Symbol::Flag);
   void CheckSourceLabel(const parser::Label &);
   void CheckLabelContext(const parser::CharBlock, const parser::CharBlock,
@@ -2225,20 +2226,7 @@ void OmpAttributeVisitor::Post(const parser::Name &name) {
         return;
     }
 
-    if (auto *stmtFunction{symbol->detailsIf<semantics::SubprogramDetails>()};
-        stmtFunction && stmtFunction->stmtFunction()) {
-      // Each non-dummy argument from a statement function must be handled too,
-      // as if it was explicitly referenced.
-      semantics::UnorderedSymbolSet symbols{
-          CollectSymbols(stmtFunction->stmtFunction().value())};
-      for (const auto &sym : symbols) {
-        if (!IsStmtFunctionDummy(sym) && !IsObjectWithDSA(*sym)) {
-          CreateImplicitSymbols(&*sym, Symbol::Flag::OmpFromStmtFunction);
-        }
-      }
-    } else {
-      CreateImplicitSymbols(symbol);
-    }
+    CreateImplicitSymbols(symbol);
   } // within OpenMP construct
 }
 
@@ -2357,7 +2345,7 @@ void OmpAttributeVisitor::ResolveOmpObject(
                     CheckMultipleAppearances(*name, *symbol, ompFlag);
                   }
                   if (privateDataSharingAttributeFlags.test(ompFlag)) {
-                    CheckObjectInNamelistOrAssociate(*name, *symbol, ompFlag);
+                    CheckObjectIsPrivatizable(*name, *symbol, ompFlag);
                   }
 
                   if (ompFlag == Symbol::Flag::OmpAllocate) {
@@ -2384,6 +2372,21 @@ void OmpAttributeVisitor::ResolveOmpObject(
                               GetContext().directive)
                               .str()));
                 }
+                if (ompFlag == Symbol::Flag::OmpReduction) {
+                  const Symbol &ultimateSymbol{symbol->GetUltimate()};
+                  // Using variables inside of a namelist in OpenMP reductions
+                  // is allowed by the standard, but is not allowed for
+                  // privatisation. This looks like an oversight. If the
+                  // namelist is hoisted to a global, we cannot apply the
+                  // mapping for the reduction variable: resulting in incorrect
+                  // results. Disabling this hoisting could make some real
+                  // production code go slower. See discussion in #109303
+                  if (ultimateSymbol.test(Symbol::Flag::InNamelist)) {
+                    context_.Say(name->source,
+                        "Variable '%s' in NAMELIST cannot be in a REDUCTION clause"_err_en_US,
+                        name->ToString());
+                  }
+                }
                 if (GetContext().directive ==
                     llvm::omp::Directive::OMPD_target_data) {
                   checkExclusivelists(symbol, Symbol::Flag::OmpUseDevicePtr,
@@ -2714,7 +2717,7 @@ void OmpAttributeVisitor::CheckDataCopyingClause(
   }
 }
 
-void OmpAttributeVisitor::CheckObjectInNamelistOrAssociate(
+void OmpAttributeVisitor::CheckObjectIsPrivatizable(
     const parser::Name &name, const Symbol &symbol, Symbol::Flag ompFlag) {
   const auto &ultimateSymbol{symbol.GetUltimate()};
   llvm::StringRef clauseName{"PRIVATE"};
@@ -2735,6 +2738,14 @@ void OmpAttributeVisitor::CheckObjectInNamelistOrAssociate(
         "Variable '%s' in ASSOCIATE cannot be in a %s clause"_err_en_US,
         name.ToString(), clauseName.str());
   }
+
+  if (stmtFunctionExprSymbols_.find(ultimateSymbol) !=
+      stmtFunctionExprSymbols_.end()) {
+    context_.Say(name.source,
+        "Variable '%s' in statement function expression cannot be in a "
+        "%s clause"_err_en_US,
+        name.ToString(), clauseName.str());
+  }
 }
 
 void OmpAttributeVisitor::CheckSourceLabel(const parser::Label &label) {
@@ -2873,8 +2884,7 @@ void OmpAttributeVisitor::IssueNonConformanceWarning(
   default:
     warnStr = "OpenMP directive '" + dirName + "' has been deprecated.";
   }
-  if (context_.ShouldWarn(common::UsageWarning::OpenMPUsage)) {
-    context_.Say(source, "%s"_warn_en_US, warnStr);
-  }
+  context_.Warn(
+      common::UsageWarning::OpenMPUsage, source, "%s"_warn_en_US, warnStr);
 }
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/resolve-labels.cpp b/flang/lib/Semantics/resolve-labels.cpp
index e5e96ec6327e2..04e4b142efedf 100644
--- a/flang/lib/Semantics/resolve-labels.cpp
+++ b/flang/lib/Semantics/resolve-labels.cpp
@@ -938,7 +938,9 @@ void CheckBranchesIntoDoBody(const SourceStmtList &branches,
           context
               .Say(
                   fromPosition, "branch into loop body from outside"_warn_en_US)
-              .Attach(body.first, "the loop branched into"_en_US);
+              .Attach(body.first, "the loop branched into"_en_US)
+              .set_languageFeature(
+                  common::LanguageFeature::BranchIntoConstruct);
         }
       }
     }
@@ -1007,7 +1009,9 @@ void CheckLabelDoConstraints(const SourceStmtList &dos,
             .Say(position,
                 "A DO loop should terminate with an END DO or CONTINUE"_port_en_US)
             .Attach(doTarget.parserCharBlock,
-                "DO loop currently ends at statement:"_en_US);
+                "DO loop currently ends at statement:"_en_US)
+            .set_languageFeature(
+                common::LanguageFeature::OldLabelDoEndStatements);
       }
     } else if (!InInclusiveScope(scopes, scope, doTarget.proxyForScope)) {
       context.Say(position, "Label '%u' is not in DO loop scope"_err_en_US,
@@ -1067,9 +1071,11 @@ void CheckScopeConstraints(const SourceStmtList &stmts,
             SayLabel(label));
       } else if (context.ShouldWarn(
                      common::LanguageFeature::BranchIntoConstruct)) {
-        context.Say(position,
-            "Label '%u' is in a construct that should not be used as a branch target here"_warn_en_US,
-            SayLabel(label));
+        context
+            .Say(position,
+                "Label '%u' is in a construct that should not be used as a branch target here"_warn_en_US,
+                SayLabel(label))
+            .set_languageFeature(common::LanguageFeature::BranchIntoConstruct);
       }
     }
   }
@@ -1097,7 +1103,8 @@ void CheckBranchTargetConstraints(const SourceStmtList &stmts,
             .Say(branchTarget.parserCharBlock,
                 "Label '%u' is not a branch target"_warn_en_US, SayLabel(label))
             .Attach(stmt.parserCharBlock, "Control flow use of '%u'"_en_US,
-                SayLabel(label));
+                SayLabel(label))
+            .set_languageFeature(common::LanguageFeature::BadBranchTarget);
       }
     }
   }
@@ -1152,9 +1159,12 @@ void CheckAssignTargetConstraints(const SourceStmtList &stmts,
             "Label '%u' is not a branch target or FORMAT"_err_en_US,
             SayLabel(label));
       } else if (context.ShouldWarn(common::LanguageFeature::BadBranchTarget)) {
-        msg = &context.Say(target.parserCharBlock,
-            "Label '%u' is not a branch target or FORMAT"_warn_en_US,
-            SayLabel(label));
+        msg =
+            &context
+                 .Say(target.parserCharBlock,
+                     "Label '%u' is not a branch target or FORMAT"_warn_en_US,
+                     SayLabel(label))
+                 .set_languageFeature(common::LanguageFeature::BadBranchTarget);
       }
       if (msg) {
         msg->Attach(stmt.parserCharBlock, "ASSIGN statement use of '%u'"_en_US,
diff --git a/flang/lib/Semantics/resolve-names-utils.cpp b/flang/lib/Semantics/resolve-names-utils.cpp
index 39392257938d6..b8ce8d14a33fa 100644
--- a/flang/lib/Semantics/resolve-names-utils.cpp
+++ b/flang/lib/Semantics/resolve-names-utils.cpp
@@ -435,6 +435,7 @@ void EquivalenceSets::FinishSet(const parser::CharBlock &source) {
 // set.
 bool EquivalenceSets::CheckCanEquivalence(
     const parser::CharBlock &source, const Symbol &sym1, const Symbol &sym2) {
+  std::optional<common::LanguageFeature> feature;
   std::optional<parser::MessageFixedText> msg;
   const DeclTypeSpec *type1{sym1.GetType()};
   const DeclTypeSpec *type2{sym2.GetType()};
@@ -453,24 +454,19 @@ bool EquivalenceSets::CheckCanEquivalence(
   } else if (!(isAnyNum1 || isChar1) &&
       !(isAnyNum2 || isChar2)) { // C8110 - C8113
     if (AreTkCompatibleTypes(type1, type2)) {
-      if (context_.ShouldWarn(LanguageFeature::EquivalenceSameNonSequence)) {
-        msg =
-            "nonstandard: Equivalence set contains '%s' and '%s' with same "
-            "type that is neither numeric nor character sequence type"_port_en_US;
-      }
+      msg =
+          "nonstandard: Equivalence set contains '%s' and '%s' with same type that is neither numeric nor character sequence type"_port_en_US;
+      feature = LanguageFeature::EquivalenceSameNonSequence;
     } else {
       msg = "Equivalence set cannot contain '%s' and '%s' with distinct types "
             "that are not both numeric or character sequence types"_err_en_US;
     }
   } else if (isAnyNum1) {
     if (isChar2) {
-      if (context_.ShouldWarn(
-              LanguageFeature::EquivalenceNumericWithCharacter)) {
-        msg = "nonstandard: Equivalence set contains '%s' that is numeric "
-              "sequence type and '%s' that is character"_port_en_US;
-      }
-    } else if (isAnyNum2 &&
-        context_.ShouldWarn(LanguageFeature::EquivalenceNonDefaultNumeric)) {
+      msg =
+          "nonstandard: Equivalence set contains '%s' that is numeric sequence type and '%s' that is character"_port_en_US;
+      feature = LanguageFeature::EquivalenceNumericWithCharacter;
+    } else if (isAnyNum2) {
       if (isDefaultNum1) {
         msg =
             "nonstandard: Equivalence set contains '%s' that is a default "
@@ -479,12 +475,16 @@ bool EquivalenceSets::CheckCanEquivalence(
         msg = "nonstandard: Equivalence set contains '%s' and '%s' that are "
               "numeric sequence types with non-default kinds"_port_en_US;
       }
+      feature = LanguageFeature::EquivalenceNonDefaultNumeric;
     }
   }
-  if (msg &&
-      (!context_.IsInModuleFile(source) ||
-          msg->severity() == parser::Severity::Error)) {
-    context_.Say(source, std::move(*msg), sym1.name(), sym2.name());
+  if (msg) {
+    if (feature) {
+      context_.Warn(
+          *feature, source, std::move(*msg), sym1.name(), sym2.name());
+    } else {
+      context_.Say(source, std::move(*msg), sym1.name(), sym2.name());
+    }
     return false;
   }
   return true;
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 7c26e39892488..e5e03f644f1b0 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -540,15 +540,16 @@ class ScopeHandler : public ImplicitRulesVisitor {
   void SayWithReason(
       const parser::Name &, Symbol &, MessageFixedText &&, Message &&);
   template <typename... A>
-  void SayWithDecl(
+  Message &SayWithDecl(
       const parser::Name &, Symbol &, MessageFixedText &&, A &&...args);
   void SayLocalMustBeVariable(const parser::Name &, Symbol &);
-  void SayDerivedType(const SourceName &, MessageFixedText &&, const Scope &);
-  void Say2(const SourceName &, MessageFixedText &&, const SourceName &,
+  Message &SayDerivedType(
+      const SourceName &, MessageFixedText &&, const Scope &);
+  Message &Say2(const SourceName &, MessageFixedText &&, const SourceName &,
       MessageFixedText &&);
-  void Say2(
+  Message &Say2(
       const SourceName &, MessageFixedText &&, Symbol &, MessageFixedText &&);
-  void Say2(
+  Message &Say2(
       const parser::Name &, MessageFixedText &&, Symbol &, MessageFixedText &&);
 
   // Search for symbol by name in current, parent derived type, and
@@ -1867,11 +1868,10 @@ bool AttrsVisitor::Pre(const parser::Pass &x) {
 bool AttrsVisitor::IsDuplicateAttr(Attr attrName) {
   CHECK(attrs_);
   if (attrs_->test(attrName)) {
-    if (context().ShouldWarn(common::LanguageFeature::RedundantAttribute)) {
-      Say(currStmtSource().value(),
-          "Attribute '%s' cannot be used more than once"_warn_en_US,
-          AttrToString(attrName));
-    }
+    context().Warn(common::LanguageFeature::RedundantAttribute,
+        currStmtSource().value(),
+        "Attribute '%s' cannot be used more than once"_warn_en_US,
+        AttrToString(attrName));
     return true;
   }
   return false;
@@ -2283,7 +2283,7 @@ void ScopeHandler::SayWithReason(const parser::Name &name, Symbol &symbol,
 }
 
 template <typename... A>
-void ScopeHandler::SayWithDecl(const parser::Name &name, Symbol &symbol,
+Message &ScopeHandler::SayWithDecl(const parser::Name &name, Symbol &symbol,
     MessageFixedText &&msg, A &&...args) {
   auto &message{
       Say(name.source, std::move(msg), symbol.name(), std::forward<A>(args)...)
@@ -2299,6 +2299,7 @@ void ScopeHandler::SayWithDecl(const parser::Name &name, Symbol &symbol,
       }
     }
   }
+  return message;
 }
 
 void ScopeHandler::SayLocalMustBeVariable(
@@ -2308,28 +2309,31 @@ void ScopeHandler::SayLocalMustBeVariable(
       " in a locality-spec"_err_en_US);
 }
 
-void ScopeHandler::SayDerivedType(
+Message &ScopeHandler::SayDerivedType(
     const SourceName &name, MessageFixedText &&msg, const Scope &type) {
   const Symbol &typeSymbol{DEREF(type.GetSymbol())};
-  Say(name, std::move(msg), name, typeSymbol.name())
+  return Say(name, std::move(msg), name, typeSymbol.name())
       .Attach(typeSymbol.name(), "Declaration of derived type '%s'"_en_US,
           typeSymbol.name());
 }
-void ScopeHandler::Say2(const SourceName &name1, MessageFixedText &&msg1,
+Message &ScopeHandler::Say2(const SourceName &name1, MessageFixedText &&msg1,
     const SourceName &name2, MessageFixedText &&msg2) {
-  Say(name1, std::move(msg1)).Attach(name2, std::move(msg2), name2);
+  return Say(name1, std::move(msg1)).Attach(name2, std::move(msg2), name2);
 }
-void ScopeHandler::Say2(const SourceName &name, MessageFixedText &&msg1,
+Message &ScopeHandler::Say2(const SourceName &name, MessageFixedText &&msg1,
     Symbol &symbol, MessageFixedText &&msg2) {
   bool isFatal{msg1.IsFatal()};
-  Say2(name, std::move(msg1), symbol.name(), std::move(msg2));
+  Message &result{Say2(name, std::move(msg1), symbol.name(), std::move(msg2))};
   context().SetError(symbol, isFatal);
+  return result;
 }
-void ScopeHandler::Say2(const parser::Name &name, MessageFixedText &&msg1,
+Message &ScopeHandler::Say2(const parser::Name &name, MessageFixedText &&msg1,
     Symbol &symbol, MessageFixedText &&msg2) {
   bool isFatal{msg1.IsFatal()};
-  Say2(name.source, std::move(msg1), symbol.name(), std::move(msg2));
+  Message &result{
+      Say2(name.source, std::move(msg1), symbol.name(), std::move(msg2))};
   context().SetError(symbol, isFatal);
+  return result;
 }
 
 // This is essentially GetProgramUnitContaining(), but it can return
@@ -2613,11 +2617,9 @@ bool ScopeHandler::ImplicitlyTypeForwardRef(Symbol &symbol) {
     return false;
   }
   // TODO: check no INTENT(OUT) if dummy?
-  if (context().ShouldWarn(common::LanguageFeature::ForwardRefImplicitNone)) {
-    Say(symbol.name(),
-        "'%s' was used without (or before) being explicitly typed"_warn_en_US,
-        symbol.name());
-  }
+  context().Warn(common::LanguageFeature::ForwardRefImplicitNone, symbol.name(),
+      "'%s' was used without (or before) being explicitly typed"_warn_en_US,
+      symbol.name());
   symbol.set(Symbol::Flag::Implicit);
   symbol.SetType(*type);
   return true;
@@ -3391,21 +3393,21 @@ void ModuleVisitor::AddAndCheckModuleUse(SourceName name, bool isIntrinsic) {
   if (isIntrinsic) {
     if (auto iter{nonIntrinsicUses_.find(name)};
         iter != nonIntrinsicUses_.end()) {
-      if (context().ShouldWarn(common::LanguageFeature::MiscUseExtensions)) {
-        Say(name,
-            "Should not USE the intrinsic module '%s' in the same scope as a USE of the non-intrinsic module"_port_en_US,
-            name)
-            .Attach(*iter, "Previous USE of '%s'"_en_US, *iter);
+      if (auto *msg{context().Warn(common::LanguageFeature::MiscUseExtensions,
+              name,
+              "Should not USE the intrinsic module '%s' in the same scope as a USE of the non-intrinsic module"_port_en_US,
+              name)}) {
+        msg->Attach(*iter, "Previous USE of '%s'"_en_US, *iter);
       }
     }
     intrinsicUses_.insert(name);
   } else {
     if (auto iter{intrinsicUses_.find(name)}; iter != intrinsicUses_.end()) {
-      if (context().ShouldWarn(common::LanguageFeature::MiscUseExtensions)) {
-        Say(name,
-            "Should not USE the non-intrinsic module '%s' in the same scope as a USE of the intrinsic module"_port_en_US,
-            name)
-            .Attach(*iter, "Previous USE of '%s'"_en_US, *iter);
+      if (auto *msg{context().Warn(common::LanguageFeature::MiscUseExtensions,
+              name,
+              "Should not USE the non-intrinsic module '%s' in the same scope as a USE of the intrinsic module"_port_en_US,
+              name)}) {
+        msg->Attach(*iter, "Previous USE of '%s'"_en_US, *iter);
       }
     }
     nonIntrinsicUses_.insert(name);
@@ -3675,11 +3677,11 @@ void InterfaceVisitor::CheckGenericProcedures(Symbol &generic) {
   ResolveSpecificsInGeneric(generic, true);
   auto &details{generic.get<GenericDetails>()};
   if (auto *proc{details.CheckSpecific()}) {
-    if (context().ShouldWarn(common::UsageWarning::HomonymousSpecific)) {
-      Say(proc->name().begin() > generic.name().begin() ? proc->name()
-                                                        : generic.name(),
-          "'%s' should not be the name of both a generic interface and a procedure unless it is a specific procedure of the generic"_warn_en_US);
-    }
+    context().Warn(common::UsageWarning::HomonymousSpecific,
+        proc->name().begin() > generic.name().begin() ? proc->name()
+                                                      : generic.name(),
+        "'%s' should not be the name of both a generic interface and a procedure unless it is a specific procedure of the generic"_warn_en_US,
+        generic.name());
   }
   auto &specifics{details.specificProcs()};
   if (specifics.empty()) {
@@ -3697,19 +3699,23 @@ void InterfaceVisitor::CheckGenericProcedures(Symbol &generic) {
       subroutine = &specific;
       if (details.derivedType() &&
           context().ShouldWarn(
-              common::LanguageFeature::SubroutineAndFunctionSpecifics)) {
+              common::LanguageFeature::SubroutineAndFunctionSpecifics) &&
+          !InModuleFile()) {
         SayDerivedType(generic.name(),
             "Generic interface '%s' should only contain functions due to derived type with same name"_warn_en_US,
-            *details.derivedType()->GetUltimate().scope());
+            *details.derivedType()->GetUltimate().scope())
+            .set_languageFeature(
+                common::LanguageFeature::SubroutineAndFunctionSpecifics);
       }
     }
-    if (function && subroutine) {
-      if (context().ShouldWarn(common::LanguageFeature::
-                  SubroutineAndFunctionSpecifics)) { // C1514
-        auto &msg{Say(generic.name(),
-            "Generic interface '%s' has both a function and a subroutine"_warn_en_US)};
-        msg.Attach(function->name(), "Function declaration"_en_US);
-        msg.Attach(subroutine->name(), "Subroutine declaration"_en_US);
+    if (function && subroutine) { // F'2023 C1514
+      if (auto *msg{context().Warn(
+              common::LanguageFeature::SubroutineAndFunctionSpecifics,
+              generic.name(),
+              "Generic interface '%s' has both a function and a subroutine"_warn_en_US,
+              generic.name())}) {
+        msg->Attach(function->name(), "Function declaration"_en_US)
+            .Attach(subroutine->name(), "Subroutine declaration"_en_US);
       }
       break;
     }
@@ -3738,11 +3744,10 @@ bool SubprogramVisitor::HandleStmtFunction(const parser::StmtFunctionStmt &x) {
       return false;
     }
     if (IsHostAssociated(*symbol, currScope())) {
-      if (context().ShouldWarn(
-              common::LanguageFeature::StatementFunctionExtensions)) {
-        Say(name,
-            "Name '%s' from host scope should have a type declaration before its local statement function definition"_port_en_US);
-      }
+      context().Warn(common::LanguageFeature::StatementFunctionExtensions,
+          name.source,
+          "Name '%s' from host scope should have a type declaration before its local statement function definition"_port_en_US,
+          name.source);
       MakeSymbol(name, Attrs{}, UnknownDetails{});
     } else if (auto *entity{ultimate.detailsIf<EntityDetails>()};
                entity && !ultimate.has<ProcEntityDetails>()) {
@@ -3844,12 +3849,10 @@ bool SubprogramVisitor::Pre(const parser::PrefixSpec::Attributes &attrs) {
             (*current == common::CUDASubprogramAttrs::HostDevice &&
                 (attr == common::CUDASubprogramAttrs::Host ||
                     attr == common::CUDASubprogramAttrs::Device))) {
-          if (context().ShouldWarn(
-                  common::LanguageFeature::RedundantAttribute)) {
-            Say(currStmtSource().value(),
-                "ATTRIBUTES(%s) appears more than once"_warn_en_US,
-                common::EnumToString(attr));
-          }
+          context().Warn(common::LanguageFeature::RedundantAttribute,
+              currStmtSource().value(),
+              "ATTRIBUTES(%s) appears more than once"_warn_en_US,
+              common::EnumToString(attr));
         } else if ((attr == common::CUDASubprogramAttrs::Host ||
                        attr == common::CUDASubprogramAttrs::Device) &&
             (*current == common::CUDASubprogramAttrs::Host ||
@@ -4054,13 +4057,12 @@ void SubprogramVisitor::Post(const parser::FunctionStmt &stmt) {
   }
   // C1560.
   if (info.resultName && !distinctResultName) {
-    if (context().ShouldWarn(common::UsageWarning::HomonymousResult)) {
-      Say(info.resultName->source,
-          "The function name should not appear in RESULT; references to '%s' "
-          "inside the function will be considered as references to the "
-          "result only"_warn_en_US,
-          name.source);
-    }
+    context().Warn(common::UsageWarning::HomonymousResult,
+        info.resultName->source,
+        "The function name should not appear in RESULT; references to '%s' "
+        "inside the function will be considered as references to the "
+        "result only"_warn_en_US,
+        name.source);
     // RESULT name was ignored above, the only side effect from doing so will be
     // the inability to make recursive calls. The related parser::Name is still
     // resolved to the created function result symbol because every parser::Name
@@ -4468,10 +4470,9 @@ bool SubprogramVisitor::HandlePreviousCalls(
       if (symbol.attrs().test(Attr::EXTERNAL) &&
           !symbol.implicitAttrs().test(Attr::EXTERNAL)) {
         // Warn if external statement previously declared.
-        if (context().ShouldWarn(common::LanguageFeature::RedundantAttribute)) {
-          Say(name,
-              "EXTERNAL attribute was already specified on '%s'"_warn_en_US);
-        }
+        context().Warn(common::LanguageFeature::RedundantAttribute, name.source,
+            "EXTERNAL attribute was already specified on '%s'"_warn_en_US,
+            name.source);
       } else if (symbol.test(other)) {
         Say2(name,
             subpFlag == Symbol::Flag::Function
@@ -4924,11 +4925,10 @@ bool DeclarationVisitor::Pre(const parser::ExternalStmt &x) {
       if (auto *details{symbol->detailsIf<SubprogramDetails>()}) {
         if (details->isInterface()) {
           // Warn if interface previously declared.
-          if (context().ShouldWarn(
-                  common::LanguageFeature::RedundantAttribute)) {
-            Say(name,
-                "EXTERNAL attribute was already specified on '%s'"_warn_en_US);
-          }
+          context().Warn(common::LanguageFeature::RedundantAttribute,
+              name.source,
+              "EXTERNAL attribute was already specified on '%s'"_warn_en_US,
+              name.source);
         }
       } else {
         SayWithDecl(
@@ -4973,14 +4973,12 @@ void DeclarationVisitor::DeclareIntrinsic(const parser::Name &name) {
     if (symbol.GetType()) {
       // These warnings are worded so that they should make sense in either
       // order.
-      if (context().ShouldWarn(
-              common::UsageWarning::IgnoredIntrinsicFunctionType)) {
-        Say(symbol.name(),
-            "Explicit type declaration ignored for intrinsic function '%s'"_warn_en_US,
-            symbol.name())
-            .Attach(name.source,
-                "INTRINSIC statement for explicitly-typed '%s'"_en_US,
-                name.source);
+      if (auto *msg{context().Warn(
+              common::UsageWarning::IgnoredIntrinsicFunctionType, symbol.name(),
+              "Explicit type declaration ignored for intrinsic function '%s'"_warn_en_US,
+              symbol.name())}) {
+        msg->Attach(name.source,
+            "INTRINSIC statement for explicitly-typed '%s'"_en_US, name.source);
       }
     }
     if (!symbol.test(Symbol::Flag::Function) &&
@@ -5047,11 +5045,10 @@ Symbol &DeclarationVisitor::HandleAttributeStmt(
     }
   } else if (symbol && symbol->has<UseDetails>()) {
     if (symbol->GetUltimate().attrs().test(attr)) {
-      if (context().ShouldWarn(common::LanguageFeature::RedundantAttribute)) {
-        Say(currStmtSource().value(),
-            "Use-associated '%s' already has '%s' attribute"_warn_en_US,
-            name.source, EnumToString(attr));
-      }
+      context().Warn(common::LanguageFeature::RedundantAttribute,
+          currStmtSource().value(),
+          "Use-associated '%s' already has '%s' attribute"_warn_en_US,
+          name.source, EnumToString(attr));
     } else {
       Say(currStmtSource().value(),
           "Cannot change %s attribute on use-associated '%s'"_err_en_US,
@@ -5193,10 +5190,9 @@ Symbol &DeclarationVisitor::DeclareObjectEntity(
           context().SetError(symbol);
         }
       } else if (MustBeScalar(symbol)) {
-        if (context().ShouldWarn(common::UsageWarning::PreviousScalarUse)) {
-          Say(name,
-              "'%s' appeared earlier as a scalar actual argument to a specification function"_warn_en_US);
-        }
+        context().Warn(common::UsageWarning::PreviousScalarUse, name.source,
+            "'%s' appeared earlier as a scalar actual argument to a specification function"_warn_en_US,
+            name.source);
       } else if (details->init() || symbol.test(Symbol::Flag::InDataStmt)) {
         Say(name, "'%s' was initialized earlier as a scalar"_err_en_US);
       } else {
@@ -5554,10 +5550,8 @@ bool DeclarationVisitor::Pre(const parser::DerivedTypeDef &x) {
     details.set_sequence(true);
     if (componentDefs.empty()) {
       // F'2023 C745 - not enforced by any compiler
-      if (context().ShouldWarn(common::LanguageFeature::EmptySequenceType)) {
-        Say(stmt.source,
-            "A sequence type should have at least one component"_warn_en_US);
-      }
+      context().Warn(common::LanguageFeature::EmptySequenceType, stmt.source,
+          "A sequence type should have at least one component"_warn_en_US);
     }
     if (!details.paramDeclOrder().empty()) { // C740
       Say(stmt.source,
@@ -5683,17 +5677,15 @@ bool DeclarationVisitor::Pre(const parser::PrivateStmt &) {
   } else if (!derivedTypeInfo_.privateComps) {
     derivedTypeInfo_.privateComps = true;
   } else { // C738
-    if (context().ShouldWarn(common::LanguageFeature::RedundantAttribute)) {
-      Say("PRIVATE should not appear more than once in derived type components"_warn_en_US);
-    }
+    context().Warn(common::LanguageFeature::RedundantAttribute,
+        "PRIVATE should not appear more than once in derived type components"_warn_en_US);
   }
   return false;
 }
 bool DeclarationVisitor::Pre(const parser::SequenceStmt &) {
   if (derivedTypeInfo_.sequence) { // C738
-    if (context().ShouldWarn(common::LanguageFeature::RedundantAttribute)) {
-      Say("SEQUENCE should not appear more than once in derived type components"_warn_en_US);
-    }
+    context().Warn(common::LanguageFeature::RedundantAttribute,
+        "SEQUENCE should not appear more than once in derived type components"_warn_en_US);
   }
   derivedTypeInfo_.sequence = true;
   return false;
@@ -5800,9 +5792,8 @@ bool DeclarationVisitor::Pre(const parser::DataComponentDefStmt &x) {
           !InModuleFile()) {
         if (GetAttrs().test(Attr::POINTER) &&
             context().IsEnabled(common::LanguageFeature::PointerInSeqType)) {
-          if (context().ShouldWarn(common::LanguageFeature::PointerInSeqType)) {
-            Say("A sequence type data component that is a pointer to a non-sequence type is not standard"_port_en_US);
-          }
+          context().Warn(common::LanguageFeature::PointerInSeqType,
+              "A sequence type data component that is a pointer to a non-sequence type is not standard"_port_en_US);
         } else {
           Say("A sequence type data component must either be of an intrinsic type or a derived sequence type"_err_en_US);
         }
@@ -6219,11 +6210,11 @@ void DeclarationVisitor::Post(const parser::BasedPointer &bp) {
     }
     if (const auto *pointeeType{pointee->GetType()}) {
       if (const auto *derived{pointeeType->AsDerived()}) {
-        if (!IsSequenceOrBindCType(derived) &&
-            context().ShouldWarn(
-                common::LanguageFeature::NonSequenceCrayPointee)) {
-          Say(pointeeName,
-              "Type of Cray pointee '%s' is a derived type that is neither SEQUENCE nor BIND(C)"_warn_en_US);
+        if (!IsSequenceOrBindCType(derived)) {
+          context().Warn(common::LanguageFeature::NonSequenceCrayPointee,
+              pointeeName.source,
+              "Type of Cray pointee '%s' is a derived type that is neither SEQUENCE nor BIND(C)"_warn_en_US,
+              pointeeName.source);
         }
       }
     }
@@ -6372,7 +6363,8 @@ void DeclarationVisitor::CheckSaveStmts() {
       if (context().ShouldWarn(common::LanguageFeature::RedundantAttribute)) {
         Say2(name,
             "Explicit SAVE of '%s' is redundant due to global SAVE statement"_warn_en_US,
-            *specPartState_.saveInfo.saveAll, "Global SAVE statement"_en_US);
+            *specPartState_.saveInfo.saveAll, "Global SAVE statement"_en_US)
+            .set_languageFeature(common::LanguageFeature::RedundantAttribute);
       }
     } else if (!IsSaved(*symbol)) {
       SetExplicitAttr(*symbol, Attr::SAVE);
@@ -6418,7 +6410,8 @@ void DeclarationVisitor::AddSaveName(
   if (!pair.second &&
       context().ShouldWarn(common::LanguageFeature::RedundantAttribute)) {
     Say2(name, "SAVE attribute was already specified on '%s'"_warn_en_US,
-        *pair.first, "Previous specification of SAVE attribute"_en_US);
+        *pair.first, "Previous specification of SAVE attribute"_en_US)
+        .set_languageFeature(common::LanguageFeature::RedundantAttribute);
   }
 }
 
@@ -6728,11 +6721,11 @@ void DeclarationVisitor::SetType(
   }
   auto *prevType{symbol.GetType()};
   if (!prevType) {
-    if (symbol.test(Symbol::Flag::InDataStmt) && isImplicitNoneType() &&
-        context().ShouldWarn(
-            common::LanguageFeature::ForwardRefImplicitNoneData)) {
-      Say(name,
-          "'%s' appeared in a DATA statement before its type was declared under IMPLICIT NONE(TYPE)"_port_en_US);
+    if (symbol.test(Symbol::Flag::InDataStmt) && isImplicitNoneType()) {
+      context().Warn(common::LanguageFeature::ForwardRefImplicitNoneData,
+          name.source,
+          "'%s' appeared in a DATA statement before its type was declared under IMPLICIT NONE(TYPE)"_port_en_US,
+          name.source);
     }
     symbol.SetType(type);
   } else if (symbol.has<UseDetails>()) {
@@ -6877,6 +6870,7 @@ bool DeclarationVisitor::OkToAddComponent(
     CHECK(scope->IsDerivedType());
     if (auto *prev{FindInScope(*scope, name.source)}) {
       std::optional<parser::MessageFixedText> msg;
+      std::optional<common::UsageWarning> warning;
       if (context().HasError(*prev)) { // don't pile on
       } else if (extends) {
         msg = "Type cannot be extended as it has a component named"
@@ -6887,24 +6881,28 @@ bool DeclarationVisitor::OkToAddComponent(
                 common::UsageWarning::RedeclaredInaccessibleComponent)) {
           msg =
               "Component '%s' is inaccessibly declared in or as a parent of this derived type"_warn_en_US;
+          warning = common::UsageWarning::RedeclaredInaccessibleComponent;
         }
       } else if (prev->test(Symbol::Flag::ParentComp)) {
-        msg = "'%s' is a parent type of this type and so cannot be"
-              " a component"_err_en_US;
+        msg =
+            "'%s' is a parent type of this type and so cannot be a component"_err_en_US;
       } else if (scope == &currScope()) {
-        msg = "Component '%s' is already declared in this"
-              " derived type"_err_en_US;
+        msg =
+            "Component '%s' is already declared in this derived type"_err_en_US;
       } else {
-        msg = "Component '%s' is already declared in a parent of this"
-              " derived type"_err_en_US;
+        msg =
+            "Component '%s' is already declared in a parent of this derived type"_err_en_US;
       }
       if (msg) {
-        Say2(
-            name, std::move(*msg), *prev, "Previous declaration of '%s'"_en_US);
+        auto &said{Say2(name, std::move(*msg), *prev,
+            "Previous declaration of '%s'"_en_US)};
         if (msg->severity() == parser::Severity::Error) {
           Resolve(name, *prev);
           return false;
         }
+        if (warning) {
+          said.set_usageWarning(*warning);
+        }
       }
     }
     if (scope == &currScope() && extends) {
@@ -6946,7 +6944,9 @@ void ConstructVisitor::ResolveIndexName(
         context().ShouldWarn(
             common::LanguageFeature::OddIndexVariableRestrictions)) {
       SayWithDecl(name, *prev,
-          "Index variable '%s' should not also be an index in an enclosing FORALL or DO CONCURRENT"_port_en_US);
+          "Index variable '%s' should not also be an index in an enclosing FORALL or DO CONCURRENT"_port_en_US)
+          .set_languageFeature(
+              common::LanguageFeature::OddIndexVariableRestrictions);
     }
     name.symbol = nullptr;
   }
@@ -6969,13 +6969,17 @@ void ConstructVisitor::ResolveIndexName(
           context().ShouldWarn(
               common::LanguageFeature::OddIndexVariableRestrictions)) {
         SayWithDecl(name, *prev,
-            "Index variable '%s' should be scalar in the enclosing scope"_port_en_US);
+            "Index variable '%s' should be scalar in the enclosing scope"_port_en_US)
+            .set_languageFeature(
+                common::LanguageFeature::OddIndexVariableRestrictions);
       }
     } else if (!prevRoot.has<CommonBlockDetails>() &&
         context().ShouldWarn(
             common::LanguageFeature::OddIndexVariableRestrictions)) {
       SayWithDecl(name, *prev,
-          "Index variable '%s' should be a scalar object or common block if it is present in the enclosing scope"_port_en_US);
+          "Index variable '%s' should be a scalar object or common block if it is present in the enclosing scope"_port_en_US)
+          .set_languageFeature(
+              common::LanguageFeature::OddIndexVariableRestrictions);
     }
   }
   EvaluateExpr(parser::Scalar{parser::Integer{common::Clone(name)}});
@@ -7022,10 +7026,9 @@ bool ConstructVisitor::Pre(const parser::LocalitySpec::Reduce &x) {
 bool ConstructVisitor::Pre(const parser::LocalitySpec::Shared &x) {
   for (const auto &name : x.v) {
     if (!FindSymbol(name)) {
-      if (context().ShouldWarn(common::UsageWarning::ImplicitShared)) {
-        Say(name,
-            "Variable '%s' with SHARED locality implicitly declared"_warn_en_US);
-      }
+      context().Warn(common::UsageWarning::ImplicitShared, name.source,
+          "Variable '%s' with SHARED locality implicitly declared"_warn_en_US,
+          name.source);
     }
     Symbol &prev{FindOrDeclareEnclosingEntity(name)};
     if (PassesSharedLocalityChecks(name, prev)) {
@@ -7403,7 +7406,8 @@ bool ConstructVisitor::CheckDef(const std::optional<parser::Name> &x) {
         if (Symbol *
             other{FindInScopeOrBlockConstructs(InclusiveScope(), x->source)}) {
           SayWithDecl(*x, *other,
-              "The construct name '%s' should be distinct at the subprogram level"_port_en_US);
+              "The construct name '%s' should be distinct at the subprogram level"_port_en_US)
+              .set_languageFeature(common::LanguageFeature::BenignNameClash);
         }
       }
       MakeSymbol(*x, MiscDetails{MiscDetails::Kind::ConstructName});
@@ -7827,10 +7831,9 @@ bool ResolveNamesVisitor::Pre(const parser::ImportStmt &x) {
       scope.add_importName(name.source);
       if (Symbol * symbol{FindInScope(name)}) {
         if (outer->GetUltimate() == symbol->GetUltimate()) {
-          if (context().ShouldWarn(common::LanguageFeature::BenignNameClash)) {
-            Say(name,
-                "The same '%s' is already present in this scope"_port_en_US);
-          }
+          context().Warn(common::LanguageFeature::BenignNameClash, name.source,
+              "The same '%s' is already present in this scope"_port_en_US,
+              name.source);
         } else {
           Say(name,
               "A distinct '%s' is already present in this scope"_err_en_US)
@@ -7924,11 +7927,9 @@ const parser::Name *DeclarationVisitor::ResolveName(const parser::Name &name) {
     }
     if (checkIndexUseInOwnBounds_ &&
         *checkIndexUseInOwnBounds_ == name.source && !InModuleFile()) {
-      if (context().ShouldWarn(common::LanguageFeature::ImpliedDoIndexScope)) {
-        Say(name,
-            "Implied DO index '%s' uses an object of the same name in its bounds expressions"_port_en_US,
-            name.source);
-      }
+      context().Warn(common::LanguageFeature::ImpliedDoIndexScope, name.source,
+          "Implied DO index '%s' uses an object of the same name in its bounds expressions"_port_en_US,
+          name.source);
     }
     return &name;
   }
@@ -8498,13 +8499,13 @@ Symbol &ModuleVisitor::SetAccess(
   if (attrs.HasAny({Attr::PUBLIC, Attr::PRIVATE})) {
     // PUBLIC/PRIVATE already set: make it a fatal error if it changed
     Attr prev{attrs.test(Attr::PUBLIC) ? Attr::PUBLIC : Attr::PRIVATE};
-    if (attr != prev ||
-        context().ShouldWarn(common::LanguageFeature::RedundantAttribute)) {
+    if (attr != prev) {
       Say(name,
-          WithSeverity(
-              "The accessibility of '%s' has already been specified as %s"_warn_en_US,
-              attr != prev ? parser::Severity::Error
-                           : parser::Severity::Warning),
+          "The accessibility of '%s' has already been specified as %s"_err_en_US,
+          MakeOpName(name), EnumToString(prev));
+    } else {
+      context().Warn(common::LanguageFeature::RedundantAttribute, name,
+          "The accessibility of '%s' has already been specified as %s"_warn_en_US,
           MakeOpName(name), EnumToString(prev));
     }
   } else {
@@ -9103,7 +9104,8 @@ void ResolveNamesVisitor::Post(const parser::CompilerDirective &x) {
       }
     }
   } else if (context().ShouldWarn(common::UsageWarning::IgnoredDirective)) {
-    Say(x.source, "Unrecognized compiler directive was ignored"_warn_en_US);
+    Say(x.source, "Unrecognized compiler directive was ignored"_warn_en_US)
+        .set_usageWarning(common::UsageWarning::IgnoredDirective);
   }
 }
 
@@ -9165,7 +9167,8 @@ bool ResolveNamesVisitor::Pre(const parser::Program &x) {
       if (auto iter{uses.find(name)}; iter != uses.end()) {
         if (context().ShouldWarn(common::LanguageFeature::MiscUseExtensions)) {
           Say(name,
-              "A USE statement referencing module '%s' appears earlier in this compilation unit"_port_en_US)
+              "A USE statement referencing module '%s' appears earlier in this compilation unit"_port_en_US,
+              name)
               .Attach(*iter, "First USE of module"_en_US);
         }
         disordered = true;
@@ -9203,7 +9206,7 @@ bool ResolveNamesVisitor::Pre(const parser::Program &x) {
       }
     }
     if (!ok) {
-      parser::Message *msg{nullptr};
+      Message *msg{nullptr};
       for (const auto &pair : modules) {
         if (msg) {
           msg->Attach(pair.first, "Module in a cycle"_en_US);
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index 1f2980b07b3e0..637088ff0171c 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -181,7 +181,7 @@ static void WarnUndefinedFunctionResult(
             }
           }
           if (!wasDefined) {
-            context.Say(
+            context.Warn(common::UsageWarning::UndefinedFunctionResult,
                 symbol->name(), "Function result is never defined"_warn_en_US);
           }
         }
@@ -222,10 +222,7 @@ static bool PerformStatementSemantics(
     SemanticsVisitor<CUDAChecker>{context}.Walk(program);
   }
   if (!context.messages().AnyFatalError()) {
-    // Do this if all messages are only warnings
-    if (context.ShouldWarn(common::UsageWarning::UndefinedFunctionResult)) {
-      WarnUndefinedFunctionResult(context, context.globalScope());
-    }
+    WarnUndefinedFunctionResult(context, context.globalScope());
   }
   if (!context.AnyFatalError()) {
     pass2.CompileDataInitializationsIntoInitializers();
@@ -285,15 +282,15 @@ class CommonBlockMap {
           info.initialization = common;
         }
       }
-      if (common.size() != info.biggestSize->size() && !common.name().empty() &&
-          context.ShouldWarn(common::LanguageFeature::DistinctCommonSizes)) {
-        context
-            .Say(common.name(),
+      if (common.size() != info.biggestSize->size() && !common.name().empty()) {
+        if (auto *msg{context.Warn(common::LanguageFeature::DistinctCommonSizes,
+                common.name(),
                 "A named COMMON block should have the same size everywhere it appears (%zd bytes here)"_port_en_US,
-                common.size())
-            .Attach(info.biggestSize->name(),
-                "Previously defined with a size of %zd bytes"_en_US,
-                info.biggestSize->size());
+                common.size())}) {
+          msg->Attach(info.biggestSize->name(),
+              "Previously defined with a size of %zd bytes"_en_US,
+              info.biggestSize->size());
+        }
       }
       if (common.size() > info.biggestSize->size()) {
         info.biggestSize = common;
@@ -473,22 +470,28 @@ void SemanticsContext::PopConstruct() {
   constructStack_.pop_back();
 }
 
-void SemanticsContext::CheckIndexVarRedefine(const parser::CharBlock &location,
-    const Symbol &variable, parser::MessageFixedText &&message) {
+parser::Message *SemanticsContext::CheckIndexVarRedefine(
+    const parser::CharBlock &location, const Symbol &variable,
+    parser::MessageFixedText &&message) {
   const Symbol &symbol{ResolveAssociations(variable)};
   auto it{activeIndexVars_.find(symbol)};
   if (it != activeIndexVars_.end()) {
     std::string kind{EnumToString(it->second.kind)};
-    Say(location, std::move(message), kind, symbol.name())
-        .Attach(it->second.location, "Enclosing %s construct"_en_US, kind);
+    return &Say(location, std::move(message), kind, symbol.name())
+                .Attach(
+                    it->second.location, "Enclosing %s construct"_en_US, kind);
+  } else {
+    return nullptr;
   }
 }
 
 void SemanticsContext::WarnIndexVarRedefine(
     const parser::CharBlock &location, const Symbol &variable) {
   if (ShouldWarn(common::UsageWarning::IndexVarRedefinition)) {
-    CheckIndexVarRedefine(location, variable,
-        "Possible redefinition of %s variable '%s'"_warn_en_US);
+    if (auto *msg{CheckIndexVarRedefine(location, variable,
+            "Possible redefinition of %s variable '%s'"_warn_en_US)}) {
+      msg->set_usageWarning(common::UsageWarning::IndexVarRedefinition);
+    }
   }
 }
 
@@ -642,8 +645,8 @@ bool Semantics::Perform() {
       CanonicalizeAcc(context_.messages(), program_) &&
       CanonicalizeOmp(context_.messages(), program_) &&
       CanonicalizeCUDA(program_) &&
-      CanonicalizeDirectives(context_.messages(), program_) &&
       PerformStatementSemantics(context_, program_) &&
+      CanonicalizeDirectives(context_.messages(), program_) &&
       ModFileWriter{context_}
           .set_hermeticModuleFileOutput(hermeticModuleFileOutput_)
           .WriteAll();
diff --git a/flang/runtime/extensions.cpp b/flang/runtime/extensions.cpp
index 4412a9cbeb6d2..50d3c72fe650d 100644
--- a/flang/runtime/extensions.cpp
+++ b/flang/runtime/extensions.cpp
@@ -58,6 +58,24 @@ extern "C" {
 
 namespace Fortran::runtime {
 
+gid_t RTNAME(GetGID)() {
+#ifdef _WIN32
+  // Group IDs don't exist on Windows, return 1 to avoid errors
+  return 1;
+#else
+  return getgid();
+#endif
+}
+
+uid_t RTNAME(GetUID)() {
+#ifdef _WIN32
+  // User IDs don't exist on Windows, return 1 to avoid errors
+  return 1;
+#else
+  return getuid();
+#endif
+}
+
 void GetUsernameEnvVar(const char *envName, char *arg, std::int64_t length) {
   Descriptor name{*Descriptor::Create(
       1, std::strlen(envName) + 1, const_cast<char *>(envName), 0)};
@@ -66,6 +84,7 @@ void GetUsernameEnvVar(const char *envName, char *arg, std::int64_t length) {
   RTNAME(GetEnvVariable)
   (name, &value, nullptr, false, nullptr, __FILE__, __LINE__);
 }
+
 namespace io {
 // SUBROUTINE FLUSH(N)
 //   FLUSH N
diff --git a/flang/test/Fir/array-copies-pointers.fir b/flang/test/Fir/array-copies-pointers.fir
index 490c1c1055e18..a304e5f84b9e1 100644
--- a/flang/test/Fir/array-copies-pointers.fir
+++ b/flang/test/Fir/array-copies-pointers.fir
@@ -202,7 +202,7 @@ func.func @derived_whose_component_may_be_aliased(%arg0: !fir.box<!fir.array<4x!
 // ALL: %[[ALLOC:.*]] = fir.allocmem !fir.array<?xf32>
 // ALL-COUNT-3: fir.do_loop
 // ALL: fir.freemem %[[ALLOC]] : !fir.heap<!fir.array<?xf32>>
-func.func @complex_real_aliasing(%arg0: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, %arg1: !fir.ref<!fir.array<4x!fir.complex<4>>> {fir.target}) {
+func.func @complex_real_aliasing(%arg0: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, %arg1: !fir.ref<!fir.array<4xcomplex<f32>>> {fir.target}) {
   %c4 = arith.constant 4 : index
   %0 = fir.load %arg0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
   %c0 = arith.constant 0 : index
@@ -213,7 +213,7 @@ func.func @complex_real_aliasing(%arg0: !fir.ref<!fir.box<!fir.ptr<!fir.array<?x
   %4 = fir.shape %c4 : (index) -> !fir.shape<1>
   %c1 = arith.constant 1 : index
   %5 = fir.slice %c1, %c4, %c1 path %c0_i32 : (index, index, index, i32) -> !fir.slice<1>
-  %6 = fir.array_load %arg1(%4) [%5] : (!fir.ref<!fir.array<4x!fir.complex<4>>>, !fir.shape<1>, !fir.slice<1>) -> !fir.array<4xf32>
+  %6 = fir.array_load %arg1(%4) [%5] : (!fir.ref<!fir.array<4xcomplex<f32>>>, !fir.shape<1>, !fir.slice<1>) -> !fir.array<4xf32>
   %7 = arith.subi %c4, %c1 : index
   %8 = fir.do_loop %arg2 = %c0 to %7 step %c1 unordered iter_args(%arg3 = %3) -> (!fir.array<?xf32>) {
     %9 = fir.array_fetch %6, %arg2 : (!fir.array<4xf32>, index) -> f32
diff --git a/flang/test/Fir/compare.fir b/flang/test/Fir/compare.fir
index 045a50d6bec71..2bd00bb85c429 100644
--- a/flang/test/Fir/compare.fir
+++ b/flang/test/Fir/compare.fir
@@ -1,17 +1,17 @@
 // RUN: fir-opt %s | tco --target=x86_64-unknown-linux-gnu | FileCheck %s
 
 // CHECK-LABEL: define i1 @cmp3(<2 x float> %0, <2 x float> %1)
-func.func @cmp3(%a : !fir.complex<4>, %b : !fir.complex<4>) -> i1 {
+func.func @cmp3(%a : complex<f32>, %b : complex<f32>) -> i1 {
   // CHECK: fcmp oeq float
-  %1 = fir.cmpc "oeq", %a, %b : !fir.complex<4>
+  %1 = fir.cmpc "oeq", %a, %b : complex<f32>
   return %1 : i1
 }
 
 // CHECK-LABEL: define { double, double } @neg3(double %0, double %1)
-func.func @neg3(%a : !fir.complex<8>) -> !fir.complex<8> {
+func.func @neg3(%a : complex<f64>) -> complex<f64> {
   // CHECK: %[[g2:.*]] = insertvalue { double, double } %
   // CHECK: %[[r3:.*]] = fneg double
   // CHECK: insertvalue { double, double } %[[g2]], double %[[r3]]
-  %1 = fir.negc %a : !fir.complex<8>
-  return %1 : !fir.complex<8>
+  %1 = fir.negc %a : complex<f64>
+  return %1 : complex<f64>
 }
diff --git a/flang/test/Fir/constant.fir b/flang/test/Fir/constant.fir
index 380c6121acf9b..c204b7c426146 100644
--- a/flang/test/Fir/constant.fir
+++ b/flang/test/Fir/constant.fir
@@ -7,15 +7,6 @@ func.func @x() -> !fir.char<1,3> {
   return %1 : !fir.char<1,3>
 }
 
-// CHECK-LABEL: define x86_fp80 @y()
-func.func @y() -> !fir.real<10> {
- %c1 = arith.constant 42.4 :f32
- %0 = fir.convert %c1 : (f32) -> !fir.real<10>
-  // CHECK: ret x86_fp80 0xK4004A9999A0000000000
-  // TODO: What's that number?
-  return %0 : !fir.real<10>
-}
-
 // CHECK-LABEL: define i16 @z()
 func.func @z() -> !fir.logical<2> {
   %1 = arith.constant true
diff --git a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
index 61f18008633d5..aa2f416063fce 100644
--- a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
+++ b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
@@ -793,9 +793,9 @@ func.func @_QPsimple_reduction(%arg0: !fir.ref<!fir.array<100x!fir.logical<4>>>
 // CHECK: llvm.func @_QPs
 // CHECK: omp.atomic.read %{{.*}} = %{{.*}}   : !llvm.ptr, !llvm.struct<(f32, f32)>
 
-func.func @_QPs(%arg0: !fir.ref<!fir.complex<4>> {fir.bindc_name = "x"}) {
-  %0 = fir.alloca !fir.complex<4> {bindc_name = "v", uniq_name = "_QFsEv"}
-  omp.atomic.read %0 = %arg0   : !fir.ref<!fir.complex<4>>, !fir.complex<4>
+func.func @_QPs(%arg0: !fir.ref<complex<f32>> {fir.bindc_name = "x"}) {
+  %0 = fir.alloca complex<f32> {bindc_name = "v", uniq_name = "_QFsEv"}
+  omp.atomic.read %0 = %arg0   : !fir.ref<complex<f32>>, complex<f32>
   return
 }
 
diff --git a/flang/test/Fir/convert-to-llvm.fir b/flang/test/Fir/convert-to-llvm.fir
index a4e8170af036c..0c17d7c25a8c8 100644
--- a/flang/test/Fir/convert-to-llvm.fir
+++ b/flang/test/Fir/convert-to-llvm.fir
@@ -507,9 +507,9 @@ func.func @test_call_return_val() -> i32 {
 // given: (x + iy) + (x' + iy')
 // result: (x + x') + i(y + y')
 
-func.func @fir_complex_add(%a: !fir.complex<16>, %b: !fir.complex<16>) -> !fir.complex<16> {
-  %c = fir.addc %a, %b {fastmath = #arith.fastmath<fast>} : !fir.complex<16>
-  return %c : !fir.complex<16>
+func.func @fir_complex_add(%a: complex<f128>, %b: complex<f128>) -> complex<f128> {
+  %c = fir.addc %a, %b {fastmath = #arith.fastmath<fast>} : complex<f128>
+  return %c : complex<f128>
 }
 
 // CHECK-LABEL: llvm.func @fir_complex_add(
@@ -532,9 +532,9 @@ func.func @fir_complex_add(%a: !fir.complex<16>, %b: !fir.complex<16>) -> !fir.c
 // given: (x + iy) - (x' + iy')
 // result: (x - x') + i(y - y')
 
-func.func @fir_complex_sub(%a: !fir.complex<16>, %b: !fir.complex<16>) -> !fir.complex<16> {
-  %c = fir.subc %a, %b {fastmath = #arith.fastmath<fast>} : !fir.complex<16>
-  return %c : !fir.complex<16>
+func.func @fir_complex_sub(%a: complex<f128>, %b: complex<f128>) -> complex<f128> {
+  %c = fir.subc %a, %b {fastmath = #arith.fastmath<fast>} : complex<f128>
+  return %c : complex<f128>
 }
 
 // CHECK-LABEL: llvm.func @fir_complex_sub(
@@ -557,9 +557,9 @@ func.func @fir_complex_sub(%a: !fir.complex<16>, %b: !fir.complex<16>) -> !fir.c
 // given: (x + iy) * (x' + iy')
 // result: (xx'-yy')+i(xy'+yx')
 
-func.func @fir_complex_mul(%a: !fir.complex<16>, %b: !fir.complex<16>) -> !fir.complex<16> {
-  %c = fir.mulc %a, %b {fastmath = #arith.fastmath<fast>} : !fir.complex<16>
-  return %c : !fir.complex<16>
+func.func @fir_complex_mul(%a: complex<f128>, %b: complex<f128>) -> complex<f128> {
+  %c = fir.mulc %a, %b {fastmath = #arith.fastmath<fast>} : complex<f128>
+  return %c : complex<f128>
 }
 
 // CHECK-LABEL: llvm.func @fir_complex_mul(
@@ -586,9 +586,9 @@ func.func @fir_complex_mul(%a: !fir.complex<16>, %b: !fir.complex<16>) -> !fir.c
 // given: (x + iy) / (x' + iy')
 // result: ((xx'+yy')/d) + i((yx'-xy')/d) where d = x'x' + y'y'
 
-func.func @fir_complex_div(%a: !fir.complex<16>, %b: !fir.complex<16>) -> !fir.complex<16> {
-  %c = fir.divc %a, %b {fastmath = #arith.fastmath<fast>} : !fir.complex<16>
-  return %c : !fir.complex<16>
+func.func @fir_complex_div(%a: complex<f128>, %b: complex<f128>) -> complex<f128> {
+  %c = fir.divc %a, %b {fastmath = #arith.fastmath<fast>} : complex<f128>
+  return %c : complex<f128>
 }
 
 // CHECK-LABEL: llvm.func @fir_complex_div(
@@ -620,9 +620,9 @@ func.func @fir_complex_div(%a: !fir.complex<16>, %b: !fir.complex<16>) -> !fir.c
 // given: -(x + iy)
 // result: -x - iy
 
-func.func @fir_complex_neg(%a: !fir.complex<16>) -> !fir.complex<16> {
-  %c = fir.negc %a : !fir.complex<16>
-  return %c : !fir.complex<16>
+func.func @fir_complex_neg(%a: complex<f128>) -> complex<f128> {
+  %c = fir.negc %a : complex<f128>
+  return %c : complex<f128>
 }
 
 // CHECK-LABEL: llvm.func @fir_complex_neg(
@@ -639,8 +639,8 @@ func.func @fir_complex_neg(%a: !fir.complex<16>) -> !fir.complex<16> {
 
 // Test FIR complex compare conversion
 
-func.func @compare_complex_eq(%a : !fir.complex<8>, %b : !fir.complex<8>) -> i1 {
-  %r = fir.cmpc "oeq", %a, %b : !fir.complex<8>
+func.func @compare_complex_eq(%a : complex<f64>, %b : complex<f64>) -> i1 {
+  %r = fir.cmpc "oeq", %a, %b : complex<f64>
   return %r : i1
 }
 
@@ -656,8 +656,8 @@ func.func @compare_complex_eq(%a : !fir.complex<8>, %b : !fir.complex<8>) -> i1
 // CHECK: [[RES:%.*]] = llvm.and [[RESR]], [[RESI]] : i1
 // CHECK: return [[RES]] : i1
 
-func.func @compare_complex_ne(%a : !fir.complex<8>, %b : !fir.complex<8>) -> i1 {
-  %r = fir.cmpc "une", %a, %b {fastmath = #arith.fastmath<fast>} : !fir.complex<8>
+func.func @compare_complex_ne(%a : complex<f64>, %b : complex<f64>) -> i1 {
+  %r = fir.cmpc "une", %a, %b {fastmath = #arith.fastmath<fast>} : complex<f64>
   return %r : i1
 }
 
@@ -673,8 +673,8 @@ func.func @compare_complex_ne(%a : !fir.complex<8>, %b : !fir.complex<8>) -> i1
 // CHECK: [[RES:%.*]] = llvm.or [[RESR]], [[RESI]] : i1
 // CHECK: return [[RES]] : i1
 
-func.func @compare_complex_other(%a : !fir.complex<8>, %b : !fir.complex<8>) -> i1 {
-  %r = fir.cmpc "ogt", %a, %b {fastmath = #arith.fastmath<fast>} : !fir.complex<8>
+func.func @compare_complex_other(%a : complex<f64>, %b : complex<f64>) -> i1 {
+  %r = fir.cmpc "ogt", %a, %b {fastmath = #arith.fastmath<fast>} : complex<f64>
   return %r : i1
 }
 
@@ -780,9 +780,9 @@ func.func @convert_from_ref(%arg0 : !fir.ref<i32>) {
 
 // Test `fir.convert` operation conversion between fir.complex types.
 
-func.func @convert_complex4(%arg0 : !fir.complex<4>) -> !fir.complex<8> {
-  %0 = fir.convert %arg0 : (!fir.complex<4>) -> !fir.complex<8>
-  return %0 : !fir.complex<8>
+func.func @convert_complex4(%arg0 : complex<f32>) -> complex<f64> {
+  %0 = fir.convert %arg0 : (complex<f32>) -> complex<f64>
+  return %0 : complex<f64>
 }
 
 // CHECK-LABEL: func @convert_complex4(
@@ -798,9 +798,9 @@ func.func @convert_complex4(%arg0 : !fir.complex<4>) -> !fir.complex<8> {
 
 // Test `fir.convert` operation conversion between fir.complex types.
 
-func.func @convert_complex16(%arg0 : !fir.complex<16>) -> !fir.complex<2> {
-  %0 = fir.convert %arg0 : (!fir.complex<16>) -> !fir.complex<2>
-  return %0 : !fir.complex<2>
+func.func @convert_complex16(%arg0 : complex<f128>) -> complex<f16> {
+  %0 = fir.convert %arg0 : (complex<f128>) -> complex<f16>
+  return %0 : complex<f16>
 }
 
 // CHECK-LABEL: func @convert_complex16(
@@ -816,38 +816,6 @@ func.func @convert_complex16(%arg0 : !fir.complex<16>) -> !fir.complex<2> {
 
 // -----
 
-// Test constc.
-
-func.func @test_constc4() -> !fir.complex<4> {
-  %0 = fir.constc (#fir.real<4, 1.4>, #fir.real<4, 2.3>) : !fir.complex<4>
-  return %0 : !fir.complex<4>
-}
-
-// CHECK-LABEL: @test_constc4
-// CHECK-SAME: () -> !llvm.struct<(f32, f32)>
-// CHECK-DAG: [[rp:%.*]] = llvm.mlir.constant(1.400000e+00 : f32) : f32
-// CHECK-DAG: [[ip:%.*]] = llvm.mlir.constant(2.300000e+00 : f32) : f32
-// CHECK: [[undef:%.*]] = llvm.mlir.undef : !llvm.struct<(f32, f32)>
-// CHECK: [[withr:%.*]] = llvm.insertvalue [[rp]], [[undef]][0] : !llvm.struct<(f32, f32)>
-// CHECK: [[full:%.*]] = llvm.insertvalue [[ip]], [[withr]][1] : !llvm.struct<(f32, f32)>
-// CHECK: return [[full]] : !llvm.struct<(f32, f32)>
-
-func.func @test_constc8() -> !fir.complex<8> {
-  %0 = fir.constc (#fir.real<8, 1.8>, #fir.real<8, 2.3>) : !fir.complex<8>
-  return %0 : !fir.complex<8>
-}
-
-// CHECK-LABEL: @test_constc8
-// CHECK-SAME: () -> !llvm.struct<(f64, f64)>
-// CHECK-DAG: [[rp:%.*]] = llvm.mlir.constant(1.800000e+00 : f64) : f64
-// CHECK-DAG: [[ip:%.*]] = llvm.mlir.constant(2.300000e+00 : f64) : f64
-// CHECK: [[undef:%.*]] = llvm.mlir.undef : !llvm.struct<(f64, f64)>
-// CHECK: [[withr:%.*]] = llvm.insertvalue [[rp]], [[undef]][0] : !llvm.struct<(f64, f64)>
-// CHECK: [[full:%.*]] = llvm.insertvalue [[ip]], [[withr]][1] : !llvm.struct<(f64, f64)>
-// CHECK: return [[full]] : !llvm.struct<(f64, f64)>
-
-// -----
-
 // Test `fir.store` --> `llvm.store` conversion
 
 func.func @test_store_index(%val_to_store : index, %addr : !fir.ref<index>) {
@@ -1748,8 +1716,8 @@ func.func @embox_typecode2(%arg0: !fir.ref<f128>) {
 // CHECK: %[[TYPE_CODE_F128_I8:.*]] = llvm.trunc %[[TYPE_CODE_F128]] : i32 to i8
 // CHECK: %{{.*}} = llvm.insertvalue %[[TYPE_CODE_F128_I8]], %{{.*}}[4] : !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}})>
 
-func.func @embox_typecode3(%arg0: !fir.ref<!fir.complex<4>>) {
-  %0 = fir.embox %arg0 : (!fir.ref<!fir.complex<4>>) -> !fir.box<!fir.ptr<!fir.complex<4>>>
+func.func @embox_typecode3(%arg0: !fir.ref<complex<f32>>) {
+  %0 = fir.embox %arg0 : (!fir.ref<complex<f32>>) -> !fir.box<!fir.ptr<complex<f32>>>
   return
 }
 
@@ -2489,9 +2457,9 @@ func.func @foo(%arg0: !fir.box<!fir.array<?x!fir.type<t{i:i32,c:!fir.char<1,10>}
 
 // 1. COMPLEX TYPE (`fir.complex` is a special case)
 // Complex type wrapped in `fir.ref`
-func.func @coordinate_ref_complex(%arg0: !fir.ref<!fir.complex<16>>) {
+func.func @coordinate_ref_complex(%arg0: !fir.ref<complex<f128>>) {
   %arg1 = llvm.mlir.constant(0 : i32) : i32
-  %p = fir.coordinate_of %arg0, %arg1 : (!fir.ref<!fir.complex<16>>, i32) -> !fir.ref<f32>
+  %p = fir.coordinate_of %arg0, %arg1 : (!fir.ref<complex<f128>>, i32) -> !fir.ref<f32>
   return
 }
 // CHECK-LABEL: llvm.func @coordinate_ref_complex
@@ -2502,9 +2470,9 @@ func.func @coordinate_ref_complex(%arg0: !fir.ref<!fir.complex<16>>) {
 // -----
 
 // Complex type wrapped in `fir.box`
-func.func @coordinate_box_complex(%arg0: !fir.box<!fir.complex<16>>) {
+func.func @coordinate_box_complex(%arg0: !fir.box<complex<f128>>) {
   %arg1 = llvm.mlir.constant(0 : i32) : i32
-  %p = fir.coordinate_of %arg0, %arg1 : (!fir.box<!fir.complex<16>>, i32) -> !fir.ref<f32>
+  %p = fir.coordinate_of %arg0, %arg1 : (!fir.box<complex<f128>>, i32) -> !fir.ref<f32>
   return
 }
 // CHECK-LABEL: llvm.func @coordinate_box_complex
diff --git a/flang/test/Fir/convert.fir b/flang/test/Fir/convert.fir
index 25a318336aa07..3c5c931e95550 100644
--- a/flang/test/Fir/convert.fir
+++ b/flang/test/Fir/convert.fir
@@ -1,15 +1,15 @@
 // RUN: tco --target=x86_64-unknown-linux-gnu %s | FileCheck %s
 
 // CHECK-LABEL: define { double, double } @c(<2 x float> %
-func.func @c(%x : !fir.complex<4>) -> !fir.complex<8> {
+func.func @c(%x : complex<f32>) -> complex<f64> {
 // CHECK: %[[R:.*]] = extractvalue { float, float } %{{.*}}, 0
 // CHECK: %[[I:.*]] = extractvalue { float, float } %{{.*}}, 1
 // CHECK: %[[CR:.*]] = fpext float %[[R]] to double
 // CHECK: %[[CI:.*]] = fpext float %[[I]] to double
 // CHECK: %[[X:.*]] = insertvalue { double, double } undef, double %[[CR]], 0
 // CHECK: insertvalue { double, double } %[[X]], double %[[CI]], 1
-  %1 = fir.convert %x : (!fir.complex<4>) -> !fir.complex<8>
-  return %1 : !fir.complex<8>
+  %1 = fir.convert %x : (complex<f32>) -> complex<f64>
+  return %1 : complex<f64>
 }
 
 func.func @convert_to_llvm_pointer(%x : !fir.llvm_ptr<i32>) -> !llvm.ptr {
diff --git a/flang/test/Fir/dispatch.f90 b/flang/test/Fir/dispatch.f90
index fc935217defa1..2ffdcd5b1884d 100644
--- a/flang/test/Fir/dispatch.f90
+++ b/flang/test/Fir/dispatch.f90
@@ -18,6 +18,7 @@ module dispatch1
     procedure :: proc_with_values => proc_p1
     procedure, nopass :: proc_nopass => proc_nopass_p1
     procedure, pass(this) :: proc_pass => proc_pass_p1
+    procedure, nopass :: z_proc_nopass_bindc => proc_nopass_bindc_p1
   end type
 
   type, extends(p1) :: p2
@@ -30,6 +31,7 @@ module dispatch1
     procedure :: proc_with_values => proc_p2
     procedure, nopass :: proc_nopass => proc_nopass_p2
     procedure, pass(this) :: proc_pass => proc_pass_p2
+    procedure, nopass :: z_proc_nopass_bindc => proc_nopass_bindc_p2
   end type
 
   type, abstract :: a1
@@ -118,16 +120,24 @@ subroutine proc_nopass_p2()
     print*, 'call proc_nopass_p2'
   end subroutine
 
+  subroutine proc_nopass_bindc_p1() bind(c)
+    print*, 'call proc_nopass_bindc_p1'
+  end subroutine
+
+  subroutine proc_nopass_bindc_p2() bind(c)
+    print*, 'call proc_nopass_bindc_p2'
+  end subroutine
+
   subroutine proc_pass_p1(i, this)
     integer :: i
     class(p1) :: this
-    print*, 'call proc_nopass_p1'
+    print*, 'call proc_pass_p1'
   end subroutine
 
   subroutine proc_pass_p2(i, this)
     integer :: i
     class(p2) :: this
-    print*, 'call proc_nopass_p2'
+    print*, 'call proc_pass_p2'
   end subroutine
 
   subroutine display_class(p)
@@ -140,6 +150,7 @@ subroutine display_class(p)
     call p%proc_with_values(2.5)
     call p%proc_nopass()
     call p%proc_pass(1)
+    call p%z_proc_nopass_bindc()
   end subroutine
 
   subroutine no_pass_array(a)
@@ -297,6 +308,10 @@ program test_type_to_class
 ! CHECK: %[[FUNC_PTR:.*]] = fir.convert %[[FUNC_ADDR]] : (i64) -> ((!fir.ref<i32>, [[CLASS]]) -> ())
 ! CHECK: fir.call %[[FUNC_PTR]](%{{.*}}, %[[ARG_DECL]]#0) : (!fir.ref<i32>, [[CLASS]]) -> ()
 
+! Test attributes are propagated from fir.dispatch to fir.call
+! for `call p%z_proc_nopass_bindc()`
+! CHECK: fir.call %{{.*}}() proc_attrs<bind_c> : () -> ()
+
 ! CHECK-LABEL: _QMdispatch1Pno_pass_array
 ! CHECK-LABEL: _QMdispatch1Pno_pass_array_allocatable
 ! CHECK-LABEL: _QMdispatch1Pno_pass_array_pointer
@@ -316,6 +331,7 @@ program test_type_to_class
 ! BT: fir.dt_entry "proc_nopass", @_QMdispatch1Pproc_nopass_p1
 ! BT: fir.dt_entry "proc_pass", @_QMdispatch1Pproc_pass_p1
 ! BT: fir.dt_entry "proc_with_values", @_QMdispatch1Pproc_p1
+! BT: fir.dt_entry "z_proc_nopass_bindc", @proc_nopass_bindc_p1
 ! BT: }
 
 ! BT-LABEL: fir.type_info @_QMdispatch1Ta1
@@ -334,5 +350,6 @@ program test_type_to_class
 ! BT:  fir.dt_entry "proc_nopass", @_QMdispatch1Pproc_nopass_p2
 ! BT:  fir.dt_entry "proc_pass", @_QMdispatch1Pproc_pass_p2
 ! BT:  fir.dt_entry "proc_with_values", @_QMdispatch1Pproc_p2
+! BT:  fir.dt_entry "z_proc_nopass_bindc", @proc_nopass_bindc_p2
 ! BT:  fir.dt_entry "display3", @_QMdispatch1Pdisplay3
 ! BT: }
diff --git a/flang/test/Fir/fir-ops.fir b/flang/test/Fir/fir-ops.fir
index b8ae566e87a1f..5a30858511f0c 100644
--- a/flang/test/Fir/fir-ops.fir
+++ b/flang/test/Fir/fir-ops.fir
@@ -468,44 +468,44 @@ fir.type_info @cpinfo : !fir.type<cpinfo{comp_i:!fir.array<10x20xi32>}> componen
 }
 
 // CHECK-LABEL: func @compare_complex(
-// CHECK-SAME: [[VAL_151:%.*]]: !fir.complex<16>, [[VAL_152:%.*]]: !fir.complex<16>) {
-func.func @compare_complex(%a : !fir.complex<16>, %b : !fir.complex<16>) {
-
-// CHECK: [[VAL_153:%.*]] = fir.cmpc "false", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : !fir.complex<16>
-// CHECK: [[VAL_154:%.*]] = fir.cmpc "oeq", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : !fir.complex<16>
-// CHECK: [[VAL_155:%.*]] = fir.cmpc "ogt", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : !fir.complex<16>
-// CHECK: [[VAL_156:%.*]] = fir.cmpc "oge", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : !fir.complex<16>
-  %d0 = fir.cmpc "false", %a, %b : !fir.complex<16>
-  %d1 = fir.cmpc "oeq", %a, %b : !fir.complex<16>
-  %d2 = fir.cmpc "ogt", %a, %b : !fir.complex<16>
-  %d3 = fir.cmpc "oge", %a, %b : !fir.complex<16>
-
-// CHECK: [[VAL_157:%.*]] = fir.cmpc "olt", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : !fir.complex<16>
-// CHECK: [[VAL_158:%.*]] = fir.cmpc "ole", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : !fir.complex<16>
-// CHECK: [[VAL_159:%.*]] = fir.cmpc "one", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : !fir.complex<16>
-// CHECK: [[VAL_160:%.*]] = fir.cmpc "ord", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : !fir.complex<16>
-  %a0 = fir.cmpc "olt", %a, %b : !fir.complex<16>
-  %a1 = fir.cmpc "ole", %a, %b : !fir.complex<16>
-  %a2 = fir.cmpc "one", %a, %b : !fir.complex<16>
-  %a3 = fir.cmpc "ord", %a, %b : !fir.complex<16>
-
-// CHECK: [[VAL_161:%.*]] = fir.cmpc "ueq", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : !fir.complex<16>
-// CHECK: [[VAL_162:%.*]] = fir.cmpc "ugt", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : !fir.complex<16>
-// CHECK: [[VAL_163:%.*]] = fir.cmpc "uge", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : !fir.complex<16>
-// CHECK: [[VAL_164:%.*]] = fir.cmpc "ult", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : !fir.complex<16>
-  %b0 = fir.cmpc "ueq", %a, %b : !fir.complex<16>
-  %b1 = fir.cmpc "ugt", %a, %b : !fir.complex<16>
-  %b2 = fir.cmpc "uge", %a, %b : !fir.complex<16>
-  %b3 = fir.cmpc "ult", %a, %b : !fir.complex<16>
-
-// CHECK: [[VAL_165:%.*]] = fir.cmpc "ule", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : !fir.complex<16>
-// CHECK: [[VAL_166:%.*]] = fir.cmpc "une", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : !fir.complex<16>
-// CHECK: [[VAL_167:%.*]] = fir.cmpc "uno", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : !fir.complex<16>
-// CHECK: [[VAL_168:%.*]] = fir.cmpc "true", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : !fir.complex<16>
-  %c0 = fir.cmpc "ule", %a, %b : !fir.complex<16>
-  %c1 = fir.cmpc "une", %a, %b : !fir.complex<16>
-  %c2 = fir.cmpc "uno", %a, %b : !fir.complex<16>
-  %c3 = fir.cmpc "true", %a, %b : !fir.complex<16>
+// CHECK-SAME: [[VAL_151:%.*]]: complex<f128>, [[VAL_152:%.*]]: complex<f128>) {
+func.func @compare_complex(%a : complex<f128>, %b : complex<f128>) {
+
+// CHECK: [[VAL_153:%.*]] = fir.cmpc "false", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : complex<f128>
+// CHECK: [[VAL_154:%.*]] = fir.cmpc "oeq", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : complex<f128>
+// CHECK: [[VAL_155:%.*]] = fir.cmpc "ogt", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : complex<f128>
+// CHECK: [[VAL_156:%.*]] = fir.cmpc "oge", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : complex<f128>
+  %d0 = fir.cmpc "false", %a, %b : complex<f128>
+  %d1 = fir.cmpc "oeq", %a, %b : complex<f128>
+  %d2 = fir.cmpc "ogt", %a, %b : complex<f128>
+  %d3 = fir.cmpc "oge", %a, %b : complex<f128>
+
+// CHECK: [[VAL_157:%.*]] = fir.cmpc "olt", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : complex<f128>
+// CHECK: [[VAL_158:%.*]] = fir.cmpc "ole", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : complex<f128>
+// CHECK: [[VAL_159:%.*]] = fir.cmpc "one", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : complex<f128>
+// CHECK: [[VAL_160:%.*]] = fir.cmpc "ord", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : complex<f128>
+  %a0 = fir.cmpc "olt", %a, %b : complex<f128>
+  %a1 = fir.cmpc "ole", %a, %b : complex<f128>
+  %a2 = fir.cmpc "one", %a, %b : complex<f128>
+  %a3 = fir.cmpc "ord", %a, %b : complex<f128>
+
+// CHECK: [[VAL_161:%.*]] = fir.cmpc "ueq", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : complex<f128>
+// CHECK: [[VAL_162:%.*]] = fir.cmpc "ugt", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : complex<f128>
+// CHECK: [[VAL_163:%.*]] = fir.cmpc "uge", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : complex<f128>
+// CHECK: [[VAL_164:%.*]] = fir.cmpc "ult", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : complex<f128>
+  %b0 = fir.cmpc "ueq", %a, %b : complex<f128>
+  %b1 = fir.cmpc "ugt", %a, %b : complex<f128>
+  %b2 = fir.cmpc "uge", %a, %b : complex<f128>
+  %b3 = fir.cmpc "ult", %a, %b : complex<f128>
+
+// CHECK: [[VAL_165:%.*]] = fir.cmpc "ule", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : complex<f128>
+// CHECK: [[VAL_166:%.*]] = fir.cmpc "une", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : complex<f128>
+// CHECK: [[VAL_167:%.*]] = fir.cmpc "uno", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : complex<f128>
+// CHECK: [[VAL_168:%.*]] = fir.cmpc "true", [[VAL_151]], [[VAL_152]] {fastmath = {{.*}}} : complex<f128>
+  %c0 = fir.cmpc "ule", %a, %b : complex<f128>
+  %c1 = fir.cmpc "une", %a, %b : complex<f128>
+  %c2 = fir.cmpc "uno", %a, %b : complex<f128>
+  %c3 = fir.cmpc "true", %a, %b : complex<f128>
 // CHECK: return
 // CHECK: }
   return
@@ -535,21 +535,21 @@ func.func @arith_real(%a : f128, %b : f128) -> f128 {
 }
 
 // CHECK-LABEL: func @arith_complex(
-// CHECK-SAME: [[VAL_179:%.*]]: !fir.complex<16>, [[VAL_180:%.*]]: !fir.complex<16>) -> !fir.complex<16> {
-func.func @arith_complex(%a : !fir.complex<16>, %b : !fir.complex<16>) -> !fir.complex<16> {
-// CHECK: [[VAL_181:%.*]] = fir.negc [[VAL_179]] : !fir.complex<16>
-// CHECK: [[VAL_182:%.*]] = fir.addc [[VAL_180]], [[VAL_181]] : !fir.complex<16>
-// CHECK: [[VAL_183:%.*]] = fir.subc [[VAL_182]], [[VAL_180]] : !fir.complex<16>
-// CHECK: [[VAL_184:%.*]] = fir.mulc [[VAL_181]], [[VAL_183]] : !fir.complex<16>
-// CHECK: [[VAL_185:%.*]] = fir.divc [[VAL_184]], [[VAL_179]] : !fir.complex<16>
-  %1 = fir.negc %a : !fir.complex<16>
-  %2 = fir.addc %b, %1 : !fir.complex<16>
-  %3 = fir.subc %2, %b : !fir.complex<16>
-  %4 = fir.mulc %1, %3 : !fir.complex<16>
-  %5 = fir.divc %4, %a : !fir.complex<16>
-// CHECK: return [[VAL_185]] : !fir.complex<16>
+// CHECK-SAME: [[VAL_179:%.*]]: complex<f128>, [[VAL_180:%.*]]: complex<f128>) -> complex<f128> {
+func.func @arith_complex(%a : complex<f128>, %b : complex<f128>) -> complex<f128> {
+// CHECK: [[VAL_181:%.*]] = fir.negc [[VAL_179]] : complex<f128>
+// CHECK: [[VAL_182:%.*]] = fir.addc [[VAL_180]], [[VAL_181]] : complex<f128>
+// CHECK: [[VAL_183:%.*]] = fir.subc [[VAL_182]], [[VAL_180]] : complex<f128>
+// CHECK: [[VAL_184:%.*]] = fir.mulc [[VAL_181]], [[VAL_183]] : complex<f128>
+// CHECK: [[VAL_185:%.*]] = fir.divc [[VAL_184]], [[VAL_179]] : complex<f128>
+  %1 = fir.negc %a : complex<f128>
+  %2 = fir.addc %b, %1 : complex<f128>
+  %3 = fir.subc %2, %b : complex<f128>
+  %4 = fir.mulc %1, %3 : complex<f128>
+  %5 = fir.divc %4, %a : complex<f128>
+// CHECK: return [[VAL_185]] : complex<f128>
 // CHECK: }
-  return %5 : !fir.complex<16>
+  return %5 : complex<f128>
 }
 
 // CHECK-LABEL: func @character_literal() -> !fir.char<1,13> {
@@ -675,23 +675,6 @@ func.func @test_misc_ops(%arr1 : !fir.ref<!fir.array<?x?xf32>>, %m : index, %n :
   return
 }
 
-// CHECK-LABEL: @test_const_complex
-func.func @test_const_complex() {
- // CHECK-DAG: {{%.*}} = fir.constc(#fir.real<2, i x3000>, #fir.real<2, i x4C40>) : !fir.complex<2>
- // CHECK-DAG: {{%.*}} = fir.constc(#fir.real<3, i x3E80>, #fir.real<3, i x4202>) : !fir.complex<3>
- // CHECK-DAG: {{%.*}} = fir.constc(#fir.real<4, i x3E800000>, #fir.real<4, i x42028000>) : !fir.complex<4>
- // CHECK-DAG: {{%.*}} = fir.constc(#fir.real<8, i x3FD0000000000000>, #fir.real<8, i x4040500000000000>) : !fir.complex<8>
- // CHECK-DAG: {{%.*}} = fir.constc(#fir.real<10, i x3FFD8000000000000000>, #fir.real<10, i x40048280000000000000>) : !fir.complex<10>
- // CHECK-DAG: {{%.*}} = fir.constc(#fir.real<16, i x3FFD0000000000000000000000000000>, #fir.real<16, i x40040500000000000000000000000000>) : !fir.complex<16>
-  %c2 = fir.constc (#fir.real<2, 0.125>, #fir.real<2, 17.0>) : !fir.complex<2>
-  %c3 = fir.constc (#fir.real<3, 0.25>, #fir.real<3, 32.625>) : !fir.complex<3>
-  %c4 = fir.constc (#fir.real<4, 0.25>, #fir.real<4, 32.625>) : !fir.complex<4>
-  %c8 = fir.constc (#fir.real<8, 0.25>, #fir.real<8, 32.625>) : !fir.complex<8>
-  %c10 = fir.constc (#fir.real<10, 0.25>, #fir.real<10, 32.625>) : !fir.complex<10>
-  %c16 = fir.constc (#fir.real<16, 0.25>, #fir.real<16, 32.625>) : !fir.complex<16>
-  return
-}
-
 // CHECK-LABEL: @insert_on_range_multi_dim
 // CHECK-SAME: %[[ARR:.*]]: !fir.array<10x20xi32>, %[[CST:.*]]: i32
 func.func @insert_on_range_multi_dim(%arr : !fir.array<10x20xi32>, %cst : i32) {
@@ -822,8 +805,8 @@ func.func private @dispatch(%arg0: !fir.class<!fir.type<dispatch_derived1{a:i32,
   fir.dispatch "proc1"(%arg0 : !fir.class<!fir.type<dispatch_derived1{a:i32,b:i32}>>) (%arg0 : !fir.class<!fir.type<dispatch_derived1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
   // CHECK: fir.dispatch "proc1"(%[[CLASS]] : !fir.class<!fir.type<dispatch_derived1{a:i32,b:i32}>>) (%[[CLASS]] : !fir.class<!fir.type<dispatch_derived1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
 
-  fir.dispatch "proc2"(%arg0 : !fir.class<!fir.type<dispatch_derived1{a:i32,b:i32}>>)
-  // CHECK: fir.dispatch "proc2"(%[[CLASS]] : !fir.class<!fir.type<dispatch_derived1{a:i32,b:i32}>>)
+  fir.dispatch "proc2"(%arg0 : !fir.class<!fir.type<dispatch_derived1{a:i32,b:i32}>>) proc_attrs<pure>
+  // CHECK: fir.dispatch "proc2"(%[[CLASS]] : !fir.class<!fir.type<dispatch_derived1{a:i32,b:i32}>>) proc_attrs <pure>
 
   fir.dispatch "proc3"(%arg0 : !fir.class<!fir.type<dispatch_derived1{a:i32,b:i32}>>) (%arg1, %arg0 : i32, !fir.class<!fir.type<dispatch_derived1{a:i32,b:i32}>>) {pass_arg_pos = 1 : i32}
   // CHECK: fir.dispatch "proc3"(%[[CLASS]] : !fir.class<!fir.type<dispatch_derived1{a:i32,b:i32}>>) (%[[INTARG]], %[[CLASS]] : i32, !fir.class<!fir.type<dispatch_derived1{a:i32,b:i32}>>) {pass_arg_pos = 1 : i32}
diff --git a/flang/test/Fir/fir-types.fir b/flang/test/Fir/fir-types.fir
index 7f9c079f1451f..083aca31d8e05 100644
--- a/flang/test/Fir/fir-types.fir
+++ b/flang/test/Fir/fir-types.fir
@@ -4,15 +4,13 @@
 
 // Fortran Intrinsic types
 // CHECK-LABEL: func private @it1() -> !fir.int<4>
-// CHECK-LABEL: func private @it2() -> !fir.real<8>
-// CHECK-LABEL: func private @it3() -> !fir.complex<8>
+// CHECK-LABEL: func private @it3() -> complex<f64>
 // CHECK-LABEL: func private @it4() -> !fir.logical<1>
 // CHECK-LABEL: func private @it5() -> !fir.char<1>
 // CHECK-LABEL: func private @it6() -> !fir.char<2,10>
 // CHECK-LABEL: func private @it7() -> !fir.char<4,?>
 func.func private @it1() -> !fir.int<4>
-func.func private @it2() -> !fir.real<8>
-func.func private @it3() -> !fir.complex<8>
+func.func private @it3() -> complex<f64>
 func.func private @it4() -> !fir.logical<1>
 func.func private @it5() -> !fir.char<1>
 func.func private @it6() -> !fir.char<2,10>
diff --git a/flang/test/Fir/invalid-types.fir b/flang/test/Fir/invalid-types.fir
index 18bc9f7b4896a..f4505097086ad 100644
--- a/flang/test/Fir/invalid-types.fir
+++ b/flang/test/Fir/invalid-types.fir
@@ -27,11 +27,6 @@ func.func private @it6() -> !fir.char<2, >
 
 // -----
 
-// expected-error@+1 {{expected integer value}}
-func.func private @it3() -> !fir.complex<>
-
-// -----
-
 // expected-error@+1 {{expected non-function type}}
 func.func private @mem3() -> !fir.heap<>
 
@@ -52,11 +47,6 @@ func.func private @mem3() -> !fir.ptr<>
 
 // -----
 
-// expected-error@+1 {{expected integer value}}
-func.func private @mem3() -> !fir.real<>
-
-// -----
-
 // expected-error@+1 {{expected valid keyword}}
 func.func private @mem3() -> !fir.type<>
 
diff --git a/flang/test/Fir/rebox.fir b/flang/test/Fir/rebox.fir
index d6aa84353b761..91c1e9da5d454 100644
--- a/flang/test/Fir/rebox.fir
+++ b/flang/test/Fir/rebox.fir
@@ -145,14 +145,14 @@ func.func private @bar_test_rebox_4(!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,1
 
 // CHECK-LABEL: define void @test_cmplx_1(
 // CHECK-SAME: ptr %[[INBOX:.*]])
-func.func @test_cmplx_1(%arg0: !fir.box<!fir.array<?x!fir.complex<4>>>) {
+func.func @test_cmplx_1(%arg0: !fir.box<!fir.array<?xcomplex<f32>>>) {
   // CHECK: %[[OUTBOX_ALLOC:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }
   %c1 = arith.constant 1 : index
   %c1_i32 = arith.constant 0 : i32
   %c0 = arith.constant 0 : index
-  %0:3 = fir.box_dims %arg0, %c0 : (!fir.box<!fir.array<?x!fir.complex<4>>>, index) -> (index, index, index)
+  %0:3 = fir.box_dims %arg0, %c0 : (!fir.box<!fir.array<?xcomplex<f32>>>, index) -> (index, index, index)
   %1 = fir.slice %c1, %0#1, %c1 path %c1_i32 : (index, index, index, i32) -> !fir.slice<1>
-  %2 = fir.rebox %arg0 [%1] : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+  %2 = fir.rebox %arg0 [%1] : (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
   // CHECK: %[[INSTRIDE_0_GEP:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[INBOX]], i32 0, i32 7, i64 0, i32 1
   // CHECK: %[[INSTRIDE_0:.*]] = load i64, ptr %[[INSTRIDE_0_GEP]]
   // CHECK: %[[INSTRIDE_1_GEP:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[INBOX]], i32 0, i32 7, i32 0, i32 2
@@ -185,14 +185,14 @@ func.func @test_cmplx_1(%arg0: !fir.box<!fir.array<?x!fir.complex<4>>>) {
 
 // CHECK-LABEL: define void @test_cmplx_2(
 // CHECK-SAME: ptr %[[INBOX:.*]])
-func.func @test_cmplx_2(%arg0: !fir.box<!fir.array<?x!fir.complex<4>>>) {
+func.func @test_cmplx_2(%arg0: !fir.box<!fir.array<?xcomplex<f32>>>) {
   // CHECK: %[[OUTBOX_ALLOC:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }
   %c7 = arith.constant 7 : index
   %c5 = arith.constant 5 : index
   %c60 = arith.constant 60 : index
   %c1_i32 = arith.constant 1 : i32
   %0 = fir.slice %c7, %c60, %c5 path %c1_i32 : (index, index, index, i32) -> !fir.slice<1>
-  %1 = fir.rebox %arg0 [%0] : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.slice<1>) -> !fir.box<!fir.array<11xf32>>
+  %1 = fir.rebox %arg0 [%0] : (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.slice<1>) -> !fir.box<!fir.array<11xf32>>
   %2 = fir.convert %1 : (!fir.box<!fir.array<11xf32>>) -> !fir.box<!fir.array<?xf32>>
   // CHECK: %[[INSTRIDE_0_GEP:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[INBOX]], i32 0, i32 7, i32 0, i32 2
   // CHECK: %[[INSTRIDE_0:.*]] = load i64, ptr %[[INSTRIDE_0_GEP]]
diff --git a/flang/test/Fir/struct-passing-x86-64-byval.fir b/flang/test/Fir/struct-passing-x86-64-byval.fir
index eec21cbdf230a..b6869014dbdde 100644
--- a/flang/test/Fir/struct-passing-x86-64-byval.fir
+++ b/flang/test/Fir/struct-passing-x86-64-byval.fir
@@ -30,10 +30,10 @@ func.func @ftakes_toobig_align16(%arg0: !fir.type<ftoobig_align16{i:f128,j:f32}>
 func.func @not_enough_sse_reg_1(%arg0: f32, %arg1: f32, %arg2: f32, %arg3: f32, %arg4: f32, %arg5: f32, %arg6: f32, %arg7: f32, %arg8: !fir.type<fits_in_1_sse_reg{i:f32,j:f32}>) {
   return
 }
-func.func @not_enough_sse_reg_1b(%arg0: !fir.complex<4>, %arg1: !fir.complex<4>, %arg2: !fir.complex<4>, %arg3: !fir.complex<4>, %arg4: !fir.complex<4>, %arg5: !fir.complex<4>, %arg6: !fir.complex<4>, %arg7: !fir.complex<4>, %arg8: !fir.type<fits_in_1_sse_reg{i:f32,j:f32}>) {
+func.func @not_enough_sse_reg_1b(%arg0: complex<f32>, %arg1: complex<f32>, %arg2: complex<f32>, %arg3: complex<f32>, %arg4: complex<f32>, %arg5: complex<f32>, %arg6: complex<f32>, %arg7: complex<f32>, %arg8: !fir.type<fits_in_1_sse_reg{i:f32,j:f32}>) {
   return
 }
-func.func @not_enough_sse_reg_1c(%arg0: !fir.complex<8>, %arg1: !fir.complex<8>, %arg2: !fir.complex<8>, %arg3: !fir.complex<8>, %arg4: !fir.type<fits_in_1_sse_reg{i:f32,j:f32}>) {
+func.func @not_enough_sse_reg_1c(%arg0: complex<f64>, %arg1: complex<f64>, %arg2: complex<f64>, %arg3: complex<f64>, %arg4: !fir.type<fits_in_1_sse_reg{i:f32,j:f32}>) {
   return
 }
 func.func @not_enough_sse_reg_2(%arg0: f32, %arg1: f32, %arg2: f32, %arg3: f32, %arg4: f32, %arg5: f32, %arg6: f32, %arg7: !fir.type<fits_in_2_sse_reg{i:f64,j:f64}>) {
@@ -42,7 +42,7 @@ func.func @not_enough_sse_reg_2(%arg0: f32, %arg1: f32, %arg2: f32, %arg3: f32,
 func.func @test_contains_x87(%arg0: !fir.type<contains_x87{i:f80}>) {
   return
 }
-func.func @test_contains_complex_x87(%arg0: !fir.type<contains_complex_x87{i:!fir.complex<10>}>) {
+func.func @test_contains_complex_x87(%arg0: !fir.type<contains_complex_x87{i:complex<f80>}>) {
   return
 }
 func.func @test_nested_toobig(%arg0: !fir.type<nested_toobig{x:!fir.array<3x!fir.type<fits_in_1_int_reg{i:i32,j:i32}>>}>) {
@@ -57,10 +57,10 @@ func.func @test_logical_toobig(%arg0: !fir.type<logical_too_big{l:!fir.array<17x
 func.func @l_not_enough_int_reg(%arg0: i32, %arg1: i32, %arg2: i32, %arg3: i32, %arg4: i32, %arg5: !fir.type<l_fits_in_2_int_reg{l:!fir.array<4x!fir.logical<4>>}>) {
   return
 }
-func.func @test_complex_toobig(%arg0: !fir.type<complex_too_big{c:!fir.array<2x!fir.complex<8>>}>) {
+func.func @test_complex_toobig(%arg0: !fir.type<complex_too_big{c:!fir.array<2xcomplex<f64>>}>) {
   return
 }
-func.func @cplx_not_enough_sse_reg_1(%arg0: f32, %arg1: f32, %arg2: f32, %arg3: f32, %arg4: f32, %arg5: f32, %arg6: f32, %arg7: f32, %arg8: !fir.type<cplx_fits_in_1_sse_reg{c:!fir.complex<4>}>) {
+func.func @cplx_not_enough_sse_reg_1(%arg0: f32, %arg1: f32, %arg2: f32, %arg3: f32, %arg4: f32, %arg5: f32, %arg6: f32, %arg7: f32, %arg8: !fir.type<cplx_fits_in_1_sse_reg{c:complex<f32>}>) {
   return
 }
 func.func @test_char_to_big(%arg0: !fir.type<char_too_big{c:!fir.array<17x!fir.char<1>>}>) {
@@ -75,9 +75,9 @@ func.func @mix_not_enough_int_reg_1(%arg0: i32, %arg1: i32, %arg2: i32, %arg3: i
 func.func @mix_not_enough_sse_reg_2(%arg0: f32, %arg1: f32, %arg2: f32, %arg3: f32, %arg4: f32, %arg5: f32, %arg6: f32, %arg7: f32, %arg8: !fir.type<mix_in_1_int_reg_1_sse_reg{i:!fir.array<2xi32>,x:!fir.array<2xf32>}>) {
   return
 }
-func.func @not_enough_int_reg_3(%arg0: i32, %arg1: i32, %arg2: i32, %arg3: i32, %arg4: i32, %arg6: !fir.type<fits_in_1_int_reg{i:i32,j:i32}>) -> !fir.complex<16> {
-  %0 = fir.zero_bits !fir.complex<16> 
-  return %0 : !fir.complex<16>
+func.func @not_enough_int_reg_3(%arg0: i32, %arg1: i32, %arg2: i32, %arg3: i32, %arg4: i32, %arg6: !fir.type<fits_in_1_int_reg{i:i32,j:i32}>) -> complex<f128> {
+  %0 = fir.zero_bits complex<f128> 
+  return %0 : complex<f128>
 }
 }
 
diff --git a/flang/test/Fir/target-complex16.f90 b/flang/test/Fir/target-complex16.f90
index 9245b205f68a0..243302ddf6c67 100644
--- a/flang/test/Fir/target-complex16.f90
+++ b/flang/test/Fir/target-complex16.f90
@@ -1,29 +1,29 @@
 // RUN: tco --target=aarch64-unknown-linux-gnu %s | FileCheck %s --check-prefix=AARCH64
 
 // AARCH64-LABEL: define { fp128, fp128 } @gen16()
-func.func @gen16() -> !fir.complex<16> {
+func.func @gen16() -> complex<f128> {
   // AARCH64: %[[VAL1:.*]] = alloca { fp128, fp128 }, i64 1, align 16
-  %1 = fir.undefined !fir.complex<16>
+  %1 = fir.undefined complex<f128>
   %2 = arith.constant 1.0 : f128
   %3 = arith.constant -4.0 : f128
   %c0 = arith.constant 0 : i32
   // AARCH64: store { fp128, fp128 } { fp128 0xL0000000000000000C001000000000000, fp128 0xL00000000000000003FFF000000000000 }, ptr %[[VAL1]], align 16
-  %4 = fir.insert_value %1, %3, [0 : index] : (!fir.complex<16>, f128) -> !fir.complex<16>
+  %4 = fir.insert_value %1, %3, [0 : index] : (complex<f128>, f128) -> complex<f128>
   %c1 = arith.constant 1 : i32
-  %5 = fir.insert_value %4, %2, [1 : index] : (!fir.complex<16>, f128) -> !fir.complex<16>
+  %5 = fir.insert_value %4, %2, [1 : index] : (complex<f128>, f128) -> complex<f128>
   // AARCH64: %[[VAL2:.*]] = load { fp128, fp128 }, ptr %[[VAL1]], align 16
   // AARCH64: ret { fp128, fp128 } %[[VAL2]]
-  return %5 : !fir.complex<16>
+  return %5 : complex<f128>
 }
 
 // AARCH64: declare void @sink16([2 x fp128])
-func.func private @sink16(!fir.complex<16>) -> ()
+func.func private @sink16(complex<f128>) -> ()
 
 // AARCH64-LABEL: define void @call16()
 func.func @call16() {
   // AARCH64: = call { fp128, fp128 } @gen16()
-  %1 = fir.call @gen16() : () -> !fir.complex<16>
+  %1 = fir.call @gen16() : () -> complex<f128>
   // AARCH64: call void @sink16([2 x fp128] %
-  fir.call @sink16(%1) : (!fir.complex<16>) -> ()
+  fir.call @sink16(%1) : (complex<f128>) -> ()
   return
 }
diff --git a/flang/test/Fir/target-rewrite-arg-position.fir b/flang/test/Fir/target-rewrite-arg-position.fir
index 44d5981fdf82d..45e783662fb01 100644
--- a/flang/test/Fir/target-rewrite-arg-position.fir
+++ b/flang/test/Fir/target-rewrite-arg-position.fir
@@ -2,21 +2,21 @@
 
 // Test with an argument shift.
 
-func.func @_QFPf(%arg0: !fir.ref<tuple<!fir.ref<i32>>> {fir.host_assoc}) -> !fir.complex<16> {
-  %0 = fir.alloca !fir.complex<16> {bindc_name = "f", uniq_name = "_QFfEf"}
+func.func @_QFPf(%arg0: !fir.ref<tuple<!fir.ref<i32>>> {fir.host_assoc}) -> complex<f128> {
+  %0 = fir.alloca complex<f128> {bindc_name = "f", uniq_name = "_QFfEf"}
   %c2_i32 = arith.constant 2 : i32
   %1 = fir.convert %c2_i32 : (i32) -> f128
   %cst = arith.constant 0.000000e+00 : f128
-  %2 = fir.undefined !fir.complex<16>
-  %3 = fir.insert_value %2, %1, [0 : index] : (!fir.complex<16>, f128) -> !fir.complex<16>
-  %4 = fir.insert_value %3, %cst, [1 : index] : (!fir.complex<16>, f128) -> !fir.complex<16>
-  fir.store %4 to %0 : !fir.ref<!fir.complex<16>>
-  %5 = fir.load %0 : !fir.ref<!fir.complex<16>>
-  return %5 : !fir.complex<16>
+  %2 = fir.undefined complex<f128>
+  %3 = fir.insert_value %2, %1, [0 : index] : (complex<f128>, f128) -> complex<f128>
+  %4 = fir.insert_value %3, %cst, [1 : index] : (complex<f128>, f128) -> complex<f128>
+  fir.store %4 to %0 : !fir.ref<complex<f128>>
+  %5 = fir.load %0 : !fir.ref<complex<f128>>
+  return %5 : complex<f128>
 }
 
 // CHECK-LABEL: func.func @_QFPf
-// CHECK-SAME:    %{{.*}}: !fir.ref<tuple<!fir.real<16>, !fir.real<16>>> {llvm.align = 16 : i32, llvm.sret = tuple<!fir.real<16>, !fir.real<16>>}, %arg1: !fir.ref<tuple<!fir.ref<i32>>> {fir.host_assoc, llvm.nest}) {
+// CHECK-SAME:    %{{.*}}: !fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.sret = tuple<f128, f128>}, %arg1: !fir.ref<tuple<!fir.ref<i32>>> {fir.host_assoc, llvm.nest}) {
 
 // -----
 
diff --git a/flang/test/Fir/target-rewrite-complex-10-x86.fir b/flang/test/Fir/target-rewrite-complex-10-x86.fir
index 8d7cc9d6ef83d..404f5f9f0615b 100644
--- a/flang/test/Fir/target-rewrite-complex-10-x86.fir
+++ b/flang/test/Fir/target-rewrite-complex-10-x86.fir
@@ -3,30 +3,30 @@
 // RUN: fir-opt --target-rewrite="target=x86_64-unknown-linux-gnu" %s | FileCheck %s --check-prefix=AMD64
 // RUN: tco -target="x86_64-unknown-linux-gnu" %s | FileCheck %s --check-prefix=AMD64_LLVM
 
-func.func @returncomplex10() -> !fir.complex<10> {
-  %1 = fir.zero_bits !fir.complex<10>
-  return %1 : !fir.complex<10>
+func.func @returncomplex10() -> complex<f80> {
+  %1 = fir.zero_bits complex<f80>
+  return %1 : complex<f80>
 }
-// AMD64-LABEL:   func.func @returncomplex10() -> tuple<!fir.real<10>, !fir.real<10>> {
-// AMD64:    %[[VAL_0:.*]] = fir.zero_bits !fir.complex<10>
-// AMD64:    %[[VAL_1:.*]] = fir.alloca tuple<!fir.real<10>, !fir.real<10>>
-// AMD64:    %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<tuple<!fir.real<10>, !fir.real<10>>>) -> !fir.ref<!fir.complex<10>>
-// AMD64:    fir.store %[[VAL_0]] to %[[VAL_2]] : !fir.ref<!fir.complex<10>>
-// AMD64:    %[[VAL_3:.*]] = fir.load %[[VAL_1]] : !fir.ref<tuple<!fir.real<10>, !fir.real<10>>>
-// AMD64:    return %[[VAL_3]] : tuple<!fir.real<10>, !fir.real<10>>
+// AMD64-LABEL:   func.func @returncomplex10() -> tuple<f80, f80> {
+// AMD64:    %[[VAL_0:.*]] = fir.zero_bits complex<f80>
+// AMD64:    %[[VAL_1:.*]] = fir.alloca tuple<f80, f80>
+// AMD64:    %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<tuple<f80, f80>>) -> !fir.ref<complex<f80>>
+// AMD64:    fir.store %[[VAL_0]] to %[[VAL_2]] : !fir.ref<complex<f80>>
+// AMD64:    %[[VAL_3:.*]] = fir.load %[[VAL_1]] : !fir.ref<tuple<f80, f80>>
+// AMD64:    return %[[VAL_3]] : tuple<f80, f80>
 
 // AMD64_LLVM: define { x86_fp80, x86_fp80 } @returncomplex10()
 
-func.func @takecomplex10(%z: !fir.complex<10>) {
-  %0 = fir.alloca !fir.complex<10>
-  fir.store %z to %0 : !fir.ref<!fir.complex<10>>
+func.func @takecomplex10(%z: complex<f80>) {
+  %0 = fir.alloca complex<f80>
+  fir.store %z to %0 : !fir.ref<complex<f80>>
   return
 }
 // AMD64-LABEL:   func.func @takecomplex10(
-// AMD64-SAME:  %[[VAL_0:.*]]: !fir.ref<tuple<!fir.real<10>, !fir.real<10>>> {llvm.align = 16 : i32, llvm.byval = tuple<!fir.real<10>, !fir.real<10>>}) {
-// AMD64:    %[[VAL_1:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<tuple<!fir.real<10>, !fir.real<10>>>) -> !fir.ref<!fir.complex<10>>
-// AMD64:    %[[VAL_2:.*]] = fir.load %[[VAL_1]] : !fir.ref<!fir.complex<10>>
-// AMD64:    %[[VAL_3:.*]] = fir.alloca !fir.complex<10>
-// AMD64:    fir.store %[[VAL_2]] to %[[VAL_3]] : !fir.ref<!fir.complex<10>>
+// AMD64-SAME:  %[[VAL_0:.*]]: !fir.ref<tuple<f80, f80>> {llvm.align = 16 : i32, llvm.byval = tuple<f80, f80>}) {
+// AMD64:    %[[VAL_1:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<tuple<f80, f80>>) -> !fir.ref<complex<f80>>
+// AMD64:    %[[VAL_2:.*]] = fir.load %[[VAL_1]] : !fir.ref<complex<f80>>
+// AMD64:    %[[VAL_3:.*]] = fir.alloca complex<f80>
+// AMD64:    fir.store %[[VAL_2]] to %[[VAL_3]] : !fir.ref<complex<f80>>
 
 // AMD64_LLVM: define void @takecomplex10(ptr byval({ x86_fp80, x86_fp80 }) align 16 %0)
diff --git a/flang/test/Fir/target-rewrite-complex.fir b/flang/test/Fir/target-rewrite-complex.fir
index 9d8f79a2f1bd3..e697abb6e3ab3 100644
--- a/flang/test/Fir/target-rewrite-complex.fir
+++ b/flang/test/Fir/target-rewrite-complex.fir
@@ -14,15 +14,15 @@
 // complex<4>.
 // I32-LABEL: func @returncomplex4() -> i64
 // I32_MINGW-LABEL: func @returncomplex4() -> i64
-// X64-LABEL: func @returncomplex4() -> !fir.vector<2:!fir.real<4>>
+// X64-LABEL: func @returncomplex4() -> !fir.vector<2:f32>
 // X64_MINGW-LABEL: func @returncomplex4() -> i64
-// AARCH64-LABEL: func @returncomplex4() -> tuple<!fir.real<4>, !fir.real<4>>
-// PPC64le-LABEL: func @returncomplex4() -> tuple<!fir.real<4>, !fir.real<4>>
-// SPARCV9-LABEL: func @returncomplex4() -> tuple<!fir.real<4>, !fir.real<4>>
-// RISCV64-LABEL: func @returncomplex4() -> tuple<!fir.real<4>, !fir.real<4>>
-// PPC64-LABEL: func @returncomplex4() -> tuple<!fir.real<4>, !fir.real<4>>
-// LOONGARCH64-LABEL: func @returncomplex4() -> tuple<!fir.real<4>, !fir.real<4>>
-func.func @returncomplex4() -> !fir.complex<4> {
+// AARCH64-LABEL: func @returncomplex4() -> tuple<f32, f32>
+// PPC64le-LABEL: func @returncomplex4() -> tuple<f32, f32>
+// SPARCV9-LABEL: func @returncomplex4() -> tuple<f32, f32>
+// RISCV64-LABEL: func @returncomplex4() -> tuple<f32, f32>
+// PPC64-LABEL: func @returncomplex4() -> tuple<f32, f32>
+// LOONGARCH64-LABEL: func @returncomplex4() -> tuple<f32, f32>
+func.func @returncomplex4() -> complex<f32> {
   // I32: fir.insert_value
   // I32: [[VAL:%[0-9A-Za-z]+]] = fir.insert_value
   // I32_MINGW: fir.insert_value
@@ -43,78 +43,78 @@ func.func @returncomplex4() -> !fir.complex<4> {
   // PPC64: [[VAL:%[0-9A-Za-z]+]] = fir.insert_value
   // LOONGARCH64: fir.insert_value
   // LOONGARCH64: [[VAL:%[0-9A-Za-z]+]] = fir.insert_value
-  %1 = fir.undefined !fir.complex<4>
+  %1 = fir.undefined complex<f32>
   %2 = arith.constant 2.0 : f32
-  %3 = fir.convert %2 : (f32) -> !fir.real<4>
+  %3 = fir.convert %2 : (f32) -> f32
   %c0 = arith.constant 0 : i32
-  %4 = fir.insert_value %1, %3, [0 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
+  %4 = fir.insert_value %1, %3, [0 : i32] : (complex<f32>, f32) -> complex<f32>
   %c1 = arith.constant 1 : i32
   %5 = arith.constant -42.0 : f32
-  %6 = fir.insert_value %4, %5, [1 : i32] : (!fir.complex<4>, f32) -> !fir.complex<4>
+  %6 = fir.insert_value %4, %5, [1 : i32] : (complex<f32>, f32) -> complex<f32>
 
   // I32: [[ADDRI64:%[0-9A-Za-z]+]] = fir.alloca i64
-  // I32: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRI64]] : (!fir.ref<i64>) -> !fir.ref<!fir.complex<4>>
-  // I32: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<!fir.complex<4>>
+  // I32: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRI64]] : (!fir.ref<i64>) -> !fir.ref<complex<f32>>
+  // I32: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f32>>
   // I32: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRI64]] : !fir.ref<i64>
   // I32: return [[RES]] : i64
   // I32_MINGW: [[ADDRV:%[0-9A-Za-z]+]] = fir.alloca i64
-  // I32_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRV]] : (!fir.ref<i64>) -> !fir.ref<!fir.complex<4>>
-  // I32_MINGW: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<!fir.complex<4>>
+  // I32_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRV]] : (!fir.ref<i64>) -> !fir.ref<complex<f32>>
+  // I32_MINGW: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f32>>
   // I32_MINGW: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRV]] : !fir.ref<i64>
   // I32_MINGW: return [[RES]] : i64
-  // X64: [[ADDRV:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:!fir.real<4>>
-  // X64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRV]] : (!fir.ref<!fir.vector<2:!fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // X64: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // X64: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRV]] : !fir.ref<!fir.vector<2:!fir.real<4>>>
-  // X64: return [[RES]] : !fir.vector<2:!fir.real<4>>
+  // X64: [[ADDRV:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:f32>
+  // X64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRV]] : (!fir.ref<!fir.vector<2:f32>>) -> !fir.ref<complex<f32>>
+  // X64: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f32>>
+  // X64: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRV]] : !fir.ref<!fir.vector<2:f32>>
+  // X64: return [[RES]] : !fir.vector<2:f32>
   // X64_MINGW: [[ADDRV:%[0-9A-Za-z]+]] = fir.alloca i64
-  // X64_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRV]] : (!fir.ref<i64>) -> !fir.ref<!fir.complex<4>>
-  // X64_MINGW: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<!fir.complex<4>>
+  // X64_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRV]] : (!fir.ref<i64>) -> !fir.ref<complex<f32>>
+  // X64_MINGW: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f32>>
   // X64_MINGW: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRV]] : !fir.ref<i64>
   // X64_MINGW: return [[RES]] : i64
-  // AARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<4>, !fir.real<4>>
-  // AARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // AARCH64: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // AARCH64: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // AARCH64: return [[RES]] : tuple<!fir.real<4>, !fir.real<4>>
-  // PPC64le: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<4>, !fir.real<4>>
-  // PPC64le: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // PPC64le: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // PPC64le: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // PPC64le: return [[RES]] : tuple<!fir.real<4>, !fir.real<4>>
-  // SPARCV9: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<4>, !fir.real<4>>
-  // SPARCV9: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // SPARCV9: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // SPARCV9: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // SPARCV9: return [[RES]] : tuple<!fir.real<4>, !fir.real<4>>
-  // RISCV64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<4>, !fir.real<4>>
-  // RISCV64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // PPC64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<4>, !fir.real<4>>
-  // PPC64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // PPC64: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // PPC64: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // PPC64: return [[RES]] : tuple<!fir.real<4>, !fir.real<4>>
-  // LOONGARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<4>, !fir.real<4>>
-  // LOONGARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  return %6 : !fir.complex<4>
+  // AARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // AARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // AARCH64: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f32>>
+  // AARCH64: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<f32, f32>>
+  // AARCH64: return [[RES]] : tuple<f32, f32>
+  // PPC64le: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // PPC64le: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // PPC64le: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f32>>
+  // PPC64le: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<f32, f32>>
+  // PPC64le: return [[RES]] : tuple<f32, f32>
+  // SPARCV9: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // SPARCV9: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // SPARCV9: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f32>>
+  // SPARCV9: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<f32, f32>>
+  // SPARCV9: return [[RES]] : tuple<f32, f32>
+  // RISCV64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // RISCV64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // PPC64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // PPC64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // PPC64: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f32>>
+  // PPC64: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<f32, f32>>
+  // PPC64: return [[RES]] : tuple<f32, f32>
+  // LOONGARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // LOONGARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  return %6 : complex<f32>
 }
 
 // Test that we rewrite the signature and body of a function that returns a
 // complex<8>.
 // I32-LABEL:func @returncomplex8
-// I32-SAME: ([[ARG0:%[0-9A-Za-z]+]]: !fir.ref<tuple<!fir.real<8>, !fir.real<8>>>  {llvm.align = 4 : i32, llvm.sret = tuple<!fir.real<8>, !fir.real<8>>})
+// I32-SAME: ([[ARG0:%[0-9A-Za-z]+]]: !fir.ref<tuple<f64, f64>>  {llvm.align = 4 : i32, llvm.sret = tuple<f64, f64>})
 // I32_MINGW-LABEL: func @returncomplex8
-// I32_MINGW-SAME: ([[ARG0:%[0-9A-Za-z]+]]: !fir.ref<tuple<!fir.real<8>, !fir.real<8>>>  {llvm.align = 8 : i32, llvm.sret = tuple<!fir.real<8>, !fir.real<8>>})
-// X64-LABEL: func @returncomplex8() -> tuple<!fir.real<8>, !fir.real<8>>
+// I32_MINGW-SAME: ([[ARG0:%[0-9A-Za-z]+]]: !fir.ref<tuple<f64, f64>>  {llvm.align = 8 : i32, llvm.sret = tuple<f64, f64>})
+// X64-LABEL: func @returncomplex8() -> tuple<f64, f64>
 // X64_MINGW-LABEL: func @returncomplex8
-// X64_MINGW-SAME: ([[ARG0:%[0-9A-Za-z]+]]: !fir.ref<tuple<!fir.real<8>, !fir.real<8>>>  {llvm.align = 8 : i32, llvm.sret = tuple<!fir.real<8>, !fir.real<8>>})
-// AARCH64-LABEL: func @returncomplex8() -> tuple<!fir.real<8>, !fir.real<8>>
-// PPC64le-LABEL: func @returncomplex8() -> tuple<!fir.real<8>, !fir.real<8>>
-// SPARCV9-LABEL: func @returncomplex8() -> tuple<!fir.real<8>, !fir.real<8>>
-// RISCV64-LABEL: func @returncomplex8() -> tuple<!fir.real<8>, !fir.real<8>>
-// PPC64-LABEL: func @returncomplex8() -> tuple<!fir.real<8>, !fir.real<8>>
-// LOONGARCH64-LABEL: func @returncomplex8() -> tuple<!fir.real<8>, !fir.real<8>>
-func.func @returncomplex8() -> !fir.complex<8> {
+// X64_MINGW-SAME: ([[ARG0:%[0-9A-Za-z]+]]: !fir.ref<tuple<f64, f64>>  {llvm.align = 8 : i32, llvm.sret = tuple<f64, f64>})
+// AARCH64-LABEL: func @returncomplex8() -> tuple<f64, f64>
+// PPC64le-LABEL: func @returncomplex8() -> tuple<f64, f64>
+// SPARCV9-LABEL: func @returncomplex8() -> tuple<f64, f64>
+// RISCV64-LABEL: func @returncomplex8() -> tuple<f64, f64>
+// PPC64-LABEL: func @returncomplex8() -> tuple<f64, f64>
+// LOONGARCH64-LABEL: func @returncomplex8() -> tuple<f64, f64>
+func.func @returncomplex8() -> complex<f64> {
   // I32: fir.insert_value
   // I32: [[VAL:%[0-9A-Za-z]+]] = fir.insert_value {{.*}}
   // I32_MINGW: fir.insert_value
@@ -135,73 +135,73 @@ func.func @returncomplex8() -> !fir.complex<8> {
   // PPC64: [[VAL:%[0-9A-Za-z]+]] = fir.insert_value {{.*}}
   // LOONGARCH64: fir.insert_value
   // LOONGARCH64: [[VAL:%[0-9A-Za-z]+]] = fir.insert_value {{.*}}
-  %1 = fir.undefined !fir.complex<8>
+  %1 = fir.undefined complex<f64>
   %2 = arith.constant 1.0 : f64
   %3 = arith.constant -4.0 : f64
   %c0 = arith.constant 0 : i32
-  %4 = fir.insert_value %1, %3, [0 : i32] : (!fir.complex<8>, f64) -> !fir.complex<8>
+  %4 = fir.insert_value %1, %3, [0 : i32] : (complex<f64>, f64) -> complex<f64>
   %c1 = arith.constant 1 : i32
-  %5 = fir.insert_value %4, %2, [1 : i32] : (!fir.complex<8>, f64) -> !fir.complex<8>
+  %5 = fir.insert_value %4, %2, [1 : i32] : (complex<f64>, f64) -> complex<f64>
 
-  // I32: [[ADDR:%[0-9A-Za-z]+]] = fir.convert [[ARG0]] : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // I32: fir.store [[VAL]] to [[ADDR]] : !fir.ref<!fir.complex<8>>
+  // I32: [[ADDR:%[0-9A-Za-z]+]] = fir.convert [[ARG0]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // I32: fir.store [[VAL]] to [[ADDR]] : !fir.ref<complex<f64>>
   // I32: return{{ *$}}
-  // I32_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ARG0]] : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // I32_MINGW: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<!fir.complex<8>>
+  // I32_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ARG0]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // I32_MINGW: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f64>>
   // I32_MINGW: return
-  // X64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<8>, !fir.real<8>>
-  // X64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // X64: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<!fir.complex<8>>
-  // X64: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<!fir.real<8>, !fir.real<8>>>
-  // X64: return [[RES]] : tuple<!fir.real<8>, !fir.real<8>>
-  // X64_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ARG0]] : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // X64_MINGW: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<!fir.complex<8>>
+  // X64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
+  // X64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // X64: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f64>>
+  // X64: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<f64, f64>>
+  // X64: return [[RES]] : tuple<f64, f64>
+  // X64_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ARG0]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // X64_MINGW: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f64>>
   // X64_MINGW: return
-  // AARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<8>, !fir.real<8>>
-  // AARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // AARCH64: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<!fir.complex<8>>
-  // AARCH64: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<!fir.real<8>, !fir.real<8>>>
-  // AARCH64: return [[RES]] : tuple<!fir.real<8>, !fir.real<8>>
-  // PPC64le: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<8>, !fir.real<8>>
-  // PPC64le: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // PPC64le: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<!fir.complex<8>>
-  // PPC64le: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<!fir.real<8>, !fir.real<8>>>
-  // PPC64le: return [[RES]] : tuple<!fir.real<8>, !fir.real<8>>
-  // SPARCV9: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<8>, !fir.real<8>>
-  // SPARCV9: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // SPARCV9: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<!fir.complex<8>>
-  // SPARCV9: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<!fir.real<8>, !fir.real<8>>>
-  // SPARCV9: return [[RES]] : tuple<!fir.real<8>, !fir.real<8>>
-  // RISCV64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<8>, !fir.real<8>>
-  // RISCV64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // RISCV64: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<!fir.complex<8>>
-  // RISCV64: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<!fir.real<8>, !fir.real<8>>>
-  // RISCV64: return [[RES]] : tuple<!fir.real<8>, !fir.real<8>>
-  // PPC64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<8>, !fir.real<8>>
-  // PPC64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // PPC64: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<!fir.complex<8>>
-  // PPC64: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<!fir.real<8>, !fir.real<8>>>
-  // PPC64: return [[RES]] : tuple<!fir.real<8>, !fir.real<8>>
-  // LOONGARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<8>, !fir.real<8>>
-  // LOONGARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // LOONGARCH64: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<!fir.complex<8>>
-  // LOONGARCH64: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<!fir.real<8>, !fir.real<8>>>
-  // LOONGARCH64: return [[RES]] : tuple<!fir.real<8>, !fir.real<8>>
-  return %5 : !fir.complex<8>
+  // AARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
+  // AARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // AARCH64: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f64>>
+  // AARCH64: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<f64, f64>>
+  // AARCH64: return [[RES]] : tuple<f64, f64>
+  // PPC64le: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
+  // PPC64le: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // PPC64le: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f64>>
+  // PPC64le: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<f64, f64>>
+  // PPC64le: return [[RES]] : tuple<f64, f64>
+  // SPARCV9: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
+  // SPARCV9: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // SPARCV9: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f64>>
+  // SPARCV9: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<f64, f64>>
+  // SPARCV9: return [[RES]] : tuple<f64, f64>
+  // RISCV64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
+  // RISCV64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // RISCV64: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f64>>
+  // RISCV64: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<f64, f64>>
+  // RISCV64: return [[RES]] : tuple<f64, f64>
+  // PPC64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
+  // PPC64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // PPC64: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f64>>
+  // PPC64: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<f64, f64>>
+  // PPC64: return [[RES]] : tuple<f64, f64>
+  // LOONGARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
+  // LOONGARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // LOONGARCH64: fir.store [[VAL]] to [[ADDRC]] : !fir.ref<complex<f64>>
+  // LOONGARCH64: [[RES:%[0-9A-Za-z]+]] = fir.load [[ADDRT]] : !fir.ref<tuple<f64, f64>>
+  // LOONGARCH64: return [[RES]] : tuple<f64, f64>
+  return %5 : complex<f64>
 }
 
 // Test that we rewrite the signature of a function that accepts a complex<4>.
-// I32-LABEL: func private @paramcomplex4(!fir.ref<tuple<!fir.real<4>, !fir.real<4>>> {llvm.align = 4 : i32, llvm.byval = tuple<!fir.real<4>, !fir.real<4>>})
-// I32_MINGW-LABEL: func private @paramcomplex4(!fir.ref<tuple<!fir.real<4>, !fir.real<4>>> {llvm.align = 4 : i32, llvm.byval = tuple<!fir.real<4>, !fir.real<4>>})
-// X64-LABEL: func private @paramcomplex4(!fir.vector<2:!fir.real<4>>)
+// I32-LABEL: func private @paramcomplex4(!fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>})
+// I32_MINGW-LABEL: func private @paramcomplex4(!fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>})
+// X64-LABEL: func private @paramcomplex4(!fir.vector<2:f32>)
 // X64_MINGW-LABEL: func private @paramcomplex4(i64)
-// AARCH64-LABEL: func private @paramcomplex4(!fir.array<2x!fir.real<4>>)
-// PPC64le-LABEL: func private @paramcomplex4(!fir.real<4>, !fir.real<4>)
-// SPARCV9-LABEL: func private @paramcomplex4(!fir.real<4>, !fir.real<4>)
-// RISCV64-LABEL: func private @paramcomplex4(!fir.real<4>, !fir.real<4>)
-// PPC64-LABEL: func private @paramcomplex4(!fir.real<4>, !fir.real<4>)
-// LOONGARCH64-LABEL: func private @paramcomplex4(!fir.real<4>, !fir.real<4>)
-func.func private @paramcomplex4(!fir.complex<4>) -> ()
+// AARCH64-LABEL: func private @paramcomplex4(!fir.array<2xf32>)
+// PPC64le-LABEL: func private @paramcomplex4(f32, f32)
+// SPARCV9-LABEL: func private @paramcomplex4(f32, f32)
+// RISCV64-LABEL: func private @paramcomplex4(f32, f32)
+// PPC64-LABEL: func private @paramcomplex4(f32, f32)
+// LOONGARCH64-LABEL: func private @paramcomplex4(f32, f32)
+func.func private @paramcomplex4(complex<f32>) -> ()
 
 // Test that we rewrite calls to functions that return or accept complex<4>.
 // I32-LABEL: func @callcomplex4
@@ -218,216 +218,216 @@ func.func @callcomplex4(%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i3
 
   // I32: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> i64
   // I32_MINGW: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> i64
-  // X64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> !fir.vector<2:!fir.real<4>>
+  // X64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> !fir.vector<2:f32>
   // X64_MINGW: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> i64
-  // AARCH64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> tuple<!fir.real<4>, !fir.real<4>>
-  // PPC64le: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> tuple<!fir.real<4>, !fir.real<4>>
-  // SPARCV9: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> tuple<!fir.real<4>, !fir.real<4>>
-  // RISCV64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> tuple<!fir.real<4>, !fir.real<4>>
-  // PPC64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> tuple<!fir.real<4>, !fir.real<4>>
-  // LOONGARCH64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> tuple<!fir.real<4>, !fir.real<4>>
-  %1 = fir.call @returncomplex4() : () -> !fir.complex<4>
+  // AARCH64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> tuple<f32, f32>
+  // PPC64le: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> tuple<f32, f32>
+  // SPARCV9: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> tuple<f32, f32>
+  // RISCV64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> tuple<f32, f32>
+  // PPC64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> tuple<f32, f32>
+  // LOONGARCH64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex4() : () -> tuple<f32, f32>
+  %1 = fir.call @returncomplex4() : () -> complex<f32>
 
   // I32: [[ADDRI64:%[0-9A-Za-z]+]] = fir.alloca i64
   // I32: fir.store [[RES]] to [[ADDRI64]] : !fir.ref<i64>
-  // I32: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRI64]] : (!fir.ref<i64>) -> !fir.ref<!fir.complex<4>>
-  // I32: [[C:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // I32: [[ADDRC2:%[0-9A-Za-z]+]] = fir.alloca !fir.complex<4>
-  // I32: fir.store [[C]] to [[ADDRC2]] : !fir.ref<!fir.complex<4>>
-  // I32: [[T:%[0-9A-Za-z]+]] = fir.convert [[ADDRC2]] : (!fir.ref<!fir.complex<4>>) -> !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // I32: fir.call @paramcomplex4([[T]]) : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> ()
+  // I32: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRI64]] : (!fir.ref<i64>) -> !fir.ref<complex<f32>>
+  // I32: [[C:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
+  // I32: [[ADDRC2:%[0-9A-Za-z]+]] = fir.alloca complex<f32>
+  // I32: fir.store [[C]] to [[ADDRC2]] : !fir.ref<complex<f32>>
+  // I32: [[T:%[0-9A-Za-z]+]] = fir.convert [[ADDRC2]] : (!fir.ref<complex<f32>>) -> !fir.ref<tuple<f32, f32>>
+  // I32: fir.call @paramcomplex4([[T]]) : (!fir.ref<tuple<f32, f32>>) -> ()
 
   // I32_MINGW: [[ADDRI64:%[0-9A-Za-z]+]] = fir.alloca i64
   // I32_MINGW: fir.store [[RES]] to [[ADDRI64]] : !fir.ref<i64>
-  // I32_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRI64]] : (!fir.ref<i64>) -> !fir.ref<!fir.complex<4>>
-  // I32_MINGW: [[C:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // I32_MINGW: [[ADDRC2:%[0-9A-Za-z]+]] = fir.alloca !fir.complex<4>
-  // I32_MINGW: fir.store [[C]] to [[ADDRC2]] : !fir.ref<!fir.complex<4>>
-  // I32_MINGW: [[T:%[0-9A-Za-z]+]] = fir.convert [[ADDRC2]] : (!fir.ref<!fir.complex<4>>) -> !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // I32_MINGW: fir.call @paramcomplex4([[T]]) : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> ()
-
-  // X64: [[ADDRV:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:!fir.real<4>>
-  // X64: fir.store [[RES]] to [[ADDRV]] : !fir.ref<!fir.vector<2:!fir.real<4>>>
-  // X64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRV]] : (!fir.ref<!fir.vector<2:!fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // X64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // X64: [[ADDRV2:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:!fir.real<4>>
-  // X64: [[ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[ADDRV2]] : (!fir.ref<!fir.vector<2:!fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // X64: fir.store [[V]] to [[ADDRC2]] : !fir.ref<!fir.complex<4>>
-  // X64: [[VRELOADED:%[0-9A-Za-z]+]] = fir.load [[ADDRV2]] : !fir.ref<!fir.vector<2:!fir.real<4>>>
-  // X64: fir.call @paramcomplex4([[VRELOADED]]) : (!fir.vector<2:!fir.real<4>>) -> ()
+  // I32_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRI64]] : (!fir.ref<i64>) -> !fir.ref<complex<f32>>
+  // I32_MINGW: [[C:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
+  // I32_MINGW: [[ADDRC2:%[0-9A-Za-z]+]] = fir.alloca complex<f32>
+  // I32_MINGW: fir.store [[C]] to [[ADDRC2]] : !fir.ref<complex<f32>>
+  // I32_MINGW: [[T:%[0-9A-Za-z]+]] = fir.convert [[ADDRC2]] : (!fir.ref<complex<f32>>) -> !fir.ref<tuple<f32, f32>>
+  // I32_MINGW: fir.call @paramcomplex4([[T]]) : (!fir.ref<tuple<f32, f32>>) -> ()
+
+  // X64: [[ADDRV:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:f32>
+  // X64: fir.store [[RES]] to [[ADDRV]] : !fir.ref<!fir.vector<2:f32>>
+  // X64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRV]] : (!fir.ref<!fir.vector<2:f32>>) -> !fir.ref<complex<f32>>
+  // X64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
+  // X64: [[ADDRV2:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:f32>
+  // X64: [[ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[ADDRV2]] : (!fir.ref<!fir.vector<2:f32>>) -> !fir.ref<complex<f32>>
+  // X64: fir.store [[V]] to [[ADDRC2]] : !fir.ref<complex<f32>>
+  // X64: [[VRELOADED:%[0-9A-Za-z]+]] = fir.load [[ADDRV2]] : !fir.ref<!fir.vector<2:f32>>
+  // X64: fir.call @paramcomplex4([[VRELOADED]]) : (!fir.vector<2:f32>) -> ()
 
   // X64_MINGW: [[ADDRV:%[0-9A-Za-z]+]] = fir.alloca i64
   // X64_MINGW: fir.store [[RES]] to [[ADDRV]] : !fir.ref<i64>
-  // X64_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRV]] :  (!fir.ref<i64>) -> !fir.ref<!fir.complex<4>>
-  // X64_MINGW: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<4>>
+  // X64_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRV]] :  (!fir.ref<i64>) -> !fir.ref<complex<f32>>
+  // X64_MINGW: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
   // X64_MINGW: [[ADDRV2:%[0-9A-Za-z]+]] = fir.alloca i64
-  // X64_MINGW: [[ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[ADDRV2]] : (!fir.ref<i64>) -> !fir.ref<!fir.complex<4>>
-  // X64_MINGW: fir.store [[V]] to [[ADDRC2]] : !fir.ref<!fir.complex<4>>
+  // X64_MINGW: [[ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[ADDRV2]] : (!fir.ref<i64>) -> !fir.ref<complex<f32>>
+  // X64_MINGW: fir.store [[V]] to [[ADDRC2]] : !fir.ref<complex<f32>>
   // X64_MINGW: [[VRELOADED:%[0-9A-Za-z]+]] = fir.load [[ADDRV2]] : !fir.ref<i64>
   // X64_MINGW: fir.call @paramcomplex4([[VRELOADED]]) : (i64) -> ()
 
-  // AARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<4>, !fir.real<4>>
-  // AARCH64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // AARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // AARCH64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // AARCH64: [[ADDRARR:%[0-9A-Za-z]+]] = fir.alloca !fir.array<2x!fir.real<4>>
-  // AARCH64: [[ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[ADDRARR]] : (!fir.ref<!fir.array<2x!fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // AARCH64: fir.store [[V]] to [[ADDRC2]] : !fir.ref<!fir.complex<4>>
-  // AARCH64: [[ARR:%[0-9A-Za-z]+]] = fir.load [[ADDRARR]] : !fir.ref<!fir.array<2x!fir.real<4>>>
-  // AARCH64: fir.call @paramcomplex4([[ARR]]) : (!fir.array<2x!fir.real<4>>) -> ()
-
-  // PPC64le: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<4>, !fir.real<4>>
-  // PPC64le: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // PPC64le: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // PPC64le: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // PPC64le: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // PPC64le: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // PPC64le: fir.call @paramcomplex4([[A]], [[B]]) : (!fir.real<4>, !fir.real<4>) -> ()
-
-  // SPARCV9: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<4>, !fir.real<4>>
-  // SPARCV9: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // SPARCV9: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // SPARCV9: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // SPARCV9: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // SPARCV9: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // SPARCV9: fir.call @paramcomplex4([[A]], [[B]]) : (!fir.real<4>, !fir.real<4>) -> ()
-
-  // RISCV64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<4>, !fir.real<4>>
-  // RISCV64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // RISCV64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // RISCV64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // RISCV64: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // RISCV64: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // RISCV64: fir.call @paramcomplex4([[A]], [[B]]) : (!fir.real<4>, !fir.real<4>) -> ()
+  // AARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // AARCH64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f32, f32>>
+  // AARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // AARCH64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
+  // AARCH64: [[ADDRARR:%[0-9A-Za-z]+]] = fir.alloca !fir.array<2xf32>
+  // AARCH64: [[ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[ADDRARR]] : (!fir.ref<!fir.array<2xf32>>) -> !fir.ref<complex<f32>>
+  // AARCH64: fir.store [[V]] to [[ADDRC2]] : !fir.ref<complex<f32>>
+  // AARCH64: [[ARR:%[0-9A-Za-z]+]] = fir.load [[ADDRARR]] : !fir.ref<!fir.array<2xf32>>
+  // AARCH64: fir.call @paramcomplex4([[ARR]]) : (!fir.array<2xf32>) -> ()
+
+  // PPC64le: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // PPC64le: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f32, f32>>
+  // PPC64le: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // PPC64le: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
+  // PPC64le: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (complex<f32>) -> f32
+  // PPC64le: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (complex<f32>) -> f32
+  // PPC64le: fir.call @paramcomplex4([[A]], [[B]]) : (f32, f32) -> ()
+
+  // SPARCV9: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // SPARCV9: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f32, f32>>
+  // SPARCV9: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // SPARCV9: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
+  // SPARCV9: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (complex<f32>) -> f32
+  // SPARCV9: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (complex<f32>) -> f32
+  // SPARCV9: fir.call @paramcomplex4([[A]], [[B]]) : (f32, f32) -> ()
+
+  // RISCV64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // RISCV64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f32, f32>>
+  // RISCV64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // RISCV64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
+  // RISCV64: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (complex<f32>) -> f32
+  // RISCV64: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (complex<f32>) -> f32
+  // RISCV64: fir.call @paramcomplex4([[A]], [[B]]) : (f32, f32) -> ()
  
-  // PPC64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<4>, !fir.real<4>>
-  // PPC64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // PPC64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // PPC64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // PPC64: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // PPC64: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // PPC64: fir.call @paramcomplex4([[A]], [[B]]) : (!fir.real<4>, !fir.real<4>) -> ()
-
-  // LOONGARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<4>, !fir.real<4>>
-  // LOONGARCH64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // LOONGARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // LOONGARCH64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // LOONGARCH64: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // LOONGARCH64: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // LOONGARCH64: fir.call @paramcomplex4([[A]], [[B]]) : (!fir.real<4>, !fir.real<4>) -> ()
-  fir.call @paramcomplex4(%1) : (!fir.complex<4>) -> ()
+  // PPC64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // PPC64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f32, f32>>
+  // PPC64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // PPC64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
+  // PPC64: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (complex<f32>) -> f32
+  // PPC64: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (complex<f32>) -> f32
+  // PPC64: fir.call @paramcomplex4([[A]], [[B]]) : (f32, f32) -> ()
+
+  // LOONGARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // LOONGARCH64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f32, f32>>
+  // LOONGARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // LOONGARCH64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
+  // LOONGARCH64: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (complex<f32>) -> f32
+  // LOONGARCH64: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (complex<f32>) -> f32
+  // LOONGARCH64: fir.call @paramcomplex4([[A]], [[B]]) : (f32, f32) -> ()
+  fir.call @paramcomplex4(%1) : (complex<f32>) -> ()
 
   // I32: [[RES:%[0-9A-Za-z]+]] = fir.dispatch "ret_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> i64 {pass_arg_pos = 0 : i32}
   // I32_MINGW: [[RES:%[0-9A-Za-z]+]] = fir.dispatch "ret_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> i64 {pass_arg_pos = 0 : i32}
-  // X64: [[RES:%[0-9A-Za-z]+]] = fir.dispatch "ret_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> !fir.vector<2:!fir.real<4>> {pass_arg_pos = 0 : i32}
+  // X64: [[RES:%[0-9A-Za-z]+]] = fir.dispatch "ret_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> !fir.vector<2:f32> {pass_arg_pos = 0 : i32}
   // X64_MINGW: [[RES:%[0-9A-Za-z]+]] = fir.dispatch "ret_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> i64 {pass_arg_pos = 0 : i32}
-  // AARCH64: [[RES:%[0-9A-Za-z]+]] = fir.dispatch "ret_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> tuple<!fir.real<4>, !fir.real<4>> {pass_arg_pos = 0 : i32}
-  // PPC64le: [[RES:%[0-9A-Za-z]+]] = fir.dispatch "ret_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> tuple<!fir.real<4>, !fir.real<4>> {pass_arg_pos = 0 : i32}
-  // SPARCV9: [[RES:%[0-9A-Za-z]+]] = fir.dispatch "ret_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> tuple<!fir.real<4>, !fir.real<4>> {pass_arg_pos = 0 : i32}
-  // PPC64: [[RES:%[0-9A-Za-z]+]] = fir.dispatch "ret_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> tuple<!fir.real<4>, !fir.real<4>> {pass_arg_pos = 0 : i32}
-  %2 = fir.dispatch "ret_complex"(%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> !fir.complex<4> {pass_arg_pos = 0 : i32}
+  // AARCH64: [[RES:%[0-9A-Za-z]+]] = fir.dispatch "ret_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> tuple<f32, f32> {pass_arg_pos = 0 : i32}
+  // PPC64le: [[RES:%[0-9A-Za-z]+]] = fir.dispatch "ret_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> tuple<f32, f32> {pass_arg_pos = 0 : i32}
+  // SPARCV9: [[RES:%[0-9A-Za-z]+]] = fir.dispatch "ret_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> tuple<f32, f32> {pass_arg_pos = 0 : i32}
+  // PPC64: [[RES:%[0-9A-Za-z]+]] = fir.dispatch "ret_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> tuple<f32, f32> {pass_arg_pos = 0 : i32}
+  %2 = fir.dispatch "ret_complex"(%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> complex<f32> {pass_arg_pos = 0 : i32}
 
   // I32: [[ADDRI64:%[0-9A-Za-z]+]] = fir.alloca i64
   // I32: fir.store [[RES]] to [[ADDRI64]] : !fir.ref<i64>
-  // I32: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRI64]] : (!fir.ref<i64>) -> !fir.ref<!fir.complex<4>>
-  // I32: [[C:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // I32: [[ADDRC2:%[0-9A-Za-z]+]] = fir.alloca !fir.complex<4>
-  // I32: fir.store [[C]] to [[ADDRC2]] : !fir.ref<!fir.complex<4>>
-  // I32: [[T:%[0-9A-Za-z]+]] = fir.convert [[ADDRC2]] : (!fir.ref<!fir.complex<4>>) -> !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // I32: fir.dispatch "with_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, [[T]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) {pass_arg_pos = 0 : i32}
+  // I32: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRI64]] : (!fir.ref<i64>) -> !fir.ref<complex<f32>>
+  // I32: [[C:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
+  // I32: [[ADDRC2:%[0-9A-Za-z]+]] = fir.alloca complex<f32>
+  // I32: fir.store [[C]] to [[ADDRC2]] : !fir.ref<complex<f32>>
+  // I32: [[T:%[0-9A-Za-z]+]] = fir.convert [[ADDRC2]] : (!fir.ref<complex<f32>>) -> !fir.ref<tuple<f32, f32>>
+  // I32: fir.dispatch "with_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, [[T]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, !fir.ref<tuple<f32, f32>>) {pass_arg_pos = 0 : i32}
 
   // I32_MINGW: [[ADDRI64:%[0-9A-Za-z]+]] = fir.alloca i64
   // I32_MINGW: fir.store [[RES]] to [[ADDRI64]] : !fir.ref<i64>
-  // I32_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRI64]] : (!fir.ref<i64>) -> !fir.ref<!fir.complex<4>>
-  // I32_MINGW: [[C:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // I32_MINGW: [[ADDRC2:%[0-9A-Za-z]+]] = fir.alloca !fir.complex<4>
-  // I32_MINGW: fir.store [[C]] to [[ADDRC2]] : !fir.ref<!fir.complex<4>>
-  // I32_MINGW: [[T:%[0-9A-Za-z]+]] = fir.convert [[ADDRC2]] : (!fir.ref<!fir.complex<4>>) -> !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // I32_MINGW: fir.dispatch "with_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, [[T]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) {pass_arg_pos = 0 : i32}
-
-  // X64: [[ADDRV:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:!fir.real<4>>
-  // X64: fir.store [[RES]] to [[ADDRV]] : !fir.ref<!fir.vector<2:!fir.real<4>>>
-  // X64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRV]] : (!fir.ref<!fir.vector<2:!fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // X64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // X64: [[ADDRV2:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:!fir.real<4>>
-  // X64: [[ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[ADDRV2]] : (!fir.ref<!fir.vector<2:!fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // X64: fir.store [[V]] to [[ADDRC2]] : !fir.ref<!fir.complex<4>>
-  // X64: [[VRELOADED:%[0-9A-Za-z]+]] = fir.load [[ADDRV2]] : !fir.ref<!fir.vector<2:!fir.real<4>>>
-  // X64: fir.dispatch "with_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, [[VRELOADED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, !fir.vector<2:!fir.real<4>>) {pass_arg_pos = 0 : i32}
+  // I32_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRI64]] : (!fir.ref<i64>) -> !fir.ref<complex<f32>>
+  // I32_MINGW: [[C:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
+  // I32_MINGW: [[ADDRC2:%[0-9A-Za-z]+]] = fir.alloca complex<f32>
+  // I32_MINGW: fir.store [[C]] to [[ADDRC2]] : !fir.ref<complex<f32>>
+  // I32_MINGW: [[T:%[0-9A-Za-z]+]] = fir.convert [[ADDRC2]] : (!fir.ref<complex<f32>>) -> !fir.ref<tuple<f32, f32>>
+  // I32_MINGW: fir.dispatch "with_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, [[T]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, !fir.ref<tuple<f32, f32>>) {pass_arg_pos = 0 : i32}
+
+  // X64: [[ADDRV:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:f32>
+  // X64: fir.store [[RES]] to [[ADDRV]] : !fir.ref<!fir.vector<2:f32>>
+  // X64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRV]] : (!fir.ref<!fir.vector<2:f32>>) -> !fir.ref<complex<f32>>
+  // X64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
+  // X64: [[ADDRV2:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:f32>
+  // X64: [[ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[ADDRV2]] : (!fir.ref<!fir.vector<2:f32>>) -> !fir.ref<complex<f32>>
+  // X64: fir.store [[V]] to [[ADDRC2]] : !fir.ref<complex<f32>>
+  // X64: [[VRELOADED:%[0-9A-Za-z]+]] = fir.load [[ADDRV2]] : !fir.ref<!fir.vector<2:f32>>
+  // X64: fir.dispatch "with_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, [[VRELOADED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, !fir.vector<2:f32>) {pass_arg_pos = 0 : i32}
 
   // X64_MINGW: [[ADDRV:%[0-9A-Za-z]+]] = fir.alloca i64
   // X64_MINGW: fir.store [[RES]] to [[ADDRV]] : !fir.ref<i64>
-  // X64_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRV]] : (!fir.ref<i64>) -> !fir.ref<!fir.complex<4>>
-  // X64_MINGW: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<4>>
+  // X64_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRV]] : (!fir.ref<i64>) -> !fir.ref<complex<f32>>
+  // X64_MINGW: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
   // X64_MINGW: [[ADDRV2:%[0-9A-Za-z]+]] = fir.alloca i64
-  // X64_MINGW: [[ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[ADDRV2]] : (!fir.ref<i64>) -> !fir.ref<!fir.complex<4>>
-  // X64_MINGW: fir.store [[V]] to [[ADDRC2]] : !fir.ref<!fir.complex<4>>
+  // X64_MINGW: [[ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[ADDRV2]] : (!fir.ref<i64>) -> !fir.ref<complex<f32>>
+  // X64_MINGW: fir.store [[V]] to [[ADDRC2]] : !fir.ref<complex<f32>>
   // X64_MINGW: [[VRELOADED:%[0-9A-Za-z]+]] = fir.load [[ADDRV2]] : !fir.ref<i64>
   // X64_MINGW: fir.dispatch "with_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, [[VRELOADED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, i64) {pass_arg_pos = 0 : i32}
 
-  // AARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<4>, !fir.real<4>>
-  // AARCH64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // AARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // AARCH64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // AARCH64: [[ADDRARR:%[0-9A-Za-z]+]] = fir.alloca !fir.array<2x!fir.real<4>>
-  // AARCH64: [[ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[ADDRARR]] : (!fir.ref<!fir.array<2x!fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // AARCH64: fir.store [[V]] to [[ADDRC2]] : !fir.ref<!fir.complex<4>>
-  // AARCH64: [[ARR:%[0-9A-Za-z]+]] = fir.load [[ADDRARR]] : !fir.ref<!fir.array<2x!fir.real<4>>>
-  // AARCH64: fir.dispatch "with_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, [[ARR]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, !fir.array<2x!fir.real<4>>) {pass_arg_pos = 0 : i32}
-
-  // PPC64le: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<4>, !fir.real<4>>
-  // PPC64le: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // PPC64le: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // PPC64le: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // PPC64le: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // PPC64le: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // PPC64le: fir.dispatch "with_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, [[A]], [[B]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, !fir.real<4>, !fir.real<4>) {pass_arg_pos = 0 : i32}
-
-  // SPARCV9: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<4>, !fir.real<4>>
-  // SPARCV9: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // SPARCV9: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // SPARCV9: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // SPARCV9: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // SPARCV9: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // SPARCV9: fir.dispatch "with_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, [[A]], [[B]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, !fir.real<4>, !fir.real<4>) {pass_arg_pos = 0 : i32}
-
-  // PPC64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<4>, !fir.real<4>>
-  // PPC64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // PPC64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // PPC64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<4>>
-  // PPC64: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // PPC64: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // PPC64: fir.dispatch "with_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, [[A]], [[B]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, !fir.real<4>, !fir.real<4>) {pass_arg_pos = 0 : i32}
-  fir.dispatch "with_complex"(%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%arg0, %2 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, !fir.complex<4>) {pass_arg_pos = 0 : i32}
-
-
-  // I32: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}} : !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 1 : i32}
-  // I32_MINGW: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}} : !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 1 : i32}
-  // X64: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}} : !fir.vector<2:!fir.real<4>>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 1 : i32}
+  // AARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // AARCH64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f32, f32>>
+  // AARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // AARCH64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
+  // AARCH64: [[ADDRARR:%[0-9A-Za-z]+]] = fir.alloca !fir.array<2xf32>
+  // AARCH64: [[ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[ADDRARR]] : (!fir.ref<!fir.array<2xf32>>) -> !fir.ref<complex<f32>>
+  // AARCH64: fir.store [[V]] to [[ADDRC2]] : !fir.ref<complex<f32>>
+  // AARCH64: [[ARR:%[0-9A-Za-z]+]] = fir.load [[ADDRARR]] : !fir.ref<!fir.array<2xf32>>
+  // AARCH64: fir.dispatch "with_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, [[ARR]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, !fir.array<2xf32>) {pass_arg_pos = 0 : i32}
+
+  // PPC64le: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // PPC64le: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f32, f32>>
+  // PPC64le: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // PPC64le: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
+  // PPC64le: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (complex<f32>) -> f32
+  // PPC64le: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (complex<f32>) -> f32
+  // PPC64le: fir.dispatch "with_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, [[A]], [[B]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, f32, f32) {pass_arg_pos = 0 : i32}
+
+  // SPARCV9: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // SPARCV9: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f32, f32>>
+  // SPARCV9: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // SPARCV9: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
+  // SPARCV9: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (complex<f32>) -> f32
+  // SPARCV9: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (complex<f32>) -> f32
+  // SPARCV9: fir.dispatch "with_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, [[A]], [[B]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, f32, f32) {pass_arg_pos = 0 : i32}
+
+  // PPC64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f32, f32>
+  // PPC64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f32, f32>>
+  // PPC64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // PPC64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f32>>
+  // PPC64: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (complex<f32>) -> f32
+  // PPC64: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (complex<f32>) -> f32
+  // PPC64: fir.dispatch "with_complex"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, [[A]], [[B]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, f32, f32) {pass_arg_pos = 0 : i32}
+  fir.dispatch "with_complex"(%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%arg0, %2 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, complex<f32>) {pass_arg_pos = 0 : i32}
+
+
+  // I32: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}} : !fir.ref<tuple<f32, f32>>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 1 : i32}
+  // I32_MINGW: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}} : !fir.ref<tuple<f32, f32>>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 1 : i32}
+  // X64: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}} : !fir.vector<2:f32>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 1 : i32}
   // X64_MINGW: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}} : i64, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 1 : i32}
-  // AARCH64: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}} : !fir.array<2x!fir.real<4>>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 1 : i32}
-  // PPC64le: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}}, %{{.*}} : !fir.real<4>, !fir.real<4>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 2 : i32}
-  // SPARCV9: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}}, %{{.*}} : !fir.real<4>, !fir.real<4>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 2 : i32}
-  // RISCV64: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}}, %{{.*}} : !fir.real<4>, !fir.real<4>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 2 : i32}
-  // PPC64: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}}, %{{.*}} : !fir.real<4>, !fir.real<4>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 2 : i32}
-  // LOONGARCH64: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}}, %{{.*}} : !fir.real<4>, !fir.real<4>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 2 : i32}
-  fir.dispatch "with_complex2"(%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%2, %arg0 : !fir.complex<4>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 1 : i32}
+  // AARCH64: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}} : !fir.array<2xf32>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 1 : i32}
+  // PPC64le: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}}, %{{.*}} : f32, f32, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 2 : i32}
+  // SPARCV9: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}}, %{{.*}} : f32, f32, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 2 : i32}
+  // RISCV64: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}}, %{{.*}} : f32, f32, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 2 : i32}
+  // PPC64: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}}, %{{.*}} : f32, f32, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 2 : i32}
+  // LOONGARCH64: fir.dispatch "with_complex2"(%{{.*}} : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %{{.*}}, %{{.*}} : f32, f32, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 2 : i32}
+  fir.dispatch "with_complex2"(%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%2, %arg0 : complex<f32>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 1 : i32}
 
   return
 }
 
 // Test that we rewrite the signature of a function that accepts a complex<8>.
-// I32-LABEL: func private @paramcomplex8(!fir.ref<tuple<!fir.real<8>, !fir.real<8>>> {llvm.align = 4 : i32, llvm.byval = tuple<!fir.real<8>, !fir.real<8>>})
-// I32_MINGW-LABEL: func private @paramcomplex8(!fir.ref<tuple<!fir.real<8>, !fir.real<8>>> {llvm.align = 4 : i32, llvm.byval = tuple<!fir.real<8>, !fir.real<8>>})
-// X64-LABEL: func private @paramcomplex8(!fir.real<8>, !fir.real<8>)
-// X64_MINGW-LABEL: func private @paramcomplex8(!fir.ref<tuple<!fir.real<8>, !fir.real<8>>> {llvm.align = 8 : i32, llvm.byval = tuple<!fir.real<8>, !fir.real<8>>})
-// AARCH64-LABEL: func private @paramcomplex8(!fir.array<2x!fir.real<8>>)
-// PPC64le-LABEL: func private @paramcomplex8(!fir.real<8>, !fir.real<8>)
-// SPARCV9-LABEL: func private @paramcomplex8(!fir.real<8>, !fir.real<8>)
-// RISCV64-LABEL: func private @paramcomplex8(!fir.real<8>, !fir.real<8>)
-// PPC64-LABEL: func private @paramcomplex8(!fir.real<8>, !fir.real<8>)
-// LOONGARCH64-LABEL: func private @paramcomplex8(!fir.real<8>, !fir.real<8>)
-func.func private @paramcomplex8(!fir.complex<8>) -> ()
+// I32-LABEL: func private @paramcomplex8(!fir.ref<tuple<f64, f64>> {llvm.align = 4 : i32, llvm.byval = tuple<f64, f64>})
+// I32_MINGW-LABEL: func private @paramcomplex8(!fir.ref<tuple<f64, f64>> {llvm.align = 4 : i32, llvm.byval = tuple<f64, f64>})
+// X64-LABEL: func private @paramcomplex8(f64, f64)
+// X64_MINGW-LABEL: func private @paramcomplex8(!fir.ref<tuple<f64, f64>> {llvm.align = 8 : i32, llvm.byval = tuple<f64, f64>})
+// AARCH64-LABEL: func private @paramcomplex8(!fir.array<2xf64>)
+// PPC64le-LABEL: func private @paramcomplex8(f64, f64)
+// SPARCV9-LABEL: func private @paramcomplex8(f64, f64)
+// RISCV64-LABEL: func private @paramcomplex8(f64, f64)
+// PPC64-LABEL: func private @paramcomplex8(f64, f64)
+// LOONGARCH64-LABEL: func private @paramcomplex8(f64, f64)
+func.func private @paramcomplex8(complex<f64>) -> ()
 
 // Test that we rewrite calls to functions that return or accept complex<8>.
 // I32-LABEL: func @callcomplex8()
@@ -441,353 +441,353 @@ func.func private @paramcomplex8(!fir.complex<8>) -> ()
 // PPC64-LABEL: func @callcomplex8()
 // LOONGARCH64-LABEL: func @callcomplex8()
 func.func @callcomplex8() {
-  // I32: [[RES:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<8>, !fir.real<8>>
-  // I32: fir.call @returncomplex8([[RES]]) : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> ()
-  // I32_MINGW: [[RES:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<8>, !fir.real<8>>
-  // I32_MINGW: fir.call @returncomplex8([[RES]]) : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> ()
-  // X64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<!fir.real<8>, !fir.real<8>>
-  // X64_MINGW: [[RES:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<8>, !fir.real<8>>
-  // X64_MINGW: fir.call @returncomplex8([[RES]]) : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> ()
-  // AARCH64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<!fir.real<8>, !fir.real<8>>
-  // PPC64le: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<!fir.real<8>, !fir.real<8>>
-  // SPARCV9: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<!fir.real<8>, !fir.real<8>>
-  // RISCV64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<!fir.real<8>, !fir.real<8>>
-  // PPC64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<!fir.real<8>, !fir.real<8>>
-  // LOONGARCH64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<!fir.real<8>, !fir.real<8>>
-  %1 = fir.call @returncomplex8() : () -> !fir.complex<8>
-
-  // I32: [[RESC:%[0-9A-Za-z]+]] = fir.convert [[RES]] : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // I32: [[V:%[0-9A-Za-z]+]]  = fir.load [[RESC]] : !fir.ref<!fir.complex<8>>
-  // I32: [[ADDRC:%[0-9A-Za-z]+]] = fir.alloca !fir.complex<8>
-  // I32: fir.store [[V]] to [[ADDRC]] : !fir.ref<!fir.complex<8>>
-  // I32: [[ADDRT:%[0-9A-Za-z]+]] = fir.convert [[ADDRC]] : (!fir.ref<!fir.complex<8>>) -> !fir.ref<tuple<!fir.real<8>, !fir.real<8>>>
-  // I32: fir.call @paramcomplex8([[ADDRT]]) : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> ()
-
-  // I32_MINGW: [[RESC:%[0-9A-Za-z]+]] = fir.convert [[RES]] : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // I32_MINGW: [[V:%[0-9A-Za-z]+]]  = fir.load [[RESC]] : !fir.ref<!fir.complex<8>>
-  // I32_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.alloca !fir.complex<8>
-  // I32_MINGW: fir.store [[V]] to [[ADDRC]] : !fir.ref<!fir.complex<8>>
-  // I32_MINGW: [[ADDRT:%[0-9A-Za-z]+]] = fir.convert [[ADDRC]] : (!fir.ref<!fir.complex<8>>) -> !fir.ref<tuple<!fir.real<8>, !fir.real<8>>>
-  // I32_MINGW: fir.call @paramcomplex8([[ADDRT]]) : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> ()
-
-  // X64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<8>, !fir.real<8>>
-  // X64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<!fir.real<8>, !fir.real<8>>>
-  // X64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // X64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<8>>
-  // X64: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (!fir.complex<8>) -> !fir.real<8>
-  // X64: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (!fir.complex<8>) -> !fir.real<8>
-  // X64: fir.call @paramcomplex8([[A]], [[B]]) : (!fir.real<8>, !fir.real<8>) -> ()
-
-  // X64_MINGW: [[RESC:%[0-9A-Za-z]+]] = fir.convert [[RES]] : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // X64_MINGW: [[V:%[0-9A-Za-z]+]]  = fir.load [[RESC]] : !fir.ref<!fir.complex<8>>
-  // X64_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.alloca !fir.complex<8>
-  // X64_MINGW: fir.store [[V]] to [[ADDRC]] : !fir.ref<!fir.complex<8>>
-  // X64_MINGW: [[ADDRT:%[0-9A-Za-z]+]] = fir.convert [[ADDRC]] : (!fir.ref<!fir.complex<8>>) -> !fir.ref<tuple<!fir.real<8>, !fir.real<8>>>
-  // X64_MINGW: fir.call @paramcomplex8([[ADDRT]]) : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> ()
-
-  // AARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<8>, !fir.real<8>>
-  // AARCH64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<!fir.real<8>, !fir.real<8>>>
-  // AARCH64: [[ADDRV:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // AARCH64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRV]] : !fir.ref<!fir.complex<8>>
-  // AARCH64: [[ADDRARR:%[0-9A-Za-z]+]] = fir.alloca !fir.array<2x!fir.real<8>>
-  // AARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRARR]] : (!fir.ref<!fir.array<2x!fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // AARCH64: fir.store [[V]] to [[ADDRC]] : !fir.ref<!fir.complex<8>>
-  // AARCH64: [[ARR:%[0-9A-Za-z]+]] = fir.load [[ADDRARR]] : !fir.ref<!fir.array<2x!fir.real<8>>>
-  // AARCH64: fir.call @paramcomplex8([[ARR]]) : (!fir.array<2x!fir.real<8>>) -> ()
-
-  // PPC64le: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<8>, !fir.real<8>>
-  // PPC64le: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<!fir.real<8>, !fir.real<8>>>
-  // PPC64le: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // PPC64le: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<8>>
-  // PPC64le: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (!fir.complex<8>) -> !fir.real<8>
-  // PPC64le: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (!fir.complex<8>) -> !fir.real<8>
-  // PPC64le: fir.call @paramcomplex8([[A]], [[B]]) : (!fir.real<8>, !fir.real<8>) -> ()
-
-  // SPARCV9: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<8>, !fir.real<8>>
-  // SPARCV9: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<!fir.real<8>, !fir.real<8>>>
-  // SPARCV9: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // SPARCV9: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<8>>
-  // SPARCV9: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (!fir.complex<8>) -> !fir.real<8>
-  // SPARCV9: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (!fir.complex<8>) -> !fir.real<8>
-  // SPARCV9: fir.call @paramcomplex8([[A]], [[B]]) : (!fir.real<8>, !fir.real<8>) -> ()
-
-  // RISCV64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<8>, !fir.real<8>>
-  // RISCV64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<!fir.real<8>, !fir.real<8>>>
-  // RISCV64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // RISCV64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<8>>
-  // RISCV64: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (!fir.complex<8>) -> !fir.real<8>
-  // RISCV64: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (!fir.complex<8>) -> !fir.real<8>
-  // RISCV64: fir.call @paramcomplex8([[A]], [[B]]) : (!fir.real<8>, !fir.real<8>) -> ()
-
-  // PPC64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<8>, !fir.real<8>>
-  // PPC64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<!fir.real<8>, !fir.real<8>>>
-  // PPC64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // PPC64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<8>>
-  // PPC64: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (!fir.complex<8>) -> !fir.real<8>
-  // PPC64: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (!fir.complex<8>) -> !fir.real<8>
-  // PPC64: fir.call @paramcomplex8([[A]], [[B]]) : (!fir.real<8>, !fir.real<8>) -> ()
-
-  // LOONGARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<!fir.real<8>, !fir.real<8>>
-  // LOONGARCH64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<!fir.real<8>, !fir.real<8>>>
-  // LOONGARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<!fir.real<8>, !fir.real<8>>>) -> !fir.ref<!fir.complex<8>>
-  // LOONGARCH64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<!fir.complex<8>>
-  // LOONGARCH64: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (!fir.complex<8>) -> !fir.real<8>
-  // LOONGARCH64: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (!fir.complex<8>) -> !fir.real<8>
-  // LOONGARCH64: fir.call @paramcomplex8([[A]], [[B]]) : (!fir.real<8>, !fir.real<8>) -> ()
+  // I32: [[RES:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
+  // I32: fir.call @returncomplex8([[RES]]) : (!fir.ref<tuple<f64, f64>>) -> ()
+  // I32_MINGW: [[RES:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
+  // I32_MINGW: fir.call @returncomplex8([[RES]]) : (!fir.ref<tuple<f64, f64>>) -> ()
+  // X64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<f64, f64>
+  // X64_MINGW: [[RES:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
+  // X64_MINGW: fir.call @returncomplex8([[RES]]) : (!fir.ref<tuple<f64, f64>>) -> ()
+  // AARCH64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<f64, f64>
+  // PPC64le: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<f64, f64>
+  // SPARCV9: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<f64, f64>
+  // RISCV64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<f64, f64>
+  // PPC64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<f64, f64>
+  // LOONGARCH64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<f64, f64>
+  %1 = fir.call @returncomplex8() : () -> complex<f64>
+
+  // I32: [[RESC:%[0-9A-Za-z]+]] = fir.convert [[RES]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // I32: [[V:%[0-9A-Za-z]+]]  = fir.load [[RESC]] : !fir.ref<complex<f64>>
+  // I32: [[ADDRC:%[0-9A-Za-z]+]] = fir.alloca complex<f64>
+  // I32: fir.store [[V]] to [[ADDRC]] : !fir.ref<complex<f64>>
+  // I32: [[ADDRT:%[0-9A-Za-z]+]] = fir.convert [[ADDRC]] : (!fir.ref<complex<f64>>) -> !fir.ref<tuple<f64, f64>>
+  // I32: fir.call @paramcomplex8([[ADDRT]]) : (!fir.ref<tuple<f64, f64>>) -> ()
+
+  // I32_MINGW: [[RESC:%[0-9A-Za-z]+]] = fir.convert [[RES]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // I32_MINGW: [[V:%[0-9A-Za-z]+]]  = fir.load [[RESC]] : !fir.ref<complex<f64>>
+  // I32_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.alloca complex<f64>
+  // I32_MINGW: fir.store [[V]] to [[ADDRC]] : !fir.ref<complex<f64>>
+  // I32_MINGW: [[ADDRT:%[0-9A-Za-z]+]] = fir.convert [[ADDRC]] : (!fir.ref<complex<f64>>) -> !fir.ref<tuple<f64, f64>>
+  // I32_MINGW: fir.call @paramcomplex8([[ADDRT]]) : (!fir.ref<tuple<f64, f64>>) -> ()
+
+  // X64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
+  // X64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f64, f64>>
+  // X64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // X64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f64>>
+  // X64: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (complex<f64>) -> f64
+  // X64: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (complex<f64>) -> f64
+  // X64: fir.call @paramcomplex8([[A]], [[B]]) : (f64, f64) -> ()
+
+  // X64_MINGW: [[RESC:%[0-9A-Za-z]+]] = fir.convert [[RES]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // X64_MINGW: [[V:%[0-9A-Za-z]+]]  = fir.load [[RESC]] : !fir.ref<complex<f64>>
+  // X64_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.alloca complex<f64>
+  // X64_MINGW: fir.store [[V]] to [[ADDRC]] : !fir.ref<complex<f64>>
+  // X64_MINGW: [[ADDRT:%[0-9A-Za-z]+]] = fir.convert [[ADDRC]] : (!fir.ref<complex<f64>>) -> !fir.ref<tuple<f64, f64>>
+  // X64_MINGW: fir.call @paramcomplex8([[ADDRT]]) : (!fir.ref<tuple<f64, f64>>) -> ()
+
+  // AARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
+  // AARCH64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f64, f64>>
+  // AARCH64: [[ADDRV:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // AARCH64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRV]] : !fir.ref<complex<f64>>
+  // AARCH64: [[ADDRARR:%[0-9A-Za-z]+]] = fir.alloca !fir.array<2xf64>
+  // AARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRARR]] : (!fir.ref<!fir.array<2xf64>>) -> !fir.ref<complex<f64>>
+  // AARCH64: fir.store [[V]] to [[ADDRC]] : !fir.ref<complex<f64>>
+  // AARCH64: [[ARR:%[0-9A-Za-z]+]] = fir.load [[ADDRARR]] : !fir.ref<!fir.array<2xf64>>
+  // AARCH64: fir.call @paramcomplex8([[ARR]]) : (!fir.array<2xf64>) -> ()
+
+  // PPC64le: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
+  // PPC64le: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f64, f64>>
+  // PPC64le: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // PPC64le: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f64>>
+  // PPC64le: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (complex<f64>) -> f64
+  // PPC64le: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (complex<f64>) -> f64
+  // PPC64le: fir.call @paramcomplex8([[A]], [[B]]) : (f64, f64) -> ()
+
+  // SPARCV9: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
+  // SPARCV9: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f64, f64>>
+  // SPARCV9: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // SPARCV9: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f64>>
+  // SPARCV9: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (complex<f64>) -> f64
+  // SPARCV9: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (complex<f64>) -> f64
+  // SPARCV9: fir.call @paramcomplex8([[A]], [[B]]) : (f64, f64) -> ()
+
+  // RISCV64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
+  // RISCV64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f64, f64>>
+  // RISCV64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // RISCV64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f64>>
+  // RISCV64: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (complex<f64>) -> f64
+  // RISCV64: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (complex<f64>) -> f64
+  // RISCV64: fir.call @paramcomplex8([[A]], [[B]]) : (f64, f64) -> ()
+
+  // PPC64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
+  // PPC64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f64, f64>>
+  // PPC64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // PPC64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f64>>
+  // PPC64: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (complex<f64>) -> f64
+  // PPC64: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (complex<f64>) -> f64
+  // PPC64: fir.call @paramcomplex8([[A]], [[B]]) : (f64, f64) -> ()
+
+  // LOONGARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
+  // LOONGARCH64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f64, f64>>
+  // LOONGARCH64: [[ADDRC:%[0-9A-Za-z]+]] = fir.convert [[ADDRT]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
+  // LOONGARCH64: [[V:%[0-9A-Za-z]+]] = fir.load [[ADDRC]] : !fir.ref<complex<f64>>
+  // LOONGARCH64: [[A:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [0 : i32] : (complex<f64>) -> f64
+  // LOONGARCH64: [[B:%[0-9A-Za-z]+]] = fir.extract_value [[V]], [1 : i32] : (complex<f64>) -> f64
+  // LOONGARCH64: fir.call @paramcomplex8([[A]], [[B]]) : (f64, f64) -> ()
   
-  fir.call @paramcomplex8(%1) : (!fir.complex<8>) -> ()
+  fir.call @paramcomplex8(%1) : (complex<f64>) -> ()
   return
 }
 
 // Test multiple complex<4> parameters and arguments
-// I32-LABEL: func private @calleemultipleparamscomplex4(!fir.ref<tuple<!fir.real<4>, !fir.real<4>>> {llvm.align = 4 : i32, llvm.byval = tuple<!fir.real<4>, !fir.real<4>>}, !fir.ref<tuple<!fir.real<4>, !fir.real<4>>> {llvm.align = 4 : i32, llvm.byval = tuple<!fir.real<4>, !fir.real<4>>}, !fir.ref<tuple<!fir.real<4>, !fir.real<4>>> {llvm.align = 4 : i32, llvm.byval = tuple<!fir.real<4>, !fir.real<4>>})
-// I32_MINGW-LABEL: func private @calleemultipleparamscomplex4(!fir.ref<tuple<!fir.real<4>, !fir.real<4>>> {llvm.align = 4 : i32, llvm.byval = tuple<!fir.real<4>, !fir.real<4>>}, !fir.ref<tuple<!fir.real<4>, !fir.real<4>>> {llvm.align = 4 : i32, llvm.byval = tuple<!fir.real<4>, !fir.real<4>>}, !fir.ref<tuple<!fir.real<4>, !fir.real<4>>> {llvm.align = 4 : i32, llvm.byval = tuple<!fir.real<4>, !fir.real<4>>})
-// X64-LABEL: func private @calleemultipleparamscomplex4(!fir.vector<2:!fir.real<4>>, !fir.vector<2:!fir.real<4>>, !fir.vector<2:!fir.real<4>>)
+// I32-LABEL: func private @calleemultipleparamscomplex4(!fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>}, !fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>}, !fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>})
+// I32_MINGW-LABEL: func private @calleemultipleparamscomplex4(!fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>}, !fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>}, !fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>})
+// X64-LABEL: func private @calleemultipleparamscomplex4(!fir.vector<2:f32>, !fir.vector<2:f32>, !fir.vector<2:f32>)
 // X64_MINGW-LABEL: func private @calleemultipleparamscomplex4(i64, i64, i64)
-// AARCH64-LABEL: func private @calleemultipleparamscomplex4(!fir.array<2x!fir.real<4>>, !fir.array<2x!fir.real<4>>, !fir.array<2x!fir.real<4>>)
-// PPC64le-LABEL: func private @calleemultipleparamscomplex4(!fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>)
-// SPARCV9-LABEL: func private @calleemultipleparamscomplex4(!fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>)
-// RISCV64-LABEL: func private @calleemultipleparamscomplex4(!fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>)
-// PPC64-LABEL: func private @calleemultipleparamscomplex4(!fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>)
-// LOONGARCH64-LABEL: func private @calleemultipleparamscomplex4(!fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>)
-func.func private @calleemultipleparamscomplex4(!fir.complex<4>, !fir.complex<4>, !fir.complex<4>) -> ()
+// AARCH64-LABEL: func private @calleemultipleparamscomplex4(!fir.array<2xf32>, !fir.array<2xf32>, !fir.array<2xf32>)
+// PPC64le-LABEL: func private @calleemultipleparamscomplex4(f32, f32, f32, f32, f32, f32)
+// SPARCV9-LABEL: func private @calleemultipleparamscomplex4(f32, f32, f32, f32, f32, f32)
+// RISCV64-LABEL: func private @calleemultipleparamscomplex4(f32, f32, f32, f32, f32, f32)
+// PPC64-LABEL: func private @calleemultipleparamscomplex4(f32, f32, f32, f32, f32, f32)
+// LOONGARCH64-LABEL: func private @calleemultipleparamscomplex4(f32, f32, f32, f32, f32, f32)
+func.func private @calleemultipleparamscomplex4(complex<f32>, complex<f32>, complex<f32>) -> ()
 
 // I32-LABEL: func @multipleparamscomplex4
-// I32-SAME: ([[Z1:%[0-9A-Za-z]+]]: !fir.ref<tuple<!fir.real<4>, !fir.real<4>>> {llvm.align = 4 : i32, llvm.byval = tuple<!fir.real<4>, !fir.real<4>>}, [[Z2:%[0-9A-Za-z]+]]: !fir.ref<tuple<!fir.real<4>, !fir.real<4>>> {llvm.align = 4 : i32, llvm.byval = tuple<!fir.real<4>, !fir.real<4>>}, [[Z3:%[0-9A-Za-z]+]]: !fir.ref<tuple<!fir.real<4>, !fir.real<4>>> {llvm.align = 4 : i32, llvm.byval = tuple<!fir.real<4>, !fir.real<4>>})
+// I32-SAME: ([[Z1:%[0-9A-Za-z]+]]: !fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>}, [[Z2:%[0-9A-Za-z]+]]: !fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>}, [[Z3:%[0-9A-Za-z]+]]: !fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>})
 // I32_MINGW-LABEL: func @multipleparamscomplex4
-// I32_MINGW-SAME: ([[Z1:%[0-9A-Za-z]+]]: !fir.ref<tuple<!fir.real<4>, !fir.real<4>>> {llvm.align = 4 : i32, llvm.byval = tuple<!fir.real<4>, !fir.real<4>>}, [[Z2:%[0-9A-Za-z]+]]: !fir.ref<tuple<!fir.real<4>, !fir.real<4>>> {llvm.align = 4 : i32, llvm.byval = tuple<!fir.real<4>, !fir.real<4>>}, [[Z3:%[0-9A-Za-z]+]]: !fir.ref<tuple<!fir.real<4>, !fir.real<4>>> {llvm.align = 4 : i32, llvm.byval = tuple<!fir.real<4>, !fir.real<4>>})
+// I32_MINGW-SAME: ([[Z1:%[0-9A-Za-z]+]]: !fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>}, [[Z2:%[0-9A-Za-z]+]]: !fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>}, [[Z3:%[0-9A-Za-z]+]]: !fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>})
 // X64-LABEL: func @multipleparamscomplex4
-// X64-SAME: ([[Z1:%[0-9A-Za-z]+]]: !fir.vector<2:!fir.real<4>>, [[Z2:%[0-9A-Za-z]+]]: !fir.vector<2:!fir.real<4>>, [[Z3:%[0-9A-Za-z]+]]: !fir.vector<2:!fir.real<4>>)
+// X64-SAME: ([[Z1:%[0-9A-Za-z]+]]: !fir.vector<2:f32>, [[Z2:%[0-9A-Za-z]+]]: !fir.vector<2:f32>, [[Z3:%[0-9A-Za-z]+]]: !fir.vector<2:f32>)
 // X64_MINGW-LABEL: func @multipleparamscomplex4
 // X64_MINGW-SAME: ([[Z1:%[0-9A-Za-z]+]]: i64, [[Z2:%[0-9A-Za-z]+]]: i64, [[Z3:%[0-9A-Za-z]+]]: i64)
 // AARCH64-LABEL: func @multipleparamscomplex4
-// AARCH64-SAME: ([[Z1:%[0-9A-Za-z]+]]: !fir.array<2x!fir.real<4>>, [[Z2:%[0-9A-Za-z]+]]: !fir.array<2x!fir.real<4>>, [[Z3:%[0-9A-Za-z]+]]: !fir.array<2x!fir.real<4>>)
+// AARCH64-SAME: ([[Z1:%[0-9A-Za-z]+]]: !fir.array<2xf32>, [[Z2:%[0-9A-Za-z]+]]: !fir.array<2xf32>, [[Z3:%[0-9A-Za-z]+]]: !fir.array<2xf32>)
 // PPC64le-LABEL: func @multipleparamscomplex4
-// PPC64le-SAME: ([[A1:%[0-9A-Za-z]+]]: !fir.real<4>, [[B1:%[0-9A-Za-z]+]]: !fir.real<4>, [[A2:%[0-9A-Za-z]+]]: !fir.real<4>, [[B2:%[0-9A-Za-z]+]]: !fir.real<4>, [[A3:%[0-9A-Za-z]+]]: !fir.real<4>, [[B3:%[0-9A-Za-z]+]]: !fir.real<4>)
+// PPC64le-SAME: ([[A1:%[0-9A-Za-z]+]]: f32, [[B1:%[0-9A-Za-z]+]]: f32, [[A2:%[0-9A-Za-z]+]]: f32, [[B2:%[0-9A-Za-z]+]]: f32, [[A3:%[0-9A-Za-z]+]]: f32, [[B3:%[0-9A-Za-z]+]]: f32)
 // SPARCV9-LABEL: func @multipleparamscomplex4
-// SPARCV9-SAME: ([[A1:%[0-9A-Za-z]+]]: !fir.real<4>, [[B1:%[0-9A-Za-z]+]]: !fir.real<4>, [[A2:%[0-9A-Za-z]+]]: !fir.real<4>, [[B2:%[0-9A-Za-z]+]]: !fir.real<4>, [[A3:%[0-9A-Za-z]+]]: !fir.real<4>, [[B3:%[0-9A-Za-z]+]]: !fir.real<4>)
+// SPARCV9-SAME: ([[A1:%[0-9A-Za-z]+]]: f32, [[B1:%[0-9A-Za-z]+]]: f32, [[A2:%[0-9A-Za-z]+]]: f32, [[B2:%[0-9A-Za-z]+]]: f32, [[A3:%[0-9A-Za-z]+]]: f32, [[B3:%[0-9A-Za-z]+]]: f32)
 // RISCV64-LABEL: func @multipleparamscomplex4
-// RISCV64-SAME: ([[A1:%[0-9A-Za-z]+]]: !fir.real<4>, [[B1:%[0-9A-Za-z]+]]: !fir.real<4>, [[A2:%[0-9A-Za-z]+]]: !fir.real<4>, [[B2:%[0-9A-Za-z]+]]: !fir.real<4>, [[A3:%[0-9A-Za-z]+]]: !fir.real<4>, [[B3:%[0-9A-Za-z]+]]: !fir.real<4>)
+// RISCV64-SAME: ([[A1:%[0-9A-Za-z]+]]: f32, [[B1:%[0-9A-Za-z]+]]: f32, [[A2:%[0-9A-Za-z]+]]: f32, [[B2:%[0-9A-Za-z]+]]: f32, [[A3:%[0-9A-Za-z]+]]: f32, [[B3:%[0-9A-Za-z]+]]: f32)
 // PPC64-LABEL: func @multipleparamscomplex4
-// PPC64-SAME: ([[A1:%[0-9A-Za-z]+]]: !fir.real<4>, [[B1:%[0-9A-Za-z]+]]: !fir.real<4>, [[A2:%[0-9A-Za-z]+]]: !fir.real<4>, [[B2:%[0-9A-Za-z]+]]: !fir.real<4>, [[A3:%[0-9A-Za-z]+]]: !fir.real<4>, [[B3:%[0-9A-Za-z]+]]: !fir.real<4>)
+// PPC64-SAME: ([[A1:%[0-9A-Za-z]+]]: f32, [[B1:%[0-9A-Za-z]+]]: f32, [[A2:%[0-9A-Za-z]+]]: f32, [[B2:%[0-9A-Za-z]+]]: f32, [[A3:%[0-9A-Za-z]+]]: f32, [[B3:%[0-9A-Za-z]+]]: f32)
 // LOONGARCH64-LABEL: func @multipleparamscomplex4
-// LOONGARCH64-SAME: ([[A1:%[0-9A-Za-z]+]]: !fir.real<4>, [[B1:%[0-9A-Za-z]+]]: !fir.real<4>, [[A2:%[0-9A-Za-z]+]]: !fir.real<4>, [[B2:%[0-9A-Za-z]+]]: !fir.real<4>, [[A3:%[0-9A-Za-z]+]]: !fir.real<4>, [[B3:%[0-9A-Za-z]+]]: !fir.real<4>)
-func.func @multipleparamscomplex4(%z1 : !fir.complex<4>, %z2 : !fir.complex<4>, %z3 : !fir.complex<4>) {
-  // I32-DAG: [[Z1_ADDR:%[0-9A-Za-z]+]] = fir.convert [[Z1]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // I32-DAG: [[Z1_VAL:%[0-9A-Za-z]+]] = fir.load [[Z1_ADDR]] : !fir.ref<!fir.complex<4>>
-  // I32-DAG: [[Z2_ADDR:%[0-9A-Za-z]+]] = fir.convert [[Z2]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // I32-DAG: [[Z2_VAL:%[0-9A-Za-z]+]] = fir.load [[Z2_ADDR]] : !fir.ref<!fir.complex<4>>
-  // I32-DAG: [[Z3_ADDR:%[0-9A-Za-z]+]] = fir.convert [[Z3]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // I32-DAG: [[Z3_VAL:%[0-9A-Za-z]+]] = fir.load [[Z3_ADDR]] : !fir.ref<!fir.complex<4>>
-
-  // I32-DAG: [[Z1_ADDRC:%[0-9A-Za-z]+]] = fir.alloca !fir.complex<4>
-  // I32-DAG: fir.store [[Z1_VAL]] to [[Z1_ADDRC]] : !fir.ref<!fir.complex<4>>
-  // I32-DAG: [[Z1_ADDRT:%[0-9A-Za-z]+]] = fir.convert [[Z1_ADDRC]] : (!fir.ref<!fir.complex<4>>) -> !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // I32-DAG: [[Z2_ADDRC:%[0-9A-Za-z]+]] = fir.alloca !fir.complex<4>
-  // I32-DAG: fir.store [[Z2_VAL]] to [[Z2_ADDRC]] : !fir.ref<!fir.complex<4>>
-  // I32-DAG: [[Z2_ADDRT:%[0-9A-Za-z]+]] = fir.convert [[Z2_ADDRC]] : (!fir.ref<!fir.complex<4>>) -> !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // I32-DAG: [[Z3_ADDRC:%[0-9A-Za-z]+]] = fir.alloca !fir.complex<4>
-  // I32-DAG: fir.store [[Z3_VAL]] to [[Z3_ADDRC]] : !fir.ref<!fir.complex<4>>
-  // I32-DAG: [[Z3_ADDRT:%[0-9A-Za-z]+]] = fir.convert [[Z3_ADDRC]] : (!fir.ref<!fir.complex<4>>) -> !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-
-  // I32: fir.call @calleemultipleparamscomplex4([[Z1_ADDRT]], [[Z2_ADDRT]], [[Z3_ADDRT]]) : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>, !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>, !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> ()
-
-  // I32_MINGW-DAG: [[Z1_ADDR:%[0-9A-Za-z]+]] = fir.convert [[Z1]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // I32_MINGW-DAG: [[Z1_VAL:%[0-9A-Za-z]+]] = fir.load [[Z1_ADDR]] : !fir.ref<!fir.complex<4>>
-  // I32_MINGW-DAG: [[Z2_ADDR:%[0-9A-Za-z]+]] = fir.convert [[Z2]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // I32_MINGW-DAG: [[Z2_VAL:%[0-9A-Za-z]+]] = fir.load [[Z2_ADDR]] : !fir.ref<!fir.complex<4>>
-  // I32_MINGW-DAG: [[Z3_ADDR:%[0-9A-Za-z]+]] = fir.convert [[Z3]] : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // I32_MINGW-DAG: [[Z3_VAL:%[0-9A-Za-z]+]] = fir.load [[Z3_ADDR]] : !fir.ref<!fir.complex<4>>
-
-  // I32_MINGW-DAG: [[Z1_ADDRC:%[0-9A-Za-z]+]] = fir.alloca !fir.complex<4>
-  // I32_MINGW-DAG: fir.store [[Z1_VAL]] to [[Z1_ADDRC]] : !fir.ref<!fir.complex<4>>
-  // I32_MINGW-DAG: [[Z1_ADDRT:%[0-9A-Za-z]+]] = fir.convert [[Z1_ADDRC]] : (!fir.ref<!fir.complex<4>>) -> !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // I32_MINGW-DAG: [[Z2_ADDRC:%[0-9A-Za-z]+]] = fir.alloca !fir.complex<4>
-  // I32_MINGW-DAG: fir.store [[Z2_VAL]] to [[Z2_ADDRC]] : !fir.ref<!fir.complex<4>>
-  // I32_MINGW-DAG: [[Z2_ADDRT:%[0-9A-Za-z]+]] = fir.convert [[Z2_ADDRC]] : (!fir.ref<!fir.complex<4>>) -> !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-  // I32_MINGW-DAG: [[Z3_ADDRC:%[0-9A-Za-z]+]] = fir.alloca !fir.complex<4>
-  // I32_MINGW-DAG: fir.store [[Z3_VAL]] to [[Z3_ADDRC]] : !fir.ref<!fir.complex<4>>
-  // I32_MINGW-DAG: [[Z3_ADDRT:%[0-9A-Za-z]+]] = fir.convert [[Z3_ADDRC]] : (!fir.ref<!fir.complex<4>>) -> !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>
-
-  // I32_MINGW: fir.call @calleemultipleparamscomplex4([[Z1_ADDRT]], [[Z2_ADDRT]], [[Z3_ADDRT]]) : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>, !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>, !fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> ()
-
-  // X64-DAG: [[Z3_ADDR:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:!fir.real<4>>
-  // X64-DAG: fir.store [[Z3]] to [[Z3_ADDR]] : !fir.ref<!fir.vector<2:!fir.real<4>>>
-  // X64-DAG: [[Z3_ADDRC:%[0-9A-Za-z]+]] = fir.convert [[Z3_ADDR]] : (!fir.ref<!fir.vector<2:!fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // X64-DAG: [[Z3_VAL:%[0-9A-Za-z]+]] = fir.load [[Z3_ADDRC]] : !fir.ref<!fir.complex<4>>
-  // X64-DAG: [[Z2_ADDR:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:!fir.real<4>>
-  // X64-DAG: fir.store [[Z2]] to [[Z2_ADDR]] : !fir.ref<!fir.vector<2:!fir.real<4>>>
-  // X64-DAG: [[Z2_ADDRC:%[0-9A-Za-z]+]] = fir.convert [[Z2_ADDR]] : (!fir.ref<!fir.vector<2:!fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // X64-DAG: [[Z2_VAL:%[0-9A-Za-z]+]] = fir.load [[Z2_ADDRC]] : !fir.ref<!fir.complex<4>>
-  // X64-DAG: [[Z1_ADDR:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:!fir.real<4>>
-  // X64-DAG: fir.store [[Z1]] to [[Z1_ADDR]] : !fir.ref<!fir.vector<2:!fir.real<4>>>
-  // X64-DAG: [[Z1_ADDRC:%[0-9A-Za-z]+]] = fir.convert [[Z1_ADDR]] : (!fir.ref<!fir.vector<2:!fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // X64-DAG: [[Z1_VAL:%[0-9A-Za-z]+]] = fir.load [[Z1_ADDRC]] : !fir.ref<!fir.complex<4>>
-
-  // X64-DAG: [[Z1_ADDRV:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:!fir.real<4>>
-  // X64-DAG: [[Z1_ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[Z1_ADDRV]] : (!fir.ref<!fir.vector<2:!fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // X64-DAG: fir.store [[Z1_VAL]] to [[Z1_ADDRC2]] : !fir.ref<!fir.complex<4>>
-  // X64-DAG: [[Z1_RELOADED:%[0-9A-Za-z]+]] = fir.load [[Z1_ADDRV]] : !fir.ref<!fir.vector<2:!fir.real<4>>>
-  // X64-DAG: [[Z2_ADDRV:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:!fir.real<4>>
-  // X64-DAG: [[Z2_ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[Z2_ADDRV]] : (!fir.ref<!fir.vector<2:!fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // X64-DAG: fir.store [[Z2_VAL]] to [[Z2_ADDRC2]] : !fir.ref<!fir.complex<4>>
-  // X64-DAG: [[Z2_RELOADED:%[0-9A-Za-z]+]] = fir.load [[Z2_ADDRV]] : !fir.ref<!fir.vector<2:!fir.real<4>>>
-  // X64-DAG: [[Z3_ADDRV:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:!fir.real<4>>
-  // X64-DAG: [[Z3_ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[Z3_ADDRV]] : (!fir.ref<!fir.vector<2:!fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // X64-DAG: fir.store [[Z3_VAL]] to [[Z3_ADDRC2]] : !fir.ref<!fir.complex<4>>
-  // X64-DAG: [[Z3_RELOADED:%[0-9A-Za-z]+]] = fir.load [[Z3_ADDRV]] : !fir.ref<!fir.vector<2:!fir.real<4>>>
-
-  // X64: fir.call @calleemultipleparamscomplex4([[Z1_RELOADED]], [[Z2_RELOADED]], [[Z3_RELOADED]]) : (!fir.vector<2:!fir.real<4>>, !fir.vector<2:!fir.real<4>>, !fir.vector<2:!fir.real<4>>) -> ()
+// LOONGARCH64-SAME: ([[A1:%[0-9A-Za-z]+]]: f32, [[B1:%[0-9A-Za-z]+]]: f32, [[A2:%[0-9A-Za-z]+]]: f32, [[B2:%[0-9A-Za-z]+]]: f32, [[A3:%[0-9A-Za-z]+]]: f32, [[B3:%[0-9A-Za-z]+]]: f32)
+func.func @multipleparamscomplex4(%z1 : complex<f32>, %z2 : complex<f32>, %z3 : complex<f32>) {
+  // I32-DAG: [[Z1_ADDR:%[0-9A-Za-z]+]] = fir.convert [[Z1]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // I32-DAG: [[Z1_VAL:%[0-9A-Za-z]+]] = fir.load [[Z1_ADDR]] : !fir.ref<complex<f32>>
+  // I32-DAG: [[Z2_ADDR:%[0-9A-Za-z]+]] = fir.convert [[Z2]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // I32-DAG: [[Z2_VAL:%[0-9A-Za-z]+]] = fir.load [[Z2_ADDR]] : !fir.ref<complex<f32>>
+  // I32-DAG: [[Z3_ADDR:%[0-9A-Za-z]+]] = fir.convert [[Z3]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // I32-DAG: [[Z3_VAL:%[0-9A-Za-z]+]] = fir.load [[Z3_ADDR]] : !fir.ref<complex<f32>>
+
+  // I32-DAG: [[Z1_ADDRC:%[0-9A-Za-z]+]] = fir.alloca complex<f32>
+  // I32-DAG: fir.store [[Z1_VAL]] to [[Z1_ADDRC]] : !fir.ref<complex<f32>>
+  // I32-DAG: [[Z1_ADDRT:%[0-9A-Za-z]+]] = fir.convert [[Z1_ADDRC]] : (!fir.ref<complex<f32>>) -> !fir.ref<tuple<f32, f32>>
+  // I32-DAG: [[Z2_ADDRC:%[0-9A-Za-z]+]] = fir.alloca complex<f32>
+  // I32-DAG: fir.store [[Z2_VAL]] to [[Z2_ADDRC]] : !fir.ref<complex<f32>>
+  // I32-DAG: [[Z2_ADDRT:%[0-9A-Za-z]+]] = fir.convert [[Z2_ADDRC]] : (!fir.ref<complex<f32>>) -> !fir.ref<tuple<f32, f32>>
+  // I32-DAG: [[Z3_ADDRC:%[0-9A-Za-z]+]] = fir.alloca complex<f32>
+  // I32-DAG: fir.store [[Z3_VAL]] to [[Z3_ADDRC]] : !fir.ref<complex<f32>>
+  // I32-DAG: [[Z3_ADDRT:%[0-9A-Za-z]+]] = fir.convert [[Z3_ADDRC]] : (!fir.ref<complex<f32>>) -> !fir.ref<tuple<f32, f32>>
+
+  // I32: fir.call @calleemultipleparamscomplex4([[Z1_ADDRT]], [[Z2_ADDRT]], [[Z3_ADDRT]]) : (!fir.ref<tuple<f32, f32>>, !fir.ref<tuple<f32, f32>>, !fir.ref<tuple<f32, f32>>) -> ()
+
+  // I32_MINGW-DAG: [[Z1_ADDR:%[0-9A-Za-z]+]] = fir.convert [[Z1]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // I32_MINGW-DAG: [[Z1_VAL:%[0-9A-Za-z]+]] = fir.load [[Z1_ADDR]] : !fir.ref<complex<f32>>
+  // I32_MINGW-DAG: [[Z2_ADDR:%[0-9A-Za-z]+]] = fir.convert [[Z2]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // I32_MINGW-DAG: [[Z2_VAL:%[0-9A-Za-z]+]] = fir.load [[Z2_ADDR]] : !fir.ref<complex<f32>>
+  // I32_MINGW-DAG: [[Z3_ADDR:%[0-9A-Za-z]+]] = fir.convert [[Z3]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
+  // I32_MINGW-DAG: [[Z3_VAL:%[0-9A-Za-z]+]] = fir.load [[Z3_ADDR]] : !fir.ref<complex<f32>>
+
+  // I32_MINGW-DAG: [[Z1_ADDRC:%[0-9A-Za-z]+]] = fir.alloca complex<f32>
+  // I32_MINGW-DAG: fir.store [[Z1_VAL]] to [[Z1_ADDRC]] : !fir.ref<complex<f32>>
+  // I32_MINGW-DAG: [[Z1_ADDRT:%[0-9A-Za-z]+]] = fir.convert [[Z1_ADDRC]] : (!fir.ref<complex<f32>>) -> !fir.ref<tuple<f32, f32>>
+  // I32_MINGW-DAG: [[Z2_ADDRC:%[0-9A-Za-z]+]] = fir.alloca complex<f32>
+  // I32_MINGW-DAG: fir.store [[Z2_VAL]] to [[Z2_ADDRC]] : !fir.ref<complex<f32>>
+  // I32_MINGW-DAG: [[Z2_ADDRT:%[0-9A-Za-z]+]] = fir.convert [[Z2_ADDRC]] : (!fir.ref<complex<f32>>) -> !fir.ref<tuple<f32, f32>>
+  // I32_MINGW-DAG: [[Z3_ADDRC:%[0-9A-Za-z]+]] = fir.alloca complex<f32>
+  // I32_MINGW-DAG: fir.store [[Z3_VAL]] to [[Z3_ADDRC]] : !fir.ref<complex<f32>>
+  // I32_MINGW-DAG: [[Z3_ADDRT:%[0-9A-Za-z]+]] = fir.convert [[Z3_ADDRC]] : (!fir.ref<complex<f32>>) -> !fir.ref<tuple<f32, f32>>
+
+  // I32_MINGW: fir.call @calleemultipleparamscomplex4([[Z1_ADDRT]], [[Z2_ADDRT]], [[Z3_ADDRT]]) : (!fir.ref<tuple<f32, f32>>, !fir.ref<tuple<f32, f32>>, !fir.ref<tuple<f32, f32>>) -> ()
+
+  // X64-DAG: [[Z3_ADDR:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:f32>
+  // X64-DAG: fir.store [[Z3]] to [[Z3_ADDR]] : !fir.ref<!fir.vector<2:f32>>
+  // X64-DAG: [[Z3_ADDRC:%[0-9A-Za-z]+]] = fir.convert [[Z3_ADDR]] : (!fir.ref<!fir.vector<2:f32>>) -> !fir.ref<complex<f32>>
+  // X64-DAG: [[Z3_VAL:%[0-9A-Za-z]+]] = fir.load [[Z3_ADDRC]] : !fir.ref<complex<f32>>
+  // X64-DAG: [[Z2_ADDR:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:f32>
+  // X64-DAG: fir.store [[Z2]] to [[Z2_ADDR]] : !fir.ref<!fir.vector<2:f32>>
+  // X64-DAG: [[Z2_ADDRC:%[0-9A-Za-z]+]] = fir.convert [[Z2_ADDR]] : (!fir.ref<!fir.vector<2:f32>>) -> !fir.ref<complex<f32>>
+  // X64-DAG: [[Z2_VAL:%[0-9A-Za-z]+]] = fir.load [[Z2_ADDRC]] : !fir.ref<complex<f32>>
+  // X64-DAG: [[Z1_ADDR:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:f32>
+  // X64-DAG: fir.store [[Z1]] to [[Z1_ADDR]] : !fir.ref<!fir.vector<2:f32>>
+  // X64-DAG: [[Z1_ADDRC:%[0-9A-Za-z]+]] = fir.convert [[Z1_ADDR]] : (!fir.ref<!fir.vector<2:f32>>) -> !fir.ref<complex<f32>>
+  // X64-DAG: [[Z1_VAL:%[0-9A-Za-z]+]] = fir.load [[Z1_ADDRC]] : !fir.ref<complex<f32>>
+
+  // X64-DAG: [[Z1_ADDRV:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:f32>
+  // X64-DAG: [[Z1_ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[Z1_ADDRV]] : (!fir.ref<!fir.vector<2:f32>>) -> !fir.ref<complex<f32>>
+  // X64-DAG: fir.store [[Z1_VAL]] to [[Z1_ADDRC2]] : !fir.ref<complex<f32>>
+  // X64-DAG: [[Z1_RELOADED:%[0-9A-Za-z]+]] = fir.load [[Z1_ADDRV]] : !fir.ref<!fir.vector<2:f32>>
+  // X64-DAG: [[Z2_ADDRV:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:f32>
+  // X64-DAG: [[Z2_ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[Z2_ADDRV]] : (!fir.ref<!fir.vector<2:f32>>) -> !fir.ref<complex<f32>>
+  // X64-DAG: fir.store [[Z2_VAL]] to [[Z2_ADDRC2]] : !fir.ref<complex<f32>>
+  // X64-DAG: [[Z2_RELOADED:%[0-9A-Za-z]+]] = fir.load [[Z2_ADDRV]] : !fir.ref<!fir.vector<2:f32>>
+  // X64-DAG: [[Z3_ADDRV:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:f32>
+  // X64-DAG: [[Z3_ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[Z3_ADDRV]] : (!fir.ref<!fir.vector<2:f32>>) -> !fir.ref<complex<f32>>
+  // X64-DAG: fir.store [[Z3_VAL]] to [[Z3_ADDRC2]] : !fir.ref<complex<f32>>
+  // X64-DAG: [[Z3_RELOADED:%[0-9A-Za-z]+]] = fir.load [[Z3_ADDRV]] : !fir.ref<!fir.vector<2:f32>>
+
+  // X64: fir.call @calleemultipleparamscomplex4([[Z1_RELOADED]], [[Z2_RELOADED]], [[Z3_RELOADED]]) : (!fir.vector<2:f32>, !fir.vector<2:f32>, !fir.vector<2:f32>) -> ()
 
   // X64_MINGW-DAG: [[Z3_ADDR:%[0-9A-Za-z]+]] = fir.alloca i64
   // X64_MINGW-DAG: fir.store [[Z3]] to [[Z3_ADDR]] : !fir.ref<i64>
-  // X64_MINGW-DAG: [[Z3_ADDRC:%[0-9A-Za-z]+]] = fir.convert [[Z3_ADDR]] : (!fir.ref<i64>) -> !fir.ref<!fir.complex<4>>
-  // X64_MINGW-DAG: [[Z3_VAL:%[0-9A-Za-z]+]] = fir.load [[Z3_ADDRC]] : !fir.ref<!fir.complex<4>>
+  // X64_MINGW-DAG: [[Z3_ADDRC:%[0-9A-Za-z]+]] = fir.convert [[Z3_ADDR]] : (!fir.ref<i64>) -> !fir.ref<complex<f32>>
+  // X64_MINGW-DAG: [[Z3_VAL:%[0-9A-Za-z]+]] = fir.load [[Z3_ADDRC]] : !fir.ref<complex<f32>>
   // X64_MINGW-DAG: [[Z2_ADDR:%[0-9A-Za-z]+]] = fir.alloca i64
   // X64_MINGW-DAG: fir.store [[Z2]] to [[Z2_ADDR]] : !fir.ref<i64>
-  // X64_MINGW-DAG: [[Z2_ADDRC:%[0-9A-Za-z]+]] = fir.convert [[Z2_ADDR]] : (!fir.ref<i64>) -> !fir.ref<!fir.complex<4>>
-  // X64_MINGW-DAG: [[Z2_VAL:%[0-9A-Za-z]+]] = fir.load [[Z2_ADDRC]] : !fir.ref<!fir.complex<4>>
+  // X64_MINGW-DAG: [[Z2_ADDRC:%[0-9A-Za-z]+]] = fir.convert [[Z2_ADDR]] : (!fir.ref<i64>) -> !fir.ref<complex<f32>>
+  // X64_MINGW-DAG: [[Z2_VAL:%[0-9A-Za-z]+]] = fir.load [[Z2_ADDRC]] : !fir.ref<complex<f32>>
   // X64_MINGW-DAG: [[Z1_ADDR:%[0-9A-Za-z]+]] = fir.alloca i64
   // X64_MINGW-DAG: fir.store [[Z1]] to [[Z1_ADDR]] : !fir.ref<i64>
-  // X64_MINGW-DAG: [[Z1_ADDRC:%[0-9A-Za-z]+]] = fir.convert [[Z1_ADDR]] : (!fir.ref<i64>) -> !fir.ref<!fir.complex<4>>
-  // X64_MINGW-DAG: [[Z1_VAL:%[0-9A-Za-z]+]] = fir.load [[Z1_ADDRC]] : !fir.ref<!fir.complex<4>>
+  // X64_MINGW-DAG: [[Z1_ADDRC:%[0-9A-Za-z]+]] = fir.convert [[Z1_ADDR]] : (!fir.ref<i64>) -> !fir.ref<complex<f32>>
+  // X64_MINGW-DAG: [[Z1_VAL:%[0-9A-Za-z]+]] = fir.load [[Z1_ADDRC]] : !fir.ref<complex<f32>>
 
   // X64_MINGW-DAG: [[Z1_ADDRV:%[0-9A-Za-z]+]] = fir.alloca i64
-  // X64_MINGW-DAG: [[Z1_ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[Z1_ADDRV]] : (!fir.ref<i64>) -> !fir.ref<!fir.complex<4>>
-  // X64_MINGW-DAG: fir.store [[Z1_VAL]] to [[Z1_ADDRC2]] : !fir.ref<!fir.complex<4>>
+  // X64_MINGW-DAG: [[Z1_ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[Z1_ADDRV]] : (!fir.ref<i64>) -> !fir.ref<complex<f32>>
+  // X64_MINGW-DAG: fir.store [[Z1_VAL]] to [[Z1_ADDRC2]] : !fir.ref<complex<f32>>
   // X64_MINGW-DAG: [[Z1_RELOADED:%[0-9A-Za-z]+]] = fir.load [[Z1_ADDRV]] : !fir.ref<i64>
   // X64_MINGW-DAG: [[Z2_ADDRV:%[0-9A-Za-z]+]] = fir.alloca i64
-  // X64_MINGW-DAG: [[Z2_ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[Z2_ADDRV]] : (!fir.ref<i64>) -> !fir.ref<!fir.complex<4>>
-  // X64_MINGW-DAG: fir.store [[Z2_VAL]] to [[Z2_ADDRC2]] : !fir.ref<!fir.complex<4>>
+  // X64_MINGW-DAG: [[Z2_ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[Z2_ADDRV]] : (!fir.ref<i64>) -> !fir.ref<complex<f32>>
+  // X64_MINGW-DAG: fir.store [[Z2_VAL]] to [[Z2_ADDRC2]] : !fir.ref<complex<f32>>
   // X64_MINGW-DAG: [[Z2_RELOADED:%[0-9A-Za-z]+]] = fir.load [[Z2_ADDRV]] : !fir.ref<i64>
   // X64_MINGW-DAG: [[Z3_ADDRV:%[0-9A-Za-z]+]] = fir.alloca i64
-  // X64_MINGW-DAG: [[Z3_ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[Z3_ADDRV]] : (!fir.ref<i64>) -> !fir.ref<!fir.complex<4>>
-  // X64_MINGW-DAG: fir.store [[Z3_VAL]] to [[Z3_ADDRC2]] : !fir.ref<!fir.complex<4>>
+  // X64_MINGW-DAG: [[Z3_ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[Z3_ADDRV]] : (!fir.ref<i64>) -> !fir.ref<complex<f32>>
+  // X64_MINGW-DAG: fir.store [[Z3_VAL]] to [[Z3_ADDRC2]] : !fir.ref<complex<f32>>
   // X64_MINGW-DAG: [[Z3_RELOADED:%[0-9A-Za-z]+]] = fir.load [[Z3_ADDRV]] : !fir.ref<i64>
 
   // X64_MINGW: fir.call @calleemultipleparamscomplex4([[Z1_RELOADED]], [[Z2_RELOADED]], [[Z3_RELOADED]]) : (i64, i64, i64) -> ()
 
-  // AARCH64-DAG: [[Z3_ARR:%[0-9A-Za-z]+]] = fir.alloca !fir.array<2x!fir.real<4>>
-  // AARCH64-DAG: fir.store [[Z3]] to [[Z3_ARR]] : !fir.ref<!fir.array<2x!fir.real<4>>>
-  // AARCH64-DAG: [[Z3_ADDRC:%[0-9A-Za-z]+]] = fir.convert [[Z3_ARR]] : (!fir.ref<!fir.array<2x!fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // AARCH64-DAG: [[Z3_VAL:%[0-9A-Za-z]+]] = fir.load [[Z3_ADDRC]] : !fir.ref<!fir.complex<4>>
-  // AARCH64-DAG: [[Z2_ARR:%[0-9A-Za-z]+]] = fir.alloca !fir.array<2x!fir.real<4>>
-  // AARCH64-DAG: fir.store [[Z2]] to [[Z2_ARR]] : !fir.ref<!fir.array<2x!fir.real<4>>>
-  // AARCH64-DAG: [[Z2_ADDRC:%[0-9A-Za-z]+]] = fir.convert [[Z2_ARR]] : (!fir.ref<!fir.array<2x!fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // AARCH64-DAG: [[Z2_VAL:%[0-9A-Za-z]+]] = fir.load [[Z2_ADDRC]] : !fir.ref<!fir.complex<4>>
-  // AARCH64-DAG: [[Z1_ARR:%[0-9A-Za-z]+]] = fir.alloca !fir.array<2x!fir.real<4>>
-  // AARCH64-DAG: fir.store [[Z1]] to [[Z1_ARR]] : !fir.ref<!fir.array<2x!fir.real<4>>>
-  // AARCH64-DAG: [[Z1_ADDRC:%[0-9A-Za-z]+]] = fir.convert [[Z1_ARR]] : (!fir.ref<!fir.array<2x!fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // AARCH64-DAG: [[Z1_VAL:%[0-9A-Za-z]+]] = fir.load [[Z1_ADDRC]] : !fir.ref<!fir.complex<4>>
-
-  // AARCH64-DAG: [[Z1_ARR2:%[0-9A-Za-z]+]] = fir.alloca !fir.array<2x!fir.real<4>>
-  // AARCH64-DAG: [[Z1_ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[Z1_ARR2]] : (!fir.ref<!fir.array<2x!fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // AARCH64-DAG: fir.store [[Z1_VAL]] to [[Z1_ADDRC2]] : !fir.ref<!fir.complex<4>>
-  // AARCH64-DAG: [[Z1_RELOADED:%[0-9A-Za-z]+]] = fir.load [[Z1_ARR2]] : !fir.ref<!fir.array<2x!fir.real<4>>>
-  // AARCH64-DAG: [[Z2_ARR2:%[0-9A-Za-z]+]] = fir.alloca !fir.array<2x!fir.real<4>>
-  // AARCH64-DAG: [[Z2_ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[Z2_ARR2]] : (!fir.ref<!fir.array<2x!fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // AARCH64-DAG: fir.store [[Z2_VAL]] to [[Z2_ADDRC2]] : !fir.ref<!fir.complex<4>>
-  // AARCH64-DAG: [[Z2_RELOADED:%[0-9A-Za-z]+]] = fir.load [[Z2_ARR2]] : !fir.ref<!fir.array<2x!fir.real<4>>>
-  // AARCH64-DAG: [[Z3_ARR2:%[0-9A-Za-z]+]] = fir.alloca !fir.array<2x!fir.real<4>>
-  // AARCH64-DAG: [[Z3_ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[Z3_ARR2]] : (!fir.ref<!fir.array<2x!fir.real<4>>>) -> !fir.ref<!fir.complex<4>>
-  // AARCH64-DAG: fir.store [[Z3_VAL]] to [[Z3_ADDRC2]] : !fir.ref<!fir.complex<4>>
-  // AARCH64-DAG: [[Z3_RELOADED:%[0-9A-Za-z]+]] = fir.load [[Z3_ARR2]] : !fir.ref<!fir.array<2x!fir.real<4>>>
-
-  // AARCH64: fir.call @calleemultipleparamscomplex4([[Z1_RELOADED]], [[Z2_RELOADED]], [[Z3_RELOADED]]) : (!fir.array<2x!fir.real<4>>, !fir.array<2x!fir.real<4>>, !fir.array<2x!fir.real<4>>) -> ()
-
-  // PPC64le-DAG: [[Z3_EMPTY:%[0-9A-Za-z]+]] = fir.undefined !fir.complex<4>
-  // PPC64le-DAG: [[Z3_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_EMPTY]], [[A3]], [0 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // PPC64le-DAG: [[Z3:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_PARTIAL]], [[B3]], [1 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // PPC64le-DAG: [[Z2_EMPTY:%[0-9A-Za-z]+]] = fir.undefined !fir.complex<4>
-  // PPC64le-DAG: [[Z2_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_EMPTY]], [[A2]], [0 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // PPC64le-DAG: [[Z2:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_PARTIAL]], [[B2]], [1 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // PPC64le-DAG: [[Z1_EMPTY:%[0-9A-Za-z]+]] = fir.undefined !fir.complex<4>
-  // PPC64le-DAG: [[Z1_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_EMPTY]], [[A1]], [0 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // PPC64le-DAG: [[Z1:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_PARTIAL]], [[B1]], [1 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-
-  // PPC64le-DAG: [[A1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // PPC64le-DAG: [[B1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // PPC64le-DAG: [[A2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // PPC64le-DAG: [[B2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // PPC64le-DAG: [[A3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // PPC64le-DAG: [[B3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-
-  // PPC64le: fir.call @calleemultipleparamscomplex4([[A1_EXTR]], [[B1_EXTR]], [[A2_EXTR]], [[B2_EXTR]], [[A3_EXTR]], [[B3_EXTR]]) : (!fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>) -> ()
-
-  // SPARCV9-DAG: [[Z3_EMPTY:%[0-9A-Za-z]+]] = fir.undefined !fir.complex<4>
-  // SPARCV9-DAG: [[Z3_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_EMPTY]], [[A3]], [0 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // SPARCV9-DAG: [[Z3:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_PARTIAL]], [[B3]], [1 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // SPARCV9-DAG: [[Z2_EMPTY:%[0-9A-Za-z]+]] = fir.undefined !fir.complex<4>
-  // SPARCV9-DAG: [[Z2_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_EMPTY]], [[A2]], [0 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // SPARCV9-DAG: [[Z2:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_PARTIAL]], [[B2]], [1 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // SPARCV9-DAG: [[Z1_EMPTY:%[0-9A-Za-z]+]] = fir.undefined !fir.complex<4>
-  // SPARCV9-DAG: [[Z1_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_EMPTY]], [[A1]], [0 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // SPARCV9-DAG: [[Z1:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_PARTIAL]], [[B1]], [1 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-
-  // SPARCV9-DAG: [[A1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // SPARCV9-DAG: [[B1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // SPARCV9-DAG: [[A2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // SPARCV9-DAG: [[B2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // SPARCV9-DAG: [[A3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // SPARCV9-DAG: [[B3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-
-  // SPARCV9: fir.call @calleemultipleparamscomplex4([[A1_EXTR]], [[B1_EXTR]], [[A2_EXTR]], [[B2_EXTR]], [[A3_EXTR]], [[B3_EXTR]]) : (!fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>) -> ()
-
-  // RISCV64-DAG: [[Z3_EMPTY:%[0-9A-Za-z]+]] = fir.undefined !fir.complex<4>
-  // RISCV64-DAG: [[Z3_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_EMPTY]], [[A3]], [0 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // RISCV64-DAG: [[Z3:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_PARTIAL]], [[B3]], [1 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // RISCV64-DAG: [[Z2_EMPTY:%[0-9A-Za-z]+]] = fir.undefined !fir.complex<4>
-  // RISCV64-DAG: [[Z2_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_EMPTY]], [[A2]], [0 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // RISCV64-DAG: [[Z2:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_PARTIAL]], [[B2]], [1 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // RISCV64-DAG: [[Z1_EMPTY:%[0-9A-Za-z]+]] = fir.undefined !fir.complex<4>
-  // RISCV64-DAG: [[Z1_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_EMPTY]], [[A1]], [0 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // RISCV64-DAG: [[Z1:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_PARTIAL]], [[B1]], [1 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-
-  // RISCV64-DAG: [[A1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // RISCV64-DAG: [[B1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // RISCV64-DAG: [[A2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // RISCV64-DAG: [[B2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // RISCV64-DAG: [[A3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // RISCV64-DAG: [[B3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-
-  // RISCV64: fir.call @calleemultipleparamscomplex4([[A1_EXTR]], [[B1_EXTR]], [[A2_EXTR]], [[B2_EXTR]], [[A3_EXTR]], [[B3_EXTR]]) : (!fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>) -> ()
-  // PPC64-DAG: [[Z3_EMPTY:%[0-9A-Za-z]+]] = fir.undefined !fir.complex<4>
-  // PPC64-DAG: [[Z3_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_EMPTY]], [[A3]], [0 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // PPC64-DAG: [[Z3:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_PARTIAL]], [[B3]], [1 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // PPC64-DAG: [[Z2_EMPTY:%[0-9A-Za-z]+]] = fir.undefined !fir.complex<4>
-  // PPC64-DAG: [[Z2_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_EMPTY]], [[A2]], [0 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // PPC64-DAG: [[Z2:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_PARTIAL]], [[B2]], [1 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // PPC64-DAG: [[Z1_EMPTY:%[0-9A-Za-z]+]] = fir.undefined !fir.complex<4>
-  // PPC64-DAG: [[Z1_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_EMPTY]], [[A1]], [0 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // PPC64-DAG: [[Z1:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_PARTIAL]], [[B1]], [1 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-
-  // PPC64-DAG: [[A1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // PPC64-DAG: [[B1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // PPC64-DAG: [[A2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // PPC64-DAG: [[B2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // PPC64-DAG: [[A3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // PPC64-DAG: [[B3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-
-  // PPC64: fir.call @calleemultipleparamscomplex4([[A1_EXTR]], [[B1_EXTR]], [[A2_EXTR]], [[B2_EXTR]], [[A3_EXTR]], [[B3_EXTR]]) : (!fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>) -> ()
-  // LOONGARCH64-DAG: [[Z3_EMPTY:%[0-9A-Za-z]+]] = fir.undefined !fir.complex<4>
-  // LOONGARCH64-DAG: [[Z3_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_EMPTY]], [[A3]], [0 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // LOONGARCH64-DAG: [[Z3:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_PARTIAL]], [[B3]], [1 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // LOONGARCH64-DAG: [[Z2_EMPTY:%[0-9A-Za-z]+]] = fir.undefined !fir.complex<4>
-  // LOONGARCH64-DAG: [[Z2_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_EMPTY]], [[A2]], [0 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // LOONGARCH64-DAG: [[Z2:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_PARTIAL]], [[B2]], [1 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // LOONGARCH64-DAG: [[Z1_EMPTY:%[0-9A-Za-z]+]] = fir.undefined !fir.complex<4>
-  // LOONGARCH64-DAG: [[Z1_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_EMPTY]], [[A1]], [0 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-  // LOONGARCH64-DAG: [[Z1:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_PARTIAL]], [[B1]], [1 : i32] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
-
-  // LOONGARCH64-DAG: [[A1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // LOONGARCH64-DAG: [[B1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // LOONGARCH64-DAG: [[A2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // LOONGARCH64-DAG: [[B2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // LOONGARCH64-DAG: [[A3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [0 : i32] : (!fir.complex<4>) -> !fir.real<4>
-  // LOONGARCH64-DAG: [[B3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [1 : i32] : (!fir.complex<4>) -> !fir.real<4>
-
-  // LOONGARCH64: fir.call @calleemultipleparamscomplex4([[A1_EXTR]], [[B1_EXTR]], [[A2_EXTR]], [[B2_EXTR]], [[A3_EXTR]], [[B3_EXTR]]) : (!fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>, !fir.real<4>) -> ()
-  fir.call @calleemultipleparamscomplex4(%z1, %z2, %z3) : (!fir.complex<4>, !fir.complex<4>, !fir.complex<4>) -> ()
+  // AARCH64-DAG: [[Z3_ARR:%[0-9A-Za-z]+]] = fir.alloca !fir.array<2xf32>
+  // AARCH64-DAG: fir.store [[Z3]] to [[Z3_ARR]] : !fir.ref<!fir.array<2xf32>>
+  // AARCH64-DAG: [[Z3_ADDRC:%[0-9A-Za-z]+]] = fir.convert [[Z3_ARR]] : (!fir.ref<!fir.array<2xf32>>) -> !fir.ref<complex<f32>>
+  // AARCH64-DAG: [[Z3_VAL:%[0-9A-Za-z]+]] = fir.load [[Z3_ADDRC]] : !fir.ref<complex<f32>>
+  // AARCH64-DAG: [[Z2_ARR:%[0-9A-Za-z]+]] = fir.alloca !fir.array<2xf32>
+  // AARCH64-DAG: fir.store [[Z2]] to [[Z2_ARR]] : !fir.ref<!fir.array<2xf32>>
+  // AARCH64-DAG: [[Z2_ADDRC:%[0-9A-Za-z]+]] = fir.convert [[Z2_ARR]] : (!fir.ref<!fir.array<2xf32>>) -> !fir.ref<complex<f32>>
+  // AARCH64-DAG: [[Z2_VAL:%[0-9A-Za-z]+]] = fir.load [[Z2_ADDRC]] : !fir.ref<complex<f32>>
+  // AARCH64-DAG: [[Z1_ARR:%[0-9A-Za-z]+]] = fir.alloca !fir.array<2xf32>
+  // AARCH64-DAG: fir.store [[Z1]] to [[Z1_ARR]] : !fir.ref<!fir.array<2xf32>>
+  // AARCH64-DAG: [[Z1_ADDRC:%[0-9A-Za-z]+]] = fir.convert [[Z1_ARR]] : (!fir.ref<!fir.array<2xf32>>) -> !fir.ref<complex<f32>>
+  // AARCH64-DAG: [[Z1_VAL:%[0-9A-Za-z]+]] = fir.load [[Z1_ADDRC]] : !fir.ref<complex<f32>>
+
+  // AARCH64-DAG: [[Z1_ARR2:%[0-9A-Za-z]+]] = fir.alloca !fir.array<2xf32>
+  // AARCH64-DAG: [[Z1_ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[Z1_ARR2]] : (!fir.ref<!fir.array<2xf32>>) -> !fir.ref<complex<f32>>
+  // AARCH64-DAG: fir.store [[Z1_VAL]] to [[Z1_ADDRC2]] : !fir.ref<complex<f32>>
+  // AARCH64-DAG: [[Z1_RELOADED:%[0-9A-Za-z]+]] = fir.load [[Z1_ARR2]] : !fir.ref<!fir.array<2xf32>>
+  // AARCH64-DAG: [[Z2_ARR2:%[0-9A-Za-z]+]] = fir.alloca !fir.array<2xf32>
+  // AARCH64-DAG: [[Z2_ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[Z2_ARR2]] : (!fir.ref<!fir.array<2xf32>>) -> !fir.ref<complex<f32>>
+  // AARCH64-DAG: fir.store [[Z2_VAL]] to [[Z2_ADDRC2]] : !fir.ref<complex<f32>>
+  // AARCH64-DAG: [[Z2_RELOADED:%[0-9A-Za-z]+]] = fir.load [[Z2_ARR2]] : !fir.ref<!fir.array<2xf32>>
+  // AARCH64-DAG: [[Z3_ARR2:%[0-9A-Za-z]+]] = fir.alloca !fir.array<2xf32>
+  // AARCH64-DAG: [[Z3_ADDRC2:%[0-9A-Za-z]+]] = fir.convert [[Z3_ARR2]] : (!fir.ref<!fir.array<2xf32>>) -> !fir.ref<complex<f32>>
+  // AARCH64-DAG: fir.store [[Z3_VAL]] to [[Z3_ADDRC2]] : !fir.ref<complex<f32>>
+  // AARCH64-DAG: [[Z3_RELOADED:%[0-9A-Za-z]+]] = fir.load [[Z3_ARR2]] : !fir.ref<!fir.array<2xf32>>
+
+  // AARCH64: fir.call @calleemultipleparamscomplex4([[Z1_RELOADED]], [[Z2_RELOADED]], [[Z3_RELOADED]]) : (!fir.array<2xf32>, !fir.array<2xf32>, !fir.array<2xf32>) -> ()
+
+  // PPC64le-DAG: [[Z3_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
+  // PPC64le-DAG: [[Z3_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_EMPTY]], [[A3]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
+  // PPC64le-DAG: [[Z3:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_PARTIAL]], [[B3]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
+  // PPC64le-DAG: [[Z2_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
+  // PPC64le-DAG: [[Z2_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_EMPTY]], [[A2]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
+  // PPC64le-DAG: [[Z2:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_PARTIAL]], [[B2]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
+  // PPC64le-DAG: [[Z1_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
+  // PPC64le-DAG: [[Z1_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_EMPTY]], [[A1]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
+  // PPC64le-DAG: [[Z1:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_PARTIAL]], [[B1]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
+
+  // PPC64le-DAG: [[A1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [0 : i32] : (complex<f32>) -> f32
+  // PPC64le-DAG: [[B1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [1 : i32] : (complex<f32>) -> f32
+  // PPC64le-DAG: [[A2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [0 : i32] : (complex<f32>) -> f32
+  // PPC64le-DAG: [[B2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [1 : i32] : (complex<f32>) -> f32
+  // PPC64le-DAG: [[A3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [0 : i32] : (complex<f32>) -> f32
+  // PPC64le-DAG: [[B3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [1 : i32] : (complex<f32>) -> f32
+
+  // PPC64le: fir.call @calleemultipleparamscomplex4([[A1_EXTR]], [[B1_EXTR]], [[A2_EXTR]], [[B2_EXTR]], [[A3_EXTR]], [[B3_EXTR]]) : (f32, f32, f32, f32, f32, f32) -> ()
+
+  // SPARCV9-DAG: [[Z3_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
+  // SPARCV9-DAG: [[Z3_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_EMPTY]], [[A3]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
+  // SPARCV9-DAG: [[Z3:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_PARTIAL]], [[B3]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
+  // SPARCV9-DAG: [[Z2_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
+  // SPARCV9-DAG: [[Z2_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_EMPTY]], [[A2]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
+  // SPARCV9-DAG: [[Z2:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_PARTIAL]], [[B2]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
+  // SPARCV9-DAG: [[Z1_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
+  // SPARCV9-DAG: [[Z1_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_EMPTY]], [[A1]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
+  // SPARCV9-DAG: [[Z1:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_PARTIAL]], [[B1]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
+
+  // SPARCV9-DAG: [[A1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [0 : i32] : (complex<f32>) -> f32
+  // SPARCV9-DAG: [[B1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [1 : i32] : (complex<f32>) -> f32
+  // SPARCV9-DAG: [[A2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [0 : i32] : (complex<f32>) -> f32
+  // SPARCV9-DAG: [[B2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [1 : i32] : (complex<f32>) -> f32
+  // SPARCV9-DAG: [[A3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [0 : i32] : (complex<f32>) -> f32
+  // SPARCV9-DAG: [[B3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [1 : i32] : (complex<f32>) -> f32
+
+  // SPARCV9: fir.call @calleemultipleparamscomplex4([[A1_EXTR]], [[B1_EXTR]], [[A2_EXTR]], [[B2_EXTR]], [[A3_EXTR]], [[B3_EXTR]]) : (f32, f32, f32, f32, f32, f32) -> ()
+
+  // RISCV64-DAG: [[Z3_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
+  // RISCV64-DAG: [[Z3_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_EMPTY]], [[A3]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
+  // RISCV64-DAG: [[Z3:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_PARTIAL]], [[B3]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
+  // RISCV64-DAG: [[Z2_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
+  // RISCV64-DAG: [[Z2_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_EMPTY]], [[A2]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
+  // RISCV64-DAG: [[Z2:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_PARTIAL]], [[B2]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
+  // RISCV64-DAG: [[Z1_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
+  // RISCV64-DAG: [[Z1_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_EMPTY]], [[A1]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
+  // RISCV64-DAG: [[Z1:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_PARTIAL]], [[B1]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
+
+  // RISCV64-DAG: [[A1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [0 : i32] : (complex<f32>) -> f32
+  // RISCV64-DAG: [[B1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [1 : i32] : (complex<f32>) -> f32
+  // RISCV64-DAG: [[A2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [0 : i32] : (complex<f32>) -> f32
+  // RISCV64-DAG: [[B2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [1 : i32] : (complex<f32>) -> f32
+  // RISCV64-DAG: [[A3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [0 : i32] : (complex<f32>) -> f32
+  // RISCV64-DAG: [[B3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [1 : i32] : (complex<f32>) -> f32
+
+  // RISCV64: fir.call @calleemultipleparamscomplex4([[A1_EXTR]], [[B1_EXTR]], [[A2_EXTR]], [[B2_EXTR]], [[A3_EXTR]], [[B3_EXTR]]) : (f32, f32, f32, f32, f32, f32) -> ()
+  // PPC64-DAG: [[Z3_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
+  // PPC64-DAG: [[Z3_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_EMPTY]], [[A3]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
+  // PPC64-DAG: [[Z3:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_PARTIAL]], [[B3]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
+  // PPC64-DAG: [[Z2_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
+  // PPC64-DAG: [[Z2_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_EMPTY]], [[A2]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
+  // PPC64-DAG: [[Z2:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_PARTIAL]], [[B2]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
+  // PPC64-DAG: [[Z1_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
+  // PPC64-DAG: [[Z1_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_EMPTY]], [[A1]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
+  // PPC64-DAG: [[Z1:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_PARTIAL]], [[B1]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
+
+  // PPC64-DAG: [[A1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [0 : i32] : (complex<f32>) -> f32
+  // PPC64-DAG: [[B1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [1 : i32] : (complex<f32>) -> f32
+  // PPC64-DAG: [[A2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [0 : i32] : (complex<f32>) -> f32
+  // PPC64-DAG: [[B2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [1 : i32] : (complex<f32>) -> f32
+  // PPC64-DAG: [[A3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [0 : i32] : (complex<f32>) -> f32
+  // PPC64-DAG: [[B3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [1 : i32] : (complex<f32>) -> f32
+
+  // PPC64: fir.call @calleemultipleparamscomplex4([[A1_EXTR]], [[B1_EXTR]], [[A2_EXTR]], [[B2_EXTR]], [[A3_EXTR]], [[B3_EXTR]]) : (f32, f32, f32, f32, f32, f32) -> ()
+  // LOONGARCH64-DAG: [[Z3_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
+  // LOONGARCH64-DAG: [[Z3_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_EMPTY]], [[A3]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
+  // LOONGARCH64-DAG: [[Z3:%[0-9A-Za-z]+]] = fir.insert_value [[Z3_PARTIAL]], [[B3]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
+  // LOONGARCH64-DAG: [[Z2_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
+  // LOONGARCH64-DAG: [[Z2_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_EMPTY]], [[A2]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
+  // LOONGARCH64-DAG: [[Z2:%[0-9A-Za-z]+]] = fir.insert_value [[Z2_PARTIAL]], [[B2]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
+  // LOONGARCH64-DAG: [[Z1_EMPTY:%[0-9A-Za-z]+]] = fir.undefined complex<f32>
+  // LOONGARCH64-DAG: [[Z1_PARTIAL:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_EMPTY]], [[A1]], [0 : i32] : (complex<f32>, f32) -> complex<f32>
+  // LOONGARCH64-DAG: [[Z1:%[0-9A-Za-z]+]] = fir.insert_value [[Z1_PARTIAL]], [[B1]], [1 : i32] : (complex<f32>, f32) -> complex<f32>
+
+  // LOONGARCH64-DAG: [[A1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [0 : i32] : (complex<f32>) -> f32
+  // LOONGARCH64-DAG: [[B1_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z1]], [1 : i32] : (complex<f32>) -> f32
+  // LOONGARCH64-DAG: [[A2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [0 : i32] : (complex<f32>) -> f32
+  // LOONGARCH64-DAG: [[B2_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z2]], [1 : i32] : (complex<f32>) -> f32
+  // LOONGARCH64-DAG: [[A3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [0 : i32] : (complex<f32>) -> f32
+  // LOONGARCH64-DAG: [[B3_EXTR:%[0-9A-Za-z]+]] = fir.extract_value [[Z3]], [1 : i32] : (complex<f32>) -> f32
+
+  // LOONGARCH64: fir.call @calleemultipleparamscomplex4([[A1_EXTR]], [[B1_EXTR]], [[A2_EXTR]], [[B2_EXTR]], [[A3_EXTR]], [[B3_EXTR]]) : (f32, f32, f32, f32, f32, f32) -> ()
+  fir.call @calleemultipleparamscomplex4(%z1, %z2, %z3) : (complex<f32>, complex<f32>, complex<f32>) -> ()
   return
 }
 
@@ -1101,25 +1101,25 @@ func.func private @mlircomplexf32(%z1: complex<f32>, %z2: complex<f32>) -> compl
 func.func @addrof() {
   // I32: {{%.*}} = fir.address_of(@returncomplex4) : () -> i64
   // I32_MINGW: {{%.*}} = fir.address_of(@returncomplex4) : () -> i64
-  // X64: {{%.*}} = fir.address_of(@returncomplex4) : () -> !fir.vector<2:!fir.real<4>>
+  // X64: {{%.*}} = fir.address_of(@returncomplex4) : () -> !fir.vector<2:f32>
   // X64_MINGW: {{%.*}} = fir.address_of(@returncomplex4) : () -> i64
-  // AARCH64: {{%.*}} = fir.address_of(@returncomplex4) : () -> tuple<!fir.real<4>, !fir.real<4>>
-  // PPC64le: {{%.*}} = fir.address_of(@returncomplex4) : () -> tuple<!fir.real<4>, !fir.real<4>>
-  // RISCV64: {{%.*}} = fir.address_of(@returncomplex4) : () -> tuple<!fir.real<4>, !fir.real<4>>
-  // PPC64: {{%.*}} = fir.address_of(@returncomplex4) : () -> tuple<!fir.real<4>, !fir.real<4>>
-  // LOONGARCH64: {{%.*}} = fir.address_of(@returncomplex4) : () -> tuple<!fir.real<4>, !fir.real<4>>
-  %r = fir.address_of(@returncomplex4) : () -> !fir.complex<4>
-
-  // I32: {{%.*}} = fir.address_of(@paramcomplex4) : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> ()
-  // I32_MINGW: {{%.*}} = fir.address_of(@paramcomplex4) : (!fir.ref<tuple<!fir.real<4>, !fir.real<4>>>) -> ()
-  // X64: {{%.*}} = fir.address_of(@paramcomplex4) : (!fir.vector<2:!fir.real<4>>) -> ()
+  // AARCH64: {{%.*}} = fir.address_of(@returncomplex4) : () -> tuple<f32, f32>
+  // PPC64le: {{%.*}} = fir.address_of(@returncomplex4) : () -> tuple<f32, f32>
+  // RISCV64: {{%.*}} = fir.address_of(@returncomplex4) : () -> tuple<f32, f32>
+  // PPC64: {{%.*}} = fir.address_of(@returncomplex4) : () -> tuple<f32, f32>
+  // LOONGARCH64: {{%.*}} = fir.address_of(@returncomplex4) : () -> tuple<f32, f32>
+  %r = fir.address_of(@returncomplex4) : () -> complex<f32>
+
+  // I32: {{%.*}} = fir.address_of(@paramcomplex4) : (!fir.ref<tuple<f32, f32>>) -> ()
+  // I32_MINGW: {{%.*}} = fir.address_of(@paramcomplex4) : (!fir.ref<tuple<f32, f32>>) -> ()
+  // X64: {{%.*}} = fir.address_of(@paramcomplex4) : (!fir.vector<2:f32>) -> ()
   // X64_MINGW: {{%.*}} = fir.address_of(@paramcomplex4) : (i64) -> ()
-  // AARCH64: {{%.*}} = fir.address_of(@paramcomplex4) : (!fir.array<2x!fir.real<4>>) -> ()
-  // PPC64le: {{%.*}} = fir.address_of(@paramcomplex4) : (!fir.real<4>, !fir.real<4>) -> ()
-  // SPARCV9: {{%.*}} = fir.address_of(@paramcomplex4) : (!fir.real<4>, !fir.real<4>) -> ()
-  // RISCV64: {{%.*}} = fir.address_of(@paramcomplex4) : (!fir.real<4>, !fir.real<4>) -> ()
-  // PPC64: {{%.*}} = fir.address_of(@paramcomplex4) : (!fir.real<4>, !fir.real<4>) -> ()
-  // LOONGARCH64: {{%.*}} = fir.address_of(@paramcomplex4) : (!fir.real<4>, !fir.real<4>) -> ()
-  %p = fir.address_of(@paramcomplex4) : (!fir.complex<4>) -> ()
+  // AARCH64: {{%.*}} = fir.address_of(@paramcomplex4) : (!fir.array<2xf32>) -> ()
+  // PPC64le: {{%.*}} = fir.address_of(@paramcomplex4) : (f32, f32) -> ()
+  // SPARCV9: {{%.*}} = fir.address_of(@paramcomplex4) : (f32, f32) -> ()
+  // RISCV64: {{%.*}} = fir.address_of(@paramcomplex4) : (f32, f32) -> ()
+  // PPC64: {{%.*}} = fir.address_of(@paramcomplex4) : (f32, f32) -> ()
+  // LOONGARCH64: {{%.*}} = fir.address_of(@paramcomplex4) : (f32, f32) -> ()
+  %p = fir.address_of(@paramcomplex4) : (complex<f32>) -> ()
   return
 }
diff --git a/flang/test/Fir/target-rewrite-complex16.fir b/flang/test/Fir/target-rewrite-complex16.fir
index 304f15a828454..4f807e828d8f1 100644
--- a/flang/test/Fir/target-rewrite-complex16.fir
+++ b/flang/test/Fir/target-rewrite-complex16.fir
@@ -2,33 +2,33 @@
 
 // Test that we rewrite the signature and body of a func.function that returns a
 // complex<16>.
-func.func @returncomplex16() -> !fir.complex<16> {
-  %1 = fir.undefined !fir.complex<16>
+func.func @returncomplex16() -> complex<f128> {
+  %1 = fir.undefined complex<f128>
   %2 = arith.constant 2.0 : f128
-  %3 = fir.convert %2 : (f128) -> !fir.real<16>
+  %3 = fir.convert %2 : (f128) -> f128
   %c0 = arith.constant 0 : i32
-  %4 = fir.insert_value %1, %3, [0 : i32] : (!fir.complex<16>, !fir.real<16>) -> !fir.complex<16>
+  %4 = fir.insert_value %1, %3, [0 : i32] : (complex<f128>, f128) -> complex<f128>
   %c1 = arith.constant 1 : i32
   %5 = arith.constant -42.0 : f128
-  %6 = fir.insert_value %4, %5, [1 : i32] : (!fir.complex<16>, f128) -> !fir.complex<16>
-  return %6 : !fir.complex<16>
+  %6 = fir.insert_value %4, %5, [1 : i32] : (complex<f128>, f128) -> complex<f128>
+  return %6 : complex<f128>
 }
 
 // Test that we rewrite the signature of a func.function that accepts a complex<16>.
-func.func private @paramcomplex16(!fir.complex<16>) -> ()
+func.func private @paramcomplex16(complex<f128>) -> ()
 
 // Test that we rewrite calls to func.functions that return or accept complex<16>.
 func.func @callcomplex16() {
-  %1 = fir.call @returncomplex16() : () -> !fir.complex<16>
-  fir.call @paramcomplex16(%1) : (!fir.complex<16>) -> ()
+  %1 = fir.call @returncomplex16() : () -> complex<f128>
+  fir.call @paramcomplex16(%1) : (complex<f128>) -> ()
   return
 }
 
 // Test multiple complex<16> parameters and arguments
-func.func private @calleemultipleparamscomplex16(!fir.complex<16>, !fir.complex<16>, !fir.complex<16>) -> ()
+func.func private @calleemultipleparamscomplex16(complex<f128>, complex<f128>, complex<f128>) -> ()
 
-func.func @multipleparamscomplex16(%z1 : !fir.complex<16>, %z2 : !fir.complex<16>, %z3 : !fir.complex<16>) {
-  fir.call @calleemultipleparamscomplex16(%z1, %z2, %z3) : (!fir.complex<16>, !fir.complex<16>, !fir.complex<16>) -> ()
+func.func @multipleparamscomplex16(%z1 : complex<f128>, %z2 : complex<f128>, %z3 : complex<f128>) {
+  fir.call @calleemultipleparamscomplex16(%z1, %z2, %z3) : (complex<f128>, complex<f128>, complex<f128>) -> ()
   return
 }
 
@@ -41,63 +41,63 @@ func.func private @mlircomplexf128(%z1: complex<f128>, %z2: complex<f128>) -> co
 
 // Test that we rewrite the fir.address_of operator.
 func.func @addrof() {
-  %r = fir.address_of(@returncomplex16) : () -> !fir.complex<16>
-  %p = fir.address_of(@paramcomplex16) : (!fir.complex<16>) -> ()
+  %r = fir.address_of(@returncomplex16) : () -> complex<f128>
+  %p = fir.address_of(@paramcomplex16) : (complex<f128>) -> ()
   return
 }
 
 // CHECK-LABEL:   func.func @returncomplex16(
-// CHECK-SAME:      %[[VAL_0:.*]]: !fir.ref<tuple<!fir.real<16>, !fir.real<16>>> {llvm.align = 16 : i32, llvm.sret = tuple<!fir.real<16>, !fir.real<16>>}) {
-// CHECK:           %[[VAL_1:.*]] = fir.undefined !fir.complex<16>
+// CHECK-SAME:      %[[VAL_0:.*]]: !fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.sret = tuple<f128, f128>}) {
+// CHECK:           %[[VAL_1:.*]] = fir.undefined complex<f128>
 // CHECK:           %[[VAL_2:.*]] = arith.constant 2.000000e+00 : f128
-// CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (f128) -> !fir.real<16>
+// CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (f128) -> f128
 // CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i32
-// CHECK:           %[[VAL_5:.*]] = fir.insert_value %[[VAL_1]], %[[VAL_3]], [0 : i32] : (!fir.complex<16>, !fir.real<16>) -> !fir.complex<16>
+// CHECK:           %[[VAL_5:.*]] = fir.insert_value %[[VAL_1]], %[[VAL_3]], [0 : i32] : (complex<f128>, f128) -> complex<f128>
 // CHECK:           %[[VAL_6:.*]] = arith.constant 1 : i32
 // CHECK:           %[[VAL_7:.*]] = arith.constant -4.200000e+01 : f128
-// CHECK:           %[[VAL_8:.*]] = fir.insert_value %[[VAL_5]], %[[VAL_7]], [1 : i32] : (!fir.complex<16>, f128) -> !fir.complex<16>
-// CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> !fir.ref<!fir.complex<16>>
-// CHECK:           fir.store %[[VAL_8]] to %[[VAL_9]] : !fir.ref<!fir.complex<16>>
+// CHECK:           %[[VAL_8:.*]] = fir.insert_value %[[VAL_5]], %[[VAL_7]], [1 : i32] : (complex<f128>, f128) -> complex<f128>
+// CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<tuple<f128, f128>>) -> !fir.ref<complex<f128>>
+// CHECK:           fir.store %[[VAL_8]] to %[[VAL_9]] : !fir.ref<complex<f128>>
 // CHECK:           return
 // CHECK:         }
-// CHECK:         func.func private @paramcomplex16(!fir.ref<tuple<!fir.real<16>, !fir.real<16>>> {llvm.align = 16 : i32, llvm.byval = tuple<!fir.real<16>, !fir.real<16>>})
+// CHECK:         func.func private @paramcomplex16(!fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.byval = tuple<f128, f128>})
 
 // CHECK-LABEL:   func.func @callcomplex16() {
 // CHECK:           %[[VAL_0:.*]] = llvm.intr.stacksave : !llvm.ptr
-// CHECK:           %[[VAL_1:.*]] = fir.alloca tuple<!fir.real<16>, !fir.real<16>>
-// CHECK:           fir.call @returncomplex16(%[[VAL_1]]) : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> ()
-// CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> !fir.ref<!fir.complex<16>>
-// CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_2]] : !fir.ref<!fir.complex<16>>
+// CHECK:           %[[VAL_1:.*]] = fir.alloca tuple<f128, f128>
+// CHECK:           fir.call @returncomplex16(%[[VAL_1]]) : (!fir.ref<tuple<f128, f128>>) -> ()
+// CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<tuple<f128, f128>>) -> !fir.ref<complex<f128>>
+// CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_2]] : !fir.ref<complex<f128>>
 // CHECK:           llvm.intr.stackrestore %[[VAL_0]] : !llvm.ptr
 // CHECK:           %[[VAL_4:.*]] = llvm.intr.stacksave : !llvm.ptr
-// CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.complex<16>
-// CHECK:           fir.store %[[VAL_3]] to %[[VAL_5]] : !fir.ref<!fir.complex<16>>
-// CHECK:           %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<!fir.complex<16>>) -> !fir.ref<tuple<!fir.real<16>, !fir.real<16>>>
-// CHECK:           fir.call @paramcomplex16(%[[VAL_6]]) : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> ()
+// CHECK:           %[[VAL_5:.*]] = fir.alloca complex<f128>
+// CHECK:           fir.store %[[VAL_3]] to %[[VAL_5]] : !fir.ref<complex<f128>>
+// CHECK:           %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<complex<f128>>) -> !fir.ref<tuple<f128, f128>>
+// CHECK:           fir.call @paramcomplex16(%[[VAL_6]]) : (!fir.ref<tuple<f128, f128>>) -> ()
 // CHECK:           llvm.intr.stackrestore %[[VAL_4]] : !llvm.ptr
 // CHECK:           return
 // CHECK:         }
-// CHECK:         func.func private @calleemultipleparamscomplex16(!fir.ref<tuple<!fir.real<16>, !fir.real<16>>> {llvm.align = 16 : i32, llvm.byval = tuple<!fir.real<16>, !fir.real<16>>}, !fir.ref<tuple<!fir.real<16>, !fir.real<16>>> {llvm.align = 16 : i32, llvm.byval = tuple<!fir.real<16>, !fir.real<16>>}, !fir.ref<tuple<!fir.real<16>, !fir.real<16>>> {llvm.align = 16 : i32, llvm.byval = tuple<!fir.real<16>, !fir.real<16>>})
+// CHECK:         func.func private @calleemultipleparamscomplex16(!fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.byval = tuple<f128, f128>}, !fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.byval = tuple<f128, f128>}, !fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.byval = tuple<f128, f128>})
 
 // CHECK-LABEL:   func.func @multipleparamscomplex16(
-// CHECK-SAME:      %[[VAL_0:.*]]: !fir.ref<tuple<!fir.real<16>, !fir.real<16>>> {llvm.align = 16 : i32, llvm.byval = tuple<!fir.real<16>, !fir.real<16>>}, %[[VAL_1:.*]]: !fir.ref<tuple<!fir.real<16>, !fir.real<16>>> {llvm.align = 16 : i32, llvm.byval = tuple<!fir.real<16>, !fir.real<16>>}, %[[VAL_2:.*]]: !fir.ref<tuple<!fir.real<16>, !fir.real<16>>> {llvm.align = 16 : i32, llvm.byval = tuple<!fir.real<16>, !fir.real<16>>}) {
-// CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> !fir.ref<!fir.complex<16>>
-// CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]] : !fir.ref<!fir.complex<16>>
-// CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> !fir.ref<!fir.complex<16>>
-// CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_5]] : !fir.ref<!fir.complex<16>>
-// CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> !fir.ref<!fir.complex<16>>
-// CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_7]] : !fir.ref<!fir.complex<16>>
+// CHECK-SAME:      %[[VAL_0:.*]]: !fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.byval = tuple<f128, f128>}, %[[VAL_1:.*]]: !fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.byval = tuple<f128, f128>}, %[[VAL_2:.*]]: !fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.byval = tuple<f128, f128>}) {
+// CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<tuple<f128, f128>>) -> !fir.ref<complex<f128>>
+// CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]] : !fir.ref<complex<f128>>
+// CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<tuple<f128, f128>>) -> !fir.ref<complex<f128>>
+// CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_5]] : !fir.ref<complex<f128>>
+// CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<tuple<f128, f128>>) -> !fir.ref<complex<f128>>
+// CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_7]] : !fir.ref<complex<f128>>
 // CHECK:           %[[VAL_9:.*]] = llvm.intr.stacksave : !llvm.ptr
-// CHECK:           %[[VAL_10:.*]] = fir.alloca !fir.complex<16>
-// CHECK:           fir.store %[[VAL_8]] to %[[VAL_10]] : !fir.ref<!fir.complex<16>>
-// CHECK:           %[[VAL_11:.*]] = fir.convert %[[VAL_10]] : (!fir.ref<!fir.complex<16>>) -> !fir.ref<tuple<!fir.real<16>, !fir.real<16>>>
-// CHECK:           %[[VAL_12:.*]] = fir.alloca !fir.complex<16>
-// CHECK:           fir.store %[[VAL_6]] to %[[VAL_12]] : !fir.ref<!fir.complex<16>>
-// CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (!fir.ref<!fir.complex<16>>) -> !fir.ref<tuple<!fir.real<16>, !fir.real<16>>>
-// CHECK:           %[[VAL_14:.*]] = fir.alloca !fir.complex<16>
-// CHECK:           fir.store %[[VAL_4]] to %[[VAL_14]] : !fir.ref<!fir.complex<16>>
-// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.ref<!fir.complex<16>>) -> !fir.ref<tuple<!fir.real<16>, !fir.real<16>>>
-// CHECK:           fir.call @calleemultipleparamscomplex16(%[[VAL_11]], %[[VAL_13]], %[[VAL_15]]) : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>, !fir.ref<tuple<!fir.real<16>, !fir.real<16>>>, !fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> ()
+// CHECK:           %[[VAL_10:.*]] = fir.alloca complex<f128>
+// CHECK:           fir.store %[[VAL_8]] to %[[VAL_10]] : !fir.ref<complex<f128>>
+// CHECK:           %[[VAL_11:.*]] = fir.convert %[[VAL_10]] : (!fir.ref<complex<f128>>) -> !fir.ref<tuple<f128, f128>>
+// CHECK:           %[[VAL_12:.*]] = fir.alloca complex<f128>
+// CHECK:           fir.store %[[VAL_6]] to %[[VAL_12]] : !fir.ref<complex<f128>>
+// CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (!fir.ref<complex<f128>>) -> !fir.ref<tuple<f128, f128>>
+// CHECK:           %[[VAL_14:.*]] = fir.alloca complex<f128>
+// CHECK:           fir.store %[[VAL_4]] to %[[VAL_14]] : !fir.ref<complex<f128>>
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.ref<complex<f128>>) -> !fir.ref<tuple<f128, f128>>
+// CHECK:           fir.call @calleemultipleparamscomplex16(%[[VAL_11]], %[[VAL_13]], %[[VAL_15]]) : (!fir.ref<tuple<f128, f128>>, !fir.ref<tuple<f128, f128>>, !fir.ref<tuple<f128, f128>>) -> ()
 // CHECK:           llvm.intr.stackrestore %[[VAL_9]] : !llvm.ptr
 // CHECK:           return
 // CHECK:         }
@@ -126,7 +126,7 @@ func.func @addrof() {
 // CHECK:         }
 
 // CHECK-LABEL:   func.func @addrof() {
-// CHECK:           %[[VAL_0:.*]] = fir.address_of(@returncomplex16) : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> ()
-// CHECK:           %[[VAL_1:.*]] = fir.address_of(@paramcomplex16) : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> ()
+// CHECK:           %[[VAL_0:.*]] = fir.address_of(@returncomplex16) : (!fir.ref<tuple<f128, f128>>) -> ()
+// CHECK:           %[[VAL_1:.*]] = fir.address_of(@paramcomplex16) : (!fir.ref<tuple<f128, f128>>) -> ()
 // CHECK:           return
 // CHECK:         }
diff --git a/flang/test/Fir/target-rewrite-selective.fir b/flang/test/Fir/target-rewrite-selective.fir
index 40225e390ff9e..bfbbafdc2e2b3 100644
--- a/flang/test/Fir/target-rewrite-selective.fir
+++ b/flang/test/Fir/target-rewrite-selective.fir
@@ -21,49 +21,49 @@
 //   r = test2(value)
 // end function
 
-// CMPLXOFF-DAG: fir.call @_QPtest1({{.*}}, {{.*}}, {{.*}}) : (!fir.ref<!fir.char<1,?>>, !fir.complex<4>, i64) -> !fir.complex<4>
-// CMPLXOFF-DAG: fir.call @_QPtest2({{.*}}, {{.*}}, {{.*}}) : (!fir.ref<!fir.char<1,10>>, index, !fir.complex<4>) -> !fir.boxchar<1>
-// CMPLXOFF-DAG: func.func private @_QPtest1(!fir.ref<!fir.char<1,?>>, !fir.complex<4>, i64) -> !fir.complex<4>
-// CMPLXOFF-DAG: func.func private @_QPtest2(!fir.ref<!fir.char<1,10>>, index, !fir.complex<4>) -> !fir.boxchar<1>
+// CMPLXOFF-DAG: fir.call @_QPtest1({{.*}}, {{.*}}, {{.*}}) : (!fir.ref<!fir.char<1,?>>, complex<f32>, i64) -> complex<f32>
+// CMPLXOFF-DAG: fir.call @_QPtest2({{.*}}, {{.*}}, {{.*}}) : (!fir.ref<!fir.char<1,10>>, index, complex<f32>) -> !fir.boxchar<1>
+// CMPLXOFF-DAG: func.func private @_QPtest1(!fir.ref<!fir.char<1,?>>, complex<f32>, i64) -> complex<f32>
+// CMPLXOFF-DAG: func.func private @_QPtest2(!fir.ref<!fir.char<1,10>>, index, complex<f32>) -> !fir.boxchar<1>
 
-// CHAROFF-DAG: fir.call @_QPtest1({{.*}}, {{.*}}) : (!fir.boxchar<1>, !fir.vector<2:!fir.real<4>>) -> !fir.vector<2:!fir.real<4>>
-// CHAROFF-DAG: fir.call @_QPtest2({{.*}}, {{.*}}, {{.*}}) : (!fir.ref<!fir.char<1,10>>, index, !fir.vector<2:!fir.real<4>>) -> !fir.boxchar<1>
-// CHAROFF-DAG: func.func private @_QPtest1(!fir.boxchar<1>, !fir.vector<2:!fir.real<4>>) -> !fir.vector<2:!fir.real<4>>
-// CHAROFF-DAG: func.func private @_QPtest2(!fir.ref<!fir.char<1,10>>, index, !fir.vector<2:!fir.real<4>>) -> !fir.boxchar<1>
+// CHAROFF-DAG: fir.call @_QPtest1({{.*}}, {{.*}}) : (!fir.boxchar<1>, !fir.vector<2:f32>) -> !fir.vector<2:f32>
+// CHAROFF-DAG: fir.call @_QPtest2({{.*}}, {{.*}}, {{.*}}) : (!fir.ref<!fir.char<1,10>>, index, !fir.vector<2:f32>) -> !fir.boxchar<1>
+// CHAROFF-DAG: func.func private @_QPtest1(!fir.boxchar<1>, !fir.vector<2:f32>) -> !fir.vector<2:f32>
+// CHAROFF-DAG: func.func private @_QPtest2(!fir.ref<!fir.char<1,10>>, index, !fir.vector<2:f32>) -> !fir.boxchar<1>
 
 module {
-  func.func @_QPtest(%arg0: !fir.complex<4> {fir.bindc_name = "value"}) -> !fir.complex<4> {
+  func.func @_QPtest(%arg0: complex<f32> {fir.bindc_name = "value"}) -> complex<f32> {
     %c10 = arith.constant 10 : index
     %false = arith.constant false
     %c0 = arith.constant 0 : index
     %0 = fir.alloca !fir.char<1,10> {bindc_name = ".result"}
-    %1 = fir.alloca !fir.complex<4>
-    fir.store %arg0 to %1 : !fir.ref<!fir.complex<4>>
+    %1 = fir.alloca complex<f32>
+    fir.store %arg0 to %1 : !fir.ref<complex<f32>>
     %2 = fir.alloca !fir.char<1,10> {bindc_name = "r", uniq_name = "_QFtestEr"}
-    %3 = fir.alloca !fir.complex<4> {bindc_name = "test", uniq_name = "_QFtestEtest"}
+    %3 = fir.alloca complex<f32> {bindc_name = "test", uniq_name = "_QFtestEtest"}
     %4 = fir.address_of(@_QQclX) : !fir.ref<!fir.char<1,0>>
     %5 = fir.convert %4 : (!fir.ref<!fir.char<1,0>>) -> !fir.ref<!fir.char<1,?>>
     %6 = fir.emboxchar %5, %c0 : (!fir.ref<!fir.char<1,?>>, index) -> !fir.boxchar<1>
-    %7 = fir.load %1 : !fir.ref<!fir.complex<4>>
-    %8 = fir.call @_QPtest1(%6, %7) fastmath<contract> : (!fir.boxchar<1>, !fir.complex<4>) -> !fir.complex<4>
-    fir.store %8 to %3 : !fir.ref<!fir.complex<4>>
-    %9 = fir.load %1 : !fir.ref<!fir.complex<4>>
+    %7 = fir.load %1 : !fir.ref<complex<f32>>
+    %8 = fir.call @_QPtest1(%6, %7) fastmath<contract> : (!fir.boxchar<1>, complex<f32>) -> complex<f32>
+    fir.store %8 to %3 : !fir.ref<complex<f32>>
+    %9 = fir.load %1 : !fir.ref<complex<f32>>
     %10 = fir.call @llvm.stacksave() fastmath<contract> : () -> !fir.ref<i8>
-    %11 = fir.call @_QPtest2(%0, %c10, %9) fastmath<contract> : (!fir.ref<!fir.char<1,10>>, index, !fir.complex<4>) -> !fir.boxchar<1>
+    %11 = fir.call @_QPtest2(%0, %c10, %9) fastmath<contract> : (!fir.ref<!fir.char<1,10>>, index, complex<f32>) -> !fir.boxchar<1>
     %12 = fir.convert %c10 : (index) -> i64
     %13 = fir.convert %2 : (!fir.ref<!fir.char<1,10>>) -> !fir.ref<i8>
     %14 = fir.convert %0 : (!fir.ref<!fir.char<1,10>>) -> !fir.ref<i8>
     fir.call @llvm.memmove.p0.p0.i64(%13, %14, %12, %false) fastmath<contract> : (!fir.ref<i8>, !fir.ref<i8>, i64, i1) -> ()
     fir.call @llvm.stackrestore(%10) fastmath<contract> : (!fir.ref<i8>) -> ()
-    %15 = fir.load %3 : !fir.ref<!fir.complex<4>>
-    return %15 : !fir.complex<4>
+    %15 = fir.load %3 : !fir.ref<complex<f32>>
+    return %15 : complex<f32>
   }
-  func.func private @_QPtest1(!fir.boxchar<1>, !fir.complex<4>) -> !fir.complex<4>
+  func.func private @_QPtest1(!fir.boxchar<1>, complex<f32>) -> complex<f32>
   fir.global linkonce @_QQclX constant : !fir.char<1,0> {
     %0 = fir.string_lit ""(0) : !fir.char<1,0>
     fir.has_value %0 : !fir.char<1,0>
   }
-  func.func private @_QPtest2(!fir.ref<!fir.char<1,10>>, index, !fir.complex<4>) -> !fir.boxchar<1>
+  func.func private @_QPtest2(!fir.ref<!fir.char<1,10>>, index, complex<f32>) -> !fir.boxchar<1>
   func.func private @llvm.stacksave() -> !fir.ref<i8>
   func.func private @llvm.memmove.p0.p0.i64(!fir.ref<i8>, !fir.ref<i8>, i64, i1)
   func.func private @llvm.stackrestore(!fir.ref<i8>)
diff --git a/flang/test/Fir/target.fir b/flang/test/Fir/target.fir
index 653b100bccbd4..788deebcb5e84 100644
--- a/flang/test/Fir/target.fir
+++ b/flang/test/Fir/target.fir
@@ -7,15 +7,14 @@
 // X64-LABEL: define <2 x float> @gen4()
 // AARCH64-LABEL: define { float, float } @gen4()
 // PPC-LABEL: define { float, float } @gen4()
-func.func @gen4() -> !fir.complex<4> {
-  %1 = fir.undefined !fir.complex<4>
+func.func @gen4() -> complex<f32> {
+  %1 = fir.undefined complex<f32>
   %2 = arith.constant 2.0 : f32
-  %3 = fir.convert %2 : (f32) -> !fir.real<4>
   %c0 = arith.constant 0 : i32
-  %4 = fir.insert_value %1, %3, [0 : index] : (!fir.complex<4>, !fir.real<4>) -> !fir.complex<4>
+  %4 = fir.insert_value %1, %2, [0 : index] : (complex<f32>, f32) -> complex<f32>
   %c1 = arith.constant 1 : i32
   %5 = arith.constant -42.0 : f32
-  %6 = fir.insert_value %4, %5, [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
+  %6 = fir.insert_value %4, %5, [1 : index] : (complex<f32>, f32) -> complex<f32>
   // I32: store { float, float } { float 2.000000e+00, float -4.200000e+01 }
   // I32: %[[load:.*]] = load i64, ptr
   // I32: ret i64 %[[load]]
@@ -24,41 +23,41 @@ func.func @gen4() -> !fir.complex<4> {
   // X64: ret <2 x float> %[[load]]
   // AARCH64: ret { float, float }
   // PPC: ret { float, float }
-  return %6 : !fir.complex<4>
+  return %6 : complex<f32>
 }
 
 // I32-LABEL: define void @gen8(ptr sret({ double, double }) align 4 %
 // X64-LABEL: define { double, double } @gen8()
 // AARCH64-LABEL: define { double, double } @gen8()
 // PPC-LABEL: define { double, double } @gen8()
-func.func @gen8() -> !fir.complex<8> {
-  %1 = fir.undefined !fir.complex<8>
+func.func @gen8() -> complex<f64> {
+  %1 = fir.undefined complex<f64>
   %2 = arith.constant 1.0 : f64
   %3 = arith.constant -4.0 : f64
   %c0 = arith.constant 0 : i32
-  %4 = fir.insert_value %1, %3, [0 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
+  %4 = fir.insert_value %1, %3, [0 : index] : (complex<f64>, f64) -> complex<f64>
   %c1 = arith.constant 1 : i32
-  %5 = fir.insert_value %4, %2, [1 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
+  %5 = fir.insert_value %4, %2, [1 : index] : (complex<f64>, f64) -> complex<f64>
   // I32: store { double, double } { double -4.000000e+00, double 1.000000e+00 }
   // I64: store { double, double } { double -4.000000e+00, double 1.000000e+00 }
   // I64: %[[load:.*]] = load { double, double }
   // I64: ret { double, double } %[[load]]
   // AARCH64: ret { double, double }
   // PPC: ret { double, double }
-  return %5 : !fir.complex<8>
+  return %5 : complex<f64>
 }
 
 // I32: declare void @sink4(ptr byval({ float, float }) align 4)
 // X64: declare void @sink4(<2 x float>)
 // AARCH64: declare void @sink4([2 x float])
 // PPC: declare void @sink4(float, float)
-func.func private @sink4(!fir.complex<4>) -> ()
+func.func private @sink4(complex<f32>) -> ()
 
 // I32: declare void @sink8(ptr byval({ double, double }) align 4)
 // X64: declare void @sink8(double, double)
 // AARCH64: declare void @sink8([2 x double])
 // PPC: declare void @sink8(double, double)
-func.func private @sink8(!fir.complex<8>) -> ()
+func.func private @sink8(complex<f64>) -> ()
 
 // I32-LABEL: define void @call4()
 // X64-LABEL: define void @call4()
@@ -68,12 +67,12 @@ func.func @call4() {
   // X64: = call <2 x float> @gen4()
   // AARCH64: = call { float, float } @gen4()
   // PPC: = call { float, float } @gen4()
-  %1 = fir.call @gen4() : () -> !fir.complex<4>
+  %1 = fir.call @gen4() : () -> complex<f32>
   // I32: call void @sink4(ptr %
   // X64: call void @sink4(<2 x float> %
   // AARCH64: call void @sink4([2 x float] %
   // PPC: call void @sink4(float %{{.*}}, float %{{.*}})
-  fir.call @sink4(%1) : (!fir.complex<4>) -> ()
+  fir.call @sink4(%1) : (complex<f32>) -> ()
   return
 }
 
@@ -85,12 +84,12 @@ func.func @call8() {
   // X64: = call { double, double } @gen8()
   // AARCH64: = call { double, double } @gen8()
   // PPC: = call { double, double } @gen8()
-  %1 = fir.call @gen8() : () -> !fir.complex<8>
+  %1 = fir.call @gen8() : () -> complex<f64>
   // I32: call void @sink8(ptr %
   // X64: call void @sink8(double %{{[0-9]*}}, double %{{[0-9]*}})
   // AARCH64: call void @sink8([2 x double] %
   // PPC: call void @sink8(double %{{.*}}, double %{{.*}})
-  fir.call @sink8(%1) : (!fir.complex<8>) -> ()
+  fir.call @sink8(%1) : (complex<f64>) -> ()
   return
 }
 
diff --git a/flang/test/Fir/types-to-llvm.fir b/flang/test/Fir/types-to-llvm.fir
index 4c3431d3426eb..80bdb6ad87c76 100644
--- a/flang/test/Fir/types-to-llvm.fir
+++ b/flang/test/Fir/types-to-llvm.fir
@@ -257,27 +257,27 @@ func.func private @foo5(%arg0: complex<f128>)
 
 // Test `!fir.complex<KIND>` conversion.
 
-func.func private @foo0(%arg0: !fir.complex<2>)
+func.func private @foo0(%arg0: complex<f16>)
 // CHECK-LABEL: foo0
 // CHECK-SAME: !llvm.struct<(f16, f16)>)
 
-func.func private @foo1(%arg0: !fir.complex<3>)
+func.func private @foo1(%arg0: complex<bf16>)
 // CHECK-LABEL: foo1
 // CHECK-SAME: !llvm.struct<(bf16, bf16)>)
 
-func.func private @foo2(%arg0: !fir.complex<4>)
+func.func private @foo2(%arg0: complex<f32>)
 // CHECK-LABEL: foo2
 // CHECK-SAME: !llvm.struct<(f32, f32)>)
 
-func.func private @foo3(%arg0: !fir.complex<8>)
+func.func private @foo3(%arg0: complex<f64>)
 // CHECK-LABEL: foo3
 // CHECK-SAME: !llvm.struct<(f64, f64)>)
 
-func.func private @foo4(%arg0: !fir.complex<10>)
+func.func private @foo4(%arg0: complex<f80>)
 // CHECK-LABEL: foo4
 // CHECK-SAME: !llvm.struct<(f80, f80)>)
 
-func.func private @foo5(%arg0: !fir.complex<16>)
+func.func private @foo5(%arg0: complex<f128>)
 // CHECK-LABEL: foo5
 // CHECK-SAME: !llvm.struct<(f128, f128)>)
 
@@ -329,30 +329,6 @@ func.func private @foo10(%arg0: !fir.vector<30:i64>)
 // CHECK-LABEL: foo10
 // CHECK-SAME: vector<30xi64>
 
-func.func private @foo11(%arg0: !fir.vector<2:!fir.real<2>>)
-// CHECK-LABEL: foo11
-// CHECK-SAME: vector<2xf16>
-
-func.func private @foo12(%arg0: !fir.vector<2:!fir.real<3>>)
-// CHECK-LABEL: foo12
-// CHECK-SAME: vector<2xbf16>
-
-func.func private @foo13(%arg0: !fir.vector<2:!fir.real<4>>)
-// CHECK-LABEL: foo13
-// CHECK-SAME: vector<2xf32>
-
-func.func private @foo14(%arg0: !fir.vector<2:!fir.real<8>>)
-// CHECK-LABEL: foo14
-// CHECK-SAME: vector<2xf64>
-
-func.func private @foo15(%arg0: !fir.vector<2:!fir.real<10>>)
-// CHECK-LABEL: foo15
-// CHECK-SAME: vector<2xf80>
-
-func.func private @foo16(%arg0: !fir.vector<2:!fir.real<16>>)
-// CHECK-LABEL: foo16
-// CHECK-SAME: vector<2xf128>
-
 // -----
 
 // Test `!fir.boxchar<n>` conversion
diff --git a/flang/test/Fir/undo-complex-pattern.fir b/flang/test/Fir/undo-complex-pattern.fir
index 853579b4bf5b9..5c5f6319d48b1 100644
--- a/flang/test/Fir/undo-complex-pattern.fir
+++ b/flang/test/Fir/undo-complex-pattern.fir
@@ -3,96 +3,96 @@
 
 
 // CHECK-LABEL: @add
-func.func @add(%z: !fir.ref<!fir.complex<8>>, %z1 : !fir.complex<8>, %z2 : !fir.complex<8>) {
+func.func @add(%z: !fir.ref<complex<f64>>, %z1 : complex<f64>, %z2 : complex<f64>) {
   %c0 = arith.constant 0 : index
   %c1 = arith.constant 1 : index
-  %real1 = fir.extract_value %z1, [0 : index] : (!fir.complex<8>) -> f64
-  %imag1 = fir.extract_value %z1, [1 : index] : (!fir.complex<8>) -> f64
-  %real2 = fir.extract_value %z2, [0 : index] : (!fir.complex<8>) -> f64
-  %imag2 = fir.extract_value %z2, [1 : index] : (!fir.complex<8>) -> f64
+  %real1 = fir.extract_value %z1, [0 : index] : (complex<f64>) -> f64
+  %imag1 = fir.extract_value %z1, [1 : index] : (complex<f64>) -> f64
+  %real2 = fir.extract_value %z2, [0 : index] : (complex<f64>) -> f64
+  %imag2 = fir.extract_value %z2, [1 : index] : (complex<f64>) -> f64
 
   // CHECK-LABEL: fir.addc
   %real = arith.addf %real1, %real2 : f64
   %imag = arith.addf %imag1, %imag2 : f64
-  %undef = fir.undefined !fir.complex<8>
-  %insert_real = fir.insert_value %undef, %real, [0 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
-  %insert_imag = fir.insert_value %insert_real, %imag, [1 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
-  fir.store %insert_imag to %z : !fir.ref<!fir.complex<8>>
+  %undef = fir.undefined complex<f64>
+  %insert_real = fir.insert_value %undef, %real, [0 : index] : (complex<f64>, f64) -> complex<f64>
+  %insert_imag = fir.insert_value %insert_real, %imag, [1 : index] : (complex<f64>, f64) -> complex<f64>
+  fir.store %insert_imag to %z : !fir.ref<complex<f64>>
   return
 }
 
 // CHECK-LABEL: @sub
-func.func @sub(%z: !fir.ref<!fir.complex<8>>, %z1 : !fir.complex<8>, %z2 : !fir.complex<8>) {
+func.func @sub(%z: !fir.ref<complex<f64>>, %z1 : complex<f64>, %z2 : complex<f64>) {
   %c0 = arith.constant 0 : index
   %c1 = arith.constant 1 : index
-  %real1 = fir.extract_value %z1, [0 : index] : (!fir.complex<8>) -> f64
-  %imag1 = fir.extract_value %z1, [1 : index] : (!fir.complex<8>) -> f64
-  %real2 = fir.extract_value %z2, [0 : index] : (!fir.complex<8>) -> f64
-  %imag2 = fir.extract_value %z2, [1 : index] : (!fir.complex<8>) -> f64
+  %real1 = fir.extract_value %z1, [0 : index] : (complex<f64>) -> f64
+  %imag1 = fir.extract_value %z1, [1 : index] : (complex<f64>) -> f64
+  %real2 = fir.extract_value %z2, [0 : index] : (complex<f64>) -> f64
+  %imag2 = fir.extract_value %z2, [1 : index] : (complex<f64>) -> f64
 
   // CHECK-LABEL: fir.subc
   %real = arith.subf %real1, %real2 : f64
   %imag = arith.subf %imag1, %imag2 : f64
-  %undef = fir.undefined !fir.complex<8>
-  %insert_real = fir.insert_value %undef, %real, [0 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
-  %insert_imag = fir.insert_value %insert_real, %imag, [1 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
-  fir.store %insert_imag to %z : !fir.ref<!fir.complex<8>>
+  %undef = fir.undefined complex<f64>
+  %insert_real = fir.insert_value %undef, %real, [0 : index] : (complex<f64>, f64) -> complex<f64>
+  %insert_imag = fir.insert_value %insert_real, %imag, [1 : index] : (complex<f64>, f64) -> complex<f64>
+  fir.store %insert_imag to %z : !fir.ref<complex<f64>>
   return
 }
 
 // CHECK-LABEL: @undefOpHiddenByBranch
-func.func @undefOpHiddenByBranch(%z: !fir.ref<!fir.complex<8>>, %b: i1) {
+func.func @undefOpHiddenByBranch(%z: !fir.ref<complex<f64>>, %b: i1) {
   %c0 = arith.constant 0 : index
   %c1 = arith.constant 1 : index
   cf.cond_br %b, ^bb1, ^bb2
 ^bb1:  // pred: ^bb0
-  %u1 = fir.undefined !fir.complex<8>
-  %z1l = fir.call @bar1() : () -> !fir.complex<8>
-  %z1r = fir.call @bar1() : () -> !fir.complex<8>
-  cf.br ^bb3(%u1, %z1l, %z1r : !fir.complex<8>, !fir.complex<8>, !fir.complex<8>)
+  %u1 = fir.undefined complex<f64>
+  %z1l = fir.call @bar1() : () -> complex<f64>
+  %z1r = fir.call @bar1() : () -> complex<f64>
+  cf.br ^bb3(%u1, %z1l, %z1r : complex<f64>, complex<f64>, complex<f64>)
 ^bb2:  // pred: ^bb0
-  %u2 = fir.undefined !fir.complex<8>
-  %z2l = fir.call @bar2() : () -> !fir.complex<8>
-  %z2r = fir.call @bar2() : () -> !fir.complex<8>
-  cf.br ^bb3(%u2, %z2l, %z2r : !fir.complex<8>, !fir.complex<8>, !fir.complex<8>)
+  %u2 = fir.undefined complex<f64>
+  %z2l = fir.call @bar2() : () -> complex<f64>
+  %z2r = fir.call @bar2() : () -> complex<f64>
+  cf.br ^bb3(%u2, %z2l, %z2r : complex<f64>, complex<f64>, complex<f64>)
 
-// CHECK: ^bb3(%[[z1:.*]]: !fir.complex<8>, %[[z2:.*]]: !fir.complex<8>):  // 2 preds: ^bb1, ^bb2
-// CHECK:  fir.addc %[[z1]], %[[z2]] : !fir.complex<8>
+// CHECK: ^bb3(%[[z1:.*]]: complex<f64>, %[[z2:.*]]: complex<f64>):  // 2 preds: ^bb1, ^bb2
+// CHECK:  fir.addc %[[z1]], %[[z2]] : complex<f64>
 
-^bb3(%undef : !fir.complex<8>, %z1 : !fir.complex<8>, %z2 : !fir.complex<8>):  // 2 preds: ^bb1, ^bb2
-  %real1 = fir.extract_value %z1, [0 : index] : (!fir.complex<8>) -> f64
-  %imag1 = fir.extract_value %z1, [1 : index] : (!fir.complex<8>) -> f64
-  %real2 = fir.extract_value %z2, [0 : index] : (!fir.complex<8>) -> f64
-  %imag2 = fir.extract_value %z2, [1 : index] : (!fir.complex<8>) -> f64
+^bb3(%undef : complex<f64>, %z1 : complex<f64>, %z2 : complex<f64>):  // 2 preds: ^bb1, ^bb2
+  %real1 = fir.extract_value %z1, [0 : index] : (complex<f64>) -> f64
+  %imag1 = fir.extract_value %z1, [1 : index] : (complex<f64>) -> f64
+  %real2 = fir.extract_value %z2, [0 : index] : (complex<f64>) -> f64
+  %imag2 = fir.extract_value %z2, [1 : index] : (complex<f64>) -> f64
   %real = arith.addf %real1, %real2 : f64
   %imag = arith.addf %imag1, %imag2 : f64
-  %insert_real = fir.insert_value %undef, %real, [0 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
-  %insert_imag = fir.insert_value %insert_real, %imag, [1 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
-  fir.store %insert_imag to %z : !fir.ref<!fir.complex<8>>
+  %insert_real = fir.insert_value %undef, %real, [0 : index] : (complex<f64>, f64) -> complex<f64>
+  %insert_imag = fir.insert_value %insert_real, %imag, [1 : index] : (complex<f64>, f64) -> complex<f64>
+  fir.store %insert_imag to %z : !fir.ref<complex<f64>>
   return
 }
-func.func private @bar1() -> !fir.complex<8>
-func.func private @bar2() -> !fir.complex<8>
+func.func private @bar1() -> complex<f64>
+func.func private @bar2() -> complex<f64>
 
 // CHECK-LABEL: @close_but_bad_pattern
-func.func @close_but_bad_pattern(%z: !fir.ref<!fir.complex<8>>, %z1 : !fir.complex<8>, %z2 : !fir.complex<8>) {
+func.func @close_but_bad_pattern(%z: !fir.ref<complex<f64>>, %z1 : complex<f64>, %z2 : complex<f64>) {
   %c0 = arith.constant 0 : index
   %c1 = arith.constant 1 : index
-  %real1 = fir.extract_value %z1, [0 : index] : (!fir.complex<8>) -> f64
+  %real1 = fir.extract_value %z1, [0 : index] : (complex<f64>) -> f64
   // extracting %c0 instead of %c1 
-  %imag1 = fir.extract_value %z1, [0 : index] : (!fir.complex<8>) -> f64
-  %real2 = fir.extract_value %z2, [0 : index] : (!fir.complex<8>) -> f64
-  %imag2 = fir.extract_value %z2, [1 : index] : (!fir.complex<8>) -> f64
+  %imag1 = fir.extract_value %z1, [0 : index] : (complex<f64>) -> f64
+  %real2 = fir.extract_value %z2, [0 : index] : (complex<f64>) -> f64
+  %imag2 = fir.extract_value %z2, [1 : index] : (complex<f64>) -> f64
   // CHECK: arith.subf
   // CHECK: subf
   %real = arith.subf %real1, %real2 : f64
   %imag = arith.subf %imag1, %imag2 : f64
-  %undef = fir.undefined !fir.complex<8>
+  %undef = fir.undefined complex<f64>
   // CHECK: %[[insert1:.*]] = fir.insert_value %{{.*}}, %{{.*}}, [0
   // CHECK: %[[insert2:.*]] = fir.insert_value %[[insert1]], %{{.*}}, [1
-  %insert_real = fir.insert_value %undef, %real, [0 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
-  %insert_imag = fir.insert_value %insert_real, %imag, [1 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
+  %insert_real = fir.insert_value %undef, %real, [0 : index] : (complex<f64>, f64) -> complex<f64>
+  %insert_imag = fir.insert_value %insert_real, %imag, [1 : index] : (complex<f64>, f64) -> complex<f64>
   // CHECK: fir.store %[[insert2]] to {{.*}}
-  fir.store %insert_imag to %z : !fir.ref<!fir.complex<8>>
+  fir.store %insert_imag to %z : !fir.ref<complex<f64>>
   return
 }
diff --git a/flang/test/HLFIR/assign-codegen.fir b/flang/test/HLFIR/assign-codegen.fir
index 9a21c02808c63..e0dcc06d75a9c 100644
--- a/flang/test/HLFIR/assign-codegen.fir
+++ b/flang/test/HLFIR/assign-codegen.fir
@@ -63,24 +63,24 @@ func.func @scalar_real_2(%arg0: !fir.ref<f32>) {
 // CHECK:  %[[VAL_1:.*]] = arith.constant 3.140000e+00 : f32
 // CHECK:  fir.store %[[VAL_1]] to %[[VAL_0]] : !fir.ref<f32>
 
-func.func @scalar_complex(%arg0: !fir.ref<!fir.complex<4>>, %arg1: !fir.ref<!fir.complex<4>>) {
-  hlfir.assign %arg0 to %arg0 : !fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>
+func.func @scalar_complex(%arg0: !fir.ref<complex<f32>>, %arg1: !fir.ref<complex<f32>>) {
+  hlfir.assign %arg0 to %arg0 : !fir.ref<complex<f32>>, !fir.ref<complex<f32>>
   return
 }
 // CHECK-LABEL:   func.func @scalar_complex(
-// CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<!fir.complex<4>>,
-// CHECK-SAME:    %[[VAL_1:.*]]: !fir.ref<!fir.complex<4>>) {
-// CHECK:  %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.complex<4>>
-// CHECK:  fir.store %[[VAL_2]] to %[[VAL_0]] : !fir.ref<!fir.complex<4>>
+// CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<complex<f32>>,
+// CHECK-SAME:    %[[VAL_1:.*]]: !fir.ref<complex<f32>>) {
+// CHECK:  %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<complex<f32>>
+// CHECK:  fir.store %[[VAL_2]] to %[[VAL_0]] : !fir.ref<complex<f32>>
 
-func.func @scalar_complex_2(%arg0: !fir.ref<!fir.complex<4>>, %arg1: !fir.complex<4>) {
-  hlfir.assign %arg1 to %arg0 : !fir.complex<4>, !fir.ref<!fir.complex<4>>
+func.func @scalar_complex_2(%arg0: !fir.ref<complex<f32>>, %arg1: complex<f32>) {
+  hlfir.assign %arg1 to %arg0 : complex<f32>, !fir.ref<complex<f32>>
   return
 }
 // CHECK-LABEL:   func.func @scalar_complex_2(
-// CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<!fir.complex<4>>,
-// CHECK-SAME:    %[[VAL_1:.*]]: !fir.complex<4>) {
-// CHECK:  fir.store %[[VAL_1]] to %[[VAL_0]] : !fir.ref<!fir.complex<4>>
+// CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<complex<f32>>,
+// CHECK-SAME:    %[[VAL_1:.*]]: complex<f32>) {
+// CHECK:  fir.store %[[VAL_1]] to %[[VAL_0]] : !fir.ref<complex<f32>>
 
 func.func @scalar_character(%arg0: !fir.boxchar<1>, %arg1: !fir.boxchar<1>) {
   %0:2 = fir.unboxchar %arg0 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
diff --git a/flang/test/HLFIR/assign.fir b/flang/test/HLFIR/assign.fir
index a416c79d56d69..eb02884d3b0f1 100644
--- a/flang/test/HLFIR/assign.fir
+++ b/flang/test/HLFIR/assign.fir
@@ -47,23 +47,23 @@ func.func @scalar_real(%arg0: !fir.ref<f16>, %arg1: !fir.ref<f16>) {
 // CHECK-SAME:    %[[VAL_1:.*]]: !fir.ref<f16>) {
 // CHECK:  hlfir.assign %[[VAL_1]] to %[[VAL_0]] : !fir.ref<f16>, !fir.ref<f16>
 
-func.func @scalar_complex(%arg0: !fir.ref<!fir.complex<8>>, %arg1: !fir.ref<!fir.complex<8>>) {
-  hlfir.assign %arg1 to %arg0 : !fir.ref<!fir.complex<8>>, !fir.ref<!fir.complex<8>>
+func.func @scalar_complex(%arg0: !fir.ref<complex<f64>>, %arg1: !fir.ref<complex<f64>>) {
+  hlfir.assign %arg1 to %arg0 : !fir.ref<complex<f64>>, !fir.ref<complex<f64>>
   return
 }
 // CHECK-LABEL:   func.func @scalar_complex(
-// CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<!fir.complex<8>>,
-// CHECK-SAME:    %[[VAL_1:.*]]: !fir.ref<!fir.complex<8>>) {
-// CHECK:  hlfir.assign %[[VAL_1]] to %[[VAL_0]] : !fir.ref<!fir.complex<8>>, !fir.ref<!fir.complex<8>>
+// CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<complex<f64>>,
+// CHECK-SAME:    %[[VAL_1:.*]]: !fir.ref<complex<f64>>) {
+// CHECK:  hlfir.assign %[[VAL_1]] to %[[VAL_0]] : !fir.ref<complex<f64>>, !fir.ref<complex<f64>>
 
-func.func @scalar_complex_2(%arg0: !fir.ref<!fir.complex<8>>, %arg1: !fir.complex<8>) {
-  hlfir.assign %arg1 to %arg0 : !fir.complex<8>, !fir.ref<!fir.complex<8>>
+func.func @scalar_complex_2(%arg0: !fir.ref<complex<f64>>, %arg1: complex<f64>) {
+  hlfir.assign %arg1 to %arg0 : complex<f64>, !fir.ref<complex<f64>>
   return
 }
 // CHECK-LABEL:   func.func @scalar_complex_2(
-// CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<!fir.complex<8>>,
-// CHECK-SAME:    %[[VAL_1:.*]]: !fir.complex<8>) {
-// CHECK:  hlfir.assign %[[VAL_1]] to %[[VAL_0]] : !fir.complex<8>, !fir.ref<!fir.complex<8>>
+// CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<complex<f64>>,
+// CHECK-SAME:    %[[VAL_1:.*]]: complex<f64>) {
+// CHECK:  hlfir.assign %[[VAL_1]] to %[[VAL_0]] : complex<f64>, !fir.ref<complex<f64>>
 
 
 func.func @scalar_character(%arg0: !fir.boxchar<1>, %arg1: !fir.boxchar<1>) {
diff --git a/flang/test/HLFIR/associate.fir b/flang/test/HLFIR/associate.fir
index d0a1e36416b23..52563398da971 100644
--- a/flang/test/HLFIR/associate.fir
+++ b/flang/test/HLFIR/associate.fir
@@ -63,18 +63,18 @@ func.func private @foo4(!fir.ref<!fir.logical<8>>)
 // CHECK:  hlfir.end_associate %[[VAL_1]]#1, %[[VAL_1]]#2 : !fir.ref<!fir.logical<8>>, i1
 
 
-func.func @test_complex(%arg0: !fir.complex<8>) {
-  %0:3 = hlfir.associate %arg0 {uniq_name = "x"} : (!fir.complex<8>) -> (!fir.ref<!fir.complex<8>>, !fir.ref<!fir.complex<8>>, i1)
-  fir.call @foo5(%0#0) : (!fir.ref<!fir.complex<8>>) -> ()
-  hlfir.end_associate %0#1, %0#2 : !fir.ref<!fir.complex<8>>, i1
+func.func @test_complex(%arg0: complex<f64>) {
+  %0:3 = hlfir.associate %arg0 {uniq_name = "x"} : (complex<f64>) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>, i1)
+  fir.call @foo5(%0#0) : (!fir.ref<complex<f64>>) -> ()
+  hlfir.end_associate %0#1, %0#2 : !fir.ref<complex<f64>>, i1
   return
 }
-func.func private @foo5(!fir.ref<!fir.complex<8>>)
+func.func private @foo5(!fir.ref<complex<f64>>)
 // CHECK-LABEL:   func.func @test_complex(
-// CHECK-SAME:    %[[VAL_0:.*]]: !fir.complex<8>) {
-// CHECK:  %[[VAL_1:.*]]:3 = hlfir.associate %[[VAL_0]] {uniq_name = "x"} : (!fir.complex<8>) -> (!fir.ref<!fir.complex<8>>, !fir.ref<!fir.complex<8>>, i1)
-// CHECK:  fir.call @foo5(%[[VAL_1]]#0) : (!fir.ref<!fir.complex<8>>) -> ()
-// CHECK:  hlfir.end_associate %[[VAL_1]]#1, %[[VAL_1]]#2 : !fir.ref<!fir.complex<8>>, i1
+// CHECK-SAME:    %[[VAL_0:.*]]: complex<f64>) {
+// CHECK:  %[[VAL_1:.*]]:3 = hlfir.associate %[[VAL_0]] {uniq_name = "x"} : (complex<f64>) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>, i1)
+// CHECK:  fir.call @foo5(%[[VAL_1]]#0) : (!fir.ref<complex<f64>>) -> ()
+// CHECK:  hlfir.end_associate %[[VAL_1]]#1, %[[VAL_1]]#2 : !fir.ref<complex<f64>>, i1
 
 
 func.func @test_array(%arg0: !hlfir.expr<!fir.array<10x?xi32>>) {
diff --git a/flang/test/HLFIR/assumed_shape_with_value_keyword.f90 b/flang/test/HLFIR/assumed_shape_with_value_keyword.f90
index 208f22badda28..feea3712a822c 100644
--- a/flang/test/HLFIR/assumed_shape_with_value_keyword.f90
+++ b/flang/test/HLFIR/assumed_shape_with_value_keyword.f90
@@ -64,12 +64,12 @@ subroutine test_complex_value1(x)
   call internal_call5(x)
 end
 ! CHECK-LABEL:  func.func @_QPtest_complex_value1(
-! CHECK-SAME:     %[[ARG0:.*]]: !fir.box<!fir.array<?x!fir.complex<4>>> {fir.bindc_name = "x"}) {
-! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFtest_complex_value1Ex"} : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
-! CHECK:          %[[VAL_1:.*]]:2 = hlfir.copy_in %[[VAL_0]]#0 to %[[TMP_BOX:.*]] : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>>) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, i1)
-! CHECK:          %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]]#0 : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> !fir.ref<!fir.array<?x!fir.complex<4>>>
-! CHECK:          fir.call @_QPinternal_call5(%[[VAL_2]]) fastmath<contract> : (!fir.ref<!fir.array<?x!fir.complex<4>>>) -> ()
-! CHECK:          hlfir.copy_out %[[TMP_BOX]], %[[VAL_1]]#1 to %[[VAL_0]]#0 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>>, i1, !fir.box<!fir.array<?x!fir.complex<4>>>) -> ()
+! CHECK-SAME:     %[[ARG0:.*]]: !fir.box<!fir.array<?xcomplex<f32>>> {fir.bindc_name = "x"}) {
+! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFtest_complex_value1Ex"} : (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.dscope) -> (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.box<!fir.array<?xcomplex<f32>>>)
+! CHECK:          %[[VAL_1:.*]]:2 = hlfir.copy_in %[[VAL_0]]#0 to %[[TMP_BOX:.*]] : (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>>) -> (!fir.box<!fir.array<?xcomplex<f32>>>, i1)
+! CHECK:          %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]]#0 : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.ref<!fir.array<?xcomplex<f32>>>
+! CHECK:          fir.call @_QPinternal_call5(%[[VAL_2]]) fastmath<contract> : (!fir.ref<!fir.array<?xcomplex<f32>>>) -> ()
+! CHECK:          hlfir.copy_out %[[TMP_BOX]], %[[VAL_1]]#1 to %[[VAL_0]]#0 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>>, i1, !fir.box<!fir.array<?xcomplex<f32>>>) -> ()
 ! CHECK:          return
 ! CHECK:        }
 
@@ -78,12 +78,12 @@ subroutine test_complex_value2(x)
   call internal_call6(x)
 end
 ! CHECK-LABEL:  func.func @_QPtest_complex_value2(
-! CHECK-SAME:     %[[ARG0:.*]]: !fir.box<!fir.array<?x?x!fir.complex<4>>> {fir.bindc_name = "x"}) {
-! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFtest_complex_value2Ex"} : (!fir.box<!fir.array<?x?x!fir.complex<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x?x!fir.complex<4>>>, !fir.box<!fir.array<?x?x!fir.complex<4>>>)
-! CHECK:          %[[VAL_1:.*]]:2 = hlfir.copy_in %[[VAL_0]]#0 to %[[TMP_BOX:.*]] : (!fir.box<!fir.array<?x?x!fir.complex<4>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x!fir.complex<4>>>>>) -> (!fir.box<!fir.array<?x?x!fir.complex<4>>>, i1)
-! CHECK:          %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]]#0 : (!fir.box<!fir.array<?x?x!fir.complex<4>>>) -> !fir.ref<!fir.array<?x?x!fir.complex<4>>>
-! CHECK:          fir.call @_QPinternal_call6(%[[VAL_2]]) fastmath<contract> : (!fir.ref<!fir.array<?x?x!fir.complex<4>>>) -> ()
-! CHECK:          hlfir.copy_out %[[TMP_BOX]], %[[VAL_1]]#1 to %[[VAL_0]]#0 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?x!fir.complex<4>>>>>, i1, !fir.box<!fir.array<?x?x!fir.complex<4>>>) -> ()
+! CHECK-SAME:     %[[ARG0:.*]]: !fir.box<!fir.array<?x?xcomplex<f32>>> {fir.bindc_name = "x"}) {
+! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFtest_complex_value2Ex"} : (!fir.box<!fir.array<?x?xcomplex<f32>>>, !fir.dscope) -> (!fir.box<!fir.array<?x?xcomplex<f32>>>, !fir.box<!fir.array<?x?xcomplex<f32>>>)
+! CHECK:          %[[VAL_1:.*]]:2 = hlfir.copy_in %[[VAL_0]]#0 to %[[TMP_BOX:.*]] : (!fir.box<!fir.array<?x?xcomplex<f32>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xcomplex<f32>>>>>) -> (!fir.box<!fir.array<?x?xcomplex<f32>>>, i1)
+! CHECK:          %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]]#0 : (!fir.box<!fir.array<?x?xcomplex<f32>>>) -> !fir.ref<!fir.array<?x?xcomplex<f32>>>
+! CHECK:          fir.call @_QPinternal_call6(%[[VAL_2]]) fastmath<contract> : (!fir.ref<!fir.array<?x?xcomplex<f32>>>) -> ()
+! CHECK:          hlfir.copy_out %[[TMP_BOX]], %[[VAL_1]]#1 to %[[VAL_0]]#0 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xcomplex<f32>>>>>, i1, !fir.box<!fir.array<?x?xcomplex<f32>>>) -> ()
 ! CHECK:          return
 ! CHECK:        }
 
diff --git a/flang/test/HLFIR/designate-codegen-complex-part.fir b/flang/test/HLFIR/designate-codegen-complex-part.fir
index 21601f2124470..6f36cf4f4c4ae 100644
--- a/flang/test/HLFIR/designate-codegen-complex-part.fir
+++ b/flang/test/HLFIR/designate-codegen-complex-part.fir
@@ -2,82 +2,82 @@
 // with complex parts.
 // RUN: fir-opt %s -convert-hlfir-to-fir | FileCheck %s
 
-func.func @test_set_scalar(%arg0: !fir.ref<!fir.complex<4>>, %arg1: !fir.ref<f32>) {
-  %0:2 = hlfir.declare %arg0 {uniq_name = "a"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+func.func @test_set_scalar(%arg0: !fir.ref<complex<f32>>, %arg1: !fir.ref<f32>) {
+  %0:2 = hlfir.declare %arg0 {uniq_name = "a"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
   %1:2 = hlfir.declare %arg1 {uniq_name = "b"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
   %2 = fir.load %1#0 : !fir.ref<f32>
-  %3 = hlfir.designate %0#0  imag : (!fir.ref<!fir.complex<4>>) -> !fir.ref<f32>
+  %3 = hlfir.designate %0#0  imag : (!fir.ref<complex<f32>>) -> !fir.ref<f32>
   hlfir.assign %2 to %3 : f32, !fir.ref<f32>
   return
 }
 // CHECK-LABEL: func.func @test_set_scalar(
-// CHECK-SAME: %[[VAL_0:.*]]: !fir.ref<!fir.complex<4>>, %[[VAL_1:.*]]: !fir.ref<f32>) {
-// CHECK: %[[VAL_2:.*]] = fir.declare %[[VAL_0]] {uniq_name = "a"} : (!fir.ref<!fir.complex<4>>) -> !fir.ref<!fir.complex<4>>
+// CHECK-SAME: %[[VAL_0:.*]]: !fir.ref<complex<f32>>, %[[VAL_1:.*]]: !fir.ref<f32>) {
+// CHECK: %[[VAL_2:.*]] = fir.declare %[[VAL_0]] {uniq_name = "a"} : (!fir.ref<complex<f32>>) -> !fir.ref<complex<f32>>
 // CHECK: %[[VAL_3:.*]] = fir.declare %[[VAL_1]] {uniq_name = "b"} : (!fir.ref<f32>) -> !fir.ref<f32>
 // CHECK: %[[VAL_4:.*]] = fir.load %[[VAL_3]] : !fir.ref<f32>
 // CHECK: %[[VAL_5:.*]] = arith.constant 1 : index
-// CHECK: %[[VAL_6:.*]] = fir.coordinate_of %[[VAL_2]], %[[VAL_5]] : (!fir.ref<!fir.complex<4>>, index) -> !fir.ref<f32>
+// CHECK: %[[VAL_6:.*]] = fir.coordinate_of %[[VAL_2]], %[[VAL_5]] : (!fir.ref<complex<f32>>, index) -> !fir.ref<f32>
 // CHECK: fir.store %[[VAL_4]] to %[[VAL_6]] : !fir.ref<f32>
 
-func.func @test_scalar_at_index(%arg0: !fir.box<!fir.array<?x!fir.complex<4>>>, %arg1: !fir.ref<i32>) {
-  %0:2 = hlfir.declare %arg0 {uniq_name = "a"} : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
+func.func @test_scalar_at_index(%arg0: !fir.box<!fir.array<?xcomplex<f32>>>, %arg1: !fir.ref<i32>) {
+  %0:2 = hlfir.declare %arg0 {uniq_name = "a"} : (!fir.box<!fir.array<?xcomplex<f32>>>) -> (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.box<!fir.array<?xcomplex<f32>>>)
   %1:2 = hlfir.declare %arg1 {uniq_name = "b"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   %2 = fir.load %1#0 : !fir.ref<i32>
   %3 = fir.convert %2 : (i32) -> i64
-  %4 = hlfir.designate %0#0 (%3) real : (!fir.box<!fir.array<?x!fir.complex<4>>>, i64) -> !fir.ref<f32>
+  %4 = hlfir.designate %0#0 (%3) real : (!fir.box<!fir.array<?xcomplex<f32>>>, i64) -> !fir.ref<f32>
   return
 }
 // CHECK-LABEL: func.func @test_scalar_at_index(
-// CHECK-SAME: %[[VAL_0:.*]]: !fir.box<!fir.array<?x!fir.complex<4>>>, %[[VAL_1:.*]]: !fir.ref<i32>) {
-// CHECK: %[[VAL_2:.*]] = fir.declare %[[VAL_0]] {uniq_name = "a"} : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> !fir.box<!fir.array<?x!fir.complex<4>>>
-// CHECK: %[[VAL_3:.*]] = fir.rebox %[[VAL_2]] : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> !fir.box<!fir.array<?x!fir.complex<4>>>
+// CHECK-SAME: %[[VAL_0:.*]]: !fir.box<!fir.array<?xcomplex<f32>>>, %[[VAL_1:.*]]: !fir.ref<i32>) {
+// CHECK: %[[VAL_2:.*]] = fir.declare %[[VAL_0]] {uniq_name = "a"} : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.box<!fir.array<?xcomplex<f32>>>
+// CHECK: %[[VAL_3:.*]] = fir.rebox %[[VAL_2]] : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.box<!fir.array<?xcomplex<f32>>>
 // CHECK: %[[VAL_4:.*]] = fir.declare %[[VAL_1]] {uniq_name = "b"} : (!fir.ref<i32>) -> !fir.ref<i32>
 // CHECK: %[[VAL_5:.*]] = fir.load %[[VAL_4]] : !fir.ref<i32>
 // CHECK: %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (i32) -> i64
-// CHECK: %[[VAL_7:.*]] = fir.array_coor %[[VAL_3]] %[[VAL_6]] : (!fir.box<!fir.array<?x!fir.complex<4>>>, i64) -> !fir.ref<!fir.complex<4>>
+// CHECK: %[[VAL_7:.*]] = fir.array_coor %[[VAL_3]] %[[VAL_6]] : (!fir.box<!fir.array<?xcomplex<f32>>>, i64) -> !fir.ref<complex<f32>>
 // CHECK: %[[VAL_8:.*]] = arith.constant 0 : index
-// CHECK: %[[VAL_9:.*]] = fir.coordinate_of %[[VAL_7]], %[[VAL_8]] : (!fir.ref<!fir.complex<4>>, index) -> !fir.ref<f32>
+// CHECK: %[[VAL_9:.*]] = fir.coordinate_of %[[VAL_7]], %[[VAL_8]] : (!fir.ref<complex<f32>>, index) -> !fir.ref<f32>
 
-func.func @test_complete_slice(%arg0: !fir.box<!fir.array<?x!fir.complex<4>>>) {
+func.func @test_complete_slice(%arg0: !fir.box<!fir.array<?xcomplex<f32>>>) {
   %c0 = arith.constant 0 : index
-  %0:2 = hlfir.declare %arg0 {uniq_name = "a"} : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
-  %1:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?x!fir.complex<4>>>, index) -> (index, index, index)
+  %0:2 = hlfir.declare %arg0 {uniq_name = "a"} : (!fir.box<!fir.array<?xcomplex<f32>>>) -> (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.box<!fir.array<?xcomplex<f32>>>)
+  %1:3 = fir.box_dims %0#0, %c0 : (!fir.box<!fir.array<?xcomplex<f32>>>, index) -> (index, index, index)
   %2 = fir.shape %1#1 : (index) -> !fir.shape<1>
-  %3 = hlfir.designate %0#0  imag shape %2 : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
+  %3 = hlfir.designate %0#0  imag shape %2 : (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
   return
 }
 // CHECK-LABEL: func.func @test_complete_slice(
-// CHECK-SAME: %[[VAL_0:.*]]: !fir.box<!fir.array<?x!fir.complex<4>>>) {
+// CHECK-SAME: %[[VAL_0:.*]]: !fir.box<!fir.array<?xcomplex<f32>>>) {
 // CHECK: %[[VAL_1:.*]] = arith.constant 0 : index
-// CHECK: %[[VAL_2:.*]] = fir.declare %[[VAL_0]] {uniq_name = "a"} : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> !fir.box<!fir.array<?x!fir.complex<4>>>
-// CHECK: %[[VAL_3:.*]] = fir.rebox %[[VAL_2]] : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> !fir.box<!fir.array<?x!fir.complex<4>>>
-// CHECK: %[[VAL_4:.*]]:3 = fir.box_dims %[[VAL_3]], %[[VAL_1]] : (!fir.box<!fir.array<?x!fir.complex<4>>>, index) -> (index, index, index)
+// CHECK: %[[VAL_2:.*]] = fir.declare %[[VAL_0]] {uniq_name = "a"} : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.box<!fir.array<?xcomplex<f32>>>
+// CHECK: %[[VAL_3:.*]] = fir.rebox %[[VAL_2]] : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.box<!fir.array<?xcomplex<f32>>>
+// CHECK: %[[VAL_4:.*]]:3 = fir.box_dims %[[VAL_3]], %[[VAL_1]] : (!fir.box<!fir.array<?xcomplex<f32>>>, index) -> (index, index, index)
 // CHECK: %[[VAL_5:.*]] = fir.shape %[[VAL_4]]#1 : (index) -> !fir.shape<1>
 // CHECK: %[[VAL_6:.*]] = arith.constant 1 : index
 // CHECK: %[[VAL_7:.*]] = arith.constant 1 : index
 // CHECK: %[[VAL_8:.*]] = arith.constant 0 : index
-// CHECK: %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_3]], %[[VAL_8]] : (!fir.box<!fir.array<?x!fir.complex<4>>>, index) -> (index, index, index)
+// CHECK: %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_3]], %[[VAL_8]] : (!fir.box<!fir.array<?xcomplex<f32>>>, index) -> (index, index, index)
 // CHECK: %[[VAL_10:.*]] = arith.constant 1 : index
 // CHECK: %[[VAL_11:.*]] = fir.slice %[[VAL_7]], %[[VAL_9]]#1, %[[VAL_6]] path %[[VAL_10]] : (index, index, index, index) -> !fir.slice<1>
-// CHECK: %[[VAL_12:.*]] = fir.rebox %[[VAL_3]] [%[[VAL_11]]] : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
+// CHECK: %[[VAL_12:.*]] = fir.rebox %[[VAL_3]] [%[[VAL_11]]] : (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.slice<1>) -> !fir.box<!fir.array<?xf32>>
 
-func.func @test_slice_steps(%arg0: !fir.box<!fir.array<?x!fir.complex<4>>>) {
+func.func @test_slice_steps(%arg0: !fir.box<!fir.array<?xcomplex<f32>>>) {
   %c3 = arith.constant 3 : index
   %c12 = arith.constant 12 : index
   %c4 = arith.constant 4 : index
-  %0:2 = hlfir.declare %arg0 {uniq_name = "a"} : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
+  %0:2 = hlfir.declare %arg0 {uniq_name = "a"} : (!fir.box<!fir.array<?xcomplex<f32>>>) -> (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.box<!fir.array<?xcomplex<f32>>>)
   %1 = fir.shape %c3 : (index) -> !fir.shape<1>
-  %2 = hlfir.designate %0#0 (%c4:%c12:%c3) real shape %1 : (!fir.box<!fir.array<?x!fir.complex<4>>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<3xf32>>
+  %2 = hlfir.designate %0#0 (%c4:%c12:%c3) real shape %1 : (!fir.box<!fir.array<?xcomplex<f32>>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<3xf32>>
   return
 }
 // CHECK-LABEL: func.func @test_slice_steps(
-// CHECK-SAME: %[[VAL_0:.*]]: !fir.box<!fir.array<?x!fir.complex<4>>>) {
+// CHECK-SAME: %[[VAL_0:.*]]: !fir.box<!fir.array<?xcomplex<f32>>>) {
 // CHECK: %[[VAL_1:.*]] = arith.constant 3 : index
 // CHECK: %[[VAL_2:.*]] = arith.constant 12 : index
 // CHECK: %[[VAL_3:.*]] = arith.constant 4 : index
-// CHECK: %[[VAL_4:.*]] = fir.declare %[[VAL_0]] {uniq_name = "a"} : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> !fir.box<!fir.array<?x!fir.complex<4>>>
-// CHECK: %[[VAL_5:.*]] = fir.rebox %[[VAL_4]] : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> !fir.box<!fir.array<?x!fir.complex<4>>>
+// CHECK: %[[VAL_4:.*]] = fir.declare %[[VAL_0]] {uniq_name = "a"} : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.box<!fir.array<?xcomplex<f32>>>
+// CHECK: %[[VAL_5:.*]] = fir.rebox %[[VAL_4]] : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.box<!fir.array<?xcomplex<f32>>>
 // CHECK: %[[VAL_6:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
 // CHECK: %[[VAL_7:.*]] = arith.constant 0 : index
 // CHECK: %[[VAL_8:.*]] = fir.slice %[[VAL_3]], %[[VAL_2]], %[[VAL_1]] path %[[VAL_7]] : (index, index, index, index) -> !fir.slice<1>
-// CHECK: %[[VAL_9:.*]] = fir.rebox %[[VAL_5]] [%[[VAL_8]]] : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.slice<1>) -> !fir.box<!fir.array<3xf32>>
+// CHECK: %[[VAL_9:.*]] = fir.rebox %[[VAL_5]] [%[[VAL_8]]] : (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.slice<1>) -> !fir.box<!fir.array<3xf32>>
diff --git a/flang/test/HLFIR/designate.fir b/flang/test/HLFIR/designate.fir
index dcc156aed4797..e477cab6cfa90 100644
--- a/flang/test/HLFIR/designate.fir
+++ b/flang/test/HLFIR/designate.fir
@@ -92,13 +92,13 @@ func.func @array_substring(%arg0 : !fir.box<!fir.array<10x!fir.char<1,?>>>) {
 // CHECK:  %[[VAL_4:.*]] = fir.undefined !fir.shape<1>
 // CHECK:  %[[VAL_5:.*]] = hlfir.designate %[[VAL_0]] substr %[[VAL_2]], %[[VAL_3]] shape %[[VAL_4]] typeparams %[[VAL_1]] : (!fir.box<!fir.array<10x!fir.char<1,?>>>, index, index, !fir.shape<1>, index) -> !fir.box<!fir.array<10x!fir.char<1,?>>>
 
-func.func @real_part(%arg0 : !fir.ref<!fir.complex<4>>) {
-  %0 = hlfir.designate %arg0 real :  (!fir.ref<!fir.complex<4>>) -> !fir.ref<!fir.real<4>>
+func.func @real_part(%arg0 : !fir.ref<complex<f32>>) {
+  %0 = hlfir.designate %arg0 real :  (!fir.ref<complex<f32>>) -> !fir.ref<f32>
   return
 }
 // CHECK-LABEL: func.func @real_part(
-// CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<!fir.complex<4>>) {
-// CHECK:  %[[VAL_1:.*]] = hlfir.designate %[[VAL_0]] real :  (!fir.ref<!fir.complex<4>>) -> !fir.ref<!fir.real<4>>
+// CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<complex<f32>>) {
+// CHECK:  %[[VAL_1:.*]] = hlfir.designate %[[VAL_0]] real :  (!fir.ref<complex<f32>>) -> !fir.ref<f32>
 
 func.func @imag_part_mlir_complex(%arg0 : !fir.ref<complex<f32>>) {
   %0 = hlfir.designate %arg0 imag :  (!fir.ref<complex<f32>>) -> !fir.ref<f32>
@@ -109,19 +109,19 @@ func.func @imag_part_mlir_complex(%arg0 : !fir.ref<complex<f32>>) {
 // CHECK:  %[[VAL_1:.*]] = hlfir.designate %[[VAL_0]] imag :  (!fir.ref<complex<f32>>) -> !fir.ref<f32>
 
 // array%array_complex_comp(1)%im
-func.func @component_array_ref_cplx_part(%arg0 : !fir.box<!fir.array<10x!fir.type<t3{array_comp:!fir.array<100x!fir.complex<8>>}>>>) {
+func.func @component_array_ref_cplx_part(%arg0 : !fir.box<!fir.array<10x!fir.type<t3{array_comp:!fir.array<100xcomplex<f64>>}>>>) {
   %c1 = arith.constant 1 : index
   %component_shape = fir.undefined !fir.shapeshift<1>
   %result_shape = fir.undefined !fir.shape<1>
-  %0 = hlfir.designate %arg0 {"array_comp"}<%component_shape>(%c1) imag shape %result_shape : (!fir.box<!fir.array<10x!fir.type<t3{array_comp:!fir.array<100x!fir.complex<8>>}>>>, !fir.shapeshift<1>, index, !fir.shape<1>) -> !fir.box<!fir.array<10x!fir.real<8>>>
+  %0 = hlfir.designate %arg0 {"array_comp"}<%component_shape>(%c1) imag shape %result_shape : (!fir.box<!fir.array<10x!fir.type<t3{array_comp:!fir.array<100xcomplex<f64>>}>>>, !fir.shapeshift<1>, index, !fir.shape<1>) -> !fir.box<!fir.array<10xf64>>
   return
 }
 // CHECK-LABEL: func.func @component_array_ref_cplx_part(
-// CHECK-SAME:    %[[VAL_0:.*]]: !fir.box<!fir.array<10x!fir.type<t3{array_comp:!fir.array<100x!fir.complex<8>>}>>>) {
+// CHECK-SAME:    %[[VAL_0:.*]]: !fir.box<!fir.array<10x!fir.type<t3{array_comp:!fir.array<100xcomplex<f64>>}>>>) {
 // CHECK:  %[[VAL_1:.*]] = arith.constant 1 : index
 // CHECK:  %[[VAL_2:.*]] = fir.undefined !fir.shapeshift<1>
 // CHECK:  %[[VAL_3:.*]] = fir.undefined !fir.shape<1>
-// CHECK:  %[[VAL_4:.*]] = hlfir.designate %[[VAL_0]]{"array_comp"} <%[[VAL_2]]> (%[[VAL_1]]) imag shape %[[VAL_3]] : (!fir.box<!fir.array<10x!fir.type<t3{array_comp:!fir.array<100x!fir.complex<8>>}>>>, !fir.shapeshift<1>, index, !fir.shape<1>) -> !fir.box<!fir.array<10x!fir.real<8>>>
+// CHECK:  %[[VAL_4:.*]] = hlfir.designate %[[VAL_0]]{"array_comp"} <%[[VAL_2]]> (%[[VAL_1]]) imag shape %[[VAL_3]] : (!fir.box<!fir.array<10x!fir.type<t3{array_comp:!fir.array<100xcomplex<f64>>}>>>, !fir.shapeshift<1>, index, !fir.shape<1>) -> !fir.box<!fir.array<10xf64>>
 
 // pdt_array(1)
 func.func @pdt(%arg0 : !fir.box<!fir.array<?x!fir.type<pdt(param:i32){field:i32}>>>) {
diff --git a/flang/test/HLFIR/invalid.fir b/flang/test/HLFIR/invalid.fir
index e88912da8c6d2..c390dddcf3f38 100644
--- a/flang/test/HLFIR/invalid.fir
+++ b/flang/test/HLFIR/invalid.fir
@@ -202,16 +202,16 @@ func.func @bad_designate_out_rank(%arg0 : !fir.ref<!fir.array<10xi32>>) {
 }
 
 // -----
-func.func @bad_designate_out_type(%arg0 : !fir.ref<!fir.complex<4>>) {
-  // expected-error@+1 {{'hlfir.designate' op result element type is not consistent with operands, expected '!fir.real<4>'}}
-  %0 = hlfir.designate %arg0 imag:  (!fir.ref<!fir.complex<4>>) -> !fir.ref<!fir.complex<4>>
+func.func @bad_designate_out_type(%arg0 : !fir.ref<complex<f32>>) {
+  // expected-error@+1 {{'hlfir.designate' op result element type is not consistent with operands, expected 'f32'}}
+  %0 = hlfir.designate %arg0 imag:  (!fir.ref<complex<f32>>) -> !fir.ref<complex<f64>>
   return
 }
 
 // -----
-func.func @bad_designate_out_type(%arg0 : !fir.ref<!fir.box<!fir.complex<4>>>) {
+func.func @bad_designate_out_type(%arg0 : !fir.ref<!fir.box<complex<f32>>>) {
   // expected-error@+1 {{'hlfir.designate' op result type must only be a box address type if it designates a component that is a fir.box or fir.class and if there are no indices, substrings, and complex part}}
-  %0 = hlfir.designate %arg0 imag:  (!fir.ref<!fir.box<!fir.complex<4>>>) -> !fir.ref<!fir.box<!fir.real<4>>>
+  %0 = hlfir.designate %arg0 imag:  (!fir.ref<!fir.box<complex<f32>>>) -> !fir.ref<!fir.box<f32>>
   return
 }
 
diff --git a/flang/test/HLFIR/opt-scalar-assign.fir b/flang/test/HLFIR/opt-scalar-assign.fir
index 2b1631a8202ab..02ab02945b042 100644
--- a/flang/test/HLFIR/opt-scalar-assign.fir
+++ b/flang/test/HLFIR/opt-scalar-assign.fir
@@ -102,30 +102,30 @@ func.func @_QPtest3(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {fir
 // CHECK:           return
 // CHECK:         }
 
-func.func @_QPtest5(%arg0: !fir.ref<!fir.array<77x!fir.complex<4>>> {fir.bindc_name = "x"}) {
+func.func @_QPtest5(%arg0: !fir.ref<!fir.array<77xcomplex<f32>>> {fir.bindc_name = "x"}) {
   %cst = arith.constant 0.000000e+00 : f32
   %c77 = arith.constant 77 : index
   %0 = fir.shape %c77 : (index) -> !fir.shape<1>
-  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFtest5Ex"} : (!fir.ref<!fir.array<77x!fir.complex<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<77x!fir.complex<4>>>, !fir.ref<!fir.array<77x!fir.complex<4>>>)
-  %2 = fir.undefined !fir.complex<4>
-  %3 = fir.insert_value %2, %cst, [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-  %4 = fir.insert_value %3, %cst, [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-  hlfir.assign %4 to %1#0 : !fir.complex<4>, !fir.ref<!fir.array<77x!fir.complex<4>>>
+  %1:2 = hlfir.declare %arg0(%0) {uniq_name = "_QFtest5Ex"} : (!fir.ref<!fir.array<77xcomplex<f32>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<77xcomplex<f32>>>, !fir.ref<!fir.array<77xcomplex<f32>>>)
+  %2 = fir.undefined complex<f32>
+  %3 = fir.insert_value %2, %cst, [0 : index] : (complex<f32>, f32) -> complex<f32>
+  %4 = fir.insert_value %3, %cst, [1 : index] : (complex<f32>, f32) -> complex<f32>
+  hlfir.assign %4 to %1#0 : complex<f32>, !fir.ref<!fir.array<77xcomplex<f32>>>
   return
 }
 // CHECK-LABEL:   func.func @_QPtest5(
-// CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<!fir.array<77x!fir.complex<4>>> {fir.bindc_name = "x"}) {
+// CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<!fir.array<77xcomplex<f32>>> {fir.bindc_name = "x"}) {
 // CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_2:.*]] = arith.constant 0.000000e+00 : f32
 // CHECK:           %[[VAL_3:.*]] = arith.constant 77 : index
 // CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
-// CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) {uniq_name = "_QFtest5Ex"} : (!fir.ref<!fir.array<77x!fir.complex<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<77x!fir.complex<4>>>, !fir.ref<!fir.array<77x!fir.complex<4>>>)
-// CHECK:           %[[VAL_6:.*]] = fir.undefined !fir.complex<4>
-// CHECK:           %[[VAL_7:.*]] = fir.insert_value %[[VAL_6]], %[[VAL_2]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-// CHECK:           %[[VAL_8:.*]] = fir.insert_value %[[VAL_7]], %[[VAL_2]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
+// CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) {uniq_name = "_QFtest5Ex"} : (!fir.ref<!fir.array<77xcomplex<f32>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<77xcomplex<f32>>>, !fir.ref<!fir.array<77xcomplex<f32>>>)
+// CHECK:           %[[VAL_6:.*]] = fir.undefined complex<f32>
+// CHECK:           %[[VAL_7:.*]] = fir.insert_value %[[VAL_6]], %[[VAL_2]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+// CHECK:           %[[VAL_8:.*]] = fir.insert_value %[[VAL_7]], %[[VAL_2]], [1 : index] : (complex<f32>, f32) -> complex<f32>
 // CHECK:           fir.do_loop %[[VAL_9:.*]] = %[[VAL_1]] to %[[VAL_3]] step %[[VAL_1]] unordered {
-// CHECK:             %[[VAL_10:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_9]])  : (!fir.ref<!fir.array<77x!fir.complex<4>>>, index) -> !fir.ref<!fir.complex<4>>
-// CHECK:             hlfir.assign %[[VAL_8]] to %[[VAL_10]] : !fir.complex<4>, !fir.ref<!fir.complex<4>>
+// CHECK:             %[[VAL_10:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_9]])  : (!fir.ref<!fir.array<77xcomplex<f32>>>, index) -> !fir.ref<complex<f32>>
+// CHECK:             hlfir.assign %[[VAL_8]] to %[[VAL_10]] : complex<f32>, !fir.ref<complex<f32>>
 // CHECK:           }
 // CHECK:           return
 // CHECK:         }
diff --git a/flang/test/Integration/OpenMP/atomic-update-complex.f90 b/flang/test/Integration/OpenMP/atomic-update-complex.f90
new file mode 100644
index 0000000000000..827e84a011f53
--- /dev/null
+++ b/flang/test/Integration/OpenMP/atomic-update-complex.f90
@@ -0,0 +1,42 @@
+!===----------------------------------------------------------------------===!
+! This directory can be used to add Integration tests involving multiple
+! stages of the compiler (for eg. from Fortran to LLVM IR). It should not
+! contain executable tests. We should only add tests here sparingly and only
+! if there is no other way to test. Repeat this message in each test that is
+! added to this directory and sub-directories.
+!===----------------------------------------------------------------------===!
+
+!RUN: %flang_fc1 -emit-llvm -fopenmp %s -o - | FileCheck %s
+
+!CHECK: define void @_QQmain() {
+!CHECK: %[[X_NEW_VAL:.*]] = alloca { float, float }, align 8
+!CHECK: {{.*}} = alloca { float, float }, i64 1, align 8
+!CHECK: %[[ORIG_VAL:.*]] = alloca { float, float }, i64 1, align 8
+!CHECK: store { float, float } { float 2.000000e+00, float 2.000000e+00 }, ptr %[[ORIG_VAL]], align 4
+!CHECK: br label %entry
+
+!CHECK: entry:
+!CHECK: %[[ATOMIC_TEMP_LOAD:.*]] = alloca { float, float }, align 8
+!CHECK: call void @__atomic_load(i64 8, ptr %[[ORIG_VAL]], ptr %[[ATOMIC_TEMP_LOAD]], i32 0)
+!CHECK: %[[PHI_NODE_ENTRY_1:.*]] = load { float, float }, ptr %[[ATOMIC_TEMP_LOAD]], align 8
+!CHECK: br label %.atomic.cont
+
+!CHECK: .atomic.cont
+!CHECK: %[[VAL_4:.*]] = phi { float, float } [ %[[PHI_NODE_ENTRY_1]], %entry ], [ %{{.*}}, %.atomic.cont ]
+!CHECK: %[[VAL_5:.*]] = extractvalue { float, float } %[[VAL_4]], 0
+!CHECK: %[[VAL_6:.*]] = extractvalue { float, float } %[[VAL_4]], 1
+!CHECK: %[[VAL_7:.*]] = fadd contract float %[[VAL_5]], 1.000000e+00
+!CHECK: %[[VAL_8:.*]] = fadd contract float %[[VAL_6]], 1.000000e+00
+!CHECK: %[[VAL_9:.*]] = insertvalue { float, float } undef, float %[[VAL_7]], 0
+!CHECK: %[[VAL_10:.*]] = insertvalue { float, float } %[[VAL_9]], float %[[VAL_8]], 1
+!CHECK: store { float, float } %[[VAL_10]], ptr %[[X_NEW_VAL]], align 4
+!CHECK: %[[VAL_11:.*]] = call i1 @__atomic_compare_exchange(i64 8, ptr %[[ORIG_VAL]], ptr %[[ATOMIC_TEMP_LOAD]], ptr %[[X_NEW_VAL]], i32 2, i32 2)
+!CHECK: %[[VAL_12:.*]] = load { float, float }, ptr %[[ATOMIC_TEMP_LOAD]], align 4
+!CHECK: br i1 %[[VAL_11]], label %.atomic.exit, label %.atomic.cont
+program main
+      complex*8 ia, ib
+      ia = (2, 2)
+      !$omp atomic update
+        ia = ia + (1, 1)
+      !$omp end atomic
+end program
diff --git a/flang/test/Integration/debug-assumed-size-array.f90 b/flang/test/Integration/debug-assumed-size-array.f90
index efa53e643ecbb..2d7edf62339b0 100644
--- a/flang/test/Integration/debug-assumed-size-array.f90
+++ b/flang/test/Integration/debug-assumed-size-array.f90
@@ -12,7 +12,7 @@ end module helper
 
 ! CHECK-DAG: ![[TY1:[0-9]+]] = !DICompositeType(tag: DW_TAG_array_type{{.*}}elements: ![[ELEMS1:[0-9]+]]{{.*}})
 ! CHECK-DAG: ![[ELEMS1]] = !{![[ELM1:[0-9]+]], ![[EMPTY:[0-9]+]]}
-! CHECK-DAG: ![[ELM1]] = !DISubrange(count: 5, lowerBound: 1)
+! CHECK-DAG: ![[ELM1]] = !DISubrange(count: 5)
 ! CHECK-DAG: ![[EMPTY]] = !DISubrange()
 ! CHECK-DAG: ![[TY2:[0-9]+]] = !DICompositeType(tag: DW_TAG_array_type{{.*}}elements: ![[ELEMS2:[0-9]+]]{{.*}})
 ! CHECK-DAG: ![[ELEMS2]] = !{![[EMPTY:[0-9]+]]}
diff --git a/flang/test/Integration/debug-fixed-array-type-2.f90 b/flang/test/Integration/debug-fixed-array-type-2.f90
index 705c1da593c70..009b99af10325 100644
--- a/flang/test/Integration/debug-fixed-array-type-2.f90
+++ b/flang/test/Integration/debug-fixed-array-type-2.f90
@@ -23,11 +23,11 @@ function fn1(a1, b1, c1) result (res)
 
 ! CHECK-DAG: ![[INT:.*]] = !DIBasicType(name: "integer", size: 32, encoding: DW_ATE_signed)
 ! CHECK-DAG: ![[REAL:.*]] = !DIBasicType(name: "real", size: 32, encoding: DW_ATE_float)
-! CHECK-DAG: ![[R1:.*]] = !DISubrange(count: 3, lowerBound: 1)
+! CHECK-DAG: ![[R1:.*]] = !DISubrange(count: 3)
 ! CHECK-DAG: ![[SUB1:.*]] = !{![[R1]]}
 ! CHECK-DAG: ![[D1TY:.*]] = !DICompositeType(tag: DW_TAG_array_type, baseType: ![[INT]], elements: ![[SUB1]])
 
-! CHECK-DAG: ![[R21:.*]] = !DISubrange(count: 4, lowerBound: 1)
+! CHECK-DAG: ![[R21:.*]] = !DISubrange(count: 4)
 ! CHECK-DAG: ![[R22:.*]] = !DISubrange(count: 5, lowerBound: -1)
 ! CHECK-DAG: ![[SUB2:.*]] = !{![[R21]], ![[R22]]}
 ! CHECK-DAG: ![[D2TY:.*]] = !DICompositeType(tag: DW_TAG_array_type, baseType: ![[INT]], elements: ![[SUB2]])
diff --git a/flang/test/Integration/debug-local-var-2.f90 b/flang/test/Integration/debug-local-var-2.f90
index 79fe1bab6e35b..525474a378ed6 100644
--- a/flang/test/Integration/debug-local-var-2.f90
+++ b/flang/test/Integration/debug-local-var-2.f90
@@ -53,7 +53,7 @@
 ! BOTH-LABEL: }
 
 program mn
-! BOTH-DAG: ![[MAIN:.*]] = distinct !DISubprogram(name: "_QQmain", {{.*}})
+! BOTH-DAG: ![[MAIN:.*]] = distinct !DISubprogram(name: "mn", {{.*}})
 
 ! BOTH-DAG: ![[TYI32:.*]] = !DIBasicType(name: "integer", size: 32, encoding: DW_ATE_signed)
 ! BOTH-DAG: ![[TYI64:.*]] = !DIBasicType(name: "integer", size: 64, encoding: DW_ATE_signed)
diff --git a/flang/test/Integration/debug-variable-array-dim.f90 b/flang/test/Integration/debug-variable-array-dim.f90
new file mode 100644
index 0000000000000..e68cb46b5f835
--- /dev/null
+++ b/flang/test/Integration/debug-variable-array-dim.f90
@@ -0,0 +1,19 @@
+! RUN: %flang_fc1 -emit-llvm -debug-info-kind=standalone %s -o - | FileCheck %s
+
+subroutine foo(a, n, m, p)
+  integer n, m, p
+  integer :: a(n:m, p)
+  a(1, 2) = 10
+  print *, a
+end subroutine foo
+
+
+! CHECK-DAG: ![[VAR0:.*]] = !DILocalVariable(name: "._QFfooEa3"{{.*}}scope: ![[SCOPE:[0-9]+]]{{.*}}flags: DIFlagArtificial)
+! CHECK-DAG: ![[VAR1:.*]] = !DILocalVariable(name: "._QFfooEa1"{{.*}}scope: ![[SCOPE]]{{.*}}flags: DIFlagArtificial)
+! CHECK-DAG: ![[VAR2:.*]] = !DILocalVariable(name: "._QFfooEa2"{{.*}}scope: ![[SCOPE]]{{.*}}flags: DIFlagArtificial)
+! CHECK-DAG: ![[SR1:.*]] = !DISubrange(count: ![[VAR1]], lowerBound: ![[VAR0]])
+! CHECK-DAG: ![[SR2:.*]] = !DISubrange(count: ![[VAR2]])
+! CHECK-DAG: ![[SR:.*]] = !{![[SR1]], ![[SR2]]}
+! CHECK-DAG: ![[TY:.*]] = !DICompositeType(tag: DW_TAG_array_type{{.*}}elements: ![[SR]])
+! CHECK-DAG: !DILocalVariable(name: "a"{{.*}}scope: ![[SCOPE]]{{.*}}type: ![[TY]])
+
diff --git a/flang/test/Intrinsics/math-codegen.fir b/flang/test/Intrinsics/math-codegen.fir
index 62d841253075e..2165a13b10de2 100644
--- a/flang/test/Intrinsics/math-codegen.fir
+++ b/flang/test/Intrinsics/math-codegen.fir
@@ -42,33 +42,33 @@ func.func @_QPtest_real16(%arg0: !fir.ref<f128> {fir.bindc_name = "x"}) -> f128
   %3 = fir.load %0 : !fir.ref<f128>
   return %3 : f128
 }
-func.func @_QPtest_complex4(%arg0: !fir.ref<!fir.complex<4>> {fir.bindc_name = "c"}) -> !fir.complex<4> {
-  %0 = fir.alloca !fir.complex<4> {bindc_name = "test_complex4", uniq_name = "_QFtest_complex4Etest_complex4"}
-  %1 = fir.load %arg0 : !fir.ref<!fir.complex<4>>
-  %2 = fir.extract_value %1, [0 : index] : (!fir.complex<4>) -> f32
-  %3 = fir.extract_value %1, [1 : index] : (!fir.complex<4>) -> f32
+func.func @_QPtest_complex4(%arg0: !fir.ref<complex<f32>> {fir.bindc_name = "c"}) -> complex<f32> {
+  %0 = fir.alloca complex<f32> {bindc_name = "test_complex4", uniq_name = "_QFtest_complex4Etest_complex4"}
+  %1 = fir.load %arg0 : !fir.ref<complex<f32>>
+  %2 = fir.extract_value %1, [0 : index] : (complex<f32>) -> f32
+  %3 = fir.extract_value %1, [1 : index] : (complex<f32>) -> f32
   %4 = fir.call @hypotf(%2, %3) : (f32, f32) -> f32
   %cst = arith.constant 0.000000e+00 : f32
-  %5 = fir.undefined !fir.complex<4>
-  %6 = fir.insert_value %5, %4, [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-  %7 = fir.insert_value %6, %cst, [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-  fir.store %7 to %0 : !fir.ref<!fir.complex<4>>
-  %8 = fir.load %0 : !fir.ref<!fir.complex<4>>
-  return %8 : !fir.complex<4>
-}
-func.func @_QPtest_complex8(%arg0: !fir.ref<!fir.complex<8>> {fir.bindc_name = "c"}) -> !fir.complex<8> {
-  %0 = fir.alloca !fir.complex<8> {bindc_name = "test_complex8", uniq_name = "_QFtest_complex8Etest_complex8"}
-  %1 = fir.load %arg0 : !fir.ref<!fir.complex<8>>
-  %2 = fir.extract_value %1, [0 : index] : (!fir.complex<8>) -> f64
-  %3 = fir.extract_value %1, [1 : index] : (!fir.complex<8>) -> f64
+  %5 = fir.undefined complex<f32>
+  %6 = fir.insert_value %5, %4, [0 : index] : (complex<f32>, f32) -> complex<f32>
+  %7 = fir.insert_value %6, %cst, [1 : index] : (complex<f32>, f32) -> complex<f32>
+  fir.store %7 to %0 : !fir.ref<complex<f32>>
+  %8 = fir.load %0 : !fir.ref<complex<f32>>
+  return %8 : complex<f32>
+}
+func.func @_QPtest_complex8(%arg0: !fir.ref<complex<f64>> {fir.bindc_name = "c"}) -> complex<f64> {
+  %0 = fir.alloca complex<f64> {bindc_name = "test_complex8", uniq_name = "_QFtest_complex8Etest_complex8"}
+  %1 = fir.load %arg0 : !fir.ref<complex<f64>>
+  %2 = fir.extract_value %1, [0 : index] : (complex<f64>) -> f64
+  %3 = fir.extract_value %1, [1 : index] : (complex<f64>) -> f64
   %4 = fir.call @hypot(%2, %3) : (f64, f64) -> f64
   %cst = arith.constant 0.000000e+00 : f64
-  %5 = fir.undefined !fir.complex<8>
-  %6 = fir.insert_value %5, %4, [0 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
-  %7 = fir.insert_value %6, %cst, [1 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
-  fir.store %7 to %0 : !fir.ref<!fir.complex<8>>
-  %8 = fir.load %0 : !fir.ref<!fir.complex<8>>
-  return %8 : !fir.complex<8>
+  %5 = fir.undefined complex<f64>
+  %6 = fir.insert_value %5, %4, [0 : index] : (complex<f64>, f64) -> complex<f64>
+  %7 = fir.insert_value %6, %cst, [1 : index] : (complex<f64>, f64) -> complex<f64>
+  fir.store %7 to %0 : !fir.ref<complex<f64>>
+  %8 = fir.load %0 : !fir.ref<complex<f64>>
+  return %8 : complex<f64>
 }
 func.func private @hypotf(f32, f32) -> f32
 func.func private @hypot(f64, f64) -> f64
@@ -114,33 +114,33 @@ func.func @_QPtest_real16(%arg0: !fir.ref<f128> {fir.bindc_name = "x"}) -> f128
   %3 = fir.load %0 : !fir.ref<f128>
   return %3 : f128
 }
-func.func @_QPtest_complex4(%arg0: !fir.ref<!fir.complex<4>> {fir.bindc_name = "c"}) -> !fir.complex<4> {
-  %0 = fir.alloca !fir.complex<4> {bindc_name = "test_complex4", uniq_name = "_QFtest_complex4Etest_complex4"}
-  %1 = fir.load %arg0 : !fir.ref<!fir.complex<4>>
-  %2 = fir.extract_value %1, [0 : index] : (!fir.complex<4>) -> f32
-  %3 = fir.extract_value %1, [1 : index] : (!fir.complex<4>) -> f32
+func.func @_QPtest_complex4(%arg0: !fir.ref<complex<f32>> {fir.bindc_name = "c"}) -> complex<f32> {
+  %0 = fir.alloca complex<f32> {bindc_name = "test_complex4", uniq_name = "_QFtest_complex4Etest_complex4"}
+  %1 = fir.load %arg0 : !fir.ref<complex<f32>>
+  %2 = fir.extract_value %1, [0 : index] : (complex<f32>) -> f32
+  %3 = fir.extract_value %1, [1 : index] : (complex<f32>) -> f32
   %4 = fir.call @hypotf(%2, %3) : (f32, f32) -> f32
   %cst = arith.constant 0.000000e+00 : f32
-  %5 = fir.undefined !fir.complex<4>
-  %6 = fir.insert_value %5, %4, [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-  %7 = fir.insert_value %6, %cst, [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-  fir.store %7 to %0 : !fir.ref<!fir.complex<4>>
-  %8 = fir.load %0 : !fir.ref<!fir.complex<4>>
-  return %8 : !fir.complex<4>
-}
-func.func @_QPtest_complex8(%arg0: !fir.ref<!fir.complex<8>> {fir.bindc_name = "c"}) -> !fir.complex<8> {
-  %0 = fir.alloca !fir.complex<8> {bindc_name = "test_complex8", uniq_name = "_QFtest_complex8Etest_complex8"}
-  %1 = fir.load %arg0 : !fir.ref<!fir.complex<8>>
-  %2 = fir.extract_value %1, [0 : index] : (!fir.complex<8>) -> f64
-  %3 = fir.extract_value %1, [1 : index] : (!fir.complex<8>) -> f64
+  %5 = fir.undefined complex<f32>
+  %6 = fir.insert_value %5, %4, [0 : index] : (complex<f32>, f32) -> complex<f32>
+  %7 = fir.insert_value %6, %cst, [1 : index] : (complex<f32>, f32) -> complex<f32>
+  fir.store %7 to %0 : !fir.ref<complex<f32>>
+  %8 = fir.load %0 : !fir.ref<complex<f32>>
+  return %8 : complex<f32>
+}
+func.func @_QPtest_complex8(%arg0: !fir.ref<complex<f64>> {fir.bindc_name = "c"}) -> complex<f64> {
+  %0 = fir.alloca complex<f64> {bindc_name = "test_complex8", uniq_name = "_QFtest_complex8Etest_complex8"}
+  %1 = fir.load %arg0 : !fir.ref<complex<f64>>
+  %2 = fir.extract_value %1, [0 : index] : (complex<f64>) -> f64
+  %3 = fir.extract_value %1, [1 : index] : (complex<f64>) -> f64
   %4 = fir.call @hypot(%2, %3) : (f64, f64) -> f64
   %cst = arith.constant 0.000000e+00 : f64
-  %5 = fir.undefined !fir.complex<8>
-  %6 = fir.insert_value %5, %4, [0 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
-  %7 = fir.insert_value %6, %cst, [1 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
-  fir.store %7 to %0 : !fir.ref<!fir.complex<8>>
-  %8 = fir.load %0 : !fir.ref<!fir.complex<8>>
-  return %8 : !fir.complex<8>
+  %5 = fir.undefined complex<f64>
+  %6 = fir.insert_value %5, %4, [0 : index] : (complex<f64>, f64) -> complex<f64>
+  %7 = fir.insert_value %6, %cst, [1 : index] : (complex<f64>, f64) -> complex<f64>
+  fir.store %7 to %0 : !fir.ref<complex<f64>>
+  %8 = fir.load %0 : !fir.ref<complex<f64>>
+  return %8 : complex<f64>
 }
 func.func private @hypotf(f32, f32) -> f32
 func.func private @hypot(f64, f64) -> f64
@@ -186,33 +186,33 @@ func.func @_QPtest_real16(%arg0: !fir.ref<f128> {fir.bindc_name = "x"}) -> f128
   %3 = fir.load %0 : !fir.ref<f128>
   return %3 : f128
 }
-func.func @_QPtest_complex4(%arg0: !fir.ref<!fir.complex<4>> {fir.bindc_name = "c"}) -> !fir.complex<4> {
-  %0 = fir.alloca !fir.complex<4> {bindc_name = "test_complex4", uniq_name = "_QFtest_complex4Etest_complex4"}
-  %1 = fir.load %arg0 : !fir.ref<!fir.complex<4>>
-  %2 = fir.extract_value %1, [0 : index] : (!fir.complex<4>) -> f32
-  %3 = fir.extract_value %1, [1 : index] : (!fir.complex<4>) -> f32
+func.func @_QPtest_complex4(%arg0: !fir.ref<complex<f32>> {fir.bindc_name = "c"}) -> complex<f32> {
+  %0 = fir.alloca complex<f32> {bindc_name = "test_complex4", uniq_name = "_QFtest_complex4Etest_complex4"}
+  %1 = fir.load %arg0 : !fir.ref<complex<f32>>
+  %2 = fir.extract_value %1, [0 : index] : (complex<f32>) -> f32
+  %3 = fir.extract_value %1, [1 : index] : (complex<f32>) -> f32
   %4 = fir.call @hypotf(%2, %3) : (f32, f32) -> f32
   %cst = arith.constant 0.000000e+00 : f32
-  %5 = fir.undefined !fir.complex<4>
-  %6 = fir.insert_value %5, %4, [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-  %7 = fir.insert_value %6, %cst, [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-  fir.store %7 to %0 : !fir.ref<!fir.complex<4>>
-  %8 = fir.load %0 : !fir.ref<!fir.complex<4>>
-  return %8 : !fir.complex<4>
-}
-func.func @_QPtest_complex8(%arg0: !fir.ref<!fir.complex<8>> {fir.bindc_name = "c"}) -> !fir.complex<8> {
-  %0 = fir.alloca !fir.complex<8> {bindc_name = "test_complex8", uniq_name = "_QFtest_complex8Etest_complex8"}
-  %1 = fir.load %arg0 : !fir.ref<!fir.complex<8>>
-  %2 = fir.extract_value %1, [0 : index] : (!fir.complex<8>) -> f64
-  %3 = fir.extract_value %1, [1 : index] : (!fir.complex<8>) -> f64
+  %5 = fir.undefined complex<f32>
+  %6 = fir.insert_value %5, %4, [0 : index] : (complex<f32>, f32) -> complex<f32>
+  %7 = fir.insert_value %6, %cst, [1 : index] : (complex<f32>, f32) -> complex<f32>
+  fir.store %7 to %0 : !fir.ref<complex<f32>>
+  %8 = fir.load %0 : !fir.ref<complex<f32>>
+  return %8 : complex<f32>
+}
+func.func @_QPtest_complex8(%arg0: !fir.ref<complex<f64>> {fir.bindc_name = "c"}) -> complex<f64> {
+  %0 = fir.alloca complex<f64> {bindc_name = "test_complex8", uniq_name = "_QFtest_complex8Etest_complex8"}
+  %1 = fir.load %arg0 : !fir.ref<complex<f64>>
+  %2 = fir.extract_value %1, [0 : index] : (complex<f64>) -> f64
+  %3 = fir.extract_value %1, [1 : index] : (complex<f64>) -> f64
   %4 = fir.call @hypot(%2, %3) : (f64, f64) -> f64
   %cst = arith.constant 0.000000e+00 : f64
-  %5 = fir.undefined !fir.complex<8>
-  %6 = fir.insert_value %5, %4, [0 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
-  %7 = fir.insert_value %6, %cst, [1 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
-  fir.store %7 to %0 : !fir.ref<!fir.complex<8>>
-  %8 = fir.load %0 : !fir.ref<!fir.complex<8>>
-  return %8 : !fir.complex<8>
+  %5 = fir.undefined complex<f64>
+  %6 = fir.insert_value %5, %4, [0 : index] : (complex<f64>, f64) -> complex<f64>
+  %7 = fir.insert_value %6, %cst, [1 : index] : (complex<f64>, f64) -> complex<f64>
+  fir.store %7 to %0 : !fir.ref<complex<f64>>
+  %8 = fir.load %0 : !fir.ref<complex<f64>>
+  return %8 : complex<f64>
 }
 func.func private @fabsf(f32) -> f32
 func.func private @fabs(f64) -> f64
diff --git a/flang/test/Lower/HLFIR/array-ctor-as-inlined-temp.f90 b/flang/test/Lower/HLFIR/array-ctor-as-inlined-temp.f90
index a7c2faa410fb8..6bbfffca47047 100644
--- a/flang/test/Lower/HLFIR/array-ctor-as-inlined-temp.f90
+++ b/flang/test/Lower/HLFIR/array-ctor-as-inlined-temp.f90
@@ -57,26 +57,26 @@ subroutine test_simple_complex(z)
 ! CHECK:  %[[VAL_2:.*]] = arith.constant 2 : index
 ! CHECK:  %[[VAL_3:.*]] = arith.constant 1 : index
 ! CHECK:  %[[VAL_3B:.*]] = arith.constant 1 : index
-! CHECK:  %[[VAL_4:.*]] = fir.allocmem !fir.array<2x!fir.complex<8>> {bindc_name = ".tmp.arrayctor", uniq_name = ""}
+! CHECK:  %[[VAL_4:.*]] = fir.allocmem !fir.array<2xcomplex<f64>> {bindc_name = ".tmp.arrayctor", uniq_name = ""}
 ! CHECK:  %[[VAL_5:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1>
-! CHECK:  %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_4]](%[[VAL_5]]) {uniq_name = ".tmp.arrayctor"} : (!fir.heap<!fir.array<2x!fir.complex<8>>>, !fir.shape<1>) -> (!fir.heap<!fir.array<2x!fir.complex<8>>>, !fir.heap<!fir.array<2x!fir.complex<8>>>)
+! CHECK:  %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_4]](%[[VAL_5]]) {uniq_name = ".tmp.arrayctor"} : (!fir.heap<!fir.array<2xcomplex<f64>>>, !fir.shape<1>) -> (!fir.heap<!fir.array<2xcomplex<f64>>>, !fir.heap<!fir.array<2xcomplex<f64>>>)
 ! CHECK:  %[[VAL_7:.*]] = arith.constant 42 : i32
 ! CHECK:  %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i32) -> f64
 ! CHECK:  %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f64
-! CHECK:  %[[VAL_10:.*]] = fir.undefined !fir.complex<8>
-! CHECK:  %[[VAL_11:.*]] = fir.insert_value %[[VAL_10]], %[[VAL_8]], [0 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
-! CHECK:  %[[VAL_12:.*]] = fir.insert_value %[[VAL_11]], %[[VAL_9]], [1 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
+! CHECK:  %[[VAL_10:.*]] = fir.undefined complex<f64>
+! CHECK:  %[[VAL_11:.*]] = fir.insert_value %[[VAL_10]], %[[VAL_8]], [0 : index] : (complex<f64>, f64) -> complex<f64>
+! CHECK:  %[[VAL_12:.*]] = fir.insert_value %[[VAL_11]], %[[VAL_9]], [1 : index] : (complex<f64>, f64) -> complex<f64>
 ! CHECK:  %[[VAL_13:.*]] = arith.addi %[[VAL_3]], %[[VAL_3B]] : index
-! CHECK:  %[[VAL_14:.*]] = hlfir.designate %[[VAL_6]]#0 (%[[VAL_3]])  : (!fir.heap<!fir.array<2x!fir.complex<8>>>, index) -> !fir.ref<!fir.complex<8>>
-! CHECK:  hlfir.assign %[[VAL_12]] to %[[VAL_14]] : !fir.complex<8>, !fir.ref<!fir.complex<8>>
-! CHECK:  %[[VAL_15:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.complex<4>>
-! CHECK:  %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.complex<4>) -> !fir.complex<8>
-! CHECK:  %[[VAL_17:.*]] = hlfir.designate %[[VAL_6]]#0 (%[[VAL_13]])  : (!fir.heap<!fir.array<2x!fir.complex<8>>>, index) -> !fir.ref<!fir.complex<8>>
-! CHECK:  hlfir.assign %[[VAL_16]] to %[[VAL_17]] : !fir.complex<8>, !fir.ref<!fir.complex<8>>
+! CHECK:  %[[VAL_14:.*]] = hlfir.designate %[[VAL_6]]#0 (%[[VAL_3]])  : (!fir.heap<!fir.array<2xcomplex<f64>>>, index) -> !fir.ref<complex<f64>>
+! CHECK:  hlfir.assign %[[VAL_12]] to %[[VAL_14]] : complex<f64>, !fir.ref<complex<f64>>
+! CHECK:  %[[VAL_15:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<complex<f32>>
+! CHECK:  %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (complex<f32>) -> complex<f64>
+! CHECK:  %[[VAL_17:.*]] = hlfir.designate %[[VAL_6]]#0 (%[[VAL_13]])  : (!fir.heap<!fir.array<2xcomplex<f64>>>, index) -> !fir.ref<complex<f64>>
+! CHECK:  hlfir.assign %[[VAL_16]] to %[[VAL_17]] : complex<f64>, !fir.ref<complex<f64>>
 ! CHECK:  %[[VAL_18:.*]] = arith.constant true
-! CHECK:  %[[VAL_19:.*]] = hlfir.as_expr %[[VAL_6]]#0 move %[[VAL_18]] : (!fir.heap<!fir.array<2x!fir.complex<8>>>, i1) -> !hlfir.expr<2x!fir.complex<8>>
+! CHECK:  %[[VAL_19:.*]] = hlfir.as_expr %[[VAL_6]]#0 move %[[VAL_18]] : (!fir.heap<!fir.array<2xcomplex<f64>>>, i1) -> !hlfir.expr<2xcomplex<f64>>
 ! CHECK:  fir.call
-! CHECK:  hlfir.destroy %[[VAL_19]] : !hlfir.expr<2x!fir.complex<8>>
+! CHECK:  hlfir.destroy %[[VAL_19]] : !hlfir.expr<2xcomplex<f64>>
 
 subroutine test_simple_logical(a, b)
   logical :: a, b
diff --git a/flang/test/Lower/HLFIR/assignment-intrinsics.f90 b/flang/test/Lower/HLFIR/assignment-intrinsics.f90
index 544815e881409..ae30ae52096f3 100644
--- a/flang/test/Lower/HLFIR/assignment-intrinsics.f90
+++ b/flang/test/Lower/HLFIR/assignment-intrinsics.f90
@@ -40,10 +40,10 @@ subroutine scalar_complex(x, y)
   x = y
 end subroutine
 ! CHECK-LABEL: func.func @_QPscalar_complex(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_complexEx"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_complexEy"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_complexEx"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_complexEy"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
 ! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0
-! CHECK:  hlfir.assign %[[VAL_4]] to %[[VAL_2]]#0 : !fir.complex<4>, !fir.ref<!fir.complex<4>>
+! CHECK:  hlfir.assign %[[VAL_4]] to %[[VAL_2]]#0 : complex<f32>, !fir.ref<complex<f32>>
 
 subroutine scalar_character(x, y)
   character(*) :: x, y
@@ -91,13 +91,13 @@ subroutine scalar_complex_2(x)
   x = (1., -1.)
 end subroutine
 ! CHECK-LABEL: func.func @_QPscalar_complex_2(
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_complex_2Ex"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_complex_2Ex"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
 ! CHECK:  %[[VAL_2:.*]] = arith.constant 1.000000e+00 : f32
 ! CHECK:  %[[VAL_3:.*]] = arith.constant -1.000000e+00 : f32
-! CHECK:  %[[VAL_4:.*]] = fir.undefined !fir.complex<4>
-! CHECK:  %[[VAL_5:.*]] = fir.insert_value %[[VAL_4]], %[[VAL_2]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:  %[[VAL_6:.*]] = fir.insert_value %[[VAL_5]], %[[VAL_3]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:  hlfir.assign %[[VAL_6]] to %[[VAL_1]]#0 : !fir.complex<4>, !fir.ref<!fir.complex<4>>
+! CHECK:  %[[VAL_4:.*]] = fir.undefined complex<f32>
+! CHECK:  %[[VAL_5:.*]] = fir.insert_value %[[VAL_4]], %[[VAL_2]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:  %[[VAL_6:.*]] = fir.insert_value %[[VAL_5]], %[[VAL_3]], [1 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:  hlfir.assign %[[VAL_6]] to %[[VAL_1]]#0 : complex<f32>, !fir.ref<complex<f32>>
 
 subroutine scalar_character_2(x)
   character(*) :: x
diff --git a/flang/test/Lower/HLFIR/assumed-rank-calls.f90 b/flang/test/Lower/HLFIR/assumed-rank-calls.f90
index afb2bbac99891..63b8d9fd81f33 100644
--- a/flang/test/Lower/HLFIR/assumed-rank-calls.f90
+++ b/flang/test/Lower/HLFIR/assumed-rank-calls.f90
@@ -1,7 +1,7 @@
 ! Test passing of assumed-ranks that require creating a
 ! a new descriptor for the dummy argument (different lower bounds,
 ! attribute, or dynamic type)
-! RUN: bbc -emit-hlfir -allow-assumed-rank -o - %s | FileCheck %s
+! RUN: bbc -emit-hlfir -o - %s | FileCheck %s
 
 subroutine test_alloc_to_nonalloc(x)
   real, allocatable ::  x(..)
diff --git a/flang/test/Lower/HLFIR/assumed-rank-entry.f90 b/flang/test/Lower/HLFIR/assumed-rank-entry.f90
index 9928f5459f71f..9ac56c4282f87 100644
--- a/flang/test/Lower/HLFIR/assumed-rank-entry.f90
+++ b/flang/test/Lower/HLFIR/assumed-rank-entry.f90
@@ -1,6 +1,6 @@
 ! Test assumed-rank dummy argument that is not present in
 ! all ENTRY statements.
-! RUN: bbc -emit-hlfir -allow-assumed-rank -o - %s | FileCheck %s
+! RUN: bbc -emit-hlfir -o - %s | FileCheck %s
 
 subroutine test_main_entry(x)
   real :: x(..)
diff --git a/flang/test/Lower/HLFIR/assumed-rank-inquiries-2.f90 b/flang/test/Lower/HLFIR/assumed-rank-inquiries-2.f90
index 6c8f5ba48d1bf..696b228bacee0 100644
--- a/flang/test/Lower/HLFIR/assumed-rank-inquiries-2.f90
+++ b/flang/test/Lower/HLFIR/assumed-rank-inquiries-2.f90
@@ -1,6 +1,6 @@
 ! Test lowering of SIZE/SIZEOF inquiry intrinsics with assumed-ranks
 ! arguments.
-! RUN: bbc -emit-hlfir -o - %s -allow-assumed-rank | FileCheck %s
+! RUN: bbc -emit-hlfir -o - %s | FileCheck %s
 
 
 subroutine test_size_1(x)
diff --git a/flang/test/Lower/HLFIR/assumed-rank-inquiries-3.f90 b/flang/test/Lower/HLFIR/assumed-rank-inquiries-3.f90
index fb44efcad3ce7..af89cb833b337 100644
--- a/flang/test/Lower/HLFIR/assumed-rank-inquiries-3.f90
+++ b/flang/test/Lower/HLFIR/assumed-rank-inquiries-3.f90
@@ -1,5 +1,5 @@
 ! Test shape lowering for assumed-rank
-! RUN: bbc -emit-hlfir -o - %s -allow-assumed-rank | FileCheck %s
+! RUN: bbc -emit-hlfir -o - %s | FileCheck %s
 
 subroutine test_shape(x)
   real :: x(..)
diff --git a/flang/test/Lower/HLFIR/assumed-rank-inquiries.f90 b/flang/test/Lower/HLFIR/assumed-rank-inquiries.f90
index a1d150a21d149..a94ae7da36593 100644
--- a/flang/test/Lower/HLFIR/assumed-rank-inquiries.f90
+++ b/flang/test/Lower/HLFIR/assumed-rank-inquiries.f90
@@ -1,5 +1,5 @@
 ! Test lowering of inquiry intrinsics with assumed-ranks arguments.
-! RUN: bbc -emit-hlfir -o - %s -allow-assumed-rank | FileCheck %s
+! RUN: bbc -emit-hlfir -o - %s | FileCheck %s
 
 subroutine test_allocated(x)
   real, allocatable :: x(..)
diff --git a/flang/test/Lower/HLFIR/assumed-rank-internal-proc.f90 b/flang/test/Lower/HLFIR/assumed-rank-internal-proc.f90
index f8d5e84696c5f..690ceb64a03cf 100644
--- a/flang/test/Lower/HLFIR/assumed-rank-internal-proc.f90
+++ b/flang/test/Lower/HLFIR/assumed-rank-internal-proc.f90
@@ -1,5 +1,5 @@
 ! Test assumed-rank capture inside internal procedures.
-! RUN: bbc -emit-hlfir -o - %s -allow-assumed-rank | FileCheck %s
+! RUN: bbc -emit-hlfir -o - %s | FileCheck %s
 
 subroutine test_assumed_rank(x)
   real :: x(..)
diff --git a/flang/test/Lower/HLFIR/binary-ops.f90 b/flang/test/Lower/HLFIR/binary-ops.f90
index 912cea0f5e0e6..5855d5ad00036 100644
--- a/flang/test/Lower/HLFIR/binary-ops.f90
+++ b/flang/test/Lower/HLFIR/binary-ops.f90
@@ -28,11 +28,11 @@ subroutine complex_add(x, y, z)
  x = y + z
 end subroutine
 ! CHECK-LABEL: func.func @_QPcomplex_add(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.complex<4>>
-! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<!fir.complex<4>>
-! CHECK:  %[[VAL_8:.*]] = fir.addc %[[VAL_6]], %[[VAL_7]] {fastmath = #arith.fastmath<contract>} : !fir.complex<4>
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<complex<f32>>
+! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f32>>
+! CHECK:  %[[VAL_8:.*]] = fir.addc %[[VAL_6]], %[[VAL_7]] {fastmath = #arith.fastmath<contract>} : complex<f32>
 
 subroutine int_sub(x, y, z)
  integer :: x, y, z
@@ -61,11 +61,11 @@ subroutine complex_sub(x, y, z)
  x = y - z
 end subroutine
 ! CHECK-LABEL: func.func @_QPcomplex_sub(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.complex<4>>
-! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<!fir.complex<4>>
-! CHECK:  %[[VAL_8:.*]] = fir.subc %[[VAL_6]], %[[VAL_7]] {fastmath = #arith.fastmath<contract>} : !fir.complex<4>
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<complex<f32>>
+! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f32>>
+! CHECK:  %[[VAL_8:.*]] = fir.subc %[[VAL_6]], %[[VAL_7]] {fastmath = #arith.fastmath<contract>} : complex<f32>
 
 subroutine int_mul(x, y, z)
  integer :: x, y, z
@@ -94,11 +94,11 @@ subroutine complex_mul(x, y, z)
  x = y * z
 end subroutine
 ! CHECK-LABEL: func.func @_QPcomplex_mul(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.complex<4>>
-! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<!fir.complex<4>>
-! CHECK:  %[[VAL_8:.*]] = fir.mulc %[[VAL_6]], %[[VAL_7]] {fastmath = #arith.fastmath<contract>} : !fir.complex<4>
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<complex<f32>>
+! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f32>>
+! CHECK:  %[[VAL_8:.*]] = fir.mulc %[[VAL_6]], %[[VAL_7]] {fastmath = #arith.fastmath<contract>} : complex<f32>
 
 subroutine int_div(x, y, z)
  integer :: x, y, z
@@ -127,15 +127,15 @@ subroutine complex_div(x, y, z)
  x = y / z
 end subroutine
 ! CHECK-LABEL: func.func @_QPcomplex_div(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.complex<4>>
-! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<!fir.complex<4>>
-! CHECK:  %[[VAL_8:.*]] = fir.extract_value %[[VAL_6]], [0 : index] : (!fir.complex<4>) -> f32
-! CHECK:  %[[VAL_9:.*]] = fir.extract_value %[[VAL_6]], [1 : index] : (!fir.complex<4>) -> f32
-! CHECK:  %[[VAL_10:.*]] = fir.extract_value %[[VAL_7]], [0 : index] : (!fir.complex<4>) -> f32
-! CHECK:  %[[VAL_11:.*]] = fir.extract_value %[[VAL_7]], [1 : index] : (!fir.complex<4>) -> f32
-! CHECK:  %[[VAL_12:.*]] = fir.call @__divsc3(%[[VAL_8]], %[[VAL_9]], %[[VAL_10]], %[[VAL_11]]) fastmath<contract> : (f32, f32, f32, f32) -> !fir.complex<4>
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<complex<f32>>
+! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f32>>
+! CHECK:  %[[VAL_8:.*]] = fir.extract_value %[[VAL_6]], [0 : index] : (complex<f32>) -> f32
+! CHECK:  %[[VAL_9:.*]] = fir.extract_value %[[VAL_6]], [1 : index] : (complex<f32>) -> f32
+! CHECK:  %[[VAL_10:.*]] = fir.extract_value %[[VAL_7]], [0 : index] : (complex<f32>) -> f32
+! CHECK:  %[[VAL_11:.*]] = fir.extract_value %[[VAL_7]], [1 : index] : (complex<f32>) -> f32
+! CHECK:  %[[VAL_12:.*]] = fir.call @__divsc3(%[[VAL_8]], %[[VAL_9]], %[[VAL_10]], %[[VAL_11]]) fastmath<contract> : (f32, f32, f32, f32) -> complex<f32>
 
 subroutine int_power(x, y, z)
   integer :: x, y, z
@@ -164,11 +164,11 @@ subroutine complex_power(x, y, z)
   x = y**z
 end subroutine
 ! CHECK-LABEL: func.func @_QPcomplex_power(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.complex<4>>
-! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<!fir.complex<4>>
-! CHECK:  %[[VAL_8:.*]] = fir.call @cpowf(%[[VAL_6]], %[[VAL_7]]) fastmath<contract> : (!fir.complex<4>, !fir.complex<4>) -> !fir.complex<4>
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<complex<f32>>
+! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f32>>
+! CHECK:  %[[VAL_8:.*]] = fir.call @cpowf(%[[VAL_6]], %[[VAL_7]]) fastmath<contract> : (complex<f32>, complex<f32>) -> complex<f32>
 
 
 subroutine real_to_int_power(x, y, z)
@@ -189,11 +189,11 @@ subroutine complex_to_int_power(x, y, z)
   x = y**z
 end subroutine
 ! CHECK-LABEL: func.func @_QPcomplex_to_int_power(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
 ! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.complex<4>>
+! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<complex<f32>>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
-! CHECK:  %[[VAL_8:.*]] = fir.call @_FortranAcpowi(%[[VAL_6]], %[[VAL_7]]) fastmath<contract> : (!fir.complex<4>, i32) -> !fir.complex<4>
+! CHECK:  %[[VAL_8:.*]] = fir.call @_FortranAcpowi(%[[VAL_6]], %[[VAL_7]]) fastmath<contract> : (complex<f32>, i32) -> complex<f32>
 
 subroutine extremum(c, n, l)
   integer(8), intent(in) :: l
@@ -271,9 +271,9 @@ subroutine cmp_cmplx(l, x, y)
 ! CHECK-LABEL: func.func @_QPcmp_cmplx(
 ! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare {{.*}}x"
 ! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare {{.*}}y"
-! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.complex<4>>
-! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<!fir.complex<4>>
-! CHECK:  %[[VAL_8:.*]] = fir.cmpc "oeq", %[[VAL_6]], %[[VAL_7]] {{.*}} : !fir.complex<4>
+! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<complex<f32>>
+! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<complex<f32>>
+! CHECK:  %[[VAL_8:.*]] = fir.cmpc "oeq", %[[VAL_6]], %[[VAL_7]] {{.*}} : complex<f32>
 
 subroutine cmp_char(l, x, y)
   logical :: l
@@ -335,9 +335,9 @@ subroutine cmplx_ctor(z, x, y)
 ! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<f32>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<f32>
-! CHECK:  %[[VAL_8:.*]] = fir.undefined !fir.complex<4>
-! CHECK:  %[[VAL_9:.*]] = fir.insert_value %[[VAL_8]], %[[VAL_6]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:  %[[VAL_10:.*]] = fir.insert_value %[[VAL_9]], %[[VAL_7]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
+! CHECK:  %[[VAL_8:.*]] = fir.undefined complex<f32>
+! CHECK:  %[[VAL_9:.*]] = fir.insert_value %[[VAL_8]], %[[VAL_6]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:  %[[VAL_10:.*]] = fir.insert_value %[[VAL_9]], %[[VAL_7]], [1 : index] : (complex<f32>, f32) -> complex<f32>
 
 subroutine cmplx_ctor_2(z, x)
   complex(8) :: z
@@ -348,6 +348,6 @@ subroutine cmplx_ctor_2(z, x)
 ! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<f64>, !fir.dscope) -> (!fir.ref<f64>, !fir.ref<f64>)
 ! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<f64>
 ! CHECK:  %[[VAL_5:.*]] = arith.constant 1.000000e+00 : f64
-! CHECK:  %[[VAL_6:.*]] = fir.undefined !fir.complex<8>
-! CHECK:  %[[VAL_7:.*]] = fir.insert_value %[[VAL_6]], %[[VAL_4]], [0 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
-! CHECK:  %[[VAL_8:.*]] = fir.insert_value %[[VAL_7]], %[[VAL_5]], [1 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
+! CHECK:  %[[VAL_6:.*]] = fir.undefined complex<f64>
+! CHECK:  %[[VAL_7:.*]] = fir.insert_value %[[VAL_6]], %[[VAL_4]], [0 : index] : (complex<f64>, f64) -> complex<f64>
+! CHECK:  %[[VAL_8:.*]] = fir.insert_value %[[VAL_7]], %[[VAL_5]], [1 : index] : (complex<f64>, f64) -> complex<f64>
diff --git a/flang/test/Lower/HLFIR/calls-f77.f90 b/flang/test/Lower/HLFIR/calls-f77.f90
index a970deb056f5a..fad32d5fd0278 100644
--- a/flang/test/Lower/HLFIR/calls-f77.f90
+++ b/flang/test/Lower/HLFIR/calls-f77.f90
@@ -147,7 +147,7 @@ subroutine return_complex()
   print *, cplxfoo()
 end subroutine
 ! CHECK-LABEL: func.func @_QPreturn_complex(
-! CHECK:  fir.call @_QPcplxfoo() fastmath<contract> : () -> !fir.complex<4>
+! CHECK:  fir.call @_QPcplxfoo() fastmath<contract> : () -> complex<f32>
 
 subroutine return_char(n)
   integer(8) :: n
diff --git a/flang/test/Lower/HLFIR/calls-percent-val-ref.f90 b/flang/test/Lower/HLFIR/calls-percent-val-ref.f90
index c8724e6d7bee4..ea50efb01a41c 100644
--- a/flang/test/Lower/HLFIR/calls-percent-val-ref.f90
+++ b/flang/test/Lower/HLFIR/calls-percent-val-ref.f90
@@ -16,12 +16,12 @@ subroutine test_val_2(x)
   call val2(%val(x))
 end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_val_2(
-! CHECK-SAME:                             %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.complex<4>>>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_val_2Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.complex<4>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.complex<4>>>>, !fir.ref<!fir.box<!fir.heap<!fir.complex<4>>>>)
-! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.complex<4>>>>
-! CHECK:           %[[VAL_3:.*]] = fir.box_addr %[[VAL_2]] : (!fir.box<!fir.heap<!fir.complex<4>>>) -> !fir.heap<!fir.complex<4>>
-! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]] : !fir.heap<!fir.complex<4>>
-! CHECK:           fir.call @_QPval2(%[[VAL_4]]) fastmath<contract> : (!fir.complex<4>) -> ()
+! CHECK-SAME:                             %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<complex<f32>>>> {fir.bindc_name = "x"}) {
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_val_2Ex"} : (!fir.ref<!fir.box<!fir.heap<complex<f32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<complex<f32>>>>, !fir.ref<!fir.box<!fir.heap<complex<f32>>>>)
+! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.heap<complex<f32>>>>
+! CHECK:           %[[VAL_3:.*]] = fir.box_addr %[[VAL_2]] : (!fir.box<!fir.heap<complex<f32>>>) -> !fir.heap<complex<f32>>
+! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]] : !fir.heap<complex<f32>>
+! CHECK:           fir.call @_QPval2(%[[VAL_4]]) fastmath<contract> : (complex<f32>) -> ()
 
 subroutine test_ref_char(x)
   ! There must be not extra length argument. Only the address is
@@ -50,12 +50,12 @@ subroutine test_ref_2(x)
   call ref2(%ref(x))
 end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_ref_2(
-! CHECK-SAME:                             %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_ref_2Ex"} : (!fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>)
-! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>
-! CHECK:           %[[VAL_3:.*]] = fir.box_addr %[[VAL_2]] : (!fir.box<!fir.ptr<!fir.complex<4>>>) -> !fir.ptr<!fir.complex<4>>
-! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.ptr<!fir.complex<4>>) -> !fir.ref<!fir.complex<4>>
-! CHECK:           fir.call @_QPref2(%[[VAL_4]]) fastmath<contract> : (!fir.ref<!fir.complex<4>>) -> ()
+! CHECK-SAME:                             %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.ptr<complex<f32>>>> {fir.bindc_name = "x"}) {
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_ref_2Ex"} : (!fir.ref<!fir.box<!fir.ptr<complex<f32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<complex<f32>>>>, !fir.ref<!fir.box<!fir.ptr<complex<f32>>>>)
+! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.ptr<complex<f32>>>>
+! CHECK:           %[[VAL_3:.*]] = fir.box_addr %[[VAL_2]] : (!fir.box<!fir.ptr<complex<f32>>>) -> !fir.ptr<complex<f32>>
+! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.ptr<complex<f32>>) -> !fir.ref<complex<f32>>
+! CHECK:           fir.call @_QPref2(%[[VAL_4]]) fastmath<contract> : (!fir.ref<complex<f32>>) -> ()
 
 subroutine test_skip_copy_in_out(x)
   real :: x(:)
diff --git a/flang/test/Lower/HLFIR/constant.f90 b/flang/test/Lower/HLFIR/constant.f90
index 86f46f932a996..82e5d6f7179d6 100644
--- a/flang/test/Lower/HLFIR/constant.f90
+++ b/flang/test/Lower/HLFIR/constant.f90
@@ -6,9 +6,9 @@ subroutine test_constant_scalar()
   print *, (10., 20.)
   ! CHECK-DAG:  %[[VAL_0:.*]] = arith.constant 2.000000e+01 : f32
   ! CHECK-DAG:  %[[VAL_1:.*]] = arith.constant 1.000000e+01 : f32
-  ! CHECK:  %[[VAL_7:.*]] = fir.undefined !fir.complex<4>
-  ! CHECK:  %[[VAL_8:.*]] = fir.insert_value %[[VAL_7]], %[[VAL_1]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-  ! CHECK:  %[[VAL_9:.*]] = fir.insert_value %[[VAL_8]], %[[VAL_0]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
+  ! CHECK:  %[[VAL_7:.*]] = fir.undefined complex<f32>
+  ! CHECK:  %[[VAL_8:.*]] = fir.insert_value %[[VAL_7]], %[[VAL_1]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+  ! CHECK:  %[[VAL_9:.*]] = fir.insert_value %[[VAL_8]], %[[VAL_0]], [1 : index] : (complex<f32>, f32) -> complex<f32>
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPtest_constant_scalar_char()
diff --git a/flang/test/Lower/HLFIR/conversion-ops.f90 b/flang/test/Lower/HLFIR/conversion-ops.f90
index d578ed1e393b3..4102199749102 100644
--- a/flang/test/Lower/HLFIR/conversion-ops.f90
+++ b/flang/test/Lower/HLFIR/conversion-ops.f90
@@ -19,10 +19,10 @@ subroutine test
   i4 = r8
 ! CHECK:  fir.convert %{{.*}} : (f64) -> i32
   i4 = z4
-! CHECK:  %[[VAL_23:.*]] = fir.extract_value %{{.*}}, [0 : index] : (!fir.complex<4>) -> f32
+! CHECK:  %[[VAL_23:.*]] = fir.extract_value %{{.*}}, [0 : index] : (complex<f32>) -> f32
 ! CHECK:  fir.convert %[[VAL_23]] : (f32) -> i32
   i4 = z8
-! CHECK:  %[[VAL_26:.*]] = fir.extract_value %{{.*}}, [0 : index] : (!fir.complex<8>) -> f64
+! CHECK:  %[[VAL_26:.*]] = fir.extract_value %{{.*}}, [0 : index] : (complex<f64>) -> f64
 ! CHECK:  fir.convert %[[VAL_26]] : (f64) -> i32
 
   r4 = i4
@@ -32,37 +32,37 @@ subroutine test
   r4 = r8
 ! CHECK:  fir.convert %{{.*}} : (f64) -> f32
   r4 = z4
-! CHECK:  fir.extract_value %{{.*}}, [0 : index] : (!fir.complex<4>) -> f32
+! CHECK:  fir.extract_value %{{.*}}, [0 : index] : (complex<f32>) -> f32
   r4 = z8
-! CHECK:  %[[VAL_36:.*]] = fir.load %{{.*}} : !fir.ref<!fir.complex<8>>
-! CHECK:  %[[VAL_37:.*]] = fir.extract_value %[[VAL_36]], [0 : index] : (!fir.complex<8>) -> f64
+! CHECK:  %[[VAL_36:.*]] = fir.load %{{.*}} : !fir.ref<complex<f64>>
+! CHECK:  %[[VAL_37:.*]] = fir.extract_value %[[VAL_36]], [0 : index] : (complex<f64>) -> f64
 ! CHECK:  fir.convert %[[VAL_37]] : (f64) -> f32
 
   z4 = i4
 ! CHECK:  %[[VAL_40:.*]] = fir.convert %{{.*}} : (i32) -> f32
 ! CHECK:  %[[VAL_41:.*]] = arith.constant 0.000000e+00 : f32
-! CHECK:  %[[VAL_42:.*]] = fir.undefined !fir.complex<4>
-! CHECK:  %[[VAL_43:.*]] = fir.insert_value %[[VAL_42]], %[[VAL_40]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:  fir.insert_value %[[VAL_43]], %[[VAL_41]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
+! CHECK:  %[[VAL_42:.*]] = fir.undefined complex<f32>
+! CHECK:  %[[VAL_43:.*]] = fir.insert_value %[[VAL_42]], %[[VAL_40]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:  fir.insert_value %[[VAL_43]], %[[VAL_41]], [1 : index] : (complex<f32>, f32) -> complex<f32>
   z4 = i8
 ! CHECK:  %[[VAL_46:.*]] = fir.convert %{{.*}} : (i64) -> f32
 ! CHECK:  %[[VAL_47:.*]] = arith.constant 0.000000e+00 : f32
-! CHECK:  %[[VAL_48:.*]] = fir.undefined !fir.complex<4>
-! CHECK:  %[[VAL_49:.*]] = fir.insert_value %[[VAL_48]], %[[VAL_46]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:  fir.insert_value %[[VAL_49]], %[[VAL_47]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
+! CHECK:  %[[VAL_48:.*]] = fir.undefined complex<f32>
+! CHECK:  %[[VAL_49:.*]] = fir.insert_value %[[VAL_48]], %[[VAL_46]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:  fir.insert_value %[[VAL_49]], %[[VAL_47]], [1 : index] : (complex<f32>, f32) -> complex<f32>
   z4 = r4
 ! CHECK:  %[[VAL_52:.*]] = arith.constant 0.000000e+00 : f32
-! CHECK:  %[[VAL_53:.*]] = fir.undefined !fir.complex<4>
-! CHECK:  %[[VAL_54:.*]] = fir.insert_value %[[VAL_53]], %{{.*}}, [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:  fir.insert_value %[[VAL_54]], %[[VAL_52]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
+! CHECK:  %[[VAL_53:.*]] = fir.undefined complex<f32>
+! CHECK:  %[[VAL_54:.*]] = fir.insert_value %[[VAL_53]], %{{.*}}, [0 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:  fir.insert_value %[[VAL_54]], %[[VAL_52]], [1 : index] : (complex<f32>, f32) -> complex<f32>
   z4 = r8
 ! CHECK:  %[[VAL_57:.*]] = fir.convert %{{.*}} : (f64) -> f32
 ! CHECK:  %[[VAL_58:.*]] = arith.constant 0.000000e+00 : f32
-! CHECK:  %[[VAL_59:.*]] = fir.undefined !fir.complex<4>
-! CHECK:  %[[VAL_60:.*]] = fir.insert_value %[[VAL_59]], %[[VAL_57]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:  fir.insert_value %[[VAL_60]], %[[VAL_58]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
+! CHECK:  %[[VAL_59:.*]] = fir.undefined complex<f32>
+! CHECK:  %[[VAL_60:.*]] = fir.insert_value %[[VAL_59]], %[[VAL_57]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:  fir.insert_value %[[VAL_60]], %[[VAL_58]], [1 : index] : (complex<f32>, f32) -> complex<f32>
   z4 = z8
-! CHECK:  fir.convert %{{.*}} : (!fir.complex<8>) -> !fir.complex<4>
+! CHECK:  fir.convert %{{.*}} : (complex<f64>) -> complex<f32>
 
   l4 = l8
 ! CHECK:  fir.convert %{{.*}} : (!fir.logical<8>) -> !fir.logical<4>
diff --git a/flang/test/Lower/HLFIR/convert-variable-assumed-rank.f90 b/flang/test/Lower/HLFIR/convert-variable-assumed-rank.f90
index 3b60b0d04e9fa..d2931eabb75c2 100644
--- a/flang/test/Lower/HLFIR/convert-variable-assumed-rank.f90
+++ b/flang/test/Lower/HLFIR/convert-variable-assumed-rank.f90
@@ -1,5 +1,5 @@
 ! Test lowering of assumed-rank variables
-! RUN: bbc -emit-hlfir %s -allow-assumed-rank -o - | FileCheck %s
+! RUN: bbc -emit-hlfir %s -o - | FileCheck %s
 
 module assumed_rank_tests
 interface
diff --git a/flang/test/Lower/HLFIR/designators-component-ref.f90 b/flang/test/Lower/HLFIR/designators-component-ref.f90
index 69cc7d2e5aa6c..653e28e0a6018 100644
--- a/flang/test/Lower/HLFIR/designators-component-ref.f90
+++ b/flang/test/Lower/HLFIR/designators-component-ref.f90
@@ -340,7 +340,7 @@ subroutine test_scalar_array_complex_chain(a)
   type(t_complex) :: a
   print *, a%array_comp%im
 ! CHECK-LABEL:   func.func @_QPtest_scalar_array_complex_chain(
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_scalar_array_complex_chainEa"} : (!fir.ref<!fir.type<_QMcomp_refTt_complex{array_comp:!fir.array<10x20x!fir.complex<4>>}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QMcomp_refTt_complex{array_comp:!fir.array<10x20x!fir.complex<4>>}>>, !fir.ref<!fir.type<_QMcomp_refTt_complex{array_comp:!fir.array<10x20x!fir.complex<4>>}>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_scalar_array_complex_chainEa"} : (!fir.ref<!fir.type<_QMcomp_refTt_complex{array_comp:!fir.array<10x20xcomplex<f32>>}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QMcomp_refTt_complex{array_comp:!fir.array<10x20xcomplex<f32>>}>>, !fir.ref<!fir.type<_QMcomp_refTt_complex{array_comp:!fir.array<10x20xcomplex<f32>>}>>)
 ! CHECK:           %[[VAL_7:.*]] = arith.constant 10 : index
 ! CHECK:           %[[VAL_8:.*]] = arith.constant 20 : index
 ! CHECK:           %[[VAL_9:.*]] = arith.constant 2 : index
@@ -365,7 +365,7 @@ subroutine test_scalar_array_complex_chain(a)
 ! CHECK:           %[[VAL_28:.*]] = arith.cmpi sgt, %[[VAL_27]], %[[VAL_24]] : index
 ! CHECK:           %[[VAL_29:.*]] = arith.select %[[VAL_28]], %[[VAL_27]], %[[VAL_24]] : index
 ! CHECK:           %[[VAL_30:.*]] = fir.shape %[[VAL_23]], %[[VAL_29]] : (index, index) -> !fir.shape<2>
-! CHECK:           %[[VAL_31:.*]] = hlfir.designate %[[VAL_1]]#0{"array_comp"} <%[[VAL_11]]> (%[[VAL_9]]:%[[VAL_15]]:%[[VAL_12]], %[[VAL_10]]:%[[VAL_17]]:%[[VAL_12]]) imag shape %[[VAL_30]] : (!fir.ref<!fir.type<_QMcomp_refTt_complex{array_comp:!fir.array<10x20x!fir.complex<4>>}>>, !fir.shapeshift<2>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.box<!fir.array<10x20xf32>>
+! CHECK:           %[[VAL_31:.*]] = hlfir.designate %[[VAL_1]]#0{"array_comp"} <%[[VAL_11]]> (%[[VAL_9]]:%[[VAL_15]]:%[[VAL_12]], %[[VAL_10]]:%[[VAL_17]]:%[[VAL_12]]) imag shape %[[VAL_30]] : (!fir.ref<!fir.type<_QMcomp_refTt_complex{array_comp:!fir.array<10x20xcomplex<f32>>}>>, !fir.shapeshift<2>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.box<!fir.array<10x20xf32>>
 end subroutine
 
 subroutine test_poly_array_vector_subscript(p, v, r)
diff --git a/flang/test/Lower/HLFIR/designators.f90 b/flang/test/Lower/HLFIR/designators.f90
index 09753d06cc278..cb1cab334c9ae 100644
--- a/flang/test/Lower/HLFIR/designators.f90
+++ b/flang/test/Lower/HLFIR/designators.f90
@@ -120,18 +120,18 @@ subroutine complex_imag_ref(x)
   print *, x%im
 end subroutine
 ! CHECK-LABEL: func.func @_QPcomplex_imag_ref(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFcomplex_imag_refEx"} : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFcomplex_imag_refEx"} : (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.dscope) -> (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.box<!fir.array<?xcomplex<f32>>>)
 ! CHECK:  %[[VAL_3:.*]] = fir.shape %[[VAL_4:.*]]#1 : (index) -> !fir.shape<1>
-! CHECK:  %[[VAL_5:.*]] = hlfir.designate %[[VAL_2]]#0  imag shape %[[VAL_3]] : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
+! CHECK:  %[[VAL_5:.*]] = hlfir.designate %[[VAL_2]]#0  imag shape %[[VAL_3]] : (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
 
 subroutine complex_real_ref(x)
   complex :: x(:)
   print *, x%re
 end subroutine
 ! CHECK-LABEL: func.func @_QPcomplex_real_ref(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFcomplex_real_refEx"} : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFcomplex_real_refEx"} : (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.dscope) -> (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.box<!fir.array<?xcomplex<f32>>>)
 ! CHECK:  %[[VAL_3:.*]] = fir.shape %[[VAL_4:.*]]#1 : (index) -> !fir.shape<1>
-! CHECK:  %[[VAL_5:.*]] = hlfir.designate %[[VAL_2]]#0  real shape %[[VAL_3]] : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
+! CHECK:  %[[VAL_5:.*]] = hlfir.designate %[[VAL_2]]#0  real shape %[[VAL_3]] : (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
 
 subroutine complex_individual_ref(x, n)
   complex :: x(:)
@@ -140,10 +140,10 @@ subroutine complex_individual_ref(x, n)
 end subroutine
 ! CHECK-LABEL: func.func @_QPcomplex_individual_ref(
 ! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFcomplex_individual_refEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFcomplex_individual_refEx"} : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFcomplex_individual_refEx"} : (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.dscope) -> (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.box<!fir.array<?xcomplex<f32>>>)
 ! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<i32>
 ! CHECK:  %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (i32) -> i64
-! CHECK:  %[[VAL_6:.*]] = hlfir.designate %{{[0-9]+}}#0 (%[[VAL_5]]) imag : (!fir.box<!fir.array<?x!fir.complex<4>>>, i64) -> !fir.ref<f32>
+! CHECK:  %[[VAL_6:.*]] = hlfir.designate %{{[0-9]+}}#0 (%[[VAL_5]]) imag : (!fir.box<!fir.array<?xcomplex<f32>>>, i64) -> !fir.ref<f32>
 
 subroutine complex_slice_ref(x, start, end)
   complex :: x(:)
@@ -153,7 +153,7 @@ subroutine complex_slice_ref(x, start, end)
 ! CHECK-LABEL: func.func @_QPcomplex_slice_ref(
 ! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFcomplex_slice_refEend"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFcomplex_slice_refEstart"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %arg0 dummy_scope %{{[0-9]+}} {uniq_name = "_QFcomplex_slice_refEx"} : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %arg0 dummy_scope %{{[0-9]+}} {uniq_name = "_QFcomplex_slice_refEx"} : (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.dscope) -> (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.box<!fir.array<?xcomplex<f32>>>)
 ! CHECK:  %[[VAL_5:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<i32>
 ! CHECK:  %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (i32) -> i64
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<i32>
@@ -166,4 +166,4 @@ subroutine complex_slice_ref(x, start, end)
 ! CHECK:  %[[VAL_14:.*]] = arith.cmpi sgt, %[[VAL_13]], %{{.*}} : index
 ! CHECK:  %[[VAL_15:.*]] = arith.select %[[VAL_14]], %[[VAL_13]], %{{.*}} : index
 ! CHECK:  %[[VAL_16:.*]] = fir.shape %[[VAL_15]] : (index) -> !fir.shape<1>
-! CHECK:  %[[VAL_17:.*]] = hlfir.designate %[[VAL_4]]#0 (%[[VAL_9]]:%[[VAL_10]]:%{{.*}}) real shape %[[VAL_16]] : (!fir.box<!fir.array<?x!fir.complex<4>>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
+! CHECK:  %[[VAL_17:.*]] = hlfir.designate %[[VAL_4]]#0 (%[[VAL_9]]:%[[VAL_10]]:%{{.*}}) real shape %[[VAL_16]] : (!fir.box<!fir.array<?xcomplex<f32>>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
diff --git a/flang/test/Lower/HLFIR/entry_return.f90 b/flang/test/Lower/HLFIR/entry_return.f90
index e184ede706681..d07be2128493f 100644
--- a/flang/test/Lower/HLFIR/entry_return.f90
+++ b/flang/test/Lower/HLFIR/entry_return.f90
@@ -6,24 +6,24 @@
   entry e1()
   e1 = .false.
 end function
-! CHECK-LABEL:   func.func @_QPf1() -> !fir.complex<4> {
-! CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.complex<4> {bindc_name = "f1", uniq_name = "_QFf1Ef1"}
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFf1Ef1"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#1 : (!fir.ref<!fir.complex<4>>) -> !fir.ref<!fir.logical<4>>
+! CHECK-LABEL:   func.func @_QPf1() -> complex<f32> {
+! CHECK:           %[[VAL_0:.*]] = fir.alloca complex<f32> {bindc_name = "f1", uniq_name = "_QFf1Ef1"}
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFf1Ef1"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#1 : (!fir.ref<complex<f32>>) -> !fir.ref<!fir.logical<4>>
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFf1Ee1"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           cf.br ^bb1
 ! CHECK:         ^bb1:
 ! CHECK:           %[[VAL_4:.*]] = arith.constant false
 ! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_5]] to %[[VAL_3]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
-! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_1]]#1 : !fir.ref<!fir.complex<4>>
-! CHECK:           return %[[VAL_6]] : !fir.complex<4>
+! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_1]]#1 : !fir.ref<complex<f32>>
+! CHECK:           return %[[VAL_6]] : complex<f32>
 ! CHECK:         }
 
 ! // CHECK-LABEL:   func.func @_QPe1() -> !fir.logical<4> {
-! CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.complex<4> {bindc_name = "f1", uniq_name = "_QFf1Ef1"}
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFf1Ef1"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#1 : (!fir.ref<!fir.complex<4>>) -> !fir.ref<!fir.logical<4>>
+! CHECK:           %[[VAL_0:.*]] = fir.alloca complex<f32> {bindc_name = "f1", uniq_name = "_QFf1Ef1"}
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFf1Ef1"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#1 : (!fir.ref<complex<f32>>) -> !fir.ref<!fir.logical<4>>
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFf1Ee1"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           cf.br ^bb1
 ! CHECK:         ^bb1:
@@ -40,9 +40,9 @@ logical function f2()
   e2 = complex(1.0, 2.0)
 end function
 ! CHECK-LABEL:   func.func @_QPf2() -> !fir.logical<4> {
-! CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.complex<4> {bindc_name = "e2", uniq_name = "_QFf2Ee2"}
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFf2Ee2"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#1 : (!fir.ref<!fir.complex<4>>) -> !fir.ref<!fir.logical<4>>
+! CHECK:           %[[VAL_0:.*]] = fir.alloca complex<f32> {bindc_name = "e2", uniq_name = "_QFf2Ee2"}
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFf2Ee2"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#1 : (!fir.ref<complex<f32>>) -> !fir.ref<!fir.logical<4>>
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFf2Ef2"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           cf.br ^bb1
 ! CHECK:         ^bb1:
@@ -54,18 +54,18 @@ logical function f2()
 ! CHECK:           hlfir.end_associate %[[VAL_6]]#1, %[[VAL_6]]#2 : !fir.ref<f32>, i1
 ! CHECK:           hlfir.end_associate %[[VAL_7]]#1, %[[VAL_7]]#2 : !fir.ref<f32>, i1
 ! CHECK:           %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f32
-! CHECK:           %[[VAL_10:.*]] = fir.undefined !fir.complex<4>
-! CHECK:           %[[VAL_11:.*]] = fir.insert_value %[[VAL_10]], %[[VAL_8]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:           %[[VAL_12:.*]] = fir.insert_value %[[VAL_11]], %[[VAL_9]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:           hlfir.assign %[[VAL_12]] to %[[VAL_1]]#0 : !fir.complex<4>, !fir.ref<!fir.complex<4>>
+! CHECK:           %[[VAL_10:.*]] = fir.undefined complex<f32>
+! CHECK:           %[[VAL_11:.*]] = fir.insert_value %[[VAL_10]], %[[VAL_8]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:           %[[VAL_12:.*]] = fir.insert_value %[[VAL_11]], %[[VAL_9]], [1 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:           hlfir.assign %[[VAL_12]] to %[[VAL_1]]#0 : complex<f32>, !fir.ref<complex<f32>>
 ! CHECK:           %[[VAL_13:.*]] = fir.load %[[VAL_3]]#1 : !fir.ref<!fir.logical<4>>
 ! CHECK:           return %[[VAL_13]] : !fir.logical<4>
 ! CHECK:         }
 
-! CHECK-LABEL:   func.func @_QPe2() -> !fir.complex<4> {
-! CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.complex<4> {bindc_name = "e2", uniq_name = "_QFf2Ee2"}
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFf2Ee2"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#1 : (!fir.ref<!fir.complex<4>>) -> !fir.ref<!fir.logical<4>>
+! CHECK-LABEL:   func.func @_QPe2() -> complex<f32> {
+! CHECK:           %[[VAL_0:.*]] = fir.alloca complex<f32> {bindc_name = "e2", uniq_name = "_QFf2Ee2"}
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFf2Ee2"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#1 : (!fir.ref<complex<f32>>) -> !fir.ref<!fir.logical<4>>
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFf2Ef2"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           cf.br ^bb1
 ! CHECK:         ^bb1:
@@ -77,10 +77,10 @@ logical function f2()
 ! CHECK:           hlfir.end_associate %[[VAL_6]]#1, %[[VAL_6]]#2 : !fir.ref<f32>, i1
 ! CHECK:           hlfir.end_associate %[[VAL_7]]#1, %[[VAL_7]]#2 : !fir.ref<f32>, i1
 ! CHECK:           %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f32
-! CHECK:           %[[VAL_10:.*]] = fir.undefined !fir.complex<4>
-! CHECK:           %[[VAL_11:.*]] = fir.insert_value %[[VAL_10]], %[[VAL_8]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:           %[[VAL_12:.*]] = fir.insert_value %[[VAL_11]], %[[VAL_9]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:           hlfir.assign %[[VAL_12]] to %[[VAL_1]]#0 : !fir.complex<4>, !fir.ref<!fir.complex<4>>
-! CHECK:           %[[VAL_13:.*]] = fir.load %[[VAL_1]]#1 : !fir.ref<!fir.complex<4>>
-! CHECK:           return %[[VAL_13]] : !fir.complex<4>
+! CHECK:           %[[VAL_10:.*]] = fir.undefined complex<f32>
+! CHECK:           %[[VAL_11:.*]] = fir.insert_value %[[VAL_10]], %[[VAL_8]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:           %[[VAL_12:.*]] = fir.insert_value %[[VAL_11]], %[[VAL_9]], [1 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:           hlfir.assign %[[VAL_12]] to %[[VAL_1]]#0 : complex<f32>, !fir.ref<complex<f32>>
+! CHECK:           %[[VAL_13:.*]] = fir.load %[[VAL_1]]#1 : !fir.ref<complex<f32>>
+! CHECK:           return %[[VAL_13]] : complex<f32>
 ! CHECK:         }
diff --git a/flang/test/Lower/HLFIR/implicit-call-mismatch.f90 b/flang/test/Lower/HLFIR/implicit-call-mismatch.f90
index 835da013c70b3..4b8ba3f33dc0c 100644
--- a/flang/test/Lower/HLFIR/implicit-call-mismatch.f90
+++ b/flang/test/Lower/HLFIR/implicit-call-mismatch.f90
@@ -84,7 +84,7 @@ subroutine pass_complex_to_real(cmplx)
 end subroutine
 ! CHECK-LABEL: func.func @_QPpass_complex_to_real(
 ! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare {{.*}}Ecmplx
-! CHECK:  %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#1 : (!fir.ref<!fir.complex<4>>) -> !fir.ref<f64>
+! CHECK:  %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#1 : (!fir.ref<complex<f32>>) -> !fir.ref<f64>
 ! CHECK:  fir.call @_QPtakes_real(%[[VAL_2]]) {{.*}}: (!fir.ref<f64>) -> ()
 
 subroutine takes_char_proc(c)
diff --git a/flang/test/Lower/HLFIR/implicit-type-conversion-allocatable.f90 b/flang/test/Lower/HLFIR/implicit-type-conversion-allocatable.f90
index 361cd61adea2d..0d5174ffed9b1 100644
--- a/flang/test/Lower/HLFIR/implicit-type-conversion-allocatable.f90
+++ b/flang/test/Lower/HLFIR/implicit-type-conversion-allocatable.f90
@@ -11,26 +11,26 @@ subroutine preserve_lbounds(x, y)
 ! CHECK-LABEL:   func.func @_QPpreserve_lbounds(
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare {{.*}}uniq_name = "_QFpreserve_lboundsEx"
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare {{.*}}uniq_name = "_QFpreserve_lboundsEy"
-! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>>
+! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>>
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
-! CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_4]], %[[VAL_5]] : (!fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>, index) -> (index, index, index)
+! CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_4]], %[[VAL_5]] : (!fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>, index) -> (index, index, index)
 ! CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_6]]#1 : (index) -> !fir.shape<1>
 ! CHECK:           %[[VAL_8:.*]] = hlfir.elemental %[[VAL_7]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
 ! CHECK:           ^bb0(%[[VAL_9:.*]]: index):
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 0 : index
-! CHECK:             %[[VAL_11:.*]]:3 = fir.box_dims %[[VAL_4]], %[[VAL_10]] : (!fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>, index) -> (index, index, index)
+! CHECK:             %[[VAL_11:.*]]:3 = fir.box_dims %[[VAL_4]], %[[VAL_10]] : (!fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>, index) -> (index, index, index)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : index
 ! CHECK:             %[[VAL_13:.*]] = arith.subi %[[VAL_11]]#0, %[[VAL_12]] : index
 ! CHECK:             %[[VAL_14:.*]] = arith.addi %[[VAL_9]], %[[VAL_13]] : index
-! CHECK:             %[[VAL_15:.*]] = hlfir.designate %[[VAL_4]] (%[[VAL_14]])  : (!fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>, index) -> !fir.ref<!fir.complex<4>>
-! CHECK:             %[[VAL_16:.*]] = fir.load %[[VAL_15]] : !fir.ref<!fir.complex<4>>
-! CHECK:             %[[VAL_17:.*]] = fir.extract_value %[[VAL_16]], [0 : index] : (!fir.complex<4>) -> f32
+! CHECK:             %[[VAL_15:.*]] = hlfir.designate %[[VAL_4]] (%[[VAL_14]])  : (!fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>, index) -> !fir.ref<complex<f32>>
+! CHECK:             %[[VAL_16:.*]] = fir.load %[[VAL_15]] : !fir.ref<complex<f32>>
+! CHECK:             %[[VAL_17:.*]] = fir.extract_value %[[VAL_16]], [0 : index] : (complex<f32>) -> f32
 ! CHECK:             %[[VAL_18:.*]] = fir.convert %[[VAL_17]] : (f32) -> i32
 ! CHECK:             hlfir.yield_element %[[VAL_18]] : i32
 ! CHECK:           }
 ! CHECK:           %[[VAL_19:.*]]:3 = hlfir.associate %[[VAL_8]](%[[VAL_7]]) {uniq_name = ".tmp.keeplbounds"} : (!hlfir.expr<?xi32>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>, i1)
 ! CHECK:           %[[VAL_20:.*]] = arith.constant 0 : index
-! CHECK:           %[[VAL_21:.*]]:3 = fir.box_dims %[[VAL_4]], %[[VAL_20]] : (!fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>, index) -> (index, index, index)
+! CHECK:           %[[VAL_21:.*]]:3 = fir.box_dims %[[VAL_4]], %[[VAL_20]] : (!fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>, index) -> (index, index, index)
 ! CHECK:           %[[VAL_22:.*]] = fir.shape_shift %[[VAL_21]]#0, %[[VAL_6]]#1 : (index, index) -> !fir.shapeshift<1>
 ! CHECK:           %[[VAL_23:.*]]:2 = hlfir.declare %[[VAL_19]]#1(%[[VAL_22]]) {uniq_name = ".tmp.keeplbounds"} : (!fir.ref<!fir.array<?xi32>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>)
 ! CHECK:           hlfir.assign %[[VAL_23]]#0 to %[[VAL_2]]#0 realloc : !fir.box<!fir.array<?xi32>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
diff --git a/flang/test/Lower/HLFIR/initial-target-component.f90 b/flang/test/Lower/HLFIR/initial-target-component.f90
index cfee3adbc70be..b6725f888c90f 100644
--- a/flang/test/Lower/HLFIR/initial-target-component.f90
+++ b/flang/test/Lower/HLFIR/initial-target-component.f90
@@ -9,9 +9,9 @@ subroutine test()
   real, pointer :: p => obj%z%re
 end subroutine
 ! CHECK-LABEL:   fir.global internal @_QFtestEp : !fir.box<!fir.ptr<f32>> {
-! CHECK-NEXT:      %[[VAL_0:.*]] = fir.address_of(@_QFtestEobj) : !fir.ref<!fir.type<_QFtestTt{z:!fir.complex<4>}>>
-! CHECK-NEXT:      %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFtestEobj"} : (!fir.ref<!fir.type<_QFtestTt{z:!fir.complex<4>}>>) -> (!fir.ref<!fir.type<_QFtestTt{z:!fir.complex<4>}>>, !fir.ref<!fir.type<_QFtestTt{z:!fir.complex<4>}>>)
-! CHECK-NEXT:      %[[VAL_2:.*]] = hlfir.designate %[[VAL_1]]#0{"z"}  real : (!fir.ref<!fir.type<_QFtestTt{z:!fir.complex<4>}>>) -> !fir.ref<f32>
+! CHECK-NEXT:      %[[VAL_0:.*]] = fir.address_of(@_QFtestEobj) : !fir.ref<!fir.type<_QFtestTt{z:complex<f32>}>>
+! CHECK-NEXT:      %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFtestEobj"} : (!fir.ref<!fir.type<_QFtestTt{z:complex<f32>}>>) -> (!fir.ref<!fir.type<_QFtestTt{z:complex<f32>}>>, !fir.ref<!fir.type<_QFtestTt{z:complex<f32>}>>)
+! CHECK-NEXT:      %[[VAL_2:.*]] = hlfir.designate %[[VAL_1]]#0{"z"}  real : (!fir.ref<!fir.type<_QFtestTt{z:complex<f32>}>>) -> !fir.ref<f32>
 ! CHECK-NEXT:      %[[VAL_3:.*]] = fir.embox %[[VAL_2]] : (!fir.ref<f32>) -> !fir.box<f32>
 ! CHECK-NEXT:      %[[VAL_4:.*]] = fir.rebox %[[VAL_3]] : (!fir.box<f32>) -> !fir.box<!fir.ptr<f32>>
 ! CHECK-NEXT:      fir.has_value %[[VAL_4]] : !fir.box<!fir.ptr<f32>>
diff --git a/flang/test/Lower/HLFIR/intrinsic-dynamically-optional.f90 b/flang/test/Lower/HLFIR/intrinsic-dynamically-optional.f90
index d1969049828c9..c422828e05dce 100644
--- a/flang/test/Lower/HLFIR/intrinsic-dynamically-optional.f90
+++ b/flang/test/Lower/HLFIR/intrinsic-dynamically-optional.f90
@@ -73,10 +73,10 @@ integer function test_optional_as_box2(x, mask)
 end function
 ! CHECK-LABEL:   func.func @_QPtest_optional_as_value(
 ! CHECK-SAME:                                         %[[REAL_ARG:.*]]: !fir.ref<f32> {fir.bindc_name = "real"},
-! CHECK-SAME:                                         %[[IMAG_ARG:.*]]: !fir.ref<f32> {fir.bindc_name = "imaginary", fir.optional}) -> !fir.complex<4> {
+! CHECK-SAME:                                         %[[IMAG_ARG:.*]]: !fir.ref<f32> {fir.bindc_name = "imaginary", fir.optional}) -> complex<f32> {
 ! CHECK:           %[[IMAG_VAR:.*]]:2 = hlfir.declare %[[IMAG_ARG]]
 ! CHECK:           %[[REAL_VAR:.*]]:2 = hlfir.declare %[[REAL_ARG]]
-! CHECK:           %[[RET_ALLOC:.*]] = fir.alloca !fir.complex<4> {bindc_name = "test_optional_as_value", uniq_name = "_QFtest_optional_as_valueEtest_optional_as_value"}
+! CHECK:           %[[RET_ALLOC:.*]] = fir.alloca complex<f32> {bindc_name = "test_optional_as_value", uniq_name = "_QFtest_optional_as_valueEtest_optional_as_value"}
 ! CHECK:           %[[RET_VAR:.*]]:2 = hlfir.declare %[[RET_ALLOC]]
 ! CHECK:           %[[IS_PRESENT:.*]] = fir.is_present %[[IMAG_VAR]]#0 : (!fir.ref<f32>) -> i1
 ! CHECK:           %[[REAL_LD:.*]] = fir.load %[[REAL_VAR]]#0 : !fir.ref<f32>
@@ -87,12 +87,12 @@ integer function test_optional_as_box2(x, mask)
 ! CHECK:             %[[IMAG_ABSENT:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:             fir.result %[[IMAG_ABSENT]] : f32
 ! CHECK:           }
-! CHECK:           %[[UNDEF:.*]] = fir.undefined !fir.complex<4>
-! CHECK:           %[[INS_REAL:.*]] = fir.insert_value %[[UNDEF]], %[[REAL_LD]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:           %[[INS_IMAG:.*]] = fir.insert_value %[[INS_REAL]], %[[IMAG_LD:.*]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
+! CHECK:           %[[UNDEF:.*]] = fir.undefined complex<f32>
+! CHECK:           %[[INS_REAL:.*]] = fir.insert_value %[[UNDEF]], %[[REAL_LD]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:           %[[INS_IMAG:.*]] = fir.insert_value %[[INS_REAL]], %[[IMAG_LD:.*]], [1 : index] : (complex<f32>, f32) -> complex<f32>
 ! CHECK:           hlfir.assign %[[INS_IMAG]] to %[[RET_VAR]]#0
-! CHECK:           %[[RET:.*]] = fir.load %[[RET_VAR]]#1 : !fir.ref<!fir.complex<4>>
-! CHECK:           return %[[RET]] : !fir.complex<4>
+! CHECK:           %[[RET:.*]] = fir.load %[[RET_VAR]]#1 : !fir.ref<complex<f32>>
+! CHECK:           return %[[RET]] : complex<f32>
 ! CHECK:         }
 
 ! stat argument is dynamically optional, lowered as an address
@@ -163,7 +163,7 @@ function test_elemental_optional_as_value(real, imaginary)
 end function
 ! CHECK-LABEL:   func.func @_QPtest_elemental_optional_as_value(
 ! CHECK-SAME:                                                   %[[VAL_0:.*]]: !fir.ref<!fir.array<3xf32>> {fir.bindc_name = "real"},
-! CHECK-SAME:                                                   %[[VAL_1:.*]]: !fir.ref<!fir.array<3xf32>> {fir.bindc_name = "imaginary", fir.optional}) -> !fir.array<3x!fir.complex<4>> {
+! CHECK-SAME:                                                   %[[VAL_1:.*]]: !fir.ref<!fir.array<3xf32>> {fir.bindc_name = "imaginary", fir.optional}) -> !fir.array<3xcomplex<f32>> {
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_3:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1>
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_3]]) dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFtest_elemental_optional_as_valueEimaginary"} : (!fir.ref<!fir.array<3xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<3xf32>>, !fir.ref<!fir.array<3xf32>>)
@@ -171,11 +171,11 @@ function test_elemental_optional_as_value(real, imaginary)
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
 ! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_elemental_optional_as_valueEreal"} : (!fir.ref<!fir.array<3xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<3xf32>>, !fir.ref<!fir.array<3xf32>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant 3 : index
-! CHECK:           %[[VAL_9:.*]] = fir.alloca !fir.array<3x!fir.complex<4>> {bindc_name = "test_elemental_optional_as_value", uniq_name = "_QFtest_elemental_optional_as_valueEtest_elemental_optional_as_value"}
+! CHECK:           %[[VAL_9:.*]] = fir.alloca !fir.array<3xcomplex<f32>> {bindc_name = "test_elemental_optional_as_value", uniq_name = "_QFtest_elemental_optional_as_valueEtest_elemental_optional_as_value"}
 ! CHECK:           %[[VAL_10:.*]] = fir.shape %[[VAL_8]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_9]](%[[VAL_10]]) {uniq_name = "_QFtest_elemental_optional_as_valueEtest_elemental_optional_as_value"} : (!fir.ref<!fir.array<3x!fir.complex<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3x!fir.complex<4>>>, !fir.ref<!fir.array<3x!fir.complex<4>>>)
+! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_9]](%[[VAL_10]]) {uniq_name = "_QFtest_elemental_optional_as_valueEtest_elemental_optional_as_value"} : (!fir.ref<!fir.array<3xcomplex<f32>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xcomplex<f32>>>, !fir.ref<!fir.array<3xcomplex<f32>>>)
 ! CHECK:           %[[VAL_12:.*]] = fir.is_present %[[VAL_4]]#0 : (!fir.ref<!fir.array<3xf32>>) -> i1
-! CHECK:           %[[VAL_13:.*]] = hlfir.elemental %[[VAL_6]] unordered : (!fir.shape<1>) -> !hlfir.expr<3x!fir.complex<4>> {
+! CHECK:           %[[VAL_13:.*]] = hlfir.elemental %[[VAL_6]] unordered : (!fir.shape<1>) -> !hlfir.expr<3xcomplex<f32>> {
 ! CHECK:           ^bb0(%[[VAL_14:.*]]: index):
 ! CHECK:             %[[VAL_15:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_14]])  : (!fir.ref<!fir.array<3xf32>>, index) -> !fir.ref<f32>
 ! CHECK:             %[[VAL_16:.*]] = fir.load %[[VAL_15]] : !fir.ref<f32>
@@ -187,15 +187,15 @@ function test_elemental_optional_as_value(real, imaginary)
 ! CHECK:               %[[VAL_20:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:               fir.result %[[VAL_20]] : f32
 ! CHECK:             }
-! CHECK:             %[[VAL_21:.*]] = fir.undefined !fir.complex<4>
-! CHECK:             %[[VAL_22:.*]] = fir.insert_value %[[VAL_21]], %[[VAL_16]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:             %[[VAL_23:.*]] = fir.insert_value %[[VAL_22]], %[[VAL_17]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:             hlfir.yield_element %[[VAL_23]] : !fir.complex<4>
+! CHECK:             %[[VAL_21:.*]] = fir.undefined complex<f32>
+! CHECK:             %[[VAL_22:.*]] = fir.insert_value %[[VAL_21]], %[[VAL_16]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:             %[[VAL_23:.*]] = fir.insert_value %[[VAL_22]], %[[VAL_17]], [1 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:             hlfir.yield_element %[[VAL_23]] : complex<f32>
 ! CHECK:           }
-! CHECK:           hlfir.assign %[[VAL_13]] to %[[VAL_11]]#0 : !hlfir.expr<3x!fir.complex<4>>, !fir.ref<!fir.array<3x!fir.complex<4>>>
-! CHECK:           hlfir.destroy %[[VAL_13]] : !hlfir.expr<3x!fir.complex<4>>
-! CHECK:           %[[VAL_24:.*]] = fir.load %[[VAL_11]]#1 : !fir.ref<!fir.array<3x!fir.complex<4>>>
-! CHECK:           return %[[VAL_24]] : !fir.array<3x!fir.complex<4>>
+! CHECK:           hlfir.assign %[[VAL_13]] to %[[VAL_11]]#0 : !hlfir.expr<3xcomplex<f32>>, !fir.ref<!fir.array<3xcomplex<f32>>>
+! CHECK:           hlfir.destroy %[[VAL_13]] : !hlfir.expr<3xcomplex<f32>>
+! CHECK:           %[[VAL_24:.*]] = fir.load %[[VAL_11]]#1 : !fir.ref<!fir.array<3xcomplex<f32>>>
+! CHECK:           return %[[VAL_24]] : !fir.array<3xcomplex<f32>>
 ! CHECK:         }
 
 ! TODO: there seem to be no intrinsics with dynamically optional arguments lowered asInquired
diff --git a/flang/test/Lower/HLFIR/select-rank.f90 b/flang/test/Lower/HLFIR/select-rank.f90
index d27a6d732ffc7..7135c248ea44d 100644
--- a/flang/test/Lower/HLFIR/select-rank.f90
+++ b/flang/test/Lower/HLFIR/select-rank.f90
@@ -1,5 +1,5 @@
 ! Test lowering of select rank to HLFIR
-! RUN: bbc -emit-hlfir -o - %s -allow-assumed-rank | FileCheck %s
+! RUN: bbc -emit-hlfir -o - %s | FileCheck %s
 
 module iface_helpers
 interface
diff --git a/flang/test/Lower/HLFIR/unary-ops.f90 b/flang/test/Lower/HLFIR/unary-ops.f90
index b04d6b4cf949a..bbe264152f2a0 100644
--- a/flang/test/Lower/HLFIR/unary-ops.f90
+++ b/flang/test/Lower/HLFIR/unary-ops.f90
@@ -69,9 +69,9 @@ subroutine test_negate_complex(res, x)
   res = -x
 end subroutine
 ! CHECK-LABEL: func.func @_QPtest_negate_complex(
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.complex<4>>
-! CHECK:  %[[VAL_5:.*]] = fir.negc %[[VAL_4]] : !fir.complex<4>
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<complex<f32>>
+! CHECK:  %[[VAL_5:.*]] = fir.negc %[[VAL_4]] : complex<f32>
 
 subroutine test_complex_component_real(res, x)
   real :: res
@@ -79,9 +79,9 @@ subroutine test_complex_component_real(res, x)
   res = real(x)
 end subroutine
 ! CHECK-LABEL: func.func @_QPtest_complex_component_real(
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.complex<4>>
-! CHECK:  %[[VAL_5:.*]] = fir.extract_value %[[VAL_4]], [0 : index] : (!fir.complex<4>) -> f32
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<complex<f32>>
+! CHECK:  %[[VAL_5:.*]] = fir.extract_value %[[VAL_4]], [0 : index] : (complex<f32>) -> f32
 
 subroutine test_complex_component_imag(res, x)
   real :: res
@@ -89,6 +89,6 @@ subroutine test_complex_component_imag(res, x)
   res = aimag(x)
 end subroutine
 ! CHECK-LABEL: func.func @_QPtest_complex_component_imag(
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.complex<4>>
-! CHECK:  %[[VAL_5:.*]] = fir.extract_value %[[VAL_4]], [1 : index] : (!fir.complex<4>) -> f32
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<complex<f32>>
+! CHECK:  %[[VAL_5:.*]] = fir.extract_value %[[VAL_4]], [1 : index] : (complex<f32>) -> f32
diff --git a/flang/test/Lower/HLFIR/user-defined-assignment.f90 b/flang/test/Lower/HLFIR/user-defined-assignment.f90
index f0e24f11c5ab0..a71e413e3226f 100644
--- a/flang/test/Lower/HLFIR/user-defined-assignment.f90
+++ b/flang/test/Lower/HLFIR/user-defined-assignment.f90
@@ -51,17 +51,17 @@ subroutine test_user_defined_elemental_array_value(z, l)
    z = l
 end subroutine
 ! CHECK-LABEL:   func.func @_QMuser_defPtest_user_defined_elemental_array_value(
-! CHECK-SAME:    %[[VAL_0:.*]]: !fir.box<!fir.array<?x!fir.complex<4>>> {fir.bindc_name = "z"},
+! CHECK-SAME:    %[[VAL_0:.*]]: !fir.box<!fir.array<?xcomplex<f32>>> {fir.bindc_name = "z"},
 ! CHECK-SAME:    %[[VAL_1:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>> {fir.bindc_name = "l"}) {
 ! CHECK:    %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMuser_defFtest_user_defined_elemental_array_valueEl"} : (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
-! CHECK:    %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMuser_defFtest_user_defined_elemental_array_valueEz"} : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
+! CHECK:    %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMuser_defFtest_user_defined_elemental_array_valueEz"} : (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.dscope) -> (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.box<!fir.array<?xcomplex<f32>>>)
 ! CHECK:    hlfir.region_assign {
 ! CHECK:      hlfir.yield %[[VAL_2]]#0 : !fir.box<!fir.array<?x!fir.logical<4>>>
 ! CHECK:    } to {
-! CHECK:      hlfir.yield %[[VAL_3]]#0 : !fir.box<!fir.array<?x!fir.complex<4>>>
-! CHECK:    } user_defined_assign  (%[[VAL_4:.*]]: !fir.ref<!fir.logical<4>>) to (%[[VAL_5:.*]]: !fir.ref<!fir.complex<4>>) {
+! CHECK:      hlfir.yield %[[VAL_3]]#0 : !fir.box<!fir.array<?xcomplex<f32>>>
+! CHECK:    } user_defined_assign  (%[[VAL_4:.*]]: !fir.ref<!fir.logical<4>>) to (%[[VAL_5:.*]]: !fir.ref<complex<f32>>) {
 ! CHECK:      %[[VAL_6:.*]] = fir.load %[[VAL_4]] : !fir.ref<!fir.logical<4>>
-! CHECK:      fir.call @_QPlogical_to_complex(%[[VAL_5]], %[[VAL_6]]) {{.*}}: (!fir.ref<!fir.complex<4>>, !fir.logical<4>) -> ()
+! CHECK:      fir.call @_QPlogical_to_complex(%[[VAL_5]], %[[VAL_6]]) {{.*}}: (!fir.ref<complex<f32>>, !fir.logical<4>) -> ()
 ! CHECK:    }
 
 subroutine test_user_defined_scalar(i, l)
diff --git a/flang/test/Lower/HLFIR/vector-subscript-as-value.f90 b/flang/test/Lower/HLFIR/vector-subscript-as-value.f90
index 7161ee088b57a..fe1f5415688bd 100644
--- a/flang/test/Lower/HLFIR/vector-subscript-as-value.f90
+++ b/flang/test/Lower/HLFIR/vector-subscript-as-value.f90
@@ -112,7 +112,7 @@ subroutine foo4(at1, vector, i, j, k, l, step)
 ! CHECK:  %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_6:[a-z0-9]*]]  {{.*}}Estep
 ! CHECK:  %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]]  {{.*}}Evector
 ! CHECK:  %[[VAL_14:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i64>
-! CHECK:  %[[VAL_15:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_14]])  : (!fir.box<!fir.array<?x!fir.type<_QFfoo4Tt1{at0:!fir.array<30x40x50x!fir.type<_QFfoo4Tt0{x:!fir.array<10x20x!fir.complex<4>>}>>}>>>, i64) -> !fir.ref<!fir.type<_QFfoo4Tt1{at0:!fir.array<30x40x50x!fir.type<_QFfoo4Tt0{x:!fir.array<10x20x!fir.complex<4>>}>>}>>
+! CHECK:  %[[VAL_15:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_14]])  : (!fir.box<!fir.array<?x!fir.type<_QFfoo4Tt1{at0:!fir.array<30x40x50x!fir.type<_QFfoo4Tt0{x:!fir.array<10x20xcomplex<f32>>}>>}>>>, i64) -> !fir.ref<!fir.type<_QFfoo4Tt1{at0:!fir.array<30x40x50x!fir.type<_QFfoo4Tt0{x:!fir.array<10x20xcomplex<f32>>}>>}>>
 ! CHECK:  %[[VAL_16:.*]] = arith.constant 30 : index
 ! CHECK:  %[[VAL_17:.*]] = arith.constant 40 : index
 ! CHECK:  %[[VAL_18:.*]] = arith.constant 50 : index
@@ -144,8 +144,8 @@ subroutine foo4(at1, vector, i, j, k, l, step)
 ! CHECK:    %[[VAL_45:.*]] = arith.addi %[[VAL_20]], %[[VAL_44]] : index
 ! CHECK:    %[[VAL_46:.*]] = hlfir.designate %[[VAL_13]]#0 (%[[VAL_41]])  : (!fir.box<!fir.array<?xi64>>, index) -> !fir.ref<i64>
 ! CHECK:    %[[VAL_47:.*]] = fir.load %[[VAL_46]] : !fir.ref<i64>
-! CHECK:    %[[VAL_48:.*]] = hlfir.designate %[[VAL_15]]{"at0"} <%[[VAL_19]]> (%[[VAL_45]], %[[VAL_30]], %[[VAL_47]])  : (!fir.ref<!fir.type<_QFfoo4Tt1{at0:!fir.array<30x40x50x!fir.type<_QFfoo4Tt0{x:!fir.array<10x20x!fir.complex<4>>}>>}>>, !fir.shape<3>, index, i64, i64) -> !fir.ref<!fir.type<_QFfoo4Tt0{x:!fir.array<10x20x!fir.complex<4>>}>>
-! CHECK:    %[[VAL_49:.*]] = hlfir.designate %[[VAL_48]]{"x"} <%[[VAL_36]]> (%[[VAL_37]], %[[VAL_38]]) imag : (!fir.ref<!fir.type<_QFfoo4Tt0{x:!fir.array<10x20x!fir.complex<4>>}>>, !fir.shape<2>, i64, i64) -> !fir.ref<f32>
+! CHECK:    %[[VAL_48:.*]] = hlfir.designate %[[VAL_15]]{"at0"} <%[[VAL_19]]> (%[[VAL_45]], %[[VAL_30]], %[[VAL_47]])  : (!fir.ref<!fir.type<_QFfoo4Tt1{at0:!fir.array<30x40x50x!fir.type<_QFfoo4Tt0{x:!fir.array<10x20xcomplex<f32>>}>>}>>, !fir.shape<3>, index, i64, i64) -> !fir.ref<!fir.type<_QFfoo4Tt0{x:!fir.array<10x20xcomplex<f32>>}>>
+! CHECK:    %[[VAL_49:.*]] = hlfir.designate %[[VAL_48]]{"x"} <%[[VAL_36]]> (%[[VAL_37]], %[[VAL_38]]) imag : (!fir.ref<!fir.type<_QFfoo4Tt0{x:!fir.array<10x20xcomplex<f32>>}>>, !fir.shape<2>, i64, i64) -> !fir.ref<f32>
 ! CHECK:    %[[VAL_50:.*]] = fir.load %[[VAL_49]] : !fir.ref<f32>
 ! CHECK:    hlfir.yield_element %[[VAL_50]] : f32
 ! CHECK:  }
diff --git a/flang/test/Lower/Intrinsics/abs.f90 b/flang/test/Lower/Intrinsics/abs.f90
index 8b977c3c0a968..e5e4b79e9f79e 100644
--- a/flang/test/Lower/Intrinsics/abs.f90
+++ b/flang/test/Lower/Intrinsics/abs.f90
@@ -96,14 +96,12 @@ subroutine abs_testr16(a, b)
 end subroutine
 
 ! CMPLX-LABEL: func @_QPabs_testzr(
-! CMPLX-SAME:  %[[VAL_0:.*]]: !fir.ref<!fir.complex<4>>{{.*}}, %[[VAL_1:.*]]: !fir.ref<f32>{{.*}}) {
+! CMPLX-SAME:  %[[VAL_0:.*]]: !fir.ref<complex<f32>>{{.*}}, %[[VAL_1:.*]]: !fir.ref<f32>{{.*}}) {
 subroutine abs_testzr(a, b)
-! CMPLX:  %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.complex<4>>
-! CMPLX-FAST: %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.complex<4>) -> complex<f32>
-! CMPLX-FAST: %[[VAL_4:.*]] = complex.abs %[[VAL_3]] fastmath<contract> : complex<f32>
-! CMPLX-APPROX: %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.complex<4>) -> complex<f32>
-! CMPLX-APPROX: %[[VAL_4:.*]] = complex.abs %[[VAL_3]] fastmath<contract,afn> : complex<f32>
-! CMPLX-PRECISE:  %[[VAL_4:.*]] = fir.call @cabsf(%[[VAL_2]]) {{.*}}: (!fir.complex<4>) -> f32
+! CMPLX:  %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<complex<f32>>
+! CMPLX-FAST: %[[VAL_4:.*]] = complex.abs %[[VAL_2]] fastmath<contract> : complex<f32>
+! CMPLX-APPROX: %[[VAL_4:.*]] = complex.abs %[[VAL_2]] fastmath<contract,afn> : complex<f32>
+! CMPLX-PRECISE:  %[[VAL_4:.*]] = fir.call @cabsf(%[[VAL_2]]) {{.*}}: (complex<f32>) -> f32
 ! CMPLX:  fir.store %[[VAL_4]] to %[[VAL_1]] : !fir.ref<f32>
 ! CMPLX:  return
   complex :: a
@@ -112,14 +110,12 @@ subroutine abs_testzr(a, b)
 end subroutine abs_testzr
 
 ! CMPLX-LABEL: func @_QPabs_testzd(
-! CMPLX-SAME:  %[[VAL_0:.*]]: !fir.ref<!fir.complex<8>>{{.*}}, %[[VAL_1:.*]]: !fir.ref<f64>{{.*}}) {
+! CMPLX-SAME:  %[[VAL_0:.*]]: !fir.ref<complex<f64>>{{.*}}, %[[VAL_1:.*]]: !fir.ref<f64>{{.*}}) {
 subroutine abs_testzd(a, b)
-! CMPLX:  %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.complex<8>>
-! CMPLX-FAST: %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.complex<8>) -> complex<f64>
-! CMPLX-FAST: %[[VAL_4:.*]] = complex.abs %[[VAL_3]] fastmath<contract> : complex<f64>
-! CMPLX-APPROX: %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.complex<8>) -> complex<f64>
-! CMPLX-APPROX: %[[VAL_4:.*]] = complex.abs %[[VAL_3]] fastmath<contract,afn> : complex<f64>
-! CMPLX-PRECISE:  %[[VAL_4:.*]] = fir.call @cabs(%[[VAL_2]]) {{.*}}: (!fir.complex<8>) -> f64
+! CMPLX:  %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<complex<f64>>
+! CMPLX-FAST: %[[VAL_4:.*]] = complex.abs %[[VAL_2]] fastmath<contract> : complex<f64>
+! CMPLX-APPROX: %[[VAL_4:.*]] = complex.abs %[[VAL_2]] fastmath<contract,afn> : complex<f64>
+! CMPLX-PRECISE:  %[[VAL_4:.*]] = fir.call @cabs(%[[VAL_2]]) {{.*}}: (complex<f64>) -> f64
 ! CMPLX:  fir.store %[[VAL_4]] to %[[VAL_1]] : !fir.ref<f64>
 ! CMPLX:  return
   complex(kind=8) :: a
diff --git a/flang/test/Lower/Intrinsics/acos.f90 b/flang/test/Lower/Intrinsics/acos.f90
index 729c7dd5dbaff..d2ef8e1cef0a8 100644
--- a/flang/test/Lower/Intrinsics/acos.f90
+++ b/flang/test/Lower/Intrinsics/acos.f90
@@ -27,7 +27,7 @@ function test_complex4(x)
 end function
 
 ! ALL-LABEL: @_QPtest_complex4
-! ALL: {{%[A-Za-z0-9._]+}} = fir.call @cacosf({{%[A-Za-z0-9._]+}}) {{.*}}: (!fir.complex<4>) -> !fir.complex<4>
+! ALL: {{%[A-Za-z0-9._]+}} = fir.call @cacosf({{%[A-Za-z0-9._]+}}) {{.*}}: (complex<f32>) -> complex<f32>
 
 function test_complex8(x)
   complex(kind=8) :: x, test_complex8
@@ -35,5 +35,5 @@ function test_complex8(x)
 end function
 
 ! ALL-LABEL: @_QPtest_complex8
-! ALL: {{%[A-Za-z0-9._]+}} = fir.call @cacos({{%[A-Za-z0-9._]+}}) {{.*}}: (!fir.complex<8>) -> !fir.complex<8>
+! ALL: {{%[A-Za-z0-9._]+}} = fir.call @cacos({{%[A-Za-z0-9._]+}}) {{.*}}: (complex<f64>) -> complex<f64>
 
diff --git a/flang/test/Lower/Intrinsics/acos_complex16.f90 b/flang/test/Lower/Intrinsics/acos_complex16.f90
index 5b3545e472e04..761a8661645e2 100644
--- a/flang/test/Lower/Intrinsics/acos_complex16.f90
+++ b/flang/test/Lower/Intrinsics/acos_complex16.f90
@@ -2,7 +2,7 @@
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
-! CHECK: fir.call @_FortranACAcosF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+! CHECK: fir.call @_FortranACAcosF128({{.*}}){{.*}}: (complex<f128>) -> complex<f128>
   complex(16) :: a, b
   b = acos(a)
 end
diff --git a/flang/test/Lower/Intrinsics/acosh.f90 b/flang/test/Lower/Intrinsics/acosh.f90
index da6654f862915..bc0d03f9d1b7f 100644
--- a/flang/test/Lower/Intrinsics/acosh.f90
+++ b/flang/test/Lower/Intrinsics/acosh.f90
@@ -27,7 +27,7 @@ function test_complex4(x)
 end function
 
 ! ALL-LABEL: @_QPtest_complex4
-! ALL: {{%[A-Za-z0-9._]+}} = fir.call @cacoshf({{%[A-Za-z0-9._]+}}) {{.*}}: (!fir.complex<4>) -> !fir.complex<4>
+! ALL: {{%[A-Za-z0-9._]+}} = fir.call @cacoshf({{%[A-Za-z0-9._]+}}) {{.*}}: (complex<f32>) -> complex<f32>
 
 function test_complex8(x)
   complex(kind=8) :: x, test_complex8
@@ -35,5 +35,5 @@ function test_complex8(x)
 end function
 
 ! ALL-LABEL: @_QPtest_complex8
-! ALL: {{%[A-Za-z0-9._]+}} = fir.call @cacosh({{%[A-Za-z0-9._]+}}) {{.*}}: (!fir.complex<8>) -> !fir.complex<8>
+! ALL: {{%[A-Za-z0-9._]+}} = fir.call @cacosh({{%[A-Za-z0-9._]+}}) {{.*}}: (complex<f64>) -> complex<f64>
 
diff --git a/flang/test/Lower/Intrinsics/acosh_complex16.f90 b/flang/test/Lower/Intrinsics/acosh_complex16.f90
index a80238d72e451..72944ed55bb7e 100644
--- a/flang/test/Lower/Intrinsics/acosh_complex16.f90
+++ b/flang/test/Lower/Intrinsics/acosh_complex16.f90
@@ -2,7 +2,7 @@
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
-! CHECK: fir.call @_FortranACAcoshF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+! CHECK: fir.call @_FortranACAcoshF128({{.*}}){{.*}}: (complex<f128>) -> complex<f128>
   complex(16) :: a, b
   b = acosh(a)
 end
diff --git a/flang/test/Lower/Intrinsics/asin.f90 b/flang/test/Lower/Intrinsics/asin.f90
index 6379c6996fb49..a2ff6d6dab757 100644
--- a/flang/test/Lower/Intrinsics/asin.f90
+++ b/flang/test/Lower/Intrinsics/asin.f90
@@ -27,7 +27,7 @@ function test_complex4(x)
 end function
 
 ! ALL-LABEL: @_QPtest_complex4
-! ALL: {{%[A-Za-z0-9._]+}} = fir.call @casinf({{%[A-Za-z0-9._]+}}) {{.*}}: (!fir.complex<4>) -> !fir.complex<4>
+! ALL: {{%[A-Za-z0-9._]+}} = fir.call @casinf({{%[A-Za-z0-9._]+}}) {{.*}}: (complex<f32>) -> complex<f32>
 
 function test_complex8(x)
   complex(kind=8) :: x, test_complex8
@@ -35,5 +35,5 @@ function test_complex8(x)
 end function
 
 ! ALL-LABEL: @_QPtest_complex8
-! ALL: {{%[A-Za-z0-9._]+}} = fir.call @casin({{%[A-Za-z0-9._]+}}) {{.*}}: (!fir.complex<8>) -> !fir.complex<8>
+! ALL: {{%[A-Za-z0-9._]+}} = fir.call @casin({{%[A-Za-z0-9._]+}}) {{.*}}: (complex<f64>) -> complex<f64>
 
diff --git a/flang/test/Lower/Intrinsics/asin_complex16.f90 b/flang/test/Lower/Intrinsics/asin_complex16.f90
index 982bf6f21b16f..e78107de8aec1 100644
--- a/flang/test/Lower/Intrinsics/asin_complex16.f90
+++ b/flang/test/Lower/Intrinsics/asin_complex16.f90
@@ -2,7 +2,7 @@
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
-! CHECK: fir.call @_FortranACAsinF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+! CHECK: fir.call @_FortranACAsinF128({{.*}}){{.*}}: (complex<f128>) -> complex<f128>
   complex(16) :: a, b
   b = asin(a)
 end
diff --git a/flang/test/Lower/Intrinsics/asinh.f90 b/flang/test/Lower/Intrinsics/asinh.f90
index 328a4ce95884b..193a8ac7a3d9e 100644
--- a/flang/test/Lower/Intrinsics/asinh.f90
+++ b/flang/test/Lower/Intrinsics/asinh.f90
@@ -27,7 +27,7 @@ function test_complex4(x)
 end function
 
 ! ALL-LABEL: @_QPtest_complex4
-! ALL: {{%[A-Za-z0-9._]+}} = fir.call @casinhf({{%[A-Za-z0-9._]+}}) {{.*}}: (!fir.complex<4>) -> !fir.complex<4>
+! ALL: {{%[A-Za-z0-9._]+}} = fir.call @casinhf({{%[A-Za-z0-9._]+}}) {{.*}}: (complex<f32>) -> complex<f32>
 
 function test_complex8(x)
   complex(kind=8) :: x, test_complex8
@@ -35,5 +35,5 @@ function test_complex8(x)
 end function
 
 ! ALL-LABEL: @_QPtest_complex8
-! ALL: {{%[A-Za-z0-9._]+}} = fir.call @casinh({{%[A-Za-z0-9._]+}}) {{.*}}: (!fir.complex<8>) -> !fir.complex<8>
+! ALL: {{%[A-Za-z0-9._]+}} = fir.call @casinh({{%[A-Za-z0-9._]+}}) {{.*}}: (complex<f64>) -> complex<f64>
 
diff --git a/flang/test/Lower/Intrinsics/asinh_complex16.f90 b/flang/test/Lower/Intrinsics/asinh_complex16.f90
index 2d658a68f8a2b..97424753ee3fc 100644
--- a/flang/test/Lower/Intrinsics/asinh_complex16.f90
+++ b/flang/test/Lower/Intrinsics/asinh_complex16.f90
@@ -2,7 +2,7 @@
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
-! CHECK: fir.call @_FortranACAsinhF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+! CHECK: fir.call @_FortranACAsinhF128({{.*}}){{.*}}: (complex<f128>) -> complex<f128>
   complex(16) :: a, b
   b = asinh(a)
 end
diff --git a/flang/test/Lower/Intrinsics/atan.f90 b/flang/test/Lower/Intrinsics/atan.f90
index cf595b4d95d1d..478988eec6e0c 100644
--- a/flang/test/Lower/Intrinsics/atan.f90
+++ b/flang/test/Lower/Intrinsics/atan.f90
@@ -27,16 +27,16 @@ function test_complex4(x)
   test_complex4 = atan(x)
 end function
 
-! CHECK-RUNTIME: {{%[A-Za-z0-9._]+}} = fir.call @catanf({{.*}}) {{.*}}: (!fir.complex<4>) -> !fir.complex<4>
-! CHECK-NORMAL: {{%[A-Za-z0-9._]+}} = fir.call @catanf({{.*}}) {{.*}}: (!fir.complex<4>) -> !fir.complex<4>
+! CHECK-RUNTIME: {{%[A-Za-z0-9._]+}} = fir.call @catanf({{.*}}) {{.*}}: (complex<f32>) -> complex<f32>
+! CHECK-NORMAL: {{%[A-Za-z0-9._]+}} = fir.call @catanf({{.*}}) {{.*}}: (complex<f32>) -> complex<f32>
 
 function test_complex8(x)
   complex(kind=8) :: x, test_complex8
   test_complex8 = atan(x)
 end function
 
-! CHECK-RUNTIME: {{%[A-Za-z0-9._]+}} = fir.call @catan({{.*}}) {{.*}}: (!fir.complex<8>) -> !fir.complex<8>
-! CHECK-NORMAL: {{%[A-Za-z0-9._]+}} = fir.call @catan({{.*}}) {{.*}}: (!fir.complex<8>) -> !fir.complex<8>
+! CHECK-RUNTIME: {{%[A-Za-z0-9._]+}} = fir.call @catan({{.*}}) {{.*}}: (complex<f64>) -> complex<f64>
+! CHECK-NORMAL: {{%[A-Za-z0-9._]+}} = fir.call @catan({{.*}}) {{.*}}: (complex<f64>) -> complex<f64>
 
 function test_real4_2(y, x)
   real :: y, x, test_real4_2
diff --git a/flang/test/Lower/Intrinsics/atan_complex16.f90 b/flang/test/Lower/Intrinsics/atan_complex16.f90
index 315928d885683..7a1e53523cf61 100644
--- a/flang/test/Lower/Intrinsics/atan_complex16.f90
+++ b/flang/test/Lower/Intrinsics/atan_complex16.f90
@@ -2,7 +2,7 @@
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
-! CHECK: fir.call @_FortranACAtanF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+! CHECK: fir.call @_FortranACAtanF128({{.*}}){{.*}}: (complex<f128>) -> complex<f128>
   complex(16) :: a, b
   b = atan(a)
 end
diff --git a/flang/test/Lower/Intrinsics/atanh.f90 b/flang/test/Lower/Intrinsics/atanh.f90
index cf2eba946b560..a23b64f3414c4 100644
--- a/flang/test/Lower/Intrinsics/atanh.f90
+++ b/flang/test/Lower/Intrinsics/atanh.f90
@@ -27,7 +27,7 @@ function test_complex4(x)
 end function
 
 ! ALL-LABEL: @_QPtest_complex4
-! ALL: {{%[A-Za-z0-9._]+}} = fir.call @catanhf({{%[A-Za-z0-9._]+}}) {{.*}}: (!fir.complex<4>) -> !fir.complex<4>
+! ALL: {{%[A-Za-z0-9._]+}} = fir.call @catanhf({{%[A-Za-z0-9._]+}}) {{.*}}: (complex<f32>) -> complex<f32>
 
 function test_complex8(x)
   complex(kind=8) :: x, test_real8
@@ -35,4 +35,4 @@ function test_complex8(x)
 end function
 
 ! ALL-LABEL: @_QPtest_complex8
-! ALL: {{%[A-Za-z0-9._]+}} = fir.call @catanh({{%[A-Za-z0-9._]+}}) {{.*}}: (!fir.complex<8>) -> !fir.complex<8>
+! ALL: {{%[A-Za-z0-9._]+}} = fir.call @catanh({{%[A-Za-z0-9._]+}}) {{.*}}: (complex<f64>) -> complex<f64>
diff --git a/flang/test/Lower/Intrinsics/atanh_complex16.f90 b/flang/test/Lower/Intrinsics/atanh_complex16.f90
index 0d9f798ea4a2b..0777fa7263d05 100644
--- a/flang/test/Lower/Intrinsics/atanh_complex16.f90
+++ b/flang/test/Lower/Intrinsics/atanh_complex16.f90
@@ -2,7 +2,7 @@
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
-! CHECK: fir.call @_FortranACAtanhF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+! CHECK: fir.call @_FortranACAtanhF128({{.*}}){{.*}}: (complex<f128>) -> complex<f128>
   complex(16) :: a, b
   b = atanh(a)
 end
diff --git a/flang/test/Lower/Intrinsics/cabs_real16.f90 b/flang/test/Lower/Intrinsics/cabs_real16.f90
index 363b154a8e7fe..c1ec0267c0f98 100644
--- a/flang/test/Lower/Intrinsics/cabs_real16.f90
+++ b/flang/test/Lower/Intrinsics/cabs_real16.f90
@@ -3,7 +3,7 @@
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
-! CHECK: fir.call @_FortranACAbsF128({{.*}}){{.*}}: (!fir.complex<16>) -> f128
+! CHECK: fir.call @_FortranACAbsF128({{.*}}){{.*}}: (complex<f128>) -> f128
   complex(16) :: a
   real(16) :: b
   b = abs(a)
diff --git a/flang/test/Lower/Intrinsics/cmplx.f90 b/flang/test/Lower/Intrinsics/cmplx.f90
index fe08920ece512..13da3efa33427 100644
--- a/flang/test/Lower/Intrinsics/cmplx.f90
+++ b/flang/test/Lower/Intrinsics/cmplx.f90
@@ -27,9 +27,9 @@ subroutine cmplx_test_scalar_ptr(x, y)
 ! CHECK:    %[[VAL_17:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:    fir.result %[[VAL_17]] : f32
 ! CHECK:  }
-! CHECK:  %[[VAL_18:.*]] = fir.undefined !fir.complex<4>
-! CHECK:  %[[VAL_19:.*]] = fir.insert_value %[[VAL_18]], %[[VAL_7]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:   fir.insert_value %[[VAL_19]], %[[VAL_21:.*]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
+! CHECK:  %[[VAL_18:.*]] = fir.undefined complex<f32>
+! CHECK:  %[[VAL_19:.*]] = fir.insert_value %[[VAL_18]], %[[VAL_7]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:   fir.insert_value %[[VAL_19]], %[[VAL_21:.*]], [1 : index] : (complex<f32>, f32) -> complex<f32>
 end subroutine
 
 ! CHECK-LABEL: func @_QPcmplx_test_scalar_optional(
@@ -48,9 +48,9 @@ subroutine cmplx_test_scalar_optional(x, y)
 ! CHECK:    %[[VAL_11:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:    fir.result %[[VAL_11]] : f32
 ! CHECK:  }
-! CHECK:  %[[VAL_12:.*]] = fir.undefined !fir.complex<4>
-! CHECK:  %[[VAL_13:.*]] = fir.insert_value %[[VAL_12]], %[[VAL_7]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:  fir.insert_value %[[VAL_13]], %[[VAL_15:.*]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
+! CHECK:  %[[VAL_12:.*]] = fir.undefined complex<f32>
+! CHECK:  %[[VAL_13:.*]] = fir.insert_value %[[VAL_12]], %[[VAL_7]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:  fir.insert_value %[[VAL_13]], %[[VAL_15:.*]], [1 : index] : (complex<f32>, f32) -> complex<f32>
 end subroutine
 
 ! CHECK-LABEL: func @_QPcmplx_test_scalar_alloc_optional(
@@ -76,9 +76,9 @@ subroutine cmplx_test_scalar_alloc_optional(x, y)
 ! CHECK:    fir.result %[[VAL_17]] : i64
 ! CHECK:  }
 ! CHECK:  %[[VAL_18:.*]] = fir.convert %[[VAL_19:.*]] : (i64) -> f32
-! CHECK:  %[[VAL_20:.*]] = fir.undefined !fir.complex<4>
-! CHECK:  %[[VAL_21:.*]] = fir.insert_value %[[VAL_20]], %[[VAL_7]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:  fir.insert_value %[[VAL_21]], %[[VAL_18]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
+! CHECK:  %[[VAL_20:.*]] = fir.undefined complex<f32>
+! CHECK:  %[[VAL_21:.*]] = fir.insert_value %[[VAL_20]], %[[VAL_7]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:  fir.insert_value %[[VAL_21]], %[[VAL_18]], [1 : index] : (complex<f32>, f32) -> complex<f32>
 end subroutine
 
 ! CHECK-LABEL: func @_QPcmplx_test_pointer_result(
@@ -108,9 +108,9 @@ function return_pointer()
 ! CHECK:    %[[VAL_19:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:    fir.result %[[VAL_19]] : f32
 ! CHECK:  }
-! CHECK:  %[[VAL_20:.*]] = fir.undefined !fir.complex<4>
-! CHECK:  %[[VAL_21:.*]] = fir.insert_value %[[VAL_20]], %[[VAL_8]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:  fir.insert_value %[[VAL_21]], %[[VAL_23:.*]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
+! CHECK:  %[[VAL_20:.*]] = fir.undefined complex<f32>
+! CHECK:  %[[VAL_21:.*]] = fir.insert_value %[[VAL_20]], %[[VAL_8]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:  fir.insert_value %[[VAL_21]], %[[VAL_23:.*]], [1 : index] : (complex<f32>, f32) -> complex<f32>
 end subroutine
 
 ! CHECK-LABEL: func @_QPcmplx_array(
@@ -132,13 +132,13 @@ subroutine cmplx_array(x, y)
 ! CHECK:  %[[VAL_14:.*]] = fir.embox %[[VAL_11]](%[[VAL_13]]) : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
 ! CHECK:  %[[VAL_15:.*]] = arith.select %[[VAL_10]], %[[VAL_1]], %[[VAL_14]] : !fir.box<!fir.array<?xf32>>
 ! CHECK:  %[[VAL_16:.*]] = fir.array_load %[[VAL_15]] {fir.optional} : (!fir.box<!fir.array<?xf32>>) -> !fir.array<?xf32>
-! CHECK:  %[[VAL_17:.*]] = fir.allocmem !fir.array<?x!fir.complex<4>>, %[[VAL_8]]#1 {uniq_name = ".array.expr"}
+! CHECK:  %[[VAL_17:.*]] = fir.allocmem !fir.array<?xcomplex<f32>>, %[[VAL_8]]#1 {uniq_name = ".array.expr"}
 ! CHECK:  %[[VAL_18:.*]] = fir.shape %[[VAL_8]]#1 : (index) -> !fir.shape<1>
-! CHECK:  %[[VAL_19:.*]] = fir.array_load %[[VAL_17]](%[[VAL_18]]) : (!fir.heap<!fir.array<?x!fir.complex<4>>>, !fir.shape<1>) -> !fir.array<?x!fir.complex<4>>
+! CHECK:  %[[VAL_19:.*]] = fir.array_load %[[VAL_17]](%[[VAL_18]]) : (!fir.heap<!fir.array<?xcomplex<f32>>>, !fir.shape<1>) -> !fir.array<?xcomplex<f32>>
 ! CHECK:  %[[VAL_20:.*]] = arith.constant 1 : index
 ! CHECK:  %[[VAL_21:.*]] = arith.constant 0 : index
 ! CHECK:  %[[VAL_22:.*]] = arith.subi %[[VAL_8]]#1, %[[VAL_20]] : index
-! CHECK:  %[[VAL_23:.*]] = fir.do_loop %[[VAL_24:.*]] = %[[VAL_21]] to %[[VAL_22]] step %[[VAL_20]] unordered iter_args(%[[VAL_25:.*]] = %[[VAL_19]]) -> (!fir.array<?x!fir.complex<4>>) {
+! CHECK:  %[[VAL_23:.*]] = fir.do_loop %[[VAL_24:.*]] = %[[VAL_21]] to %[[VAL_22]] step %[[VAL_20]] unordered iter_args(%[[VAL_25:.*]] = %[[VAL_19]]) -> (!fir.array<?xcomplex<f32>>) {
   ! CHECK:  %[[VAL_26:.*]] = fir.array_fetch %[[VAL_9]], %[[VAL_24]] : (!fir.array<?xf32>, index) -> f32
   ! CHECK:  %[[VAL_27:.*]] = fir.if %[[VAL_10]] -> (f32) {
     ! CHECK:  %[[VAL_28:.*]] = fir.array_fetch %[[VAL_16]], %[[VAL_24]] : (!fir.array<?xf32>, index) -> f32
@@ -147,11 +147,11 @@ subroutine cmplx_array(x, y)
     ! CHECK:  %[[VAL_29:.*]] = arith.constant 0.000000e+00 : f32
     ! CHECK:  fir.result %[[VAL_29]] : f32
   ! CHECK:  }
-  ! CHECK:  %[[VAL_30:.*]] = fir.undefined !fir.complex<4>
-  ! CHECK:  %[[VAL_31:.*]] = fir.insert_value %[[VAL_30]], %[[VAL_26]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-  ! CHECK:  %[[VAL_32:.*]] = fir.insert_value %[[VAL_31]], %[[VAL_33:.*]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-  ! CHECK:  %[[VAL_34:.*]] = fir.array_update %[[VAL_25]], %[[VAL_32]], %[[VAL_24]] : (!fir.array<?x!fir.complex<4>>, !fir.complex<4>, index) -> !fir.array<?x!fir.complex<4>>
-  ! CHECK:  fir.result %[[VAL_34]] : !fir.array<?x!fir.complex<4>>
+  ! CHECK:  %[[VAL_30:.*]] = fir.undefined complex<f32>
+  ! CHECK:  %[[VAL_31:.*]] = fir.insert_value %[[VAL_30]], %[[VAL_26]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+  ! CHECK:  %[[VAL_32:.*]] = fir.insert_value %[[VAL_31]], %[[VAL_33:.*]], [1 : index] : (complex<f32>, f32) -> complex<f32>
+  ! CHECK:  %[[VAL_34:.*]] = fir.array_update %[[VAL_25]], %[[VAL_32]], %[[VAL_24]] : (!fir.array<?xcomplex<f32>>, complex<f32>, index) -> !fir.array<?xcomplex<f32>>
+  ! CHECK:  fir.result %[[VAL_34]] : !fir.array<?xcomplex<f32>>
 ! CHECK:  }
 ! CHECK:  fir.array_merge_store
 end subroutine
diff --git a/flang/test/Lower/Intrinsics/cos_complex16.f90 b/flang/test/Lower/Intrinsics/cos_complex16.f90
index 79b89ce057e12..05b046deb73df 100644
--- a/flang/test/Lower/Intrinsics/cos_complex16.f90
+++ b/flang/test/Lower/Intrinsics/cos_complex16.f90
@@ -2,7 +2,7 @@
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
-! CHECK: fir.call @_FortranACCosF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+! CHECK: fir.call @_FortranACCosF128({{.*}}){{.*}}: (complex<f128>) -> complex<f128>
   complex(16) :: a, b
   b = cos(a)
 end
diff --git a/flang/test/Lower/Intrinsics/cosh_complex16.f90 b/flang/test/Lower/Intrinsics/cosh_complex16.f90
index dc1723b458204..8756672f72b3a 100644
--- a/flang/test/Lower/Intrinsics/cosh_complex16.f90
+++ b/flang/test/Lower/Intrinsics/cosh_complex16.f90
@@ -2,7 +2,7 @@
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
-! CHECK: fir.call @_FortranACCoshF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+! CHECK: fir.call @_FortranACCoshF128({{.*}}){{.*}}: (complex<f128>) -> complex<f128>
   complex(16) :: a, b
   b = cosh(a)
 end
diff --git a/flang/test/Lower/Intrinsics/dconjg.f90 b/flang/test/Lower/Intrinsics/dconjg.f90
index dcd3ff06736c1..1362b4ccf8c03 100644
--- a/flang/test/Lower/Intrinsics/dconjg.f90
+++ b/flang/test/Lower/Intrinsics/dconjg.f90
@@ -5,13 +5,13 @@ subroutine test_dconjg(r, c)
   complex(8), intent(in) :: c
 
 ! CHECK-LABEL: func @_QPtest_dconjg(
-! CHECK-SAME: %[[ARG_0:.*]]: !fir.ref<!fir.complex<8>> {fir.bindc_name = "r"},
-! CHECK-SAME: %[[ARG_1:.*]]: !fir.ref<!fir.complex<8>> {fir.bindc_name = "c"}) {
-! CHECK:   %[[VAL_0:.*]] = fir.load %[[ARG_1]] : !fir.ref<!fir.complex<8>>
-! CHECK:   %[[VAL_1:.*]] = fir.extract_value %[[VAL_0]], [1 : index] : (!fir.complex<8>) -> f64
+! CHECK-SAME: %[[ARG_0:.*]]: !fir.ref<complex<f64>> {fir.bindc_name = "r"},
+! CHECK-SAME: %[[ARG_1:.*]]: !fir.ref<complex<f64>> {fir.bindc_name = "c"}) {
+! CHECK:   %[[VAL_0:.*]] = fir.load %[[ARG_1]] : !fir.ref<complex<f64>>
+! CHECK:   %[[VAL_1:.*]] = fir.extract_value %[[VAL_0]], [1 : index] : (complex<f64>) -> f64
 ! CHECK:   %[[VAL_2:.*]] = arith.negf %[[VAL_1]] {{.*}}: f64
-! CHECK:   %[[VAL_3:.*]] = fir.insert_value %[[VAL_0]], %[[VAL_2]], [1 : index] : (!fir.complex<8>, f64) -> !fir.complex<8>
-! CHECK:   fir.store %[[VAL_3]] to %[[ARG_0]] : !fir.ref<!fir.complex<8>>
+! CHECK:   %[[VAL_3:.*]] = fir.insert_value %[[VAL_0]], %[[VAL_2]], [1 : index] : (complex<f64>, f64) -> complex<f64>
+! CHECK:   fir.store %[[VAL_3]] to %[[ARG_0]] : !fir.ref<complex<f64>>
 ! CHECK:   return
 ! CHECK: }
 
diff --git a/flang/test/Lower/Intrinsics/dimag.f90 b/flang/test/Lower/Intrinsics/dimag.f90
index 03ec711b8f355..cc0bd3167d7f7 100644
--- a/flang/test/Lower/Intrinsics/dimag.f90
+++ b/flang/test/Lower/Intrinsics/dimag.f90
@@ -6,9 +6,9 @@ subroutine test_dimag(r, c)
 
 ! CHECK-LABEL: func @_QPtest_dimag(
 ! CHECK-SAME: %[[ARG_0:.*]]: !fir.ref<f64> {fir.bindc_name = "r"},
-! CHECK-SAME: %[[ARG_1:.*]]: !fir.ref<!fir.complex<8>> {fir.bindc_name = "c"}) {
-! CHECK:   %[[VAL_0:.*]] = fir.load %[[ARG_1]] : !fir.ref<!fir.complex<8>>
-! CHECK:   %[[VAL_1:.*]] = fir.extract_value %[[VAL_0]], [1 : index] : (!fir.complex<8>) -> f64
+! CHECK-SAME: %[[ARG_1:.*]]: !fir.ref<complex<f64>> {fir.bindc_name = "c"}) {
+! CHECK:   %[[VAL_0:.*]] = fir.load %[[ARG_1]] : !fir.ref<complex<f64>>
+! CHECK:   %[[VAL_1:.*]] = fir.extract_value %[[VAL_0]], [1 : index] : (complex<f64>) -> f64
 ! CHECK:   fir.store %[[VAL_1]] to %[[ARG_0]] : !fir.ref<f64>
 ! CHECK:   return
 ! CHECK: }
diff --git a/flang/test/Lower/Intrinsics/dot_product.f90 b/flang/test/Lower/Intrinsics/dot_product.f90
index 83d6fc4b3f759..e67e9d598cd84 100644
--- a/flang/test/Lower/Intrinsics/dot_product.f90
+++ b/flang/test/Lower/Intrinsics/dot_product.f90
@@ -159,77 +159,72 @@ subroutine dot_prod_double_default (x, y, z)
 end subroutine
 
 ! CHECK-LABEL: dot_prod_complex_default
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?x!fir.complex<4>>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?x!fir.complex<4>>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?x!fir.complex<4>>>
+! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xcomplex<f32>>>
+! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xcomplex<f32>>>
+! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xcomplex<f32>>>
 subroutine dot_prod_complex_default (x, y, z)
   complex, dimension(1:) :: x,y
   complex, dimension(1:) :: z
-  ! CHECK-DAG: %0 = fir.alloca !fir.complex<4>
-  ! CHECK-DAG: %[[res_conv:[0-9]+]] = fir.convert %0 : (!fir.ref<!fir.complex<4>>) -> !fir.ref<complex<f32>>
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> !fir.box<none>
-  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex4(%[[res_conv]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f32>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
+  ! CHECK-DAG: %[[res:.*]] = fir.alloca complex<f32>
+  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.box<none>
+  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.box<none>
+  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex4(%[[res]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f32>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
   z = dot_product(x,y)
 end subroutine
 
 ! CHECK-LABEL: dot_prod_complex_kind_4
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?x!fir.complex<4>>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?x!fir.complex<4>>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?x!fir.complex<4>>>
+! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xcomplex<f32>>>
+! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xcomplex<f32>>>
+! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xcomplex<f32>>>
 subroutine dot_prod_complex_kind_4 (x, y, z)
   complex(kind=4), dimension(1:) :: x,y
   complex(kind=4), dimension(1:) :: z
-  ! CHECK-DAG: %0 = fir.alloca !fir.complex<4>
-  ! CHECK-DAG: %[[res_conv:[0-9]+]] = fir.convert %0 : (!fir.ref<!fir.complex<4>>) -> !fir.ref<complex<f32>>
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> !fir.box<none>
-  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex4(%[[res_conv]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f32>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
+  ! CHECK-DAG: %[[res:.*]] = fir.alloca complex<f32>
+  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.box<none>
+  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.box<none>
+  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex4(%[[res]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f32>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
   z = dot_product(x,y)
 end subroutine
 
 ! CHECK-LABEL: dot_prod_complex_kind_8
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?x!fir.complex<8>>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?x!fir.complex<8>>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?x!fir.complex<8>>>
+! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xcomplex<f64>>>
+! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xcomplex<f64>>>
+! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xcomplex<f64>>>
 subroutine dot_prod_complex_kind_8 (x, y, z)
   complex(kind=8), dimension(1:) :: x,y
   complex(kind=8), dimension(1:) :: z
-  ! CHECK-DAG: %0 = fir.alloca !fir.complex<8>
-  ! CHECK-DAG: %[[res_conv:[0-9]+]] = fir.convert %0 : (!fir.ref<!fir.complex<8>>) -> !fir.ref<complex<f64>>
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?x!fir.complex<8>>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?x!fir.complex<8>>>) -> !fir.box<none>
-  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex8(%[[res_conv]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f64>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
+  ! CHECK-DAG: %[[res:.*]] = fir.alloca complex<f64>
+  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xcomplex<f64>>>) -> !fir.box<none>
+  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xcomplex<f64>>>) -> !fir.box<none>
+  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex8(%[[res]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f64>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
   z = dot_product(x,y)
 end subroutine
 
 ! CHECK-LABEL: dot_prod_complex_kind_10
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?x!fir.complex<10>>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?x!fir.complex<10>>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?x!fir.complex<10>>>
+! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xcomplex<f80>>>
+! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xcomplex<f80>>>
+! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xcomplex<f80>>>
 subroutine dot_prod_complex_kind_10 (x, y, z)
   complex(kind=10), dimension(1:) :: x,y
   complex(kind=10), dimension(1:) :: z
-  ! CHECK-DAG: %0 = fir.alloca !fir.complex<10>
-  ! CHECK-DAG: %[[res_conv:[0-9]+]] = fir.convert %0 : (!fir.ref<!fir.complex<10>>) -> !fir.ref<complex<f80>>
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?x!fir.complex<10>>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?x!fir.complex<10>>>) -> !fir.box<none>
-  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex10(%[[res_conv]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f80>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+  ! CHECK-DAG: %[[res:.*]] = fir.alloca complex<f80>
+  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xcomplex<f80>>>) -> !fir.box<none>
+  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xcomplex<f80>>>) -> !fir.box<none>
+  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex10(%[[res]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f80>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
   z = dot_product(x,y)
 end subroutine
 
 ! CHECK-LABEL: dot_prod_complex_kind_16
-! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?x!fir.complex<16>>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?x!fir.complex<16>>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?x!fir.complex<16>>>
+! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xcomplex<f128>>>
+! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xcomplex<f128>>>
+! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xcomplex<f128>>>
 subroutine dot_prod_complex_kind_16 (x, y, z)
   complex(kind=16), dimension(1:) :: x,y
   complex(kind=16), dimension(1:) :: z
-  ! CHECK-DAG: %0 = fir.alloca !fir.complex<16>
-  ! CHECK-DAG: %[[res_conv:[0-9]+]] = fir.convert %0 : (!fir.ref<!fir.complex<16>>) -> !fir.ref<complex<f128>>
-  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?x!fir.complex<16>>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?x!fir.complex<16>>>) -> !fir.box<none>
-  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex16(%[[res_conv]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f128>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+  ! CHECK-DAG: %[[res:.*]] = fir.alloca complex<f128>
+  ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xcomplex<f128>>>) -> !fir.box<none>
+  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xcomplex<f128>>>) -> !fir.box<none>
+  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex16(%[[res]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f128>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> ()
   z = dot_product(x,y)
 end subroutine
 
@@ -261,30 +256,28 @@ subroutine dot_product_mixed_int_real(x, y, z)
 
 ! CHECK-LABEL: dot_product_mixed_int_complex
 ! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xi32>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?x!fir.complex<4>>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?x!fir.complex<4>>>
+! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xcomplex<f32>>>
+! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xcomplex<f32>>>
 subroutine dot_product_mixed_int_complex(x, y, z)
   integer, dimension(1:) :: x
   complex, dimension(1:) :: y, z
-  ! CHECK-DAG: %[[res:.*]] = fir.alloca !fir.complex<4>
-  ! CHECK-DAG: %[[res_conv:.*]] = fir.convert %[[res]] : (!fir.ref<!fir.complex<4>>) -> !fir.ref<complex<f32>>
+  ! CHECK-DAG: %[[res:.*]] = fir.alloca complex<f32>
   ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> !fir.box<none>
-  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex4(%[[res_conv]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f32>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
+  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.box<none>
+  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex4(%[[res]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f32>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
   z = dot_product(x,y)
 end subroutine
 
 ! CHECK-LABEL: dot_product_mixed_real_complex
 ! CHECK-SAME: %[[x:arg0]]: !fir.box<!fir.array<?xf32>>
-! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?x!fir.complex<4>>>
-! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?x!fir.complex<4>>>
+! CHECK-SAME: %[[y:arg1]]: !fir.box<!fir.array<?xcomplex<f32>>>
+! CHECK-SAME: %[[z:arg2]]: !fir.box<!fir.array<?xcomplex<f32>>>
 subroutine dot_product_mixed_real_complex(x, y, z)
   real, dimension(1:) :: x
   complex, dimension(1:) :: y, z
-  ! CHECK-DAG: %[[res:.*]] = fir.alloca !fir.complex<4>
-  ! CHECK-DAG: %[[res_conv:.*]] = fir.convert %[[res]] : (!fir.ref<!fir.complex<4>>) -> !fir.ref<complex<f32>>
+  ! CHECK-DAG: %[[res:.*]] = fir.alloca complex<f32>
   ! CHECK-DAG: %[[x_conv:.*]] = fir.convert %[[x]] : (!fir.box<!fir.array<?xf32>>) -> !fir.box<none>
-  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> !fir.box<none>
-  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex4(%[[res_conv]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f32>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
+  ! CHECK-DAG: %[[y_conv:.*]] = fir.convert %[[y]] : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.box<none>
+  ! CHECK-DAG: fir.call @_FortranACppDotProductComplex4(%[[res]], %[[x_conv]], %[[y_conv]], %{{[0-9]+}}, %{{.*}}) {{.*}}: (!fir.ref<complex<f32>>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
   z = dot_product(x,y)
 end subroutine
diff --git a/flang/test/Lower/Intrinsics/dreal.f90 b/flang/test/Lower/Intrinsics/dreal.f90
index 2794262a3e2d3..c911bc9fea59d 100644
--- a/flang/test/Lower/Intrinsics/dreal.f90
+++ b/flang/test/Lower/Intrinsics/dreal.f90
@@ -6,9 +6,9 @@ subroutine test_dreal(r, c)
 
 ! CHECK-LABEL: func @_QPtest_dreal(
 ! CHECK-SAME: %[[ARG_0:.*]]: !fir.ref<f64> {fir.bindc_name = "r"},
-! CHECK-SAME: %[[ARG_1:.*]]: !fir.ref<!fir.complex<8>> {fir.bindc_name = "c"}) {
-! CHECK:   %[[VAL_0:.*]] = fir.load %[[ARG_1]] : !fir.ref<!fir.complex<8>>
-! CHECK:   %[[VAL_1:.*]] = fir.extract_value %[[VAL_0]], [0 : index] : (!fir.complex<8>) -> f64
+! CHECK-SAME: %[[ARG_1:.*]]: !fir.ref<complex<f64>> {fir.bindc_name = "c"}) {
+! CHECK:   %[[VAL_0:.*]] = fir.load %[[ARG_1]] : !fir.ref<complex<f64>>
+! CHECK:   %[[VAL_1:.*]] = fir.extract_value %[[VAL_0]], [0 : index] : (complex<f64>) -> f64
 ! CHECK:   fir.store %[[VAL_1]] to %[[ARG_0]] : !fir.ref<f64>
 ! CHECK:   return
 ! CHECK: }
diff --git a/flang/test/Lower/Intrinsics/exp.f90 b/flang/test/Lower/Intrinsics/exp.f90
index 15c6ebe2db71d..9822a4feba662 100644
--- a/flang/test/Lower/Intrinsics/exp.f90
+++ b/flang/test/Lower/Intrinsics/exp.f90
@@ -27,22 +27,22 @@ subroutine exp_testd(a, b)
 end subroutine
 
 ! CHECK-LABEL: exp_testc
-! CHECK-SAME: (%[[AREF:.*]]: !fir.ref<!fir.complex<4>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<4>> {{.*}})
+! CHECK-SAME: (%[[AREF:.*]]: !fir.ref<complex<f32>> {{.*}}, %[[BREF:.*]]: !fir.ref<complex<f32>> {{.*}})
 subroutine exp_testc(a, b)
   complex :: a, b
-! CHECK:  %[[A:.*]] = fir.load %[[AREF:.*]] : !fir.ref<!fir.complex<4>>
-! CHECK:  %[[RES:.*]] = fir.call @fir.exp.contract.z4.z4(%[[A]]) {{.*}}: (!fir.complex<4>) -> !fir.complex<4>
-! CHECK:  fir.store %[[RES]] to %[[BREF]] : !fir.ref<!fir.complex<4>>
+! CHECK:  %[[A:.*]] = fir.load %[[AREF:.*]] : !fir.ref<complex<f32>>
+! CHECK:  %[[RES:.*]] = fir.call @fir.exp.contract.z32.z32(%[[A]]) {{.*}}: (complex<f32>) -> complex<f32>
+! CHECK:  fir.store %[[RES]] to %[[BREF]] : !fir.ref<complex<f32>>
   b = exp(a)
 end subroutine
 
 ! CHECK-LABEL: exp_testcd
-! CHECK-SAME: (%[[AREF:.*]]: !fir.ref<!fir.complex<8>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<8>> {{.*}})
+! CHECK-SAME: (%[[AREF:.*]]: !fir.ref<complex<f64>> {{.*}}, %[[BREF:.*]]: !fir.ref<complex<f64>> {{.*}})
 subroutine exp_testcd(a, b)
   complex(kind=8) :: a, b
-! CHECK:  %[[A:.*]] = fir.load %[[AREF:.*]] : !fir.ref<!fir.complex<8>>
-! CHECK:  %[[RES:.*]] = fir.call @fir.exp.contract.z8.z8(%[[A]]) {{.*}}: (!fir.complex<8>) -> !fir.complex<8>
-! CHECK:  fir.store %[[RES]] to %[[BREF]] : !fir.ref<!fir.complex<8>>
+! CHECK:  %[[A:.*]] = fir.load %[[AREF:.*]] : !fir.ref<complex<f64>>
+! CHECK:  %[[RES:.*]] = fir.call @fir.exp.contract.z64.z64(%[[A]]) {{.*}}: (complex<f64>) -> complex<f64>
+! CHECK:  fir.store %[[RES]] to %[[BREF]] : !fir.ref<complex<f64>>
   b = exp(a)
 end subroutine
 
@@ -56,28 +56,20 @@ subroutine exp_testcd(a, b)
 ! CHECK: %[[RESULT64_OUTLINE:.*]] = math.exp %[[ARG64_OUTLINE]] fastmath<contract> : f64
 ! CHECK: return %[[RESULT64_OUTLINE]] : f64
 
-! CMPLX-APPROX-LABEL: private @fir.exp.contract_afn.z4.z4
-! CMPLX-PRECISE-LABEL: private @fir.exp.contract.z4.z4
-! CMPLX-MLIR-LABEL: private @fir.exp.contract.z4.z4
-! CMPLX-SAME: (%[[ARG32_OUTLINE:.*]]: !fir.complex<4>) -> !fir.complex<4>
-! CMPLX-FAST: %[[C:.*]] = fir.convert %[[ARG32_OUTLINE]] : (!fir.complex<4>) -> complex<f32>
+! CMPLX-APPROX-LABEL: private @fir.exp.contract_afn.z32.z32
+! CMPLX-PRECISE-LABEL: private @fir.exp.contract.z32.z32
+! CMPLX-MLIR-LABEL: private @fir.exp.contract.z32.z32
+! CMPLX-SAME: (%[[C:.*]]: complex<f32>) -> complex<f32>
 ! CMPLX-FAST: %[[E:.*]] = complex.exp %[[C]] fastmath<contract> : complex<f32>
-! CMPLX-FAST: %[[RESULT32_OUTLINE:.*]] = fir.convert %[[E]] : (complex<f32>) -> !fir.complex<4>
-! CMPLX-APPROX: %[[C:.*]] = fir.convert %[[ARG32_OUTLINE]] : (!fir.complex<4>) -> complex<f32>
 ! CMPLX-APPROX: %[[E:.*]] = complex.exp %[[C]] fastmath<contract,afn> : complex<f32>
-! CMPLX-APPROX: %[[RESULT32_OUTLINE:.*]] = fir.convert %[[E]] : (complex<f32>) -> !fir.complex<4>
-! CMPLX-PRECISE: %[[RESULT32_OUTLINE:.*]] = fir.call @cexpf(%[[ARG32_OUTLINE]]) fastmath<contract> : (!fir.complex<4>) -> !fir.complex<4>
-! CMPLX: return %[[RESULT32_OUTLINE]] : !fir.complex<4>
+! CMPLX-PRECISE: %[[E:.*]] = fir.call @cexpf(%[[C]]) fastmath<contract> : (complex<f32>) -> complex<f32>
+! CMPLX: return %[[E]] : complex<f32>
 
-! CMPLX-APPROX-LABEL: private @fir.exp.contract_afn.z8.z8
-! CMPLX-PRECISE-LABEL: private @fir.exp.contract.z8.z8
-! CMPLX-MLIR-LABEL: private @fir.exp.contract.z8.z8
-! CMPLX-SAME: (%[[ARG64_OUTLINE:.*]]: !fir.complex<8>) -> !fir.complex<8>
-! CMPLX-FAST: %[[C:.*]] = fir.convert %[[ARG64_OUTLINE]] : (!fir.complex<8>) -> complex<f64>
+! CMPLX-APPROX-LABEL: private @fir.exp.contract_afn.z64.z64
+! CMPLX-PRECISE-LABEL: private @fir.exp.contract.z64.z64
+! CMPLX-MLIR-LABEL: private @fir.exp.contract.z64.z64
+! CMPLX-SAME: (%[[C:.*]]: complex<f64>) -> complex<f64>
 ! CMPLX-FAST: %[[E:.*]] = complex.exp %[[C]] fastmath<contract> : complex<f64>
-! CMPLX-FAST: %[[RESULT64_OUTLINE:.*]] = fir.convert %[[E]] : (complex<f64>) -> !fir.complex<8>
-! CMPLX-APPROX: %[[C:.*]] = fir.convert %[[ARG64_OUTLINE]] : (!fir.complex<8>) -> complex<f64>
 ! CMPLX-APPROX: %[[E:.*]] = complex.exp %[[C]] fastmath<contract,afn> : complex<f64>
-! CMPLX-APPROX: %[[RESULT64_OUTLINE:.*]] = fir.convert %[[E]] : (complex<f64>) -> !fir.complex<8>
-! CMPLX-PRECISE: %[[RESULT64_OUTLINE:.*]] = fir.call @cexp(%[[ARG64_OUTLINE]]) fastmath<contract> : (!fir.complex<8>) -> !fir.complex<8>
-! CMPLX: return %[[RESULT64_OUTLINE]] : !fir.complex<8>
+! CMPLX-PRECISE: %[[E:.*]] = fir.call @cexp(%[[C]]) fastmath<contract> : (complex<f64>) -> complex<f64>
+! CMPLX: return %[[E]] : complex<f64>
diff --git a/flang/test/Lower/Intrinsics/exp_complex16.f90 b/flang/test/Lower/Intrinsics/exp_complex16.f90
index 972285c654bfe..40a91b1beb8a5 100644
--- a/flang/test/Lower/Intrinsics/exp_complex16.f90
+++ b/flang/test/Lower/Intrinsics/exp_complex16.f90
@@ -2,7 +2,7 @@
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
-! CHECK: fir.call @_FortranACExpF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+! CHECK: fir.call @_FortranACExpF128({{.*}}){{.*}}: (complex<f128>) -> complex<f128>
   complex(16) :: a, b
   b = exp(a)
 end
diff --git a/flang/test/Lower/Intrinsics/log.f90 b/flang/test/Lower/Intrinsics/log.f90
index 63af1c5e9e556..5ede3f38e7016 100644
--- a/flang/test/Lower/Intrinsics/log.f90
+++ b/flang/test/Lower/Intrinsics/log.f90
@@ -27,22 +27,22 @@ subroutine log_testd(a, b)
 end subroutine
 
 ! CHECK-LABEL: log_testc
-! CHECK-SAME: (%[[AREF:.*]]: !fir.ref<!fir.complex<4>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<4>> {{.*}})
+! CHECK-SAME: (%[[AREF:.*]]: !fir.ref<complex<f32>> {{.*}}, %[[BREF:.*]]: !fir.ref<complex<f32>> {{.*}})
 subroutine log_testc(a, b)
   complex :: a, b
-! CHECK:  %[[A:.*]] = fir.load %[[AREF:.*]] : !fir.ref<!fir.complex<4>>
-! CHECK:  %[[RES:.*]] = fir.call @fir.log.contract.z4.z4(%[[A]]) {{.*}}: (!fir.complex<4>) -> !fir.complex<4>
-! CHECK:  fir.store %[[RES]] to %[[BREF]] : !fir.ref<!fir.complex<4>>
+! CHECK:  %[[A:.*]] = fir.load %[[AREF:.*]] : !fir.ref<complex<f32>>
+! CHECK:  %[[RES:.*]] = fir.call @fir.log.contract.z32.z32(%[[A]]) {{.*}}: (complex<f32>) -> complex<f32>
+! CHECK:  fir.store %[[RES]] to %[[BREF]] : !fir.ref<complex<f32>>
   b = log(a)
 end subroutine
 
 ! CHECK-LABEL: log_testcd
-! CHECK-SAME: (%[[AREF:.*]]: !fir.ref<!fir.complex<8>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<8>> {{.*}})
+! CHECK-SAME: (%[[AREF:.*]]: !fir.ref<complex<f64>> {{.*}}, %[[BREF:.*]]: !fir.ref<complex<f64>> {{.*}})
 subroutine log_testcd(a, b)
   complex(kind=8) :: a, b
-! CHECK:  %[[A:.*]] = fir.load %[[AREF:.*]] : !fir.ref<!fir.complex<8>>
-! CHECK:  %[[RES:.*]] = fir.call @fir.log.contract.z8.z8(%[[A]]) {{.*}}: (!fir.complex<8>) -> !fir.complex<8>
-! CHECK:  fir.store %[[RES]] to %[[BREF]] : !fir.ref<!fir.complex<8>>
+! CHECK:  %[[A:.*]] = fir.load %[[AREF:.*]] : !fir.ref<complex<f64>>
+! CHECK:  %[[RES:.*]] = fir.call @fir.log.contract.z64.z64(%[[A]]) {{.*}}: (complex<f64>) -> complex<f64>
+! CHECK:  fir.store %[[RES]] to %[[BREF]] : !fir.ref<complex<f64>>
   b = log(a)
 end subroutine
 
@@ -76,31 +76,23 @@ subroutine log10_testd(a, b)
 ! CHECK: %[[RESULT64_OUTLINE:.*]] = math.log %[[ARG64_OUTLINE]] fastmath<contract> : f64
 ! CHECK: return %[[RESULT64_OUTLINE]] : f64
 
-! CMPLX-APPROX-LABEL: private @fir.log.contract_afn.z4.z4
-! CMPLX-PRECISE-LABEL: private @fir.log.contract.z4.z4
-! CMPLX-MLIR-LABEL: private @fir.log.contract.z4.z4
-! CMPLX-SAME: (%[[ARG32_OUTLINE:.*]]: !fir.complex<4>) -> !fir.complex<4>
-! CMPLX-FAST: %[[C:.*]] = fir.convert %[[ARG32_OUTLINE]] : (!fir.complex<4>) -> complex<f32>
+! CMPLX-APPROX-LABEL: private @fir.log.contract_afn.z32.z32
+! CMPLX-PRECISE-LABEL: private @fir.log.contract.z32.z32
+! CMPLX-MLIR-LABEL: private @fir.log.contract.z32.z32
+! CMPLX-SAME: (%[[C:.*]]: complex<f32>) -> complex<f32>
 ! CMPLX-FAST: %[[E:.*]] = complex.log %[[C]] fastmath<contract> : complex<f32>
-! CMPLX-FAST: %[[RESULT32_OUTLINE:.*]] = fir.convert %[[E]] : (complex<f32>) -> !fir.complex<4>
-! CMPLX-APPROX: %[[C:.*]] = fir.convert %[[ARG32_OUTLINE]] : (!fir.complex<4>) -> complex<f32>
 ! CMPLX-APPROX: %[[E:.*]] = complex.log %[[C]] fastmath<contract,afn> : complex<f32>
-! CMPLX-APPROX: %[[RESULT32_OUTLINE:.*]] = fir.convert %[[E]] : (complex<f32>) -> !fir.complex<4>
-! CMPLX-PRECISE: %[[RESULT32_OUTLINE:.*]] = fir.call @clogf(%[[ARG32_OUTLINE]]) fastmath<contract> : (!fir.complex<4>) -> !fir.complex<4>
-! CMPLX: return %[[RESULT32_OUTLINE]] : !fir.complex<4>
+! CMPLX-PRECISE: %[[E:.*]] = fir.call @clogf(%[[C]]) fastmath<contract> : (complex<f32>) -> complex<f32>
+! CMPLX: return %[[E]] : complex<f32>
 
-! CMPLX-APPROX-LABEL: private @fir.log.contract_afn.z8.z8
-! CMPLX-PRECISE-LABEL: private @fir.log.contract.z8.z8
-! CMPLX-MLIR-LABEL: private @fir.log.contract.z8.z8
-! CMPLX-SAME: (%[[ARG64_OUTLINE:.*]]: !fir.complex<8>) -> !fir.complex<8>
-! CMPLX-FAST: %[[C:.*]] = fir.convert %[[ARG64_OUTLINE]] : (!fir.complex<8>) -> complex<f64>
+! CMPLX-APPROX-LABEL: private @fir.log.contract_afn.z64.z64
+! CMPLX-PRECISE-LABEL: private @fir.log.contract.z64.z64
+! CMPLX-MLIR-LABEL: private @fir.log.contract.z64.z64
+! CMPLX-SAME: (%[[C:.*]]: complex<f64>) -> complex<f64>
 ! CMPLX-FAST: %[[E:.*]] = complex.log %[[C]] fastmath<contract> : complex<f64>
-! CMPLX-FAST: %[[RESULT64_OUTLINE:.*]] = fir.convert %[[E]] : (complex<f64>) -> !fir.complex<8>
-! CMPLX-APPROX: %[[C:.*]] = fir.convert %[[ARG64_OUTLINE]] : (!fir.complex<8>) -> complex<f64>
 ! CMPLX-APPROX: %[[E:.*]] = complex.log %[[C]] fastmath<contract,afn> : complex<f64>
-! CMPLX-APPROX: %[[RESULT64_OUTLINE:.*]] = fir.convert %[[E]] : (complex<f64>) -> !fir.complex<8>
-! CMPLX-PRECISE: %[[RESULT64_OUTLINE:.*]] = fir.call @clog(%[[ARG64_OUTLINE]]) fastmath<contract> : (!fir.complex<8>) -> !fir.complex<8>
-! CMPLX: return %[[RESULT64_OUTLINE]] : !fir.complex<8>
+! CMPLX-PRECISE: %[[E:.*]] = fir.call @clog(%[[C]]) fastmath<contract> : (complex<f64>) -> complex<f64>
+! CMPLX: return %[[E]] : complex<f64>
 
 ! CHECK-LABEL: private @fir.log10.contract.f32.f32
 ! CHECK-SAME: (%[[ARG32_OUTLINE:.*]]: f32) -> f32
diff --git a/flang/test/Lower/Intrinsics/log_complex16.f90 b/flang/test/Lower/Intrinsics/log_complex16.f90
index d5d0eb388ca2f..2425f5bc5ab6e 100644
--- a/flang/test/Lower/Intrinsics/log_complex16.f90
+++ b/flang/test/Lower/Intrinsics/log_complex16.f90
@@ -2,7 +2,7 @@
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
-! CHECK: fir.call @_FortranACLogF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+! CHECK: fir.call @_FortranACLogF128({{.*}}){{.*}}: (complex<f128>) -> complex<f128>
   complex(16) :: a, b
   b = log(a)
 end
diff --git a/flang/test/Lower/Intrinsics/pow_complex16.f90 b/flang/test/Lower/Intrinsics/pow_complex16.f90
index db6b207aea907..b96a8b0f2665e 100644
--- a/flang/test/Lower/Intrinsics/pow_complex16.f90
+++ b/flang/test/Lower/Intrinsics/pow_complex16.f90
@@ -2,7 +2,7 @@
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
-! CHECK: fir.call @_FortranACPowF128({{.*}}){{.*}}: (!fir.complex<16>, !fir.complex<16>) -> !fir.complex<16>
+! CHECK: fir.call @_FortranACPowF128({{.*}}){{.*}}: (complex<f128>, complex<f128>) -> complex<f128>
   complex(16) :: a, b
   b = a ** b
 end
diff --git a/flang/test/Lower/Intrinsics/pow_complex16i.f90 b/flang/test/Lower/Intrinsics/pow_complex16i.f90
index 1cabaf94f8a60..c93600b588fbf 100644
--- a/flang/test/Lower/Intrinsics/pow_complex16i.f90
+++ b/flang/test/Lower/Intrinsics/pow_complex16i.f90
@@ -2,7 +2,7 @@
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
-! CHECK: fir.call @_FortranAcqpowi({{.*}}){{.*}}: (!fir.complex<16>, i32) -> !fir.complex<16>
+! CHECK: fir.call @_FortranAcqpowi({{.*}}){{.*}}: (complex<f128>, i32) -> complex<f128>
   complex(16) :: a
   integer(4) :: b
   b = a ** b
diff --git a/flang/test/Lower/Intrinsics/pow_complex16k.f90 b/flang/test/Lower/Intrinsics/pow_complex16k.f90
index e3b1b4d31afa7..618501fdc3a78 100644
--- a/flang/test/Lower/Intrinsics/pow_complex16k.f90
+++ b/flang/test/Lower/Intrinsics/pow_complex16k.f90
@@ -2,7 +2,7 @@
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
-! CHECK: fir.call @_FortranAcqpowk({{.*}}){{.*}}: (!fir.complex<16>, i64) -> !fir.complex<16>
+! CHECK: fir.call @_FortranAcqpowk({{.*}}){{.*}}: (complex<f128>, i64) -> complex<f128>
   complex(16) :: a
   integer(8) :: b
   b = a ** b
diff --git a/flang/test/Lower/Intrinsics/product.f90 b/flang/test/Lower/Intrinsics/product.f90
index ddefa7a37184d..77b8ab8e7f5a9 100644
--- a/flang/test/Lower/Intrinsics/product.f90
+++ b/flang/test/Lower/Intrinsics/product.f90
@@ -32,33 +32,31 @@ subroutine product_test2(a,r)
 end subroutine
 
 ! CHECK-LABEL: func @_QPproduct_test3(
-! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?x!fir.complex<4>>>{{.*}}) -> !fir.complex<4>
+! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?xcomplex<f32>>>{{.*}}) -> complex<f32>
 complex function product_test3(a)
 complex :: a(:)
 ! CHECK-DAG:  %[[c0:.*]] = arith.constant 0 : index
-! CHECK-DAG:  %[[a0:.*]] = fir.alloca !fir.complex<4>
+! CHECK-DAG:  %[[a0:.*]] = fir.alloca complex<f32>
 ! CHECK-DAG:  %[[a3:.*]] = fir.absent !fir.box<i1>
-! CHECK-DAG: %[[a5:.*]] = fir.convert %[[a0]] : (!fir.ref<!fir.complex<4>>) -> !fir.ref<complex<f32>>
-! CHECK-DAG:  %[[a6:.*]] = fir.convert %[[arg0]] : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> !fir.box<none>
+! CHECK-DAG:  %[[a6:.*]] = fir.convert %[[arg0]] : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.box<none>
 ! CHECK-DAG:  %[[a8:.*]] = fir.convert %[[c0]] : (index) -> i32
 ! CHECK-DAG:  %[[a9:.*]] = fir.convert %[[a3]] : (!fir.box<i1>) -> !fir.box<none>
 product_test3 = product(a)
-! CHECK:  %{{.*}} = fir.call @_FortranACppProductComplex4(%[[a5]], %[[a6]], %{{.*}}, %{{.*}}, %[[a8]], %[[a9]]) {{.*}}: (!fir.ref<complex<f32>>, !fir.box<none>, !fir.ref<i8>, i32, i32, !fir.box<none>) -> none
+! CHECK:  %{{.*}} = fir.call @_FortranACppProductComplex4(%[[a0]], %[[a6]], %{{.*}}, %{{.*}}, %[[a8]], %[[a9]]) {{.*}}: (!fir.ref<complex<f32>>, !fir.box<none>, !fir.ref<i8>, i32, i32, !fir.box<none>) -> none
 end function
 
 ! CHECK-LABEL: func @_QPproduct_test4(
-! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?x!fir.complex<10>>>{{.*}}) -> !fir.complex<10>
+! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?xcomplex<f80>>>{{.*}}) -> complex<f80>
 complex(10) function product_test4(x)
 complex(10):: x(:)
 ! CHECK-DAG:  %[[c0:.*]] = arith.constant 0 : index
-! CHECK-DAG:  %[[a0:.*]] = fir.alloca !fir.complex<10>
+! CHECK-DAG:  %[[a0:.*]] = fir.alloca complex<f80>
 product_test4 = product(x)
 ! CHECK-DAG: %[[a2:.*]] = fir.absent !fir.box<i1>
-! CHECK-DAG: %[[a4:.*]] = fir.convert %[[a0]] : (!fir.ref<!fir.complex<10>>) -> !fir.ref<complex<f80>>
-! CHECK-DAG: %[[a5:.*]] = fir.convert %[[arg0]] : (!fir.box<!fir.array<?x!fir.complex<10>>>) -> !fir.box<none>
+! CHECK-DAG: %[[a5:.*]] = fir.convert %[[arg0]] : (!fir.box<!fir.array<?xcomplex<f80>>>) -> !fir.box<none>
 ! CHECK-DAG:  %[[a7:.*]] = fir.convert %[[c0]] : (index) -> i32
 ! CHECK-DAG:  %[[a8:.*]] = fir.convert %[[a2]] : (!fir.box<i1>) -> !fir.box<none>
-! CHECK: fir.call @_FortranACppProductComplex10(%[[a4]], %[[a5]], %{{.*}}, %{{.*}}, %[[a7]], %{{[0-9]+}}) {{.*}}: (!fir.ref<complex<f80>>, !fir.box<none>, !fir.ref<i8>, i32, i32, !fir.box<none>) -> ()
+! CHECK: fir.call @_FortranACppProductComplex10(%[[a0]], %[[a5]], %{{.*}}, %{{.*}}, %[[a7]], %{{[0-9]+}}) {{.*}}: (!fir.ref<complex<f80>>, !fir.box<none>, !fir.ref<i8>, i32, i32, !fir.box<none>) -> ()
 end
 
 ! CHECK-LABEL: func @_QPproduct_test_optional(
diff --git a/flang/test/Lower/Intrinsics/sin_complex16.f90 b/flang/test/Lower/Intrinsics/sin_complex16.f90
index 5114501e48dac..c56791d78da3b 100644
--- a/flang/test/Lower/Intrinsics/sin_complex16.f90
+++ b/flang/test/Lower/Intrinsics/sin_complex16.f90
@@ -2,7 +2,7 @@
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
-! CHECK: fir.call @_FortranACSinF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+! CHECK: fir.call @_FortranACSinF128({{.*}}){{.*}}: (complex<f128>) -> complex<f128>
   complex(16) :: a, b
   b = sin(a)
 end
diff --git a/flang/test/Lower/Intrinsics/sinh_complex16.f90 b/flang/test/Lower/Intrinsics/sinh_complex16.f90
index 5baf9ea8052a2..62110388938ae 100644
--- a/flang/test/Lower/Intrinsics/sinh_complex16.f90
+++ b/flang/test/Lower/Intrinsics/sinh_complex16.f90
@@ -2,7 +2,7 @@
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
-! CHECK: fir.call @_FortranACSinhF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+! CHECK: fir.call @_FortranACSinhF128({{.*}}){{.*}}: (complex<f128>) -> complex<f128>
   complex(16) :: a, b
   b = sinh(a)
 end
diff --git a/flang/test/Lower/Intrinsics/sqrt_complex16.f90 b/flang/test/Lower/Intrinsics/sqrt_complex16.f90
index 75ffa22a6e2ed..ebe0cf007cdef 100644
--- a/flang/test/Lower/Intrinsics/sqrt_complex16.f90
+++ b/flang/test/Lower/Intrinsics/sqrt_complex16.f90
@@ -2,7 +2,7 @@
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
-! CHECK: fir.call @_FortranACSqrtF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+! CHECK: fir.call @_FortranACSqrtF128({{.*}}){{.*}}: (complex<f128>) -> complex<f128>
   complex(16) :: a, b
   b = sqrt(a)
 end
diff --git a/flang/test/Lower/Intrinsics/sum.f90 b/flang/test/Lower/Intrinsics/sum.f90
index 696892d291268..ab5da34b3c7bb 100644
--- a/flang/test/Lower/Intrinsics/sum.f90
+++ b/flang/test/Lower/Intrinsics/sum.f90
@@ -32,33 +32,31 @@ subroutine sum_test2(a,r)
 end subroutine
 
 ! CHECK-LABEL: func @_QPsum_test3(
-! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?x!fir.complex<4>>>{{.*}}) -> !fir.complex<4> {
+! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?xcomplex<f32>>>{{.*}}) -> complex<f32> {
 complex function sum_test3(a)
 complex :: a(:)
 ! CHECK-DAG:  %[[c0:.*]] = arith.constant 0 : index
-! CHECK-DAG:  %[[a0:.*]] = fir.alloca !fir.complex<4>
+! CHECK-DAG:  %[[a0:.*]] = fir.alloca complex<f32>
 ! CHECK-DAG:  %[[a3:.*]] = fir.absent !fir.box<i1>
-! CHECK-DAG: %[[a5:.*]] = fir.convert %[[a0]] : (!fir.ref<!fir.complex<4>>) -> !fir.ref<complex<f32>>
-! CHECK-DAG:  %[[a6:.*]] = fir.convert %[[arg0]] : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> !fir.box<none>
+! CHECK-DAG:  %[[a6:.*]] = fir.convert %[[arg0]] : (!fir.box<!fir.array<?xcomplex<f32>>>) -> !fir.box<none>
 ! CHECK-DAG:  %[[a8:.*]] = fir.convert %[[c0]] : (index) -> i32
 ! CHECK-DAG:  %[[a9:.*]] = fir.convert %[[a3]] : (!fir.box<i1>) -> !fir.box<none>
 sum_test3 = sum(a)
-! CHECK:  %{{.*}} = fir.call @_FortranACppSumComplex4(%[[a5]], %[[a6]], %{{.*}}, %{{.*}}, %[[a8]], %[[a9]]) {{.*}}: (!fir.ref<complex<f32>>, !fir.box<none>, !fir.ref<i8>, i32, i32, !fir.box<none>) -> none
+! CHECK:  %{{.*}} = fir.call @_FortranACppSumComplex4(%[[a0]], %[[a6]], %{{.*}}, %{{.*}}, %[[a8]], %[[a9]]) {{.*}}: (!fir.ref<complex<f32>>, !fir.box<none>, !fir.ref<i8>, i32, i32, !fir.box<none>) -> none
 end function
 
 ! CHECK-LABEL: func @_QPsum_test4(
-! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?x!fir.complex<10>>>{{.*}}) -> !fir.complex<10> {
+! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?xcomplex<f80>>>{{.*}}) -> complex<f80> {
 complex(10) function sum_test4(x)
 complex(10):: x(:)
 ! CHECK-DAG:  %[[c0:.*]] = arith.constant 0 : index
-! CHECK-DAG:  %[[a0:.*]] = fir.alloca !fir.complex<10>
+! CHECK-DAG:  %[[a0:.*]] = fir.alloca complex<f80>
 sum_test4 = sum(x)
 ! CHECK-DAG: %[[a2:.*]] = fir.absent !fir.box<i1>
-! CHECK-DAG: %[[a4:.*]] = fir.convert %[[a0]] : (!fir.ref<!fir.complex<10>>) -> !fir.ref<complex<f80>>
-! CHECK-DAG: %[[a5:.*]] = fir.convert %[[arg0]] : (!fir.box<!fir.array<?x!fir.complex<10>>>) -> !fir.box<none>
+! CHECK-DAG: %[[a5:.*]] = fir.convert %[[arg0]] : (!fir.box<!fir.array<?xcomplex<f80>>>) -> !fir.box<none>
 ! CHECK-DAG:  %[[a7:.*]] = fir.convert %[[c0]] : (index) -> i32
 ! CHECK-DAG:  %[[a8:.*]] = fir.convert %[[a2]] : (!fir.box<i1>) -> !fir.box<none>
-! CHECK: fir.call @_FortranACppSumComplex10(%[[a4]], %[[a5]], %{{.*}}, %{{.*}}, %[[a7]], %{{[0-9]+}}) {{.*}}: (!fir.ref<complex<f80>>, !fir.box<none>, !fir.ref<i8>, i32, i32, !fir.box<none>) -> ()
+! CHECK: fir.call @_FortranACppSumComplex10(%[[a0]], %[[a5]], %{{.*}}, %{{.*}}, %[[a7]], %{{[0-9]+}}) {{.*}}: (!fir.ref<complex<f80>>, !fir.box<none>, !fir.ref<i8>, i32, i32, !fir.box<none>) -> ()
 end
 
 ! CHECK-LABEL: func @_QPsum_test_optional(
diff --git a/flang/test/Lower/Intrinsics/tan_complex16.f90 b/flang/test/Lower/Intrinsics/tan_complex16.f90
index 7217145f81b00..396b4353658d7 100644
--- a/flang/test/Lower/Intrinsics/tan_complex16.f90
+++ b/flang/test/Lower/Intrinsics/tan_complex16.f90
@@ -2,7 +2,7 @@
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
-! CHECK: fir.call @_FortranACTanF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+! CHECK: fir.call @_FortranACTanF128({{.*}}){{.*}}: (complex<f128>) -> complex<f128>
   complex(16) :: a, b
   b = tan(a)
 end
diff --git a/flang/test/Lower/Intrinsics/tanh_complex16.f90 b/flang/test/Lower/Intrinsics/tanh_complex16.f90
index 1965094c5bce0..b0a2847b004b4 100644
--- a/flang/test/Lower/Intrinsics/tanh_complex16.f90
+++ b/flang/test/Lower/Intrinsics/tanh_complex16.f90
@@ -2,7 +2,7 @@
 ! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
-! CHECK: fir.call @_FortranACTanhF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+! CHECK: fir.call @_FortranACTanhF128({{.*}}){{.*}}: (complex<f128>) -> complex<f128>
   complex(16) :: a, b
   b = tanh(a)
 end
diff --git a/flang/test/Lower/OpenACC/acc-reduction.f90 b/flang/test/Lower/OpenACC/acc-reduction.f90
index 545c4f217577b..88c60a22b0fe8 100644
--- a/flang/test/Lower/OpenACC/acc-reduction.f90
+++ b/flang/test/Lower/OpenACC/acc-reduction.f90
@@ -148,44 +148,44 @@
 ! CHECK:   acc.yield %arg0 : !fir.box<!fir.array<?xi32>>
 ! CHECK: }
 
-! CHECK-LABEL: acc.reduction.recipe @reduction_mul_ref_z32 : !fir.ref<!fir.complex<4>> reduction_operator <mul> init {
-! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.complex<4>>):
+! CHECK-LABEL: acc.reduction.recipe @reduction_mul_ref_z32 : !fir.ref<complex<f32>> reduction_operator <mul> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<complex<f32>>):
 ! CHECK:   %[[REAL:.*]] = arith.constant 1.000000e+00 : f32
 ! CHECK:   %[[IMAG:.*]] = arith.constant 0.000000e+00 : f32
-! CHECK:   %[[UNDEF:.*]] = fir.undefined !fir.complex<4>
-! CHECK:   %[[UNDEF1:.*]] = fir.insert_value %[[UNDEF]], %[[REAL]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:   %[[UNDEF2:.*]] = fir.insert_value %[[UNDEF1]], %[[IMAG]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.complex<4>
-! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:   fir.store %[[UNDEF2]] to %[[DECLARE]]#0 : !fir.ref<!fir.complex<4>>
-! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.complex<4>>
+! CHECK:   %[[UNDEF:.*]] = fir.undefined complex<f32>
+! CHECK:   %[[UNDEF1:.*]] = fir.insert_value %[[UNDEF]], %[[REAL]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:   %[[UNDEF2:.*]] = fir.insert_value %[[UNDEF1]], %[[IMAG]], [1 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca complex<f32>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:   fir.store %[[UNDEF2]] to %[[DECLARE]]#0 : !fir.ref<complex<f32>>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<complex<f32>>
 ! CHECK: } combiner {
-! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.complex<4>>, %[[ARG1:.*]]: !fir.ref<!fir.complex<4>>):
-! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.complex<4>>
-! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<!fir.complex<4>>
-! CHECK:   %[[COMBINED:.*]] = fir.mulc %[[LOAD0]], %[[LOAD1]] {fastmath = #arith.fastmath<contract>} : !fir.complex<4>
-! CHECK:   fir.store %[[COMBINED]] to %[[ARG0]] : !fir.ref<!fir.complex<4>>
-! CHECK:   acc.yield %[[ARG0]] : !fir.ref<!fir.complex<4>>
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<complex<f32>>, %[[ARG1:.*]]: !fir.ref<complex<f32>>):
+! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<complex<f32>>
+! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<complex<f32>>
+! CHECK:   %[[COMBINED:.*]] = fir.mulc %[[LOAD0]], %[[LOAD1]] {fastmath = #arith.fastmath<contract>} : complex<f32>
+! CHECK:   fir.store %[[COMBINED]] to %[[ARG0]] : !fir.ref<complex<f32>>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<complex<f32>>
 ! CHECK: }
 
-! CHECK-LABEL: acc.reduction.recipe @reduction_add_ref_z32 : !fir.ref<!fir.complex<4>> reduction_operator <add> init {
-! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.complex<4>>):
+! CHECK-LABEL: acc.reduction.recipe @reduction_add_ref_z32 : !fir.ref<complex<f32>> reduction_operator <add> init {
+! CHECK: ^bb0(%{{.*}}: !fir.ref<complex<f32>>):
 ! CHECK:   %[[REAL:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:   %[[IMAG:.*]] = arith.constant 0.000000e+00 : f32
-! CHECK:   %[[UNDEF:.*]] = fir.undefined !fir.complex<4>
-! CHECK:   %[[UNDEF1:.*]] = fir.insert_value %[[UNDEF]], %[[REAL]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:   %[[UNDEF2:.*]] = fir.insert_value %[[UNDEF1]], %[[IMAG]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.complex<4>
-! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:   fir.store %[[UNDEF2]] to %[[DECLARE]]#0 : !fir.ref<!fir.complex<4>>
-! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.complex<4>>
+! CHECK:   %[[UNDEF:.*]] = fir.undefined complex<f32>
+! CHECK:   %[[UNDEF1:.*]] = fir.insert_value %[[UNDEF]], %[[REAL]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:   %[[UNDEF2:.*]] = fir.insert_value %[[UNDEF1]], %[[IMAG]], [1 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca complex<f32>
+! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+! CHECK:   fir.store %[[UNDEF2]] to %[[DECLARE]]#0 : !fir.ref<complex<f32>>
+! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<complex<f32>>
 ! CHECK: } combiner {
-! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.complex<4>>, %[[ARG1:.*]]: !fir.ref<!fir.complex<4>>):
-! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.complex<4>>
-! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<!fir.complex<4>>
-! CHECK:   %[[COMBINED:.*]] = fir.addc %[[LOAD0]], %[[LOAD1]] {fastmath = #arith.fastmath<contract>} : !fir.complex<4>
-! CHECK:   fir.store %[[COMBINED]] to %[[ARG0]] : !fir.ref<!fir.complex<4>>
-! CHECK:   acc.yield %[[ARG0]] : !fir.ref<!fir.complex<4>>
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<complex<f32>>, %[[ARG1:.*]]: !fir.ref<complex<f32>>):
+! CHECK:   %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref<complex<f32>>
+! CHECK:   %[[LOAD1:.*]] = fir.load %[[ARG1]] : !fir.ref<complex<f32>>
+! CHECK:   %[[COMBINED:.*]] = fir.addc %[[LOAD0]], %[[LOAD1]] {fastmath = #arith.fastmath<contract>} : complex<f32>
+! CHECK:   fir.store %[[COMBINED]] to %[[ARG0]] : !fir.ref<complex<f32>>
+! CHECK:   acc.yield %[[ARG0]] : !fir.ref<complex<f32>>
 ! CHECK: }
 
 ! CHECK-LABEL: acc.reduction.recipe @reduction_neqv_ref_l32 : !fir.ref<!fir.logical<4>> reduction_operator <neqv> init {
@@ -1071,8 +1071,8 @@ subroutine acc_reduction_add_cmplx()
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_cmplx()
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.complex<4>>) -> !fir.ref<!fir.complex<4>> {name = "c"}
-! CHECK: acc.parallel reduction(@reduction_add_ref_z32 -> %[[RED]] : !fir.ref<!fir.complex<4>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<complex<f32>>) -> !fir.ref<complex<f32>> {name = "c"}
+! CHECK: acc.parallel reduction(@reduction_add_ref_z32 -> %[[RED]] : !fir.ref<complex<f32>>)
 
 subroutine acc_reduction_mul_cmplx()
   complex :: c
@@ -1081,8 +1081,8 @@ subroutine acc_reduction_mul_cmplx()
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_mul_cmplx()
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.complex<4>>) -> !fir.ref<!fir.complex<4>> {name = "c"}
-! CHECK: acc.parallel reduction(@reduction_mul_ref_z32 -> %[[RED]] : !fir.ref<!fir.complex<4>>)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<complex<f32>>) -> !fir.ref<complex<f32>> {name = "c"}
+! CHECK: acc.parallel reduction(@reduction_mul_ref_z32 -> %[[RED]] : !fir.ref<complex<f32>>)
 
 subroutine acc_reduction_add_alloc()
   integer, allocatable :: i
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
index e3c1dc805d07b..ce98f518581a4 100644
--- a/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/target-private-multiple-variables.f90
@@ -51,10 +51,10 @@ end subroutine target_allocatable
 !
 ! CHECK:      omp.private {type = private}
 ! CHECK-SAME:   @[[COMP_PRIVATIZER_SYM:[^[:space:]]+comp_var[^[:space:]]+]]
-! CHECK-SAME:   : [[COMP_TYPE:!fir.ref<!fir.complex<4>>]] alloc {
+! CHECK-SAME:   : [[COMP_TYPE:!fir.ref<complex<f32>>]] alloc {
 !
 ! CHECK-NEXT: ^bb0(%[[PRIV_ARG:.*]]: [[COMP_TYPE]]):
-! CHECK-NEXT:   %[[PRIV_ALLOC:.*]] = fir.alloca !fir.complex<4>
+! CHECK-NEXT:   %[[PRIV_ALLOC:.*]] = fir.alloca complex<f32>
 ! CHECK-NEXT:   %[[PRIV_DECL:.*]]:2 = hlfir.declare %[[PRIV_ALLOC]]
 ! CHECK-NEXT:   omp.yield(%[[PRIV_DECL]]#0 : [[COMP_TYPE]])
 ! CHECK-NEXT: }
@@ -160,7 +160,7 @@ end subroutine target_allocatable
 ! CHECK-SAME:       @[[ARR_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[ARR_ARG:[^,]+]],
 ! CHECK-SAME:       @[[COMP_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[COMP_ARG:[^,]+]],
 ! CHECK-SAME:       @[[CHAR_PRIVATIZER_SYM]] %{{[^[:space:]]+}}#0 -> %[[CHAR_ARG:[^,]+]] :
-! CHECK-SAME:       !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<f32>, !fir.ref<i64>, !fir.box<!fir.array<?xf32>>, !fir.ref<!fir.complex<4>>, !fir.boxchar<1>) {
+! CHECK-SAME:       !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<f32>, !fir.ref<i64>, !fir.box<!fir.array<?xf32>>, !fir.ref<complex<f32>>, !fir.boxchar<1>) {
 ! CHECK-NOT:      fir.alloca
 ! CHECK:          hlfir.declare %[[MAPPED_ARG]]
 ! CHECK:          hlfir.declare %[[ALLOC_ARG]]
diff --git a/flang/test/Lower/OpenMP/Todo/atomic-complex.f90 b/flang/test/Lower/OpenMP/Todo/atomic-complex.f90
deleted file mode 100644
index 6d6e4399ee192..0000000000000
--- a/flang/test/Lower/OpenMP/Todo/atomic-complex.f90
+++ /dev/null
@@ -1,8 +0,0 @@
-! RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
-
-! CHECK: not yet implemented: Unsupported atomic type
-subroutine complex_atomic
-  complex :: l, r
-  !$omp atomic read
-    l = r
-end subroutine
diff --git a/flang/test/Lower/OpenMP/Todo/lastprivate-conditional.f90 b/flang/test/Lower/OpenMP/Todo/lastprivate-conditional.f90
new file mode 100644
index 0000000000000..2b96093da3a8f
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/lastprivate-conditional.f90
@@ -0,0 +1,12 @@
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -fopenmp-version=50 -o - %s 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -fopenmp-version=50 -o - %s 2>&1 | FileCheck %s
+
+! CHECK: not yet implemented: lastprivate clause with CONDITIONAL modifier
+subroutine foo()
+  integer :: x, i
+  x = 1
+  !$omp parallel do lastprivate(conditional: x)
+  do i = 1, 100
+    x = x + 1
+  enddo
+end
diff --git a/flang/test/Lower/OpenMP/copyprivate.f90 b/flang/test/Lower/OpenMP/copyprivate.f90
index bb86aa5356b1e..3f72343be666a 100644
--- a/flang/test/Lower/OpenMP/copyprivate.f90
+++ b/flang/test/Lower/OpenMP/copyprivate.f90
@@ -5,8 +5,8 @@
 !CHECK-DAG: func private @_copy_i64(%{{.*}}: !fir.ref<i64>, %{{.*}}: !fir.ref<i64>)
 !CHECK-DAG: func private @_copy_f32(%{{.*}}: !fir.ref<f32>, %{{.*}}: !fir.ref<f32>)
 !CHECK-DAG: func private @_copy_f64(%{{.*}}: !fir.ref<f64>, %{{.*}}: !fir.ref<f64>)
-!CHECK-DAG: func private @_copy_z32(%{{.*}}: !fir.ref<!fir.complex<4>>, %{{.*}}: !fir.ref<!fir.complex<4>>)
-!CHECK-DAG: func private @_copy_z64(%{{.*}}: !fir.ref<!fir.complex<8>>, %{{.*}}: !fir.ref<!fir.complex<8>>)
+!CHECK-DAG: func private @_copy_z32(%{{.*}}: !fir.ref<complex<f32>>, %{{.*}}: !fir.ref<complex<f32>>)
+!CHECK-DAG: func private @_copy_z64(%{{.*}}: !fir.ref<complex<f64>>, %{{.*}}: !fir.ref<complex<f64>>)
 !CHECK-DAG: func private @_copy_l32(%{{.*}}: !fir.ref<!fir.logical<4>>, %{{.*}}: !fir.ref<!fir.logical<4>>)
 !CHECK-DAG: func private @_copy_l64(%{{.*}}: !fir.ref<!fir.logical<8>>, %{{.*}}: !fir.ref<!fir.logical<8>>)
 !CHECK-DAG: func private @_copy_c8x3(%{{.*}}: !fir.ref<!fir.char<1,3>>, %{{.*}}: !fir.ref<!fir.char<1,3>>)
@@ -17,7 +17,7 @@
 !CHECK-DAG: func private @_copy_10xi32(%{{.*}}: !fir.ref<!fir.array<10xi32>>, %{{.*}}: !fir.ref<!fir.array<10xi32>>)
 !CHECK-DAG: func private @_copy_3x4xi32(%{{.*}}: !fir.ref<!fir.array<3x4xi32>>, %{{.*}}: !fir.ref<!fir.array<3x4xi32>>)
 !CHECK-DAG: func private @_copy_10xf32(%{{.*}}: !fir.ref<!fir.array<10xf32>>, %{{.*}}: !fir.ref<!fir.array<10xf32>>)
-!CHECK-DAG: func private @_copy_3x4xz32(%{{.*}}: !fir.ref<!fir.array<3x4x!fir.complex<4>>>, %{{.*}}: !fir.ref<!fir.array<3x4x!fir.complex<4>>>)
+!CHECK-DAG: func private @_copy_3x4xz32(%{{.*}}: !fir.ref<!fir.array<3x4xcomplex<f32>>>, %{{.*}}: !fir.ref<!fir.array<3x4xcomplex<f32>>>)
 !CHECK-DAG: func private @_copy_10xl32(%{{.*}}: !fir.ref<!fir.array<10x!fir.logical<4>>>, %{{.*}}: !fir.ref<!fir.array<10x!fir.logical<4>>>)
 !CHECK-DAG: func private @_copy_3xc8x8(%{{.*}}: !fir.ref<!fir.array<3x!fir.char<1,8>>>, %{{.*}}: !fir.ref<!fir.array<3x!fir.char<1,8>>>)
 !CHECK-DAG: func private @_copy_3xc16x5(%{{.*}}: !fir.ref<!fir.array<3x!fir.char<2,5>>>, %{{.*}}: !fir.ref<!fir.array<3x!fir.char<2,5>>>)
@@ -66,14 +66,14 @@ subroutine test_tp()
 !CHECK:           %[[I3:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_scalarEi3"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 !CHECK:           %[[R1:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_scalarEr1"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 !CHECK:           %[[R2:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_scalarEr2"} : (!fir.ref<f64>) -> (!fir.ref<f64>, !fir.ref<f64>)
-!CHECK:           %[[C1:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_scalarEc1"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-!CHECK:           %[[C2:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_scalarEc2"} : (!fir.ref<!fir.complex<8>>) -> (!fir.ref<!fir.complex<8>>, !fir.ref<!fir.complex<8>>)
+!CHECK:           %[[C1:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_scalarEc1"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+!CHECK:           %[[C2:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_scalarEc2"} : (!fir.ref<complex<f64>>) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
 !CHECK:           %[[L1:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_scalarEl1"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 !CHECK:           %[[L2:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_scalarEl2"} : (!fir.ref<!fir.logical<8>>) -> (!fir.ref<!fir.logical<8>>, !fir.ref<!fir.logical<8>>)
 !CHECK:           %[[S1:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_scalarEs1"} : (!fir.ref<!fir.char<1,3>>, index) -> (!fir.ref<!fir.char<1,3>>, !fir.ref<!fir.char<1,3>>)
 !CHECK:           %[[S2:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_scalarEs2"} : (!fir.ref<!fir.char<1,8>>, index) -> (!fir.ref<!fir.char<1,8>>, !fir.ref<!fir.char<1,8>>)
 !CHECK:           %[[S3:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_scalarEs3"} : (!fir.ref<!fir.char<2,8>>, index) -> (!fir.ref<!fir.char<2,8>>, !fir.ref<!fir.char<2,8>>)
-!CHECK:           omp.single copyprivate(%[[I1]]#0 -> @_copy_i32 : !fir.ref<i32>, %[[I2]]#0 -> @_copy_i64 : !fir.ref<i64>, %[[I3]]#0 -> @_copy_i64 : !fir.ref<i64>, %[[R1]]#0 -> @_copy_f32 : !fir.ref<f32>, %[[R2]]#0 -> @_copy_f64 : !fir.ref<f64>, %[[C1]]#0 -> @_copy_z32 : !fir.ref<!fir.complex<4>>, %[[C2]]#0 -> @_copy_z64 : !fir.ref<!fir.complex<8>>, %[[L1]]#0 -> @_copy_l32 : !fir.ref<!fir.logical<4>>, %[[L2]]#0 -> @_copy_l64 : !fir.ref<!fir.logical<8>>, %[[S1]]#0 -> @_copy_c8x3 : !fir.ref<!fir.char<1,3>>, %[[S2]]#0 -> @_copy_c8x8 : !fir.ref<!fir.char<1,8>>, %[[S3]]#0 -> @_copy_c16x8 : !fir.ref<!fir.char<2,8>>)
+!CHECK:           omp.single copyprivate(%[[I1]]#0 -> @_copy_i32 : !fir.ref<i32>, %[[I2]]#0 -> @_copy_i64 : !fir.ref<i64>, %[[I3]]#0 -> @_copy_i64 : !fir.ref<i64>, %[[R1]]#0 -> @_copy_f32 : !fir.ref<f32>, %[[R2]]#0 -> @_copy_f64 : !fir.ref<f64>, %[[C1]]#0 -> @_copy_z32 : !fir.ref<complex<f32>>, %[[C2]]#0 -> @_copy_z64 : !fir.ref<complex<f64>>, %[[L1]]#0 -> @_copy_l32 : !fir.ref<!fir.logical<4>>, %[[L2]]#0 -> @_copy_l64 : !fir.ref<!fir.logical<8>>, %[[S1]]#0 -> @_copy_c8x3 : !fir.ref<!fir.char<1,3>>, %[[S2]]#0 -> @_copy_c8x8 : !fir.ref<!fir.char<1,8>>, %[[S3]]#0 -> @_copy_c16x8 : !fir.ref<!fir.char<2,8>>)
 subroutine test_scalar()
   integer(4) :: i1
   integer(8) :: i2, i3
@@ -100,7 +100,7 @@ subroutine test_scalar()
 !CHECK:           %[[I2:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFtest_arrayEi2"} : (!fir.ref<!fir.array<3x4xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<3x4xi32>>, !fir.ref<!fir.array<3x4xi32>>)
 !CHECK:           %[[I3:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFtest_arrayEi3"} : (!fir.ref<!fir.array<?xi32>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>)
 !CHECK:           %[[R1:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFtest_arrayEr1"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
-!CHECK:           %[[C1:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFtest_arrayEc1"} : (!fir.ref<!fir.array<3x4x!fir.complex<4>>>, !fir.shape<2>) -> (!fir.ref<!fir.array<3x4x!fir.complex<4>>>, !fir.ref<!fir.array<3x4x!fir.complex<4>>>)
+!CHECK:           %[[C1:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFtest_arrayEc1"} : (!fir.ref<!fir.array<3x4xcomplex<f32>>>, !fir.shape<2>) -> (!fir.ref<!fir.array<3x4xcomplex<f32>>>, !fir.ref<!fir.array<3x4xcomplex<f32>>>)
 !CHECK:           %[[L1:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFtest_arrayEl1"} : (!fir.ref<!fir.array<10x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10x!fir.logical<4>>>, !fir.ref<!fir.array<10x!fir.logical<4>>>)
 !CHECK:           %[[S1:.*]]:2 = hlfir.declare {{.*}} {uniq_name = "_QFtest_arrayEs1"} : (!fir.ref<!fir.array<3x!fir.char<1,8>>>, !fir.shape<1>, index) -> (!fir.ref<!fir.array<3x!fir.char<1,8>>>, !fir.ref<!fir.array<3x!fir.char<1,8>>>)
 !CHECK:           %[[S2:.*]]:2 = hlfir.declare {{.*}} {uniq_name = "_QFtest_arrayEs2"} : (!fir.ref<!fir.array<3x!fir.char<2,5>>>, !fir.shape<1>, index) -> (!fir.ref<!fir.array<3x!fir.char<2,5>>>, !fir.ref<!fir.array<3x!fir.char<2,5>>>)
@@ -108,7 +108,7 @@ subroutine test_scalar()
 !CHECK:           fir.store %[[A]]#0 to %[[A_REF]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
 !CHECK:           %[[I3_REF:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
 !CHECK:           fir.store %[[I3]]#0 to %[[I3_REF]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
-!CHECK:           omp.single copyprivate(%[[A_REF]] -> @_copy_box_Uxi32 : !fir.ref<!fir.box<!fir.array<?xi32>>>, %[[I1]]#0 -> @_copy_10xi32 : !fir.ref<!fir.array<10xi32>>, %[[I2]]#0 -> @_copy_3x4xi32 : !fir.ref<!fir.array<3x4xi32>>, %[[I3_REF]] -> @_copy_box_Uxi32 : !fir.ref<!fir.box<!fir.array<?xi32>>>, %[[R1]]#0 -> @_copy_10xf32 : !fir.ref<!fir.array<10xf32>>, %[[C1]]#0 -> @_copy_3x4xz32 : !fir.ref<!fir.array<3x4x!fir.complex<4>>>, %[[L1]]#0 -> @_copy_10xl32 : !fir.ref<!fir.array<10x!fir.logical<4>>>, %[[S1]]#0 -> @_copy_3xc8x8 : !fir.ref<!fir.array<3x!fir.char<1,8>>>, %[[S2]]#0 -> @_copy_3xc16x5 : !fir.ref<!fir.array<3x!fir.char<2,5>>>)
+!CHECK:           omp.single copyprivate(%[[A_REF]] -> @_copy_box_Uxi32 : !fir.ref<!fir.box<!fir.array<?xi32>>>, %[[I1]]#0 -> @_copy_10xi32 : !fir.ref<!fir.array<10xi32>>, %[[I2]]#0 -> @_copy_3x4xi32 : !fir.ref<!fir.array<3x4xi32>>, %[[I3_REF]] -> @_copy_box_Uxi32 : !fir.ref<!fir.box<!fir.array<?xi32>>>, %[[R1]]#0 -> @_copy_10xf32 : !fir.ref<!fir.array<10xf32>>, %[[C1]]#0 -> @_copy_3x4xz32 : !fir.ref<!fir.array<3x4xcomplex<f32>>>, %[[L1]]#0 -> @_copy_10xl32 : !fir.ref<!fir.array<10x!fir.logical<4>>>, %[[S1]]#0 -> @_copy_3xc8x8 : !fir.ref<!fir.array<3x!fir.char<1,8>>>, %[[S2]]#0 -> @_copy_3xc16x5 : !fir.ref<!fir.array<3x!fir.char<2,5>>>)
 subroutine test_array(a, n)
   integer :: a(:), n
   integer :: i1(10), i2(3, 4), i3(n)
diff --git a/flang/test/Lower/OpenMP/lastprivate-allocatable.f90 b/flang/test/Lower/OpenMP/lastprivate-allocatable.f90
index 0dce4d514bd10..6b7d849fde93c 100644
--- a/flang/test/Lower/OpenMP/lastprivate-allocatable.f90
+++ b/flang/test/Lower/OpenMP/lastprivate-allocatable.f90
@@ -41,17 +41,17 @@ program lastprivate_allocatable
 
 ! CHECK-LABEL:  func @_QPlastprivate_realloc()
 ! CHECK:          %[[A:.*]]:2 = hlfir.declare %{{.*}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFlastprivate_reallocEa"} :
-! CHECK-SAME:       (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>>) ->
-! CHECK-SAME:       (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>>)
+! CHECK-SAME:       (!fir.ref<!fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>>) ->
+! CHECK-SAME:       (!fir.ref<!fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>>)
 ! CHECK:          omp.parallel {
 ! CHECK:            %[[A_PRIV:.*]]:2 = hlfir.declare %{{.*}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFlastprivate_reallocEa"} :
-! CHECK-SAME:         (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>>) ->
-! CHECK-SAME:         (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>>)
+! CHECK-SAME:         (!fir.ref<!fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>>) ->
+! CHECK-SAME:         (!fir.ref<!fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>>)
 ! CHECK:            omp.sections {
 ! CHECK:              omp.section {
-! CHECK:                %[[TEMP:.*]] = fir.load %[[A_PRIV:.*]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>>
-! CHECK:                hlfir.assign %[[TEMP]] to %[[A]]#0 realloc : !fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>,
-! CHECK-SAME:             !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>>
+! CHECK:                %[[TEMP:.*]] = fir.load %[[A_PRIV:.*]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>>
+! CHECK:                hlfir.assign %[[TEMP]] to %[[A]]#0 realloc : !fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>,
+! CHECK-SAME:             !fir.ref<!fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>>
 ! CHECK:              }
 ! CHECK:            }
 ! CHECK:          }
diff --git a/flang/test/Lower/OpenMP/loop-compound.f90 b/flang/test/Lower/OpenMP/loop-compound.f90
index e76edfe052f74..8c025ab237920 100644
--- a/flang/test/Lower/OpenMP/loop-compound.f90
+++ b/flang/test/Lower/OpenMP/loop-compound.f90
@@ -5,7 +5,7 @@
 ! RUN: %flang_fc1 -fopenmp -emit-hlfir %s -o - | FileCheck %s
 
 program main
-  integer :: i
+  integer :: i,j
 
   ! TODO When composite constructs are supported add:
   ! - TASKLOOP SIMD
@@ -231,4 +231,38 @@ program main
   do i = 1, 10
   end do
   !$omp end teams distribute simd
+
+  ! ----------------------------------------------------------------------------
+  ! Unstructured control-flow in loop
+  ! ----------------------------------------------------------------------------
+  ! CHECK:      omp.target
+  ! CHECK:      omp.teams
+  ! CHECK:      omp.parallel
+  ! CHECK:      omp.distribute
+  ! CHECK-NEXT: omp.wsloop
+  ! CHECK-NEXT: omp.loop_nest
+  !
+  ! Verify the conrol-flow of the unstructured inner loop.
+  ! CHECK:        cf.br ^[[BB1:.*]]
+  ! CHECK:      ^[[BB1]]:
+  ! CHECK:        cf.br ^[[BB2:.*]]
+  ! CHECK:      ^[[BB2]]:
+  ! CHECK:        cf.cond_br %{{.*}}, ^[[BB3:.*]], ^[[BB6:.*]]
+  ! CHECK:      ^[[BB3]]:
+  ! CHECK:        cf.cond_br %{{.*}}, ^[[BB4:.*]], ^[[BB5:.*]]
+  ! CHECK:      ^[[BB4]]:
+  ! CHECK:        cf.br ^[[BB6]]
+  ! CHECK:      ^[[BB5]]:
+  ! CHECK:        cf.br ^[[BB2]]
+  ! CHECK:      ^[[BB6]]:
+  ! CHECK-NEXT:   omp.yield
+  !$omp target teams distribute parallel do
+  do i = 1, 10
+    outerloop: do j = i-1, i+i
+      if (j == i) then
+        exit outerloop
+      end if
+    end do outerloop
+  end do
+  !$omp end target teams distribute parallel do
 end program main
diff --git a/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90 b/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
index 4cee01488f4a5..f0bee355543a6 100644
--- a/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
+++ b/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
@@ -18,27 +18,27 @@
 !CHECK:    omp.yield(%[[PVT_REF]] : !fir.ref<!fir.logical<4>>)
 !CHECK:  }
 
-!CHECK:  omp.private {type = firstprivate} @[[ARG2_COMPLEX_PRIVATIZER:_QFfirstprivate_complexEarg2_firstprivate_ref_z64]] : !fir.ref<!fir.complex<8>> alloc
+!CHECK:  omp.private {type = firstprivate} @[[ARG2_COMPLEX_PRIVATIZER:_QFfirstprivate_complexEarg2_firstprivate_ref_z64]] : !fir.ref<complex<f64>> alloc
 
-!CHECK:  omp.private {type = firstprivate} @[[ARG1_COMPLEX_PRIVATIZER:_QFfirstprivate_complexEarg1_firstprivate_ref_z32]] : !fir.ref<!fir.complex<4>> alloc {
-!CHECK:  ^bb0(%{{.*}}: !fir.ref<!fir.complex<4>>):
-!CHECK:    %[[PVT_ALLOC:.*]] = fir.alloca !fir.complex<4> {bindc_name = "arg1", {{.*}}}
+!CHECK:  omp.private {type = firstprivate} @[[ARG1_COMPLEX_PRIVATIZER:_QFfirstprivate_complexEarg1_firstprivate_ref_z32]] : !fir.ref<complex<f32>> alloc {
+!CHECK:  ^bb0(%{{.*}}: !fir.ref<complex<f32>>):
+!CHECK:    %[[PVT_ALLOC:.*]] = fir.alloca complex<f32> {bindc_name = "arg1", {{.*}}}
 !CHECK:    %[[PVT_DECL:.*]]:2 = hlfir.declare %[[PVT_ALLOC]] {{.*}}
-!CHECK:    omp.yield(%[[PVT_DECL]]#0 : !fir.ref<!fir.complex<4>>)
+!CHECK:    omp.yield(%[[PVT_DECL]]#0 : !fir.ref<complex<f32>>)
 !CHECK:  } copy {
-!CHECK:  ^bb0(%[[ORIG_REF:.*]]: !fir.ref<!fir.complex<4>>, %[[PVT_REF:.*]]: !fir.ref<!fir.complex<4>>):
+!CHECK:  ^bb0(%[[ORIG_REF:.*]]: !fir.ref<complex<f32>>, %[[PVT_REF:.*]]: !fir.ref<complex<f32>>):
 !CHECK:    %[[ORIG_VAL:.*]] = fir.load %[[ORIG_REF]] : {{.*}}
 !CHECK:    hlfir.assign %[[ORIG_VAL]] to %[[PVT_REF]] {{.*}}
-!CHECK:    omp.yield(%[[PVT_REF]] : !fir.ref<!fir.complex<4>>)
+!CHECK:    omp.yield(%[[PVT_REF]] : !fir.ref<complex<f32>>)
 !CHECK:  }
 
-!CHECK-DAG: func @_QPfirstprivate_complex(%[[ARG1:.*]]: !fir.ref<!fir.complex<4>>{{.*}}, %[[ARG2:.*]]: !fir.ref<!fir.complex<8>>{{.*}}) {
-!CHECK:    %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_complexEarg1"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-!CHECK:    %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_complexEarg2"} : (!fir.ref<!fir.complex<8>>, !fir.dscope) -> (!fir.ref<!fir.complex<8>>, !fir.ref<!fir.complex<8>>)
+!CHECK-DAG: func @_QPfirstprivate_complex(%[[ARG1:.*]]: !fir.ref<complex<f32>>{{.*}}, %[[ARG2:.*]]: !fir.ref<complex<f64>>{{.*}}) {
+!CHECK:    %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_complexEarg1"} : (!fir.ref<complex<f32>>, !fir.dscope) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+!CHECK:    %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_complexEarg2"} : (!fir.ref<complex<f64>>, !fir.dscope) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
 !CHECK:   omp.parallel private(@[[ARG1_COMPLEX_PRIVATIZER]] %{{.*}}#0 -> %[[ARG1_PVT:.*]], @[[ARG2_COMPLEX_PRIVATIZER]] %{{.*}}#0 -> %[[ARG2_PVT:.*]] : {{.*}}) {
-!CHECK:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_complexEarg1"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-!CHECK:     %[[ARG2_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG2_PVT]] {uniq_name = "_QFfirstprivate_complexEarg2"} : (!fir.ref<!fir.complex<8>>) -> (!fir.ref<!fir.complex<8>>, !fir.ref<!fir.complex<8>>)
-!CHECK:     fir.call @_QPfoo(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1) {{.*}}: (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<8>>) -> ()
+!CHECK:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_complexEarg1"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+!CHECK:     %[[ARG2_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG2_PVT]] {uniq_name = "_QFfirstprivate_complexEarg2"} : (!fir.ref<complex<f64>>) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
+!CHECK:     fir.call @_QPfoo(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1) {{.*}}: (!fir.ref<complex<f32>>, !fir.ref<complex<f64>>) -> ()
 !CHECK:     omp.terminator
 !CHECK:   }
 
diff --git a/flang/test/Lower/OpenMP/parallel-private-clause.f90 b/flang/test/Lower/OpenMP/parallel-private-clause.f90
index 3d807b83e5060..edb8f9193b827 100644
--- a/flang/test/Lower/OpenMP/parallel-private-clause.f90
+++ b/flang/test/Lower/OpenMP/parallel-private-clause.f90
@@ -55,8 +55,8 @@ subroutine private_clause(arg1, arg2, arg3, arg4)
 end subroutine
 
 !FIRDialect: func @_QPprivate_clause_scalar() {
-!FIRDialect-DAG:  %[[C:.*]] = fir.alloca !fir.complex<4> {bindc_name = "c", uniq_name = "_QFprivate_clause_scalarEc"}
-!FIRDialect-DAG:  %[[C_DECL:.*]]:2 = hlfir.declare %[[C]] {uniq_name = "_QFprivate_clause_scalarEc"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+!FIRDialect-DAG:  %[[C:.*]] = fir.alloca complex<f32> {bindc_name = "c", uniq_name = "_QFprivate_clause_scalarEc"}
+!FIRDialect-DAG:  %[[C_DECL:.*]]:2 = hlfir.declare %[[C]] {uniq_name = "_QFprivate_clause_scalarEc"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
 !FIRDialect-DAG:  %[[I1:.*]] = fir.alloca i8 {bindc_name = "i1", uniq_name = "_QFprivate_clause_scalarEi1"}
 !FIRDialect-DAG:  %[[I1_DECL:.*]]:2 = hlfir.declare %[[I1]] {uniq_name = "_QFprivate_clause_scalarEi1"} : (!fir.ref<i8>) -> (!fir.ref<i8>, !fir.ref<i8>)
 !FIRDialect-DAG:  %[[I16:.*]] = fir.alloca i128 {bindc_name = "i16", uniq_name = "_QFprivate_clause_scalarEi16"}
@@ -78,7 +78,7 @@ subroutine private_clause(arg1, arg2, arg3, arg4)
 !FIRDialect-DAG:    %[[I4_PVT_DECL:.*]]:2 = hlfir.declare %[[I4_PVT]] {uniq_name = "_QFprivate_clause_scalarEi4"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !FIRDialect-DAG:    %[[I8_PVT_DECL:.*]]:2 = hlfir.declare %[[I8_PVT]] {uniq_name = "_QFprivate_clause_scalarEi8"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 !FIRDialect-DAG:    %[[I16_PVT_DECL:.*]]:2 = hlfir.declare %[[I16_PVT]] {uniq_name = "_QFprivate_clause_scalarEi16"} : (!fir.ref<i128>) -> (!fir.ref<i128>, !fir.ref<i128>)
-!FIRDialect-DAG:    %[[C_PVT_DECL:.*]]:2 = hlfir.declare %[[C_PVT]] {uniq_name = "_QFprivate_clause_scalarEc"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+!FIRDialect-DAG:    %[[C_PVT_DECL:.*]]:2 = hlfir.declare %[[C_PVT]] {uniq_name = "_QFprivate_clause_scalarEc"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
 !FIRDialect-DAG:    %[[L_PVT_DECL:.*]]:2 = hlfir.declare %[[L_PVT]] {uniq_name = "_QFprivate_clause_scalarEl"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 !FIRDialect-DAG:    %[[R_PVT_DECL:.*]]:2 = hlfir.declare %[[R_PVT]] {uniq_name = "_QFprivate_clause_scalarEr"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 !FIRDialect:        omp.terminator
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-complex-mul.f90 b/flang/test/Lower/OpenMP/parallel-reduction-complex-mul.f90
index 376defb823581..b53295076daad 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-complex-mul.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-complex-mul.f90
@@ -2,39 +2,39 @@
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
 
 !CHECK-LABEL: omp.declare_reduction
-!CHECK-SAME: @[[RED_NAME:.*]] : !fir.complex<8> init {
-!CHECK: ^bb0(%{{.*}}: !fir.complex<8>):
+!CHECK-SAME: @[[RED_NAME:.*]] : complex<f64> init {
+!CHECK: ^bb0(%{{.*}}: complex<f64>):
 !CHECK:  %[[C0_1:.*]] = arith.constant 1.000000e+00 : f64
 !CHECK:  %[[C0_2:.*]] = arith.constant 0.000000e+00 : f64
-!CHECK:  %[[UNDEF:.*]] = fir.undefined !fir.complex<8>
+!CHECK:  %[[UNDEF:.*]] = fir.undefined complex<f64>
 !CHECK:  %[[RES_1:.*]] = fir.insert_value %[[UNDEF]], %[[C0_1]], [0 : index]
 !CHECK:  %[[RES_2:.*]] = fir.insert_value %[[RES_1]], %[[C0_2]], [1 : index]
-!CHECK:  omp.yield(%[[RES_2]] : !fir.complex<8>)
+!CHECK:  omp.yield(%[[RES_2]] : complex<f64>)
 !CHECK: } combiner {
-!CHECK: ^bb0(%[[ARG0:.*]]: !fir.complex<8>, %[[ARG1:.*]]: !fir.complex<8>):
-!CHECK:  %[[RES:.*]] = fir.mulc %[[ARG0]], %[[ARG1]] {{.*}}: !fir.complex<8>
-!CHECK:  omp.yield(%[[RES]] : !fir.complex<8>)
+!CHECK: ^bb0(%[[ARG0:.*]]: complex<f64>, %[[ARG1:.*]]: complex<f64>):
+!CHECK:  %[[RES:.*]] = fir.mulc %[[ARG0]], %[[ARG1]] {{.*}}: complex<f64>
+!CHECK:  omp.yield(%[[RES]] : complex<f64>)
 !CHECK: }
 
 !CHECK-LABEL: func.func @_QPsimple_complex_mul
-!CHECK:  %[[CREF:.*]] = fir.alloca !fir.complex<8> {bindc_name = "c", {{.*}}}
-!CHECK:  %[[C_DECL:.*]]:2 = hlfir.declare %[[CREF]] {uniq_name = "_QFsimple_complex_mulEc"} : (!fir.ref<!fir.complex<8>>) -> (!fir.ref<!fir.complex<8>>, !fir.ref<!fir.complex<8>>)
+!CHECK:  %[[CREF:.*]] = fir.alloca complex<f64> {bindc_name = "c", {{.*}}}
+!CHECK:  %[[C_DECL:.*]]:2 = hlfir.declare %[[CREF]] {uniq_name = "_QFsimple_complex_mulEc"} : (!fir.ref<complex<f64>>) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
 !CHECK:  %[[C_START_RE:.*]] = arith.constant 0.000000e+00 : f64
 !CHECK:  %[[C_START_IM:.*]] = arith.constant 0.000000e+00 : f64
-!CHECK:  %[[UNDEF_1:.*]] = fir.undefined !fir.complex<8>
+!CHECK:  %[[UNDEF_1:.*]] = fir.undefined complex<f64>
 !CHECK:  %[[VAL_1:.*]] = fir.insert_value %[[UNDEF_1]], %[[C_START_RE]], [0 : index]
 !CHECK:  %[[VAL_2:.*]] = fir.insert_value %[[VAL_1]], %[[C_START_IM]], [1 : index]
-!CHECK:  hlfir.assign %[[VAL_2]] to %[[C_DECL]]#0 : !fir.complex<8>, !fir.ref<!fir.complex<8>>
-!CHECK:  omp.parallel reduction(@[[RED_NAME]] %[[C_DECL]]#0 -> %[[PRV:.+]] : !fir.ref<!fir.complex<8>>) {
-!CHECK:    %[[P_DECL:.+]]:2 = hlfir.declare %[[PRV]] {{.*}} : (!fir.ref<!fir.complex<8>>) -> (!fir.ref<!fir.complex<8>>, !fir.ref<!fir.complex<8>>)
-!CHECK:    %[[LPRV:.+]] = fir.load %[[P_DECL]]#0 : !fir.ref<!fir.complex<8>>
+!CHECK:  hlfir.assign %[[VAL_2]] to %[[C_DECL]]#0 : complex<f64>, !fir.ref<complex<f64>>
+!CHECK:  omp.parallel reduction(@[[RED_NAME]] %[[C_DECL]]#0 -> %[[PRV:.+]] : !fir.ref<complex<f64>>) {
+!CHECK:    %[[P_DECL:.+]]:2 = hlfir.declare %[[PRV]] {{.*}} : (!fir.ref<complex<f64>>) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
+!CHECK:    %[[LPRV:.+]] = fir.load %[[P_DECL]]#0 : !fir.ref<complex<f64>>
 !CHECK:    %[[C_INCR_RE:.*]] = arith.constant 1.000000e+00 : f64
 !CHECK:    %[[C_INCR_IM:.*]] = arith.constant -2.000000e+00 : f64
-!CHECK:    %[[UNDEF_2:.*]] = fir.undefined !fir.complex<8>
+!CHECK:    %[[UNDEF_2:.*]] = fir.undefined complex<f64>
 !CHECK:    %[[INCR_1:.*]] = fir.insert_value %[[UNDEF_2]], %[[C_INCR_RE]], [0 : index]
 !CHECK:    %[[INCR_2:.*]] = fir.insert_value %[[INCR_1]], %[[C_INCR_IM]], [1 : index]
-!CHECK:    %[[RES:.+]] = fir.mulc %[[LPRV]], %[[INCR_2]] {{.*}} : !fir.complex<8>
-!CHECK:    hlfir.assign %[[RES]] to %[[P_DECL]]#0 : !fir.complex<8>, !fir.ref<!fir.complex<8>>
+!CHECK:    %[[RES:.+]] = fir.mulc %[[LPRV]], %[[INCR_2]] {{.*}} : complex<f64>
+!CHECK:    hlfir.assign %[[RES]] to %[[P_DECL]]#0 : complex<f64>, !fir.ref<complex<f64>>
 !CHECK:    omp.terminator
 !CHECK:  }
 !CHECK: return
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-complex.f90 b/flang/test/Lower/OpenMP/parallel-reduction-complex.f90
index bc5a6b475e256..9d5aef4158b36 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-complex.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-complex.f90
@@ -2,39 +2,39 @@
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
 
 !CHECK-LABEL: omp.declare_reduction
-!CHECK-SAME: @[[RED_NAME:.*]] : !fir.complex<8> init {
-!CHECK: ^bb0(%{{.*}}: !fir.complex<8>):
+!CHECK-SAME: @[[RED_NAME:.*]] : complex<f64> init {
+!CHECK: ^bb0(%{{.*}}: complex<f64>):
 !CHECK:  %[[C0_1:.*]] = arith.constant 0.000000e+00 : f64
 !CHECK:  %[[C0_2:.*]] = arith.constant 0.000000e+00 : f64
-!CHECK:  %[[UNDEF:.*]] = fir.undefined !fir.complex<8>
+!CHECK:  %[[UNDEF:.*]] = fir.undefined complex<f64>
 !CHECK:  %[[RES_1:.*]] = fir.insert_value %[[UNDEF]], %[[C0_1]], [0 : index]
 !CHECK:  %[[RES_2:.*]] = fir.insert_value %[[RES_1]], %[[C0_2]], [1 : index]
-!CHECK:  omp.yield(%[[RES_2]] : !fir.complex<8>)
+!CHECK:  omp.yield(%[[RES_2]] : complex<f64>)
 !CHECK: } combiner {
-!CHECK: ^bb0(%[[ARG0:.*]]: !fir.complex<8>, %[[ARG1:.*]]: !fir.complex<8>):
-!CHECK:  %[[RES:.*]] = fir.addc %[[ARG0]], %[[ARG1]] {{.*}}: !fir.complex<8>
-!CHECK:  omp.yield(%[[RES]] : !fir.complex<8>)
+!CHECK: ^bb0(%[[ARG0:.*]]: complex<f64>, %[[ARG1:.*]]: complex<f64>):
+!CHECK:  %[[RES:.*]] = fir.addc %[[ARG0]], %[[ARG1]] {{.*}}: complex<f64>
+!CHECK:  omp.yield(%[[RES]] : complex<f64>)
 !CHECK: }
 
 !CHECK-LABEL: func.func @_QPsimple_complex_add
-!CHECK:  %[[CREF:.*]] = fir.alloca !fir.complex<8> {bindc_name = "c", {{.*}}}
-!CHECK:  %[[C_DECL:.*]]:2 = hlfir.declare %[[CREF]] {uniq_name = "_QFsimple_complex_addEc"} : (!fir.ref<!fir.complex<8>>) -> (!fir.ref<!fir.complex<8>>, !fir.ref<!fir.complex<8>>)
+!CHECK:  %[[CREF:.*]] = fir.alloca complex<f64> {bindc_name = "c", {{.*}}}
+!CHECK:  %[[C_DECL:.*]]:2 = hlfir.declare %[[CREF]] {uniq_name = "_QFsimple_complex_addEc"} : (!fir.ref<complex<f64>>) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
 !CHECK:  %[[C_START_RE:.*]] = arith.constant 0.000000e+00 : f64
 !CHECK:  %[[C_START_IM:.*]] = arith.constant 0.000000e+00 : f64
-!CHECK:  %[[UNDEF_1:.*]] = fir.undefined !fir.complex<8>
+!CHECK:  %[[UNDEF_1:.*]] = fir.undefined complex<f64>
 !CHECK:  %[[VAL_1:.*]] = fir.insert_value %[[UNDEF_1]], %[[C_START_RE]], [0 : index]
 !CHECK:  %[[VAL_2:.*]] = fir.insert_value %[[VAL_1]], %[[C_START_IM]], [1 : index]
-!CHECK:  hlfir.assign %[[VAL_2]] to %[[C_DECL]]#0 : !fir.complex<8>, !fir.ref<!fir.complex<8>>
-!CHECK:  omp.parallel reduction(@[[RED_NAME]] %[[C_DECL]]#0 -> %[[PRV:.+]] : !fir.ref<!fir.complex<8>>) {
-!CHECK:    %[[P_DECL:.+]]:2 = hlfir.declare %[[PRV]] {{.*}} : (!fir.ref<!fir.complex<8>>) -> (!fir.ref<!fir.complex<8>>, !fir.ref<!fir.complex<8>>)
-!CHECK:    %[[LPRV:.+]] = fir.load %[[P_DECL]]#0 : !fir.ref<!fir.complex<8>>
+!CHECK:  hlfir.assign %[[VAL_2]] to %[[C_DECL]]#0 : complex<f64>, !fir.ref<complex<f64>>
+!CHECK:  omp.parallel reduction(@[[RED_NAME]] %[[C_DECL]]#0 -> %[[PRV:.+]] : !fir.ref<complex<f64>>) {
+!CHECK:    %[[P_DECL:.+]]:2 = hlfir.declare %[[PRV]] {{.*}} : (!fir.ref<complex<f64>>) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
+!CHECK:    %[[LPRV:.+]] = fir.load %[[P_DECL]]#0 : !fir.ref<complex<f64>>
 !CHECK:    %[[C_INCR_RE:.*]] = arith.constant 1.000000e+00 : f64
 !CHECK:    %[[C_INCR_IM:.*]] = arith.constant 0.000000e+00 : f64
-!CHECK:    %[[UNDEF_2:.*]] = fir.undefined !fir.complex<8>
+!CHECK:    %[[UNDEF_2:.*]] = fir.undefined complex<f64>
 !CHECK:    %[[INCR_1:.*]] = fir.insert_value %[[UNDEF_2]], %[[C_INCR_RE]], [0 : index]
 !CHECK:    %[[INCR_2:.*]] = fir.insert_value %[[INCR_1]], %[[C_INCR_IM]], [1 : index]
-!CHECK:    %[[RES:.+]] = fir.addc %[[LPRV]], %[[INCR_2]] {{.*}} : !fir.complex<8>
-!CHECK:    hlfir.assign %[[RES]] to %[[P_DECL]]#0 : !fir.complex<8>, !fir.ref<!fir.complex<8>>
+!CHECK:    %[[RES:.+]] = fir.addc %[[LPRV]], %[[INCR_2]] {{.*}} : complex<f64>
+!CHECK:    hlfir.assign %[[RES]] to %[[P_DECL]]#0 : complex<f64>, !fir.ref<complex<f64>>
 !CHECK:    omp.terminator
 !CHECK:  }
 !CHECK: return
diff --git a/flang/test/Lower/OpenMP/private-commonblock.f90 b/flang/test/Lower/OpenMP/private-commonblock.f90
index 59c55ea6bf922..f6d285a3b011e 100644
--- a/flang/test/Lower/OpenMP/private-commonblock.f90
+++ b/flang/test/Lower/OpenMP/private-commonblock.f90
@@ -88,25 +88,25 @@ subroutine private_clause_commonblock()
 !CHECK:    %[[BLK_I8_REF:.*]] = fir.convert %[[BLK_ADDR]] : (!fir.ref<!fir.array<74xi8>>) -> !fir.ref<!fir.array<?xi8>>
 !CHECK:    %[[C0:.*]] = arith.constant 0 : index
 !CHECK:    %[[C_I8_REF:.*]] = fir.coordinate_of %[[BLK_I8_REF]], %[[C0]] : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
-!CHECK:    %[[C_REF:.*]] = fir.convert %[[C_I8_REF]] : (!fir.ref<i8>) -> !fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>
-!CHECK:    %[[C_DECL:.*]]:2 = hlfir.declare %[[C_REF]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFprivate_clause_commonblock_pointerEc"} : (!fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>)
-!CHECK:    %[[C_BOX:.*]] = fir.load %[[C_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>
-!CHECK:    %[[C_ADDR:.*]] = fir.box_addr %[[C_BOX]] : (!fir.box<!fir.ptr<!fir.complex<4>>>) -> !fir.ptr<!fir.complex<4>>
-!CHECK:    %[[C_REF:.*]] = fir.convert %[[C_ADDR]] : (!fir.ptr<!fir.complex<4>>) -> !fir.ref<!fir.complex<4>>
-!CHECK:    fir.call @_QPsub4(%[[C_REF]], %[[A_DECL]]#1) fastmath<contract> : (!fir.ref<!fir.complex<4>>, !fir.ref<i32>) -> ()
+!CHECK:    %[[C_REF:.*]] = fir.convert %[[C_I8_REF]] : (!fir.ref<i8>) -> !fir.ref<!fir.box<!fir.ptr<complex<f32>>>>
+!CHECK:    %[[C_DECL:.*]]:2 = hlfir.declare %[[C_REF]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFprivate_clause_commonblock_pointerEc"} : (!fir.ref<!fir.box<!fir.ptr<complex<f32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<complex<f32>>>>, !fir.ref<!fir.box<!fir.ptr<complex<f32>>>>)
+!CHECK:    %[[C_BOX:.*]] = fir.load %[[C_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<complex<f32>>>>
+!CHECK:    %[[C_ADDR:.*]] = fir.box_addr %[[C_BOX]] : (!fir.box<!fir.ptr<complex<f32>>>) -> !fir.ptr<complex<f32>>
+!CHECK:    %[[C_REF:.*]] = fir.convert %[[C_ADDR]] : (!fir.ptr<complex<f32>>) -> !fir.ref<complex<f32>>
+!CHECK:    fir.call @_QPsub4(%[[C_REF]], %[[A_DECL]]#1) fastmath<contract> : (!fir.ref<complex<f32>>, !fir.ref<i32>) -> ()
 !CHECK: omp.parallel private(@{{.*}} %{{.*}}#0 -> %[[C_PVT_REF:.*]], @{{.*}} %{{.*}}#0 -> %[[A_PVT_REF:.*]] : {{.*}}) {
-!CHECK:      %[[C_PVT_DECL:.*]]:2 = hlfir.declare %[[C_PVT_REF]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFprivate_clause_commonblock_pointerEc"} : (!fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>)
+!CHECK:      %[[C_PVT_DECL:.*]]:2 = hlfir.declare %[[C_PVT_REF]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFprivate_clause_commonblock_pointerEc"} : (!fir.ref<!fir.box<!fir.ptr<complex<f32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<complex<f32>>>>, !fir.ref<!fir.box<!fir.ptr<complex<f32>>>>)
 !CHECK:      %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT_REF]] {uniq_name = "_QFprivate_clause_commonblock_pointerEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:      %[[C_PVT_BOX:.*]] = fir.load %[[C_PVT_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>
-!CHECK:      %[[C_PVT_ADDR:.*]] = fir.box_addr %[[C_PVT_BOX]] : (!fir.box<!fir.ptr<!fir.complex<4>>>) -> !fir.ptr<!fir.complex<4>>
-!CHECK:      %[[C_PVT_REF:.*]] = fir.convert %[[C_PVT_ADDR]] : (!fir.ptr<!fir.complex<4>>) -> !fir.ref<!fir.complex<4>>
-!CHECK:      fir.call @_QPsub5(%[[C_PVT_REF]], %[[A_PVT_DECL]]#1) fastmath<contract> : (!fir.ref<!fir.complex<4>>, !fir.ref<i32>) -> ()
+!CHECK:      %[[C_PVT_BOX:.*]] = fir.load %[[C_PVT_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<complex<f32>>>>
+!CHECK:      %[[C_PVT_ADDR:.*]] = fir.box_addr %[[C_PVT_BOX]] : (!fir.box<!fir.ptr<complex<f32>>>) -> !fir.ptr<complex<f32>>
+!CHECK:      %[[C_PVT_REF:.*]] = fir.convert %[[C_PVT_ADDR]] : (!fir.ptr<complex<f32>>) -> !fir.ref<complex<f32>>
+!CHECK:      fir.call @_QPsub5(%[[C_PVT_REF]], %[[A_PVT_DECL]]#1) fastmath<contract> : (!fir.ref<complex<f32>>, !fir.ref<i32>) -> ()
 !CHECK:      omp.terminator
 !CHECK:    }
-!CHECK:    %[[C_BOX:.*]] = fir.load %[[C_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>
-!CHECK:    %[[C_ADDR:.*]] = fir.box_addr %[[C_BOX]] : (!fir.box<!fir.ptr<!fir.complex<4>>>) -> !fir.ptr<!fir.complex<4>>
-!CHECK:    %[[C_REF:.*]] = fir.convert %[[C_ADDR]] : (!fir.ptr<!fir.complex<4>>) -> !fir.ref<!fir.complex<4>>
-!CHECK:    fir.call @_QPsub6(%[[C_REF]], %[[A_DECL]]#1) fastmath<contract> : (!fir.ref<!fir.complex<4>>, !fir.ref<i32>) -> ()
+!CHECK:    %[[C_BOX:.*]] = fir.load %[[C_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<complex<f32>>>>
+!CHECK:    %[[C_ADDR:.*]] = fir.box_addr %[[C_BOX]] : (!fir.box<!fir.ptr<complex<f32>>>) -> !fir.ptr<complex<f32>>
+!CHECK:    %[[C_REF:.*]] = fir.convert %[[C_ADDR]] : (!fir.ptr<complex<f32>>) -> !fir.ref<complex<f32>>
+!CHECK:    fir.call @_QPsub6(%[[C_REF]], %[[A_DECL]]#1) fastmath<contract> : (!fir.ref<complex<f32>>, !fir.ref<i32>) -> ()
 !CHECK: return
 !CHECK: }
 subroutine private_clause_commonblock_pointer()
diff --git a/flang/test/Lower/OpenMP/simd.f90 b/flang/test/Lower/OpenMP/simd.f90
index bdc6a1eef0156..3b2aeceb4c3f9 100644
--- a/flang/test/Lower/OpenMP/simd.f90
+++ b/flang/test/Lower/OpenMP/simd.f90
@@ -241,3 +241,36 @@ subroutine simd_with_nontemporal_clause(n)
   end do
   !$OMP END SIMD
 end subroutine
+
+!CHECK-LABEL: func.func @_QPlastprivate_with_simd() {
+subroutine lastprivate_with_simd
+
+!CHECK: %[[VAR_SUM:.*]] = fir.alloca f32 {bindc_name = "sum", uniq_name = "_QFlastprivate_with_simdEsum"}
+!CHECK: %[[VAR_SUM_DECLARE:.*]]:2 = hlfir.declare %[[VAR_SUM]] {{.*}}
+!CHECK: %[[VAR_SUM_PINNED:.*]] = fir.alloca f32 {bindc_name = "sum", pinned, uniq_name = "_QFlastprivate_with_simdEsum"}
+!CHECK: %[[VAR_SUM_PINNED_DECLARE:.*]]:2 = hlfir.declare %[[VAR_SUM_PINNED]] {{.*}}
+
+  implicit none
+  integer :: i
+  real :: sum
+
+  
+!CHECK: omp.simd {
+!CHECK: omp.loop_nest (%[[ARG:.*]]) : i32 = ({{.*}} to ({{.*}}) inclusive step ({{.*}}) {
+!CHECK: %[[ADD_RESULT:.*]] = arith.addi {{.*}}
+!CHECK: %[[ADD_RESULT_CONVERT:.*]] = fir.convert %[[ADD_RESULT]] : (i32) -> f32
+!CHECK: hlfir.assign %[[ADD_RESULT_CONVERT]] to %[[VAR_SUM_PINNED_DECLARE]]#0 : f32, !fir.ref<f32>
+!CHECK: %[[SELECT_RESULT:.*]] = arith.select {{.*}}, {{.*}}, {{.*}} : i1
+!CHECK: fir.if %[[SELECT_RESULT]] {
+!CHECK: %[[LOADED_SUM:.*]] = fir.load %[[VAR_SUM_PINNED_DECLARE]]#0 : !fir.ref<f32>
+!CHECK: hlfir.assign %[[LOADED_SUM]] to %[[VAR_SUM_DECLARE]]#0 : f32, !fir.ref<f32>
+!CHECK: }
+!CHECK: omp.yield
+!CHECK: }
+!CHECK: omp.terminator
+!CHECK: }
+  !$omp simd lastprivate(sum)
+  do i = 1, 100
+    sum = i + 1
+  end do
+end subroutine
diff --git a/flang/test/Lower/OpenMP/statement-function.f90 b/flang/test/Lower/OpenMP/statement-function.f90
index 8d30450161d7d..56601de2f4f0b 100644
--- a/flang/test/Lower/OpenMP/statement-function.f90
+++ b/flang/test/Lower/OpenMP/statement-function.f90
@@ -1,13 +1,11 @@
-! Test privatization within OpenMP constructs containing statement functions.
+! Test statement functions are not privatised
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
 
 !CHECK-LABEL: func @_QPtest_implicit_use
 !CHECK:         %[[IEXP:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_implicit_useEiexp"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:         %[[IIMP:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_implicit_useEiimp"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:         omp.parallel private({{.*firstprivate.*}} %[[IEXP]]#0 -> %[[PRIV_IEXP:[^,]+]],
-!CHECK-SAME:                         {{.*firstprivate.*}} %[[IIMP]]#0 -> %[[PRIV_IIMP:.*]] : !fir.ref<i32>, !fir.ref<i32>)
+!CHECK:         omp.parallel private({{.*firstprivate.*}} %[[IEXP]]#0 -> %[[PRIV_IEXP:[^,]+]] : !fir.ref<i32>) {
 !CHECK:           %{{.*}}:2 = hlfir.declare %[[PRIV_IEXP]] {uniq_name = "_QFtest_implicit_useEiexp"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:           %{{.*}}:2 = hlfir.declare %[[PRIV_IIMP]] {uniq_name = "_QFtest_implicit_useEiimp"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 subroutine test_implicit_use()
   implicit none
   integer :: iexp, iimp
@@ -27,9 +25,7 @@ subroutine test_implicit_use()
 !CHECK:           %[[PRIV_IEXP:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_implicit_use2Eiexp"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:           %[[TEMP0:.*]] = fir.load %[[IEXP]]#0 : !fir.ref<i32>
 !CHECK:           hlfir.assign %[[TEMP0]] to %[[PRIV_IEXP]]#0 : i32, !fir.ref<i32>
-!CHECK:           %[[PRIV_IIMP:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_implicit_use2Eiimp"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:           %[[TEMP1:.*]] = fir.load %[[IIMP]]#0 : !fir.ref<i32>
-!CHECK:           hlfir.assign %[[TEMP1]] to %[[PRIV_IIMP]]#0 : i32, !fir.ref<i32>
+!CHECK-NOT:       %[[PRIV_IIMP:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest_implicit_use2Eiimp"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 subroutine test_implicit_use2()
   implicit none
   integer :: iexp, iimp
diff --git a/flang/test/Lower/OpenMP/task.f90 b/flang/test/Lower/OpenMP/task.f90
index 28e438d342d35..4f00f261fe57d 100644
--- a/flang/test/Lower/OpenMP/task.f90
+++ b/flang/test/Lower/OpenMP/task.f90
@@ -93,7 +93,7 @@ subroutine task_depend_non_int()
   character(len = 15) :: x
   integer, allocatable :: y
   complex :: z
-  !CHECK: omp.task depend(taskdependin -> %{{.+}} : !fir.ref<!fir.char<1,15>>, taskdependin -> %{{.+}} : !fir.ref<!fir.box<!fir.heap<i32>>>, taskdependin ->  %{{.+}} : !fir.ref<!fir.complex<4>>) {
+  !CHECK: omp.task depend(taskdependin -> %{{.+}} : !fir.ref<!fir.char<1,15>>, taskdependin -> %{{.+}} : !fir.ref<!fir.box<!fir.heap<i32>>>, taskdependin ->  %{{.+}} : !fir.ref<complex<f32>>) {
   !$omp task depend(in : x, y, z)
   !CHECK: omp.terminator
   !$omp end task
diff --git a/flang/test/Lower/OpenMP/threadprivate-commonblock.f90 b/flang/test/Lower/OpenMP/threadprivate-commonblock.f90
index eb6abe4d2cacd..588294c9728b9 100644
--- a/flang/test/Lower/OpenMP/threadprivate-commonblock.f90
+++ b/flang/test/Lower/OpenMP/threadprivate-commonblock.f90
@@ -32,12 +32,12 @@ subroutine sub()
   !CHECK-DAG:  %[[B_DECL:.*]]:2 = hlfir.declare %[[B_ADDR_CVT]]({{.*}}) {uniq_name = "_QMtestEb"} : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<2xf32>>, !fir.ref<!fir.array<2xf32>>)
   !CHECK-DAG:  %[[OMP_CBLK_ADDR:.*]] = fir.convert %[[OMP_CBLK]] : (!fir.ref<!fir.array<103xi8>>) -> !fir.ref<!fir.array<?xi8>>
   !CHECK-DAG:  %[[C_ADDR:.*]] = fir.coordinate_of %[[OMP_CBLK_ADDR]], {{.*}} : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
-  !CHECK-DAG:  %[[C_ADDR_CVT:.*]] = fir.convert %[[C_ADDR]] : (!fir.ref<i8>) -> !fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>
-  !CHECK-DAG:  %[[C_DECL:.*]]:2 = hlfir.declare %[[C_ADDR_CVT]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMtestEc"} : (!fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>)
+  !CHECK-DAG:  %[[C_ADDR_CVT:.*]] = fir.convert %[[C_ADDR]] : (!fir.ref<i8>) -> !fir.ref<!fir.box<!fir.ptr<complex<f32>>>>
+  !CHECK-DAG:  %[[C_DECL:.*]]:2 = hlfir.declare %[[C_ADDR_CVT]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMtestEc"} : (!fir.ref<!fir.box<!fir.ptr<complex<f32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<complex<f32>>>>, !fir.ref<!fir.box<!fir.ptr<complex<f32>>>>)
   !CHECK-DAG:  %[[OMP_CBLK_ADDR:.*]] = fir.convert %[[OMP_CBLK]] : (!fir.ref<!fir.array<103xi8>>) -> !fir.ref<!fir.array<?xi8>>
   !CHECK-DAG:  %[[D_ADDR:.*]] = fir.coordinate_of %[[OMP_CBLK_ADDR]], {{.*}} : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
-  !CHECK-DAG:  %[[D_ADDR_CVT:.*]] = fir.convert %[[D_ADDR]] : (!fir.ref<i8>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>>
-  !CHECK-DAG:  %[[D_DECL:.*]]:2 = hlfir.declare %[[D_ADDR_CVT]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMtestEd"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>>)
+  !CHECK-DAG:  %[[D_ADDR_CVT:.*]] = fir.convert %[[D_ADDR]] : (!fir.ref<i8>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>>
+  !CHECK-DAG:  %[[D_DECL:.*]]:2 = hlfir.declare %[[D_ADDR_CVT]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMtestEd"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>>)
   !CHECK-DAG:  %[[OMP_CBLK_ADDR:.*]] = fir.convert %[[OMP_CBLK]] : (!fir.ref<!fir.array<103xi8>>) -> !fir.ref<!fir.array<?xi8>>
   !CHECK-DAG:  %[[E_ADDR:.*]] = fir.coordinate_of %[[OMP_CBLK_ADDR]], {{.*}} : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
   !CHECK-DAG:  %[[E_ADDR_CVT:.*]] = fir.convert %[[E_ADDR]] : (!fir.ref<i8>) -> !fir.ref<!fir.char<1,5>>
@@ -48,8 +48,8 @@ subroutine sub()
   !CHECK-DAG:  %[[F_DECL:.*]]:2 = hlfir.declare %[[F_ADDR_CVT]]({{.*}}) typeparams {{.*}} {uniq_name = "_QMtestEf"} : (!fir.ref<!fir.array<2x!fir.char<1,5>>>, !fir.shape<1>, index) -> (!fir.ref<!fir.array<2x!fir.char<1,5>>>, !fir.ref<!fir.array<2x!fir.char<1,5>>>)
   !CHECK-DAG:  {{.*}} = fir.load %[[A_DECL]]#0 : !fir.ref<i32>
   !CHECK-DAG:  {{.*}} = fir.embox %[[B_DECL]]#1({{.*}}) : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<2xf32>>
-  !CHECK-DAG:  {{.*}} = fir.load %[[C_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>
-  !CHECK-DAG:  {{.*}} = fir.load %[[D_DECL]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>>
+  !CHECK-DAG:  {{.*}} = fir.load %[[C_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<complex<f32>>>>
+  !CHECK-DAG:  {{.*}} = fir.load %[[D_DECL]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>>
   !CHECK-DAG:  {{.*}} = fir.convert %[[E_DECL]]#1 : (!fir.ref<!fir.char<1,5>>) -> !fir.ref<i8>
   !CHECK-DAG:  {{.*}} = fir.embox %[[F_DECL]]#1({{.*}}) : (!fir.ref<!fir.array<2x!fir.char<1,5>>>, !fir.shape<1>) -> !fir.box<!fir.array<2x!fir.char<1,5>>>
     print *, a, b, c, d, e, f
@@ -67,12 +67,12 @@ subroutine sub()
     !CHECK-DAG:  %[[TP_B_DECL:.*]]:2 = hlfir.declare %[[TP_B_ADDR_CVT]](%92) {uniq_name = "_QMtestEb"} : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<2xf32>>, !fir.ref<!fir.array<2xf32>>)
     !CHECK-DAG:  %[[TP_PARALLEL_ADDR:.*]] = fir.convert %[[TP_PARALLEL]] : (!fir.ref<!fir.array<103xi8>>) -> !fir.ref<!fir.array<?xi8>>
     !CHECK-DAG:  %[[TP_C_ADDR:.*]] = fir.coordinate_of %[[TP_PARALLEL_ADDR]], {{.*}} : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
-    !CHECK-DAG:  %[[TP_C_ADDR_CVT:.*]] = fir.convert %[[TP_C_ADDR]] : (!fir.ref<i8>) -> !fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>
-    !CHECK-DAG:  %[[TP_C_DECL:.*]]:2 = hlfir.declare %[[TP_C_ADDR_CVT]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMtestEc"} : (!fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>)
+    !CHECK-DAG:  %[[TP_C_ADDR_CVT:.*]] = fir.convert %[[TP_C_ADDR]] : (!fir.ref<i8>) -> !fir.ref<!fir.box<!fir.ptr<complex<f32>>>>
+    !CHECK-DAG:  %[[TP_C_DECL:.*]]:2 = hlfir.declare %[[TP_C_ADDR_CVT]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMtestEc"} : (!fir.ref<!fir.box<!fir.ptr<complex<f32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<complex<f32>>>>, !fir.ref<!fir.box<!fir.ptr<complex<f32>>>>)
     !CHECK-DAG:  %[[TP_PARALLEL_ADDR:.*]] = fir.convert %[[TP_PARALLEL]] : (!fir.ref<!fir.array<103xi8>>) -> !fir.ref<!fir.array<?xi8>>
     !CHECK-DAG:  %[[TP_D_ADDR:.*]] = fir.coordinate_of %[[TP_PARALLEL_ADDR]], {{.*}} : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
-    !CHECK-DAG:  %[[TP_D_ADDR_CVT:.*]] = fir.convert %[[TP_D_ADDR]] : (!fir.ref<i8>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>>
-    !CHECK-DAG:  %[[TP_D_DECL:.*]]:2 = hlfir.declare %[[TP_D_ADDR_CVT]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMtestEd"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>>)
+    !CHECK-DAG:  %[[TP_D_ADDR_CVT:.*]] = fir.convert %[[TP_D_ADDR]] : (!fir.ref<i8>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>>
+    !CHECK-DAG:  %[[TP_D_DECL:.*]]:2 = hlfir.declare %[[TP_D_ADDR_CVT]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMtestEd"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>>)
     !CHECK-DAG:  %[[TP_PARALLEL_ADDR:.*]] = fir.convert %[[TP_PARALLEL]] : (!fir.ref<!fir.array<103xi8>>) -> !fir.ref<!fir.array<?xi8>>
     !CHECK-DAG:  %[[TP_E_ADDR:.*]] = fir.coordinate_of %[[TP_PARALLEL_ADDR]], {{.*}} : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
     !CHECK-DAG:  %[[TP_E_ADDR_CVT:.*]] = fir.convert %[[TP_E_ADDR]] : (!fir.ref<i8>) -> !fir.ref<!fir.char<1,5>>
@@ -83,8 +83,8 @@ subroutine sub()
     !CHECK-DAG:  %[[TP_F_DECL:.*]]:2 = hlfir.declare %[[TP_F_ADDR_CVT]]({{.*}}) typeparams {{.*}} {uniq_name = "_QMtestEf"} : (!fir.ref<!fir.array<2x!fir.char<1,5>>>, !fir.shape<1>, index) -> (!fir.ref<!fir.array<2x!fir.char<1,5>>>, !fir.ref<!fir.array<2x!fir.char<1,5>>>)
     !CHECK-DAG:  {{.*}} = fir.load %[[TP_A_DECL]]#0 : !fir.ref<i32>
     !CHECK-DAG:  {{.*}} = fir.embox %[[TP_B_DECL]]#1({{.*}}) : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<2xf32>>
-    !CHECK-DAG:  {{.*}} = fir.load %[[TP_C_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>
-    !CHECK-DAG:  {{.*}} = fir.load %[[TP_D_DECL]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>>
+    !CHECK-DAG:  {{.*}} = fir.load %[[TP_C_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<complex<f32>>>>
+    !CHECK-DAG:  {{.*}} = fir.load %[[TP_D_DECL]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>>
     !CHECK-DAG:  {{.*}} = fir.convert %[[TP_E_DECL]]#1 : (!fir.ref<!fir.char<1,5>>) -> !fir.ref<i8>
     !CHECK-DAG:  {{.*}} = fir.embox %[[TP_F_DECL]]#1(%{{.*}}) : (!fir.ref<!fir.array<2x!fir.char<1,5>>>, !fir.shape<1>) -> !fir.box<!fir.array<2x!fir.char<1,5>>>
     !CHECK-DAG: omp.terminator
@@ -94,8 +94,8 @@ subroutine sub()
 
   !CHECK-DAG:  %{{.*}} = fir.load %[[A_DECL]]#0 : !fir.ref<i32>
   !CHECK-DAG:  %{{.*}} = fir.embox %[[B_DECL]]#1(%63) : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<2xf32>>
-  !CHECK-DAG:  %{{.*}} = fir.load %[[C_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>
-  !CHECK-DAG:  %{{.*}} = fir.load %[[D_DECL]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>>
+  !CHECK-DAG:  %{{.*}} = fir.load %[[C_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<complex<f32>>>>
+  !CHECK-DAG:  %{{.*}} = fir.load %[[D_DECL]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>>
   !CHECK-DAG:  %{{.*}} = fir.convert %[[E_DECL]]#1 : (!fir.ref<!fir.char<1,5>>) -> !fir.ref<i8>
   !CHECK-DAG:  %{{.*}} = fir.embox %[[F_DECL]]#1(%79) : (!fir.ref<!fir.array<2x!fir.char<1,5>>>, !fir.shape<1>) -> !fir.box<!fir.array<2x!fir.char<1,5>>>
     print *, a, b, c, d, e, f
diff --git a/flang/test/Lower/OpenMP/threadprivate-non-global.f90 b/flang/test/Lower/OpenMP/threadprivate-non-global.f90
index 7cdadabe2247a..0b9abd1d0bcf4 100644
--- a/flang/test/Lower/OpenMP/threadprivate-non-global.f90
+++ b/flang/test/Lower/OpenMP/threadprivate-non-global.f90
@@ -18,9 +18,9 @@ program test
 !CHECK-DAG:  %[[B:.*]] = fir.address_of(@_QFEb) : !fir.ref<!fir.box<!fir.heap<f32>>>
 !CHECK-DAG:  %[[OMP_B:.*]] = omp.threadprivate %[[B]] : !fir.ref<!fir.box<!fir.heap<f32>>> -> !fir.ref<!fir.box<!fir.heap<f32>>>
 !CHECK-DAG:  %[[OMP_B_DECL:.*]]:2 = hlfir.declare %[[OMP_B]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFEb"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
-!CHECK-DAG:  %[[W:.*]] = fir.address_of(@_QFEw) : !fir.ref<!fir.complex<4>>
-!CHECK-DAG:  %[[OMP_W:.*]] = omp.threadprivate %[[W]] : !fir.ref<!fir.complex<4>> -> !fir.ref<!fir.complex<4>>
-!CHECK-DAG:  %[[OMP_W_DECL:.*]]:2 = hlfir.declare %[[OMP_W]] {uniq_name = "_QFEw"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+!CHECK-DAG:  %[[W:.*]] = fir.address_of(@_QFEw) : !fir.ref<complex<f32>>
+!CHECK-DAG:  %[[OMP_W:.*]] = omp.threadprivate %[[W]] : !fir.ref<complex<f32>> -> !fir.ref<complex<f32>>
+!CHECK-DAG:  %[[OMP_W_DECL:.*]]:2 = hlfir.declare %[[OMP_W]] {uniq_name = "_QFEw"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
 !CHECK-DAG:  %[[X:.*]] = fir.address_of(@_QFEx) : !fir.ref<i32>
 !CHECK-DAG:  %[[OMP_X:.*]] = omp.threadprivate %[[X]] : !fir.ref<i32> -> !fir.ref<i32>
 !CHECK-DAG:  %[[OMP_X_DECL:.*]]:2 = hlfir.declare %[[OMP_X]] {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
@@ -37,7 +37,7 @@ program test
 !CHECK-DAG:  %{{.*}} = fir.load %[[OMP_X_DECL]]#0 : !fir.ref<i32>
 !CHECK-DAG:  %{{.*}} = fir.load %[[OMP_Y_DECL]]#0 : !fir.ref<f32>
 !CHECK-DAG:  %{{.*}} = fir.load %[[OMP_Z_DECL]]#0 : !fir.ref<!fir.logical<4>>
-!CHECK-DAG:  %{{.*}} = fir.load %[[OMP_W_DECL]]#0 : !fir.ref<!fir.complex<4>>
+!CHECK-DAG:  %{{.*}} = fir.load %[[OMP_W_DECL]]#0 : !fir.ref<complex<f32>>
 !CHECK-DAG:  %{{.*}} = fir.load %[[OMP_A_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>
 !CHECK-DAG:  %{{.*}} = fir.load %[[OMP_B_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<f32>>>
   print *, x, y, z, w, a, b
@@ -49,8 +49,8 @@ program test
 !CHECK-DAG:  %[[Y_PVT_DECL:.*]]:2 = hlfir.declare %[[Y_PVT]] {uniq_name = "_QFEy"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 !CHECK-DAG:  %[[Z_PVT:.*]] = omp.threadprivate %[[Z]] : !fir.ref<!fir.logical<4>> -> !fir.ref<!fir.logical<4>>
 !CHECK-DAG:  %[[Z_PVT_DECL:.*]]:2 = hlfir.declare %[[Z_PVT]] {uniq_name = "_QFEz"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-!CHECK-DAG:  %[[W_PVT:.*]] = omp.threadprivate %[[W]] : !fir.ref<!fir.complex<4>> -> !fir.ref<!fir.complex<4>>
-!CHECK-DAG:  %[[W_PVT_DECL:.*]]:2 = hlfir.declare %[[W_PVT]] {uniq_name = "_QFEw"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+!CHECK-DAG:  %[[W_PVT:.*]] = omp.threadprivate %[[W]] : !fir.ref<complex<f32>> -> !fir.ref<complex<f32>>
+!CHECK-DAG:  %[[W_PVT_DECL:.*]]:2 = hlfir.declare %[[W_PVT]] {uniq_name = "_QFEw"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
 !CHECK-DAG:  %[[A_PVT:.*]] = omp.threadprivate %[[A]] : !fir.ref<!fir.box<!fir.ptr<i32>>> -> !fir.ref<!fir.box<!fir.ptr<i32>>>
 !CHECK-DAG:  %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFEa"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
 !CHECK-DAG:  %[[B_PVT:.*]] = omp.threadprivate %[[B]] : !fir.ref<!fir.box<!fir.heap<f32>>> -> !fir.ref<!fir.box<!fir.heap<f32>>>
@@ -58,7 +58,7 @@ program test
 !CHECK-DAG:  %{{.*}} = fir.load %[[X_PVT_DECL]]#0 : !fir.ref<i32>
 !CHECK-DAG:  %{{.*}} = fir.load %[[Y_PVT_DECL]]#0 : !fir.ref<f32>
 !CHECK-DAG:  %{{.*}} = fir.load %[[Z_PVT_DECL]]#0 : !fir.ref<!fir.logical<4>>
-!CHECK-DAG:  %{{.*}} = fir.load %[[W_PVT_DECL]]#0 : !fir.ref<!fir.complex<4>>
+!CHECK-DAG:  %{{.*}} = fir.load %[[W_PVT_DECL]]#0 : !fir.ref<complex<f32>>
 !CHECK-DAG:  %{{.*}} = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>
 !CHECK-DAG:  %{{.*}} = fir.load %[[B_PVT_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<f32>>>
   print *, x, y, z, w, a, b
@@ -67,7 +67,7 @@ program test
 !CHECK-DAG:  %{{.*}} = fir.load %[[OMP_X_DECL]]#0 : !fir.ref<i32>
 !CHECK-DAG:  %{{.*}} = fir.load %[[OMP_Y_DECL]]#0 : !fir.ref<f32>
 !CHECK-DAG:  %{{.*}} = fir.load %[[OMP_Z_DECL]]#0 : !fir.ref<!fir.logical<4>>
-!CHECK-DAG:  %{{.*}} = fir.load %[[OMP_W_DECL]]#0 : !fir.ref<!fir.complex<4>>
+!CHECK-DAG:  %{{.*}} = fir.load %[[OMP_W_DECL]]#0 : !fir.ref<complex<f32>>
 !CHECK-DAG:  %{{.*}} = fir.load %[[OMP_A_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>
 !CHECK-DAG:  %{{.*}} = fir.load %[[OMP_B_DECL]]#0 : !fir.ref<!fir.box<!fir.heap<f32>>>
   print *, x, y, z, w, a, b
@@ -84,9 +84,9 @@ program test
 !CHECK-DAG:   [[E1:%.*]] = fir.embox [[Z1]] : (!fir.heap<f32>) -> !fir.box<!fir.heap<f32>>
 !CHECK-DAG:   fir.has_value [[E1]] : !fir.box<!fir.heap<f32>>
 !CHECK-DAG: }
-!CHECK-DAG: fir.global internal @_QFEw : !fir.complex<4> {
-!CHECK-DAG:   [[Z2:%.*]] = fir.undefined !fir.complex<4>
-!CHECK-DAG:   fir.has_value [[Z2]] : !fir.complex<4>
+!CHECK-DAG: fir.global internal @_QFEw : complex<f32> {
+!CHECK-DAG:   [[Z2:%.*]] = fir.undefined complex<f32>
+!CHECK-DAG:   fir.has_value [[Z2]] : complex<f32>
 !CHECK-DAG: }
 !CHECK-DAG: fir.global internal @_QFEx : i32 {
 !CHECK-DAG:   [[Z3:%.*]] = fir.undefined i32
diff --git a/flang/test/Lower/OpenMP/threadprivate-real-logical-complex-derivedtype.f90 b/flang/test/Lower/OpenMP/threadprivate-real-logical-complex-derivedtype.f90
index 0a249ff101a08..ce8f686b09326 100644
--- a/flang/test/Lower/OpenMP/threadprivate-real-logical-complex-derivedtype.f90
+++ b/flang/test/Lower/OpenMP/threadprivate-real-logical-complex-derivedtype.f90
@@ -17,7 +17,7 @@ module test
 
 !CHECK-DAG: fir.global @_QMtestEt : !fir.type<_QMtestTmy_type{t_i:i32,t_arr:!fir.array<5xf32>}> {
 !CHECK-DAG: fir.global @_QMtestEx : f32 {
-!CHECK-DAG: fir.global @_QMtestEy : !fir.complex<4> {
+!CHECK-DAG: fir.global @_QMtestEy : complex<f32> {
 !CHECK-DAG: fir.global @_QMtestEz : !fir.logical<4> {
 
 contains
@@ -31,16 +31,16 @@ subroutine sub()
 !CHECK-DAG:  %[[X_DECL:.*]]:2 = hlfir.declare %[[X]] {uniq_name = "_QMtestEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 !CHECK-DAG:  %[[OMP_X:.*]] = omp.threadprivate %[[X_DECL]]#1 : !fir.ref<f32> -> !fir.ref<f32>
 !CHECK-DAG:  %[[OMP_X_DECL:.*]]:2 = hlfir.declare %[[OMP_X]] {uniq_name = "_QMtestEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-!CHECK-DAG:  %[[Y:.*]] = fir.address_of(@_QMtestEy) : !fir.ref<!fir.complex<4>>
-!CHECK-DAG:  %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y]] {uniq_name = "_QMtestEy"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-!CHECK-DAG:  %[[OMP_Y:.*]] = omp.threadprivate %[[Y_DECL]]#1 : !fir.ref<!fir.complex<4>> -> !fir.ref<!fir.complex<4>>
-!CHECK-DAG:  %[[OMP_Y_DECL:.*]]:2 = hlfir.declare %[[OMP_Y]] {uniq_name = "_QMtestEy"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+!CHECK-DAG:  %[[Y:.*]] = fir.address_of(@_QMtestEy) : !fir.ref<complex<f32>>
+!CHECK-DAG:  %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y]] {uniq_name = "_QMtestEy"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
+!CHECK-DAG:  %[[OMP_Y:.*]] = omp.threadprivate %[[Y_DECL]]#1 : !fir.ref<complex<f32>> -> !fir.ref<complex<f32>>
+!CHECK-DAG:  %[[OMP_Y_DECL:.*]]:2 = hlfir.declare %[[OMP_Y]] {uniq_name = "_QMtestEy"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
 !CHECK-DAG:  %[[Z:.*]] = fir.address_of(@_QMtestEz) : !fir.ref<!fir.logical<4>>
 !CHECK-DAG:  %[[Z_DECL:.*]]:2 = hlfir.declare %[[Z]] {uniq_name = "_QMtestEz"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 !CHECK-DAG:  %[[OMP_Z:.*]] = omp.threadprivate %[[Z_DECL]]#1 : !fir.ref<!fir.logical<4>> -> !fir.ref<!fir.logical<4>>
 !CHECK-DAG:  %[[OMP_Z_DECL:.*]]:2 = hlfir.declare %[[OMP_Z]] {uniq_name = "_QMtestEz"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 !CHECK-DAG:  %{{.*}} = fir.load %[[OMP_X_DECL]]#0 : !fir.ref<f32>
-!CHECK-DAG:  %{{.*}} = fir.load %[[OMP_Y_DECL]]#0 : !fir.ref<!fir.complex<4>>
+!CHECK-DAG:  %{{.*}} = fir.load %[[OMP_Y_DECL]]#0 : !fir.ref<complex<f32>>
 !CHECK-DAG:  %{{.*}} = fir.load %[[OMP_Z_DECL]]#0 : !fir.ref<!fir.logical<4>>
 !CHECK-DAG:  %{{.*}} = hlfir.designate %[[OMP_T_DECL]]#0{"t_i"}   : (!fir.ref<!fir.type<_QMtestTmy_type{t_i:i32,t_arr:!fir.array<5xf32>}>>) -> !fir.ref<i32>
     print *, x, y, z, t%t_i
@@ -50,18 +50,18 @@ subroutine sub()
 !CHECK-DAG:  %[[T_PVT_DECL:.*]]:2 = hlfir.declare %[[T_PVT]] {uniq_name = "_QMtestEt"} : (!fir.ref<!fir.type<_QMtestTmy_type{t_i:i32,t_arr:!fir.array<5xf32>}>>) -> (!fir.ref<!fir.type<_QMtestTmy_type{t_i:i32,t_arr:!fir.array<5xf32>}>>, !fir.ref<!fir.type<_QMtestTmy_type{t_i:i32,t_arr:!fir.array<5xf32>}>>)
 !CHECK-DAG:  %[[X_PVT:.*]] = omp.threadprivate %[[X_DECL]]#1 : !fir.ref<f32> -> !fir.ref<f32>
 !CHECK-DAG:  %[[X_PVT_DECL:.*]]:2 = hlfir.declare %[[X_PVT]] {uniq_name = "_QMtestEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-!CHECK-DAG:  %[[Y_PVT:.*]] = omp.threadprivate %[[Y_DECL]]#1 : !fir.ref<!fir.complex<4>> -> !fir.ref<!fir.complex<4>>
-!CHECK-DAG:  %[[Y_PVT_DECL:.*]]:2 = hlfir.declare %[[Y_PVT]] {uniq_name = "_QMtestEy"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+!CHECK-DAG:  %[[Y_PVT:.*]] = omp.threadprivate %[[Y_DECL]]#1 : !fir.ref<complex<f32>> -> !fir.ref<complex<f32>>
+!CHECK-DAG:  %[[Y_PVT_DECL:.*]]:2 = hlfir.declare %[[Y_PVT]] {uniq_name = "_QMtestEy"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
 !CHECK-DAG:  %[[Z_PVT:.*]] = omp.threadprivate %[[Z_DECL]]#1 : !fir.ref<!fir.logical<4>> -> !fir.ref<!fir.logical<4>>
 !CHECK-DAG:  %[[Z_PVT_DECL:.*]]:2 = hlfir.declare %[[Z_PVT]] {uniq_name = "_QMtestEz"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 !CHECK-DAG:  %{{.*}} = fir.load %[[X_PVT_DECL]]#0 : !fir.ref<f32>
-!CHECK-DAG:  %{{.*}} = fir.load %[[Y_PVT_DECL]]#0 : !fir.ref<!fir.complex<4>>
+!CHECK-DAG:  %{{.*}} = fir.load %[[Y_PVT_DECL]]#0 : !fir.ref<complex<f32>>
 !CHECK-DAG:  %{{.*}} = fir.load %[[Z_PVT_DECL]]#0 : !fir.ref<!fir.logical<4>>
 !CHECK-DAG:  %{{.*}} = hlfir.designate %[[T_PVT_DECL]]#0{"t_i"}   : (!fir.ref<!fir.type<_QMtestTmy_type{t_i:i32,t_arr:!fir.array<5xf32>}>>) -> !fir.ref<i32>
 print *, x, y, z, t%t_i
     !$omp end parallel
 !CHECK-DAG:  %{{.*}} = fir.load %[[OMP_X_DECL]]#0 : !fir.ref<f32>
-!CHECK-DAG:  %{{.*}} = fir.load %[[OMP_Y_DECL]]#0 : !fir.ref<!fir.complex<4>>
+!CHECK-DAG:  %{{.*}} = fir.load %[[OMP_Y_DECL]]#0 : !fir.ref<complex<f32>>
 !CHECK-DAG:  %{{.*}} = fir.load %[[OMP_Z_DECL]]#0 : !fir.ref<!fir.logical<4>>
 !CHECK-DAG:  %{{.*}} = hlfir.designate %[[OMP_T_DECL]]#0{"t_i"}   : (!fir.ref<!fir.type<_QMtestTmy_type{t_i:i32,t_arr:!fir.array<5xf32>}>>) -> !fir.ref<i32>
     print *, x, y, z, t%t_i
diff --git a/flang/test/Lower/array-constructor-1.f90 b/flang/test/Lower/array-constructor-1.f90
index 9dc7f7fb31541..a766357f588a6 100644
--- a/flang/test/Lower/array-constructor-1.f90
+++ b/flang/test/Lower/array-constructor-1.f90
@@ -29,7 +29,7 @@ subroutine check_units
 subroutine zero
   complex, parameter :: a(0) = [(((k,k=1,10),j=-2,2,-1),i=2,-2,-2)]
   complex, parameter :: b(0) = [(7,i=3,-3)]
-  ! CHECK: fir.address_of(@_QQro.0xz4.null.0) : !fir.ref<!fir.array<0x!fir.complex<4>>>
+  ! CHECK: fir.address_of(@_QQro.0xz4.null.0) : !fir.ref<!fir.array<0xcomplex<f32>>>
   ! CHECK-NOT: _QQro
   print*, '>', a, '<'
   print*, '>', b, '<'
@@ -41,6 +41,6 @@ program prog
   call zero
 end
 
-! CHECK: fir.global internal @_QFzeroECa constant : !fir.array<0x!fir.complex<4>>
-! CHECK:   %0 = fir.undefined !fir.array<0x!fir.complex<4>>
-! CHECK:   fir.has_value %0 : !fir.array<0x!fir.complex<4>>
+! CHECK: fir.global internal @_QFzeroECa constant : !fir.array<0xcomplex<f32>>
+! CHECK:   %0 = fir.undefined !fir.array<0xcomplex<f32>>
+! CHECK:   fir.has_value %0 : !fir.array<0xcomplex<f32>>
diff --git a/flang/test/Lower/array-elemental-calls-2.f90 b/flang/test/Lower/array-elemental-calls-2.f90
index c3e40f3f6d4f5..2243bfdd0b289 100644
--- a/flang/test/Lower/array-elemental-calls-2.f90
+++ b/flang/test/Lower/array-elemental-calls-2.f90
@@ -141,11 +141,11 @@ subroutine check_pow()
 ! CHECK-LABEL: func @_QMtest_opsPcheck_cmplx_part() {
 subroutine check_cmplx_part()
   print *,  elem_func_real(AIMAG(z1 + z2))
-! CHECK:  %[[VAL_13:.*]] = fir.load %{{.*}} : !fir.ref<!fir.complex<8>>
+! CHECK:  %[[VAL_13:.*]] = fir.load %{{.*}} : !fir.ref<complex<f64>>
 ! CHECK:  fir.do_loop
-! CHECK:  %[[VAL_23:.*]] = fir.array_fetch %{{.*}}, %{{.*}} : (!fir.array<10x!fir.complex<8>>, index) -> !fir.complex<8>
-! CHECK:  %[[VAL_24:.*]] = fir.addc %[[VAL_23]], %[[VAL_13]] {fastmath = #arith.fastmath<contract>} : !fir.complex<8>
-! CHECK:  %[[VAL_25:.*]] = fir.extract_value %[[VAL_24]], [1 : index] : (!fir.complex<8>) -> f64
+! CHECK:  %[[VAL_23:.*]] = fir.array_fetch %{{.*}}, %{{.*}} : (!fir.array<10xcomplex<f64>>, index) -> complex<f64>
+! CHECK:  %[[VAL_24:.*]] = fir.addc %[[VAL_23]], %[[VAL_13]] {fastmath = #arith.fastmath<contract>} : complex<f64>
+! CHECK:  %[[VAL_25:.*]] = fir.extract_value %[[VAL_24]], [1 : index] : (complex<f64>) -> f64
 ! CHECK:  fir.call @_QPelem_func_real(%[[VAL_25]]) {{.*}}: (f64) -> i32
 end subroutine
 
diff --git a/flang/test/Lower/array-elemental-subroutines.f90 b/flang/test/Lower/array-elemental-subroutines.f90
index 84be5cb792f01..b68d78fc3a800 100644
--- a/flang/test/Lower/array-elemental-subroutines.f90
+++ b/flang/test/Lower/array-elemental-subroutines.f90
@@ -2,7 +2,7 @@
 ! RUN: bbc -o - -emit-fir -hlfir=false %s | FileCheck %s
 
 ! CHECK-LABEL: func @_QPtest_elem_sub(
-! CHECK-SAME:    %[[VAL_0:.*]]: !fir.box<!fir.array<?xf32>>{{.*}}, %[[VAL_1:.*]]: !fir.box<!fir.array<?x!fir.char<1,?>>>{{.*}}, %[[VAL_2:.*]]: !fir.ref<i32>{{.*}}, %[[VAL_3:.*]]: !fir.ref<!fir.complex<4>>{{.*}}) {
+! CHECK-SAME:    %[[VAL_0:.*]]: !fir.box<!fir.array<?xf32>>{{.*}}, %[[VAL_1:.*]]: !fir.box<!fir.array<?x!fir.char<1,?>>>{{.*}}, %[[VAL_2:.*]]: !fir.ref<i32>{{.*}}, %[[VAL_3:.*]]: !fir.ref<complex<f32>>{{.*}}) {
 ! CHECK:         %[[VAL_5:.*]] = arith.constant 0 : index
 ! CHECK:         %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_5]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
 ! CHECK:         %[[VAL_7:.*]] = arith.constant 10 : i64
@@ -12,7 +12,7 @@
 ! CHECK:         %[[VAL_11:.*]] = arith.constant 1 : i64
 ! CHECK:         %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (i64) -> index
 ! CHECK:         %[[VAL_13:.*]] = fir.slice %[[VAL_8]], %[[VAL_12]], %[[VAL_10]] : (index, index, index) -> !fir.slice<1>
-! CHECK:         %[[VAL_14:.*]] = fir.load %[[VAL_3]] : !fir.ref<!fir.complex<4>>
+! CHECK:         %[[VAL_14:.*]] = fir.load %[[VAL_3]] : !fir.ref<complex<f32>>
 ! CHECK:         %[[VAL_15:.*]] = arith.constant 1 : index
 ! CHECK:         %[[VAL_16:.*]] = arith.constant 0 : index
 ! CHECK:         %[[VAL_17:.*]] = arith.subi %[[VAL_6]]#1, %[[VAL_15]] : index
@@ -25,7 +25,7 @@
 ! CHECK:           %[[VAL_24:.*]] = fir.array_coor %[[VAL_1]] {{\[}}%[[VAL_13]]] %[[VAL_23]] : (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.slice<1>, index) -> !fir.ref<!fir.char<1,?>>
 ! CHECK:           %[[VAL_25:.*]] = fir.box_elesize %[[VAL_1]] : (!fir.box<!fir.array<?x!fir.char<1,?>>>) -> index
 ! CHECK:           %[[VAL_26:.*]] = fir.emboxchar %[[VAL_24]], %[[VAL_25]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.boxchar<1>
-! CHECK:           fir.call @_QPfoo(%[[VAL_21]], %[[VAL_26]], %[[VAL_2]], %[[VAL_14]]) {{.*}}: (!fir.ref<f32>, !fir.boxchar<1>, !fir.ref<i32>, !fir.complex<4>) -> ()
+! CHECK:           fir.call @_QPfoo(%[[VAL_21]], %[[VAL_26]], %[[VAL_2]], %[[VAL_14]]) {{.*}}: (!fir.ref<f32>, !fir.boxchar<1>, !fir.ref<i32>, complex<f32>) -> ()
 ! CHECK:         }
 ! CHECK:         return
 ! CHECK:       }
diff --git a/flang/test/Lower/array.f90 b/flang/test/Lower/array.f90
index a2ab3d1384955..ec65a3c4071ec 100644
--- a/flang/test/Lower/array.f90
+++ b/flang/test/Lower/array.f90
@@ -115,7 +115,7 @@ subroutine range()
 end subroutine range
 
 ! c1 data
-! CHECK: fir.global internal @_QFrangeEc1(dense<(0.000000e+00,0.000000e+00)> : tensor<3x2xcomplex<f32>>) : !fir.array<2x3x!fir.complex<4>>
+! CHECK: fir.global internal @_QFrangeEc1(dense<(0.000000e+00,0.000000e+00)> : tensor<3x2xcomplex<f32>>) : !fir.array<2x3xcomplex<f32>>
 
 ! a0 array constructor
 ! CHECK: fir.global internal @_QQro.10xi4.{{.*}}(dense<[1, 2, 3, 3, 3, 3, 3, 3, 3, 3]> : tensor<10xi32>) constant : !fir.array<10xi32>
@@ -130,7 +130,7 @@ end subroutine range
 ! CHECK: fir.global internal @_QQro.2x3x4xi4.{{.*}}(dense<{{\[\[\[1, 1], \[2, 2], \[3, 3]], \[\[4, 4], \[5, 5], \[6, 6]], \[\[7, 7], \[8, 8], \[9, 9]], \[\[10, 10], \[11, 11], \[12, 12]]]}}> : tensor<4x3x2xi32>) constant : !fir.array<2x3x4xi32>
 
 ! c0 array constructor
-! CHECK: fir.global internal @_QQro.2x3xz4.{{.*}}(dense<{{\[}}[(1.000000e+00,1.500000e+00), (2.000000e+00,2.500000e+00)], [(3.000000e+00,3.500000e+00), (4.000000e+00,4.500000e+00)], [(5.000000e+00,5.500000e+00), (6.000000e+00,6.500000e+00)]]> : tensor<3x2xcomplex<f32>>) constant : !fir.array<2x3x!fir.complex<4>>
+! CHECK: fir.global internal @_QQro.2x3xz4.{{.*}}(dense<{{\[}}[(1.000000e+00,1.500000e+00), (2.000000e+00,2.500000e+00)], [(3.000000e+00,3.500000e+00), (4.000000e+00,4.500000e+00)], [(5.000000e+00,5.500000e+00), (6.000000e+00,6.500000e+00)]]> : tensor<3x2xcomplex<f32>>) constant : !fir.array<2x3xcomplex<f32>>
 
 ! CHECK-LABEL rangeGlobal
 subroutine rangeGlobal()
diff --git a/flang/test/Lower/assignment.f90 b/flang/test/Lower/assignment.f90
index 6be53463a1c71..dde5110bdfa9a 100644
--- a/flang/test/Lower/assignment.f90
+++ b/flang/test/Lower/assignment.f90
@@ -58,11 +58,11 @@ real function negr(a)
 end 
 
 ! CHECK-LABEL: func @_QPnegc(
-! CHECK-SAME:    %[[A:.*]]: !fir.ref<!fir.complex<4>> {fir.bindc_name = "a"}) -> !fir.complex<4> {
-! CHECK:         %[[FCTRES:.*]] = fir.alloca !fir.complex<4> {bindc_name = "negc", uniq_name = "_QFnegcEnegc"}
-! CHECK:         %[[A_VAL:.*]] = fir.load %[[A]] : !fir.ref<!fir.complex<4>>
-! CHECK:         %[[NEG:.*]] = fir.negc %[[A_VAL]] : !fir.complex<4>
-! CHECK:         fir.store %[[NEG]] to %[[FCTRES]] : !fir.ref<!fir.complex<4>>
+! CHECK-SAME:    %[[A:.*]]: !fir.ref<complex<f32>> {fir.bindc_name = "a"}) -> complex<f32> {
+! CHECK:         %[[FCTRES:.*]] = fir.alloca complex<f32> {bindc_name = "negc", uniq_name = "_QFnegcEnegc"}
+! CHECK:         %[[A_VAL:.*]] = fir.load %[[A]] : !fir.ref<complex<f32>>
+! CHECK:         %[[NEG:.*]] = fir.negc %[[A_VAL]] : complex<f32>
+! CHECK:         fir.store %[[NEG]] to %[[FCTRES]] : !fir.ref<complex<f32>>
 
 integer function addi(a, b)
   integer :: a, b
@@ -198,15 +198,15 @@ real function divf(a, b)
 end
 
 ! CHECK-LABEL: func @_QPaddc(
-! CHECK-SAME:    %[[A:.*]]: !fir.ref<!fir.complex<4>> {fir.bindc_name = "a"},
-! CHECK-SAME:    %[[B:.*]]: !fir.ref<!fir.complex<4>> {fir.bindc_name = "b"}
-! CHECK:         %[[FCTRES:.*]] = fir.alloca !fir.complex<4>
-! CHECK:         %[[A_VAL:.*]] = fir.load %[[A]] : !fir.ref<!fir.complex<4>>
-! CHECK:         %[[B_VAL:.*]] = fir.load %[[B]] : !fir.ref<!fir.complex<4>>
-! CHECK:         %[[ADD:.*]] = fir.addc %[[A_VAL]], %[[B_VAL]] {fastmath = #arith.fastmath<contract>} : !fir.complex<4>
-! CHECK:         fir.store %[[ADD]] to %[[FCTRES]] : !fir.ref<!fir.complex<4>>
-! CHECK:         %[[RET:.*]] = fir.load %[[FCTRES]] : !fir.ref<!fir.complex<4>>
-! CHECK:         return %[[RET]] : !fir.complex<4>
+! CHECK-SAME:    %[[A:.*]]: !fir.ref<complex<f32>> {fir.bindc_name = "a"},
+! CHECK-SAME:    %[[B:.*]]: !fir.ref<complex<f32>> {fir.bindc_name = "b"}
+! CHECK:         %[[FCTRES:.*]] = fir.alloca complex<f32>
+! CHECK:         %[[A_VAL:.*]] = fir.load %[[A]] : !fir.ref<complex<f32>>
+! CHECK:         %[[B_VAL:.*]] = fir.load %[[B]] : !fir.ref<complex<f32>>
+! CHECK:         %[[ADD:.*]] = fir.addc %[[A_VAL]], %[[B_VAL]] {fastmath = #arith.fastmath<contract>} : complex<f32>
+! CHECK:         fir.store %[[ADD]] to %[[FCTRES]] : !fir.ref<complex<f32>>
+! CHECK:         %[[RET:.*]] = fir.load %[[FCTRES]] : !fir.ref<complex<f32>>
+! CHECK:         return %[[RET]] : complex<f32>
 
 complex function subc(a, b)
   complex :: a, b
@@ -214,15 +214,15 @@ real function divf(a, b)
 end
 
 ! CHECK-LABEL: func @_QPsubc(
-! CHECK-SAME:    %[[A:.*]]: !fir.ref<!fir.complex<4>> {fir.bindc_name = "a"},
-! CHECK-SAME:    %[[B:.*]]: !fir.ref<!fir.complex<4>> {fir.bindc_name = "b"}
-! CHECK:         %[[FCTRES:.*]] = fir.alloca !fir.complex<4>
-! CHECK:         %[[A_VAL:.*]] = fir.load %[[A]] : !fir.ref<!fir.complex<4>>
-! CHECK:         %[[B_VAL:.*]] = fir.load %[[B]] : !fir.ref<!fir.complex<4>>
-! CHECK:         %[[SUB:.*]] = fir.subc %[[A_VAL]], %[[B_VAL]] {fastmath = #arith.fastmath<contract>} : !fir.complex<4>
-! CHECK:         fir.store %[[SUB]] to %[[FCTRES]] : !fir.ref<!fir.complex<4>>
-! CHECK:         %[[RET:.*]] = fir.load %[[FCTRES]] : !fir.ref<!fir.complex<4>>
-! CHECK:         return %[[RET]] : !fir.complex<4>
+! CHECK-SAME:    %[[A:.*]]: !fir.ref<complex<f32>> {fir.bindc_name = "a"},
+! CHECK-SAME:    %[[B:.*]]: !fir.ref<complex<f32>> {fir.bindc_name = "b"}
+! CHECK:         %[[FCTRES:.*]] = fir.alloca complex<f32>
+! CHECK:         %[[A_VAL:.*]] = fir.load %[[A]] : !fir.ref<complex<f32>>
+! CHECK:         %[[B_VAL:.*]] = fir.load %[[B]] : !fir.ref<complex<f32>>
+! CHECK:         %[[SUB:.*]] = fir.subc %[[A_VAL]], %[[B_VAL]] {fastmath = #arith.fastmath<contract>} : complex<f32>
+! CHECK:         fir.store %[[SUB]] to %[[FCTRES]] : !fir.ref<complex<f32>>
+! CHECK:         %[[RET:.*]] = fir.load %[[FCTRES]] : !fir.ref<complex<f32>>
+! CHECK:         return %[[RET]] : complex<f32>
 
 complex function mulc(a, b)
   complex :: a, b
@@ -230,15 +230,15 @@ real function divf(a, b)
 end
 
 ! CHECK-LABEL: func @_QPmulc(
-! CHECK-SAME:    %[[A:.*]]: !fir.ref<!fir.complex<4>> {fir.bindc_name = "a"},
-! CHECK-SAME:    %[[B:.*]]: !fir.ref<!fir.complex<4>> {fir.bindc_name = "b"}
-! CHECK:         %[[FCTRES:.*]] = fir.alloca !fir.complex<4>
-! CHECK:         %[[A_VAL:.*]] = fir.load %[[A]] : !fir.ref<!fir.complex<4>>
-! CHECK:         %[[B_VAL:.*]] = fir.load %[[B]] : !fir.ref<!fir.complex<4>>
-! CHECK:         %[[MUL:.*]] = fir.mulc %[[A_VAL]], %[[B_VAL]] {fastmath = #arith.fastmath<contract>} : !fir.complex<4>
-! CHECK:         fir.store %[[MUL]] to %[[FCTRES]] : !fir.ref<!fir.complex<4>>
-! CHECK:         %[[RET:.*]] = fir.load %[[FCTRES]] : !fir.ref<!fir.complex<4>>
-! CHECK:         return %[[RET]] : !fir.complex<4>
+! CHECK-SAME:    %[[A:.*]]: !fir.ref<complex<f32>> {fir.bindc_name = "a"},
+! CHECK-SAME:    %[[B:.*]]: !fir.ref<complex<f32>> {fir.bindc_name = "b"}
+! CHECK:         %[[FCTRES:.*]] = fir.alloca complex<f32>
+! CHECK:         %[[A_VAL:.*]] = fir.load %[[A]] : !fir.ref<complex<f32>>
+! CHECK:         %[[B_VAL:.*]] = fir.load %[[B]] : !fir.ref<complex<f32>>
+! CHECK:         %[[MUL:.*]] = fir.mulc %[[A_VAL]], %[[B_VAL]] {fastmath = #arith.fastmath<contract>} : complex<f32>
+! CHECK:         fir.store %[[MUL]] to %[[FCTRES]] : !fir.ref<complex<f32>>
+! CHECK:         %[[RET:.*]] = fir.load %[[FCTRES]] : !fir.ref<complex<f32>>
+! CHECK:         return %[[RET]] : complex<f32>
 
 complex function divc(a, b)
   complex :: a, b
@@ -246,19 +246,19 @@ real function divf(a, b)
 end
 
 ! CHECK-LABEL: func @_QPdivc(
-! CHECK-SAME:    %[[A:.*]]: !fir.ref<!fir.complex<4>> {fir.bindc_name = "a"},
-! CHECK-SAME:    %[[B:.*]]: !fir.ref<!fir.complex<4>> {fir.bindc_name = "b"}
-! CHECK:         %[[FCTRES:.*]] = fir.alloca !fir.complex<4>
-! CHECK:         %[[A_VAL:.*]] = fir.load %[[A]] : !fir.ref<!fir.complex<4>>
-! CHECK:         %[[B_VAL:.*]] = fir.load %[[B]] : !fir.ref<!fir.complex<4>>
-! CHECK:         %[[A_REAL:.*]] = fir.extract_value %[[A_VAL]], [0 : index] : (!fir.complex<4>) -> f32
-! CHECK:         %[[A_IMAG:.*]] = fir.extract_value %[[A_VAL]], [1 : index] : (!fir.complex<4>) -> f32
-! CHECK:         %[[B_REAL:.*]] = fir.extract_value %[[B_VAL]], [0 : index] : (!fir.complex<4>) -> f32
-! CHECK:         %[[B_IMAG:.*]] = fir.extract_value %[[B_VAL]], [1 : index] : (!fir.complex<4>) -> f32
-! CHECK:         %[[DIV:.*]] = fir.call @__divsc3(%[[A_REAL]], %[[A_IMAG]], %[[B_REAL]], %[[B_IMAG]]) fastmath<contract> : (f32, f32, f32, f32) -> !fir.complex<4>
-! CHECK:         fir.store %[[DIV]] to %[[FCTRES]] : !fir.ref<!fir.complex<4>>
-! CHECK:         %[[RET:.*]] = fir.load %[[FCTRES]] : !fir.ref<!fir.complex<4>>
-! CHECK:         return %[[RET]] : !fir.complex<4>
+! CHECK-SAME:    %[[A:.*]]: !fir.ref<complex<f32>> {fir.bindc_name = "a"},
+! CHECK-SAME:    %[[B:.*]]: !fir.ref<complex<f32>> {fir.bindc_name = "b"}
+! CHECK:         %[[FCTRES:.*]] = fir.alloca complex<f32>
+! CHECK:         %[[A_VAL:.*]] = fir.load %[[A]] : !fir.ref<complex<f32>>
+! CHECK:         %[[B_VAL:.*]] = fir.load %[[B]] : !fir.ref<complex<f32>>
+! CHECK:         %[[A_REAL:.*]] = fir.extract_value %[[A_VAL]], [0 : index] : (complex<f32>) -> f32
+! CHECK:         %[[A_IMAG:.*]] = fir.extract_value %[[A_VAL]], [1 : index] : (complex<f32>) -> f32
+! CHECK:         %[[B_REAL:.*]] = fir.extract_value %[[B_VAL]], [0 : index] : (complex<f32>) -> f32
+! CHECK:         %[[B_IMAG:.*]] = fir.extract_value %[[B_VAL]], [1 : index] : (complex<f32>) -> f32
+! CHECK:         %[[DIV:.*]] = fir.call @__divsc3(%[[A_REAL]], %[[A_IMAG]], %[[B_REAL]], %[[B_IMAG]]) fastmath<contract> : (f32, f32, f32, f32) -> complex<f32>
+! CHECK:         fir.store %[[DIV]] to %[[FCTRES]] : !fir.ref<complex<f32>>
+! CHECK:         %[[RET:.*]] = fir.load %[[FCTRES]] : !fir.ref<complex<f32>>
+! CHECK:         return %[[RET]] : complex<f32>
 
 subroutine real_constant()
   real(2) :: a
@@ -299,13 +299,13 @@ subroutine complex_constant()
 end
 
 ! CHECK-LABEL: func @_QPcomplex_constant()
-! CHECK:         %[[A:.*]] = fir.alloca !fir.complex<4> {bindc_name = "a", uniq_name = "_QFcomplex_constantEa"}
+! CHECK:         %[[A:.*]] = fir.alloca complex<f32> {bindc_name = "a", uniq_name = "_QFcomplex_constantEa"}
 ! CHECK:         %[[C0:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:         %[[C1:.*]] = arith.constant 1.000000e+00 : f32
-! CHECK:         %[[UNDEF:.*]] = fir.undefined !fir.complex<4>
-! CHECK:         %[[INS0:.*]] = fir.insert_value %[[UNDEF]], %[[C0]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:         %[[INS1:.*]] = fir.insert_value %[[INS0]], %[[C1]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
-! CHECK:         fir.store %[[INS1]] to %[[A]] : !fir.ref<!fir.complex<4>>
+! CHECK:         %[[UNDEF:.*]] = fir.undefined complex<f32>
+! CHECK:         %[[INS0:.*]] = fir.insert_value %[[UNDEF]], %[[C0]], [0 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:         %[[INS1:.*]] = fir.insert_value %[[INS0]], %[[C1]], [1 : index] : (complex<f32>, f32) -> complex<f32>
+! CHECK:         fir.store %[[INS1]] to %[[A]] : !fir.ref<complex<f32>>
 
 subroutine sub1_arr(a)
   integer :: a(10)
diff --git a/flang/test/Lower/basic-function.f90 b/flang/test/Lower/basic-function.f90
index ed661ca8378aa..f5f81545c899d 100644
--- a/flang/test/Lower/basic-function.f90
+++ b/flang/test/Lower/basic-function.f90
@@ -114,33 +114,33 @@ real(16) function rfct6()
 
 complex(2) function cplxfct1()
 end
-! CHECK-LABEL: func @_QPcplxfct1() -> !fir.complex<2>
-! CHECK:         return %{{.*}} : !fir.complex<2>
+! CHECK-LABEL: func @_QPcplxfct1() -> complex<f16>
+! CHECK:         return %{{.*}} : complex<f16>
 
 complex(3) function cplxfct2()
 end
-! CHECK-LABEL: func @_QPcplxfct2() -> !fir.complex<3>
-! CHECK:         return %{{.*}} : !fir.complex<3>
+! CHECK-LABEL: func @_QPcplxfct2() -> complex<bf16>
+! CHECK:         return %{{.*}} : complex<bf16>
 
 complex(4) function cplxfct3()
 end
-! CHECK-LABEL: func @_QPcplxfct3() -> !fir.complex<4>
-! CHECK:         return %{{.*}} : !fir.complex<4>
+! CHECK-LABEL: func @_QPcplxfct3() -> complex<f32>
+! CHECK:         return %{{.*}} : complex<f32>
 
 complex(8) function cplxfct4()
 end
-! CHECK-LABEL: func @_QPcplxfct4() -> !fir.complex<8>
-! CHECK:         return %{{.*}} : !fir.complex<8>
+! CHECK-LABEL: func @_QPcplxfct4() -> complex<f64>
+! CHECK:         return %{{.*}} : complex<f64>
 
 complex(10) function cplxfct5()
 end
-! CHECK-LABEL: func @_QPcplxfct5() -> !fir.complex<10>
-! CHECK:         return %{{.*}} : !fir.complex<10>
+! CHECK-LABEL: func @_QPcplxfct5() -> complex<f80>
+! CHECK:         return %{{.*}} : complex<f80>
 
 complex(16) function cplxfct6()
 end
-! CHECK-LABEL: func @_QPcplxfct6() -> !fir.complex<16>
-! CHECK:         return %{{.*}} : !fir.complex<16>
+! CHECK-LABEL: func @_QPcplxfct6() -> complex<f128>
+! CHECK:         return %{{.*}} : complex<f128>
 
 function fct_with_character_return(i)
   character(10) :: fct_with_character_return
diff --git a/flang/test/Lower/call-bindc.f90 b/flang/test/Lower/call-bindc.f90
index f8cafffa3867d..806f0ab62dafc 100644
--- a/flang/test/Lower/call-bindc.f90
+++ b/flang/test/Lower/call-bindc.f90
@@ -18,4 +18,4 @@ program main
 end
 
 ! CHECK-LABEL: func.func @_QQmain()
-! CHECK: fir.call %{{.*}}(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32) -> !fir.complex<4>
+! CHECK: fir.call %{{.*}}(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32) -> complex<f32>
diff --git a/flang/test/Lower/call-by-value.f90 b/flang/test/Lower/call-by-value.f90
index d4694f4575d08..32b9c79e11e68 100644
--- a/flang/test/Lower/call-by-value.f90
+++ b/flang/test/Lower/call-by-value.f90
@@ -52,11 +52,11 @@ subroutine test_real_value(x) bind(c)
   call internal_call2(x)
 end
 ! CHECK-LABEL:   func.func @test_complex_value(
-! CHECK-SAME:                                  %[[VAL_0:.*]]: !fir.complex<4>
-! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.complex<4>
-! CHECK:           fir.store %[[VAL_0]] to %[[VAL_1]] : !fir.ref<!fir.complex<4>>
+! CHECK-SAME:                                  %[[VAL_0:.*]]: complex<f32>
+! CHECK:           %[[VAL_1:.*]] = fir.alloca complex<f32>
+! CHECK:           fir.store %[[VAL_0]] to %[[VAL_1]] : !fir.ref<complex<f32>>
 ! CHECK:           %[[VAL_2:.*]] = fir.declare %[[VAL_1]]
-! CHECK:           fir.call @_QPinternal_call3(%[[VAL_2]]) {{.*}}: (!fir.ref<!fir.complex<4>>) -> ()
+! CHECK:           fir.call @_QPinternal_call3(%[[VAL_2]]) {{.*}}: (!fir.ref<complex<f32>>) -> ()
 ! CHECK:           return
 ! CHECK:         }
 
diff --git a/flang/test/Lower/complex-operations.f90 b/flang/test/Lower/complex-operations.f90
index 2a353216978f0..09f922528af57 100644
--- a/flang/test/Lower/complex-operations.f90
+++ b/flang/test/Lower/complex-operations.f90
@@ -5,7 +5,7 @@ subroutine add_test(a,b,c)
   complex :: a, b, c
   ! CHECK-NOT: fir.extract_value
   ! CHECK-NOT: fir.insert_value
-  ! CHECK: fir.addc {{.*}}: !fir.complex
+  ! CHECK: fir.addc {{.*}}: complex
   a = b + c
 end subroutine add_test
 
@@ -14,7 +14,7 @@ subroutine sub_test(a,b,c)
   complex :: a, b, c
   ! CHECK-NOT: fir.extract_value
   ! CHECK-NOT: fir.insert_value
-  ! CHECK: fir.subc {{.*}}: !fir.complex
+  ! CHECK: fir.subc {{.*}}: complex
   a = b - c
 end subroutine sub_test
 
@@ -23,93 +23,87 @@ subroutine mul_test(a,b,c)
   complex :: a, b, c
   ! CHECK-NOT: fir.extract_value
   ! CHECK-NOT: fir.insert_value
-  ! CHECK: fir.mulc {{.*}}: !fir.complex
+  ! CHECK: fir.mulc {{.*}}: complex
   a = b * c
 end subroutine mul_test
 
 ! CHECK-LABEL: @_QPdiv_test_half
-! CHECK-SAME: %[[AREF:.*]]: !fir.ref<!fir.complex<2>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<2>> {{.*}}, %[[CREF:.*]]: !fir.ref<!fir.complex<2>> {{.*}})
-! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<!fir.complex<2>>
-! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<!fir.complex<2>>
-! CHECK: %[[BVAL_CVT:.*]] = fir.convert %[[BVAL]] : (!fir.complex<2>) -> complex<f16>
-! CHECK: %[[CVAL_CVT:.*]] = fir.convert %[[CVAL]] : (!fir.complex<2>) -> complex<f16>
-! CHECK: %[[AVAL_CVT:.*]] = complex.div %[[BVAL_CVT]], %[[CVAL_CVT]] fastmath<contract> : complex<f16>
-! CHECK: %[[AVAL:.*]] = fir.convert %[[AVAL_CVT]] : (complex<f16>) -> !fir.complex<2>
-! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<!fir.complex<2>>
+! CHECK-SAME: %[[AREF:.*]]: !fir.ref<complex<f16>> {{.*}}, %[[BREF:.*]]: !fir.ref<complex<f16>> {{.*}}, %[[CREF:.*]]: !fir.ref<complex<f16>> {{.*}})
+! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<complex<f16>>
+! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<complex<f16>>
+! CHECK: %[[AVAL:.*]] = complex.div %[[BVAL]], %[[CVAL]] fastmath<contract> : complex<f16>
+! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<complex<f16>>
 subroutine div_test_half(a,b,c)
   complex(kind=2) :: a, b, c
   a = b / c
 end subroutine div_test_half
 
 ! CHECK-LABEL: @_QPdiv_test_bfloat
-! CHECK-SAME: %[[AREF:.*]]: !fir.ref<!fir.complex<3>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<3>> {{.*}}, %[[CREF:.*]]: !fir.ref<!fir.complex<3>> {{.*}})
-! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<!fir.complex<3>>
-! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<!fir.complex<3>>
-! CHECK: %[[BVAL_CVT:.*]] = fir.convert %[[BVAL]] : (!fir.complex<3>) -> complex<bf16>
-! CHECK: %[[CVAL_CVT:.*]] = fir.convert %[[CVAL]] : (!fir.complex<3>) -> complex<bf16>
-! CHECK: %[[AVAL_CVT:.*]] = complex.div %[[BVAL_CVT]], %[[CVAL_CVT]] fastmath<contract> : complex<bf16>
-! CHECK: %[[AVAL:.*]] = fir.convert %[[AVAL_CVT]] : (complex<bf16>) -> !fir.complex<3>
-! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<!fir.complex<3>>
+! CHECK-SAME: %[[AREF:.*]]: !fir.ref<complex<bf16>> {{.*}}, %[[BREF:.*]]: !fir.ref<complex<bf16>> {{.*}}, %[[CREF:.*]]: !fir.ref<complex<bf16>> {{.*}})
+! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<complex<bf16>>
+! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<complex<bf16>>
+! CHECK: %[[AVAL:.*]] = complex.div %[[BVAL]], %[[CVAL]] fastmath<contract> : complex<bf16>
+! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<complex<bf16>>
 subroutine div_test_bfloat(a,b,c)
   complex(kind=3) :: a, b, c
   a = b / c
 end subroutine div_test_bfloat
 
 ! CHECK-LABEL: @_QPdiv_test_single
-! CHECK-SAME: %[[AREF:.*]]: !fir.ref<!fir.complex<4>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<4>> {{.*}}, %[[CREF:.*]]: !fir.ref<!fir.complex<4>> {{.*}})
-! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<!fir.complex<4>>
-! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<!fir.complex<4>>
-! CHECK: %[[BREAL:.*]] = fir.extract_value %[[BVAL]], [0 : index] : (!fir.complex<4>) -> f32
-! CHECK: %[[BIMAG:.*]] = fir.extract_value %[[BVAL]], [1 : index] : (!fir.complex<4>) -> f32
-! CHECK: %[[CREAL:.*]] = fir.extract_value %[[CVAL]], [0 : index] : (!fir.complex<4>) -> f32
-! CHECK: %[[CIMAG:.*]] = fir.extract_value %[[CVAL]], [1 : index] : (!fir.complex<4>) -> f32
-! CHECK: %[[AVAL:.*]] = fir.call @__divsc3(%[[BREAL]], %[[BIMAG]], %[[CREAL]], %[[CIMAG]]) fastmath<contract> : (f32, f32, f32, f32) -> !fir.complex<4>
-! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<!fir.complex<4>>
+! CHECK-SAME: %[[AREF:.*]]: !fir.ref<complex<f32>> {{.*}}, %[[BREF:.*]]: !fir.ref<complex<f32>> {{.*}}, %[[CREF:.*]]: !fir.ref<complex<f32>> {{.*}})
+! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<complex<f32>>
+! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<complex<f32>>
+! CHECK: %[[BREAL:.*]] = fir.extract_value %[[BVAL]], [0 : index] : (complex<f32>) -> f32
+! CHECK: %[[BIMAG:.*]] = fir.extract_value %[[BVAL]], [1 : index] : (complex<f32>) -> f32
+! CHECK: %[[CREAL:.*]] = fir.extract_value %[[CVAL]], [0 : index] : (complex<f32>) -> f32
+! CHECK: %[[CIMAG:.*]] = fir.extract_value %[[CVAL]], [1 : index] : (complex<f32>) -> f32
+! CHECK: %[[AVAL:.*]] = fir.call @__divsc3(%[[BREAL]], %[[BIMAG]], %[[CREAL]], %[[CIMAG]]) fastmath<contract> : (f32, f32, f32, f32) -> complex<f32>
+! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<complex<f32>>
 subroutine div_test_single(a,b,c)
   complex(kind=4) :: a, b, c
   a = b / c
 end subroutine div_test_single
 
 ! CHECK-LABEL: @_QPdiv_test_double
-! CHECK-SAME: %[[AREF:.*]]: !fir.ref<!fir.complex<8>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<8>> {{.*}}, %[[CREF:.*]]: !fir.ref<!fir.complex<8>> {{.*}})
-! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<!fir.complex<8>>
-! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<!fir.complex<8>>
-! CHECK: %[[BREAL:.*]] = fir.extract_value %[[BVAL]], [0 : index] : (!fir.complex<8>) -> f64
-! CHECK: %[[BIMAG:.*]] = fir.extract_value %[[BVAL]], [1 : index] : (!fir.complex<8>) -> f64
-! CHECK: %[[CREAL:.*]] = fir.extract_value %[[CVAL]], [0 : index] : (!fir.complex<8>) -> f64
-! CHECK: %[[CIMAG:.*]] = fir.extract_value %[[CVAL]], [1 : index] : (!fir.complex<8>) -> f64
-! CHECK: %[[AVAL:.*]] = fir.call @__divdc3(%[[BREAL]], %[[BIMAG]], %[[CREAL]], %[[CIMAG]]) fastmath<contract> : (f64, f64, f64, f64) -> !fir.complex<8>
-! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<!fir.complex<8>>
+! CHECK-SAME: %[[AREF:.*]]: !fir.ref<complex<f64>> {{.*}}, %[[BREF:.*]]: !fir.ref<complex<f64>> {{.*}}, %[[CREF:.*]]: !fir.ref<complex<f64>> {{.*}})
+! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<complex<f64>>
+! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<complex<f64>>
+! CHECK: %[[BREAL:.*]] = fir.extract_value %[[BVAL]], [0 : index] : (complex<f64>) -> f64
+! CHECK: %[[BIMAG:.*]] = fir.extract_value %[[BVAL]], [1 : index] : (complex<f64>) -> f64
+! CHECK: %[[CREAL:.*]] = fir.extract_value %[[CVAL]], [0 : index] : (complex<f64>) -> f64
+! CHECK: %[[CIMAG:.*]] = fir.extract_value %[[CVAL]], [1 : index] : (complex<f64>) -> f64
+! CHECK: %[[AVAL:.*]] = fir.call @__divdc3(%[[BREAL]], %[[BIMAG]], %[[CREAL]], %[[CIMAG]]) fastmath<contract> : (f64, f64, f64, f64) -> complex<f64>
+! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<complex<f64>>
 subroutine div_test_double(a,b,c)
   complex(kind=8) :: a, b, c
   a = b / c
 end subroutine div_test_double
 
 ! CHECK-LABEL: @_QPdiv_test_extended
-! CHECK-SAME: %[[AREF:.*]]: !fir.ref<!fir.complex<10>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<10>> {{.*}}, %[[CREF:.*]]: !fir.ref<!fir.complex<10>> {{.*}})
-! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<!fir.complex<10>>
-! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<!fir.complex<10>>
-! CHECK: %[[BREAL:.*]] = fir.extract_value %[[BVAL]], [0 : index] : (!fir.complex<10>) -> f80
-! CHECK: %[[BIMAG:.*]] = fir.extract_value %[[BVAL]], [1 : index] : (!fir.complex<10>) -> f80
-! CHECK: %[[CREAL:.*]] = fir.extract_value %[[CVAL]], [0 : index] : (!fir.complex<10>) -> f80
-! CHECK: %[[CIMAG:.*]] = fir.extract_value %[[CVAL]], [1 : index] : (!fir.complex<10>) -> f80
-! CHECK: %[[AVAL:.*]] = fir.call @__divxc3(%[[BREAL]], %[[BIMAG]], %[[CREAL]], %[[CIMAG]]) fastmath<contract> : (f80, f80, f80, f80) -> !fir.complex<10>
-! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<!fir.complex<10>>
+! CHECK-SAME: %[[AREF:.*]]: !fir.ref<complex<f80>> {{.*}}, %[[BREF:.*]]: !fir.ref<complex<f80>> {{.*}}, %[[CREF:.*]]: !fir.ref<complex<f80>> {{.*}})
+! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<complex<f80>>
+! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<complex<f80>>
+! CHECK: %[[BREAL:.*]] = fir.extract_value %[[BVAL]], [0 : index] : (complex<f80>) -> f80
+! CHECK: %[[BIMAG:.*]] = fir.extract_value %[[BVAL]], [1 : index] : (complex<f80>) -> f80
+! CHECK: %[[CREAL:.*]] = fir.extract_value %[[CVAL]], [0 : index] : (complex<f80>) -> f80
+! CHECK: %[[CIMAG:.*]] = fir.extract_value %[[CVAL]], [1 : index] : (complex<f80>) -> f80
+! CHECK: %[[AVAL:.*]] = fir.call @__divxc3(%[[BREAL]], %[[BIMAG]], %[[CREAL]], %[[CIMAG]]) fastmath<contract> : (f80, f80, f80, f80) -> complex<f80>
+! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<complex<f80>>
 subroutine div_test_extended(a,b,c)
   complex(kind=10) :: a, b, c
   a = b / c
 end subroutine div_test_extended
 
 ! CHECK-LABEL: @_QPdiv_test_quad
-! CHECK-SAME: %[[AREF:.*]]: !fir.ref<!fir.complex<16>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<16>> {{.*}}, %[[CREF:.*]]: !fir.ref<!fir.complex<16>> {{.*}})
-! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<!fir.complex<16>>
-! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<!fir.complex<16>>
-! CHECK: %[[BREAL:.*]] = fir.extract_value %[[BVAL]], [0 : index] : (!fir.complex<16>) -> f128
-! CHECK: %[[BIMAG:.*]] = fir.extract_value %[[BVAL]], [1 : index] : (!fir.complex<16>) -> f128
-! CHECK: %[[CREAL:.*]] = fir.extract_value %[[CVAL]], [0 : index] : (!fir.complex<16>) -> f128
-! CHECK: %[[CIMAG:.*]] = fir.extract_value %[[CVAL]], [1 : index] : (!fir.complex<16>) -> f128
-! CHECK: %[[AVAL:.*]] = fir.call @__divtc3(%[[BREAL]], %[[BIMAG]], %[[CREAL]], %[[CIMAG]]) fastmath<contract> : (f128, f128, f128, f128) -> !fir.complex<16>
-! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<!fir.complex<16>>
+! CHECK-SAME: %[[AREF:.*]]: !fir.ref<complex<f128>> {{.*}}, %[[BREF:.*]]: !fir.ref<complex<f128>> {{.*}}, %[[CREF:.*]]: !fir.ref<complex<f128>> {{.*}})
+! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<complex<f128>>
+! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<complex<f128>>
+! CHECK: %[[BREAL:.*]] = fir.extract_value %[[BVAL]], [0 : index] : (complex<f128>) -> f128
+! CHECK: %[[BIMAG:.*]] = fir.extract_value %[[BVAL]], [1 : index] : (complex<f128>) -> f128
+! CHECK: %[[CREAL:.*]] = fir.extract_value %[[CVAL]], [0 : index] : (complex<f128>) -> f128
+! CHECK: %[[CIMAG:.*]] = fir.extract_value %[[CVAL]], [1 : index] : (complex<f128>) -> f128
+! CHECK: %[[AVAL:.*]] = fir.call @__divtc3(%[[BREAL]], %[[BIMAG]], %[[CREAL]], %[[CIMAG]]) fastmath<contract> : (f128, f128, f128, f128) -> complex<f128>
+! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<complex<f128>>
 subroutine div_test_quad(a,b,c)
   complex(kind=16) :: a, b, c
   a = b / c
diff --git a/flang/test/Lower/complex-real.f90 b/flang/test/Lower/complex-real.f90
index a6c59ced5b717..a28703db27bdc 100644
--- a/flang/test/Lower/complex-real.f90
+++ b/flang/test/Lower/complex-real.f90
@@ -2,11 +2,11 @@
 ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s
 
 ! CHECK-LABEL: @_QPsb_complex_bfloat
-! CHECK: %[[C_REF:.*]] = fir.alloca !fir.complex<3> {bindc_name = "c", uniq_name = "_QFsb_complex_bfloatEc"}
+! CHECK: %[[C_REF:.*]] = fir.alloca complex<bf16> {bindc_name = "c", uniq_name = "_QFsb_complex_bfloatEc"}
 ! CHECK: %[[R_REF:.*]] = fir.alloca bf16 {bindc_name = "r", uniq_name = "_QFsb_complex_bfloatEr"}
 ! CHECK: %[[R_VAL:.*]] = fir.load %[[R_REF]] : !fir.ref<bf16>
 ! CHECK: %[[C0:.*]] = arith.constant 0 : i32
-! CHECK: %[[CREAL_REF:.*]] = fir.coordinate_of %[[C_REF]], %[[C0]] : (!fir.ref<!fir.complex<3>>, i32) -> !fir.ref<bf16>
+! CHECK: %[[CREAL_REF:.*]] = fir.coordinate_of %[[C_REF]], %[[C0]] : (!fir.ref<complex<bf16>>, i32) -> !fir.ref<bf16>
 ! CHECK: fir.store %[[R_VAL]] to %[[CREAL_REF]] : !fir.ref<bf16>
 subroutine sb_complex_bfloat
   complex(kind=3) :: c
diff --git a/flang/test/Lower/derived-assignments.f90 b/flang/test/Lower/derived-assignments.f90
index c3ceacd4abd69..4465a7eecc886 100644
--- a/flang/test/Lower/derived-assignments.f90
+++ b/flang/test/Lower/derived-assignments.f90
@@ -131,7 +131,7 @@ subroutine test_array_comp(t1, t2)
 end subroutine
 
 ! CHECK-LABEL: func @_QPtest_ptr_comp(
-! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref<!fir.type<_QFtest_ptr_compTt{ptr:!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>,m_i:i32}>>{{.*}}, %[[VAL_1]]: !fir.ref<!fir.type<_QFtest_ptr_compTt{ptr:!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>,m_i:i32}>>{{.*}}) {
+! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref<!fir.type<_QFtest_ptr_compTt{ptr:!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>,m_i:i32}>>{{.*}}, %[[VAL_1]]: !fir.ref<!fir.type<_QFtest_ptr_compTt{ptr:!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>,m_i:i32}>>{{.*}}) {
 subroutine test_ptr_comp(t1, t2)
   type t
      complex, pointer :: ptr(:)
@@ -139,16 +139,16 @@ subroutine test_ptr_comp(t1, t2)
   end type t
   type(t) :: t1, t2
 
-  ! CHECK:  %[[VAL_2:.*]] = fir.field_index ptr, !fir.type<_QFtest_ptr_compTt{ptr:!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>,m_i:i32}>
-  ! CHECK:  %[[VAL_3:.*]] = fir.coordinate_of %[[VAL_1]], %[[VAL_2]] : (!fir.ref<!fir.type<_QFtest_ptr_compTt{ptr:!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>,m_i:i32}>>, !fir.field) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>>
-  ! CHECK:  %[[VAL_2b:.*]] = fir.field_index ptr, !fir.type<_QFtest_ptr_compTt{ptr:!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>,m_i:i32}>
-  ! CHECK:  %[[VAL_4:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_2b]] : (!fir.ref<!fir.type<_QFtest_ptr_compTt{ptr:!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>,m_i:i32}>>, !fir.field) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>>
-  ! CHECK:  %[[VAL_5:.*]] = fir.load %[[VAL_3]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>>
-  ! CHECK:  fir.store %[[VAL_5]] to %[[VAL_4]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>>
-  ! CHECK:  %[[VAL_6:.*]] = fir.field_index m_i, !fir.type<_QFtest_ptr_compTt{ptr:!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>,m_i:i32}>
-  ! CHECK:  %[[VAL_7:.*]] = fir.coordinate_of %[[VAL_1]], %[[VAL_6]] : (!fir.ref<!fir.type<_QFtest_ptr_compTt{ptr:!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>,m_i:i32}>>, !fir.field) -> !fir.ref<i32>
-  ! CHECK:  %[[VAL_6b:.*]] = fir.field_index m_i, !fir.type<_QFtest_ptr_compTt{ptr:!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>,m_i:i32}>
-  ! CHECK:  %[[VAL_8:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_6b]] : (!fir.ref<!fir.type<_QFtest_ptr_compTt{ptr:!fir.box<!fir.ptr<!fir.array<?x!fir.complex<4>>>>,m_i:i32}>>, !fir.field) -> !fir.ref<i32>
+  ! CHECK:  %[[VAL_2:.*]] = fir.field_index ptr, !fir.type<_QFtest_ptr_compTt{ptr:!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>,m_i:i32}>
+  ! CHECK:  %[[VAL_3:.*]] = fir.coordinate_of %[[VAL_1]], %[[VAL_2]] : (!fir.ref<!fir.type<_QFtest_ptr_compTt{ptr:!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>,m_i:i32}>>, !fir.field) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>>
+  ! CHECK:  %[[VAL_2b:.*]] = fir.field_index ptr, !fir.type<_QFtest_ptr_compTt{ptr:!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>,m_i:i32}>
+  ! CHECK:  %[[VAL_4:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_2b]] : (!fir.ref<!fir.type<_QFtest_ptr_compTt{ptr:!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>,m_i:i32}>>, !fir.field) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>>
+  ! CHECK:  %[[VAL_5:.*]] = fir.load %[[VAL_3]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>>
+  ! CHECK:  fir.store %[[VAL_5]] to %[[VAL_4]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>>
+  ! CHECK:  %[[VAL_6:.*]] = fir.field_index m_i, !fir.type<_QFtest_ptr_compTt{ptr:!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>,m_i:i32}>
+  ! CHECK:  %[[VAL_7:.*]] = fir.coordinate_of %[[VAL_1]], %[[VAL_6]] : (!fir.ref<!fir.type<_QFtest_ptr_compTt{ptr:!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>,m_i:i32}>>, !fir.field) -> !fir.ref<i32>
+  ! CHECK:  %[[VAL_6b:.*]] = fir.field_index m_i, !fir.type<_QFtest_ptr_compTt{ptr:!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>,m_i:i32}>
+  ! CHECK:  %[[VAL_8:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_6b]] : (!fir.ref<!fir.type<_QFtest_ptr_compTt{ptr:!fir.box<!fir.ptr<!fir.array<?xcomplex<f32>>>>,m_i:i32}>>, !fir.field) -> !fir.ref<i32>
   ! CHECK:  %[[VAL_9:.*]] = fir.load %[[VAL_7]] : !fir.ref<i32>
   ! CHECK:  fir.store %[[VAL_9]] to %[[VAL_8]] : !fir.ref<i32>
   t1 = t2
diff --git a/flang/test/Lower/dummy-procedure.f90 b/flang/test/Lower/dummy-procedure.f90
index caa078d3205c1..6874e8eca90b9 100644
--- a/flang/test/Lower/dummy-procedure.f90
+++ b/flang/test/Lower/dummy-procedure.f90
@@ -115,8 +115,8 @@ subroutine test_atan2()
 ! CHECK-LABEL: func @_QPtest_aimag
 subroutine test_aimag()
   intrinsic :: aimag
-  !CHECK: %[[f:.*]] = fir.address_of(@fir.aimag.f32.ref_z4) : (!fir.ref<!fir.complex<4>>) -> f32
-  !CHECK: %[[fcast:.*]] = fir.emboxproc %[[f]] : ((!fir.ref<!fir.complex<4>>) -> f32) -> !fir.boxproc<() -> ()>
+  !CHECK: %[[f:.*]] = fir.address_of(@fir.aimag.f32.ref_z32) : (!fir.ref<complex<f32>>) -> f32
+  !CHECK: %[[fcast:.*]] = fir.emboxproc %[[f]] : ((!fir.ref<complex<f32>>) -> f32) -> !fir.boxproc<() -> ()>
   !CHECK: fir.call @_QPfoo_aimag(%[[fcast]]) {{.*}}: (!fir.boxproc<() -> ()>) -> ()
   call foo_aimag(aimag)
 end subroutine
@@ -164,9 +164,9 @@ subroutine todo3(dummy_proc)
   ! CHECK: %[[atan2:.*]] = math.atan2 %[[xload]], %[[yload]] fastmath<contract> : f32
   ! CHECK: return %[[atan2]] : f32
 
-!CHECK-LABEL: func private @fir.aimag.f32.ref_z4(%arg0: !fir.ref<!fir.complex<4>>)
+!CHECK-LABEL: func private @fir.aimag.f32.ref_z32(%arg0: !fir.ref<complex<f32>>)
   !CHECK: %[[load:.*]] = fir.load %arg0
-  !CHECK: %[[imag:.*]] = fir.extract_value %[[load]], [1 : index] : (!fir.complex<4>) -> f32
+  !CHECK: %[[imag:.*]] = fir.extract_value %[[load]], [1 : index] : (complex<f32>) -> f32
   !CHECK: return %[[imag]] : f32
 
 !CHECK-LABEL: func private @fir.len.i32.bc1(%arg0: !fir.boxchar<1>)
diff --git a/flang/test/Lower/entry-statement.f90 b/flang/test/Lower/entry-statement.f90
index 0ec650f8699a5..bf706d2d84000 100644
--- a/flang/test/Lower/entry-statement.f90
+++ b/flang/test/Lower/entry-statement.f90
@@ -182,16 +182,16 @@ subroutine dd1
 
 ! CHECK-LABEL: func @_QPashapec(
 subroutine ashapec(asc)
-  ! CHECK: %[[asx:[0-9]*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>
+  ! CHECK: %[[asx:[0-9]*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>
   ! CHECK: %[[asi:[0-9]*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xi32>>>
   ! CHECK: %[[zeroi:[0-9]*]] = fir.zero_bits !fir.heap<!fir.array<?xi32>>
   ! CHECK: %[[shapei:[0-9]*]] = fir.shape %c0{{.*}} : (index) -> !fir.shape<1>
   ! CHECK: %[[boxi:[0-9]*]] = fir.embox %[[zeroi]](%[[shapei]]) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
   ! CHECK: fir.store %[[boxi]] to %[[asi]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-  ! CHECK: %[[zerox:[0-9]*]] = fir.zero_bits !fir.heap<!fir.array<?x!fir.complex<4>>>
+  ! CHECK: %[[zerox:[0-9]*]] = fir.zero_bits !fir.heap<!fir.array<?xcomplex<f32>>>
   ! CHECK: %[[shapex:[0-9]*]] = fir.shape %c0{{.*}} : (index) -> !fir.shape<1>
-  ! CHECK: %[[boxx:[0-9].*]] = fir.embox %[[zerox]](%[[shapex]]) : (!fir.heap<!fir.array<?x!fir.complex<4>>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>
-  ! CHECK: fir.store %[[boxx]] to %[[asx]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>>
+  ! CHECK: %[[boxx:[0-9].*]] = fir.embox %[[zerox]](%[[shapex]]) : (!fir.heap<!fir.array<?xcomplex<f32>>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>
+  ! CHECK: fir.store %[[boxx]] to %[[asx]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>>
   character asc(:)
   integer asi(:)
   complex asx(:)
@@ -199,16 +199,16 @@ subroutine ashapec(asc)
   return
 ! CHECK-LABEL: func @_QPashapei(
 entry ashapei(asi)
-  ! CHECK: %[[asx:[0-9]*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>
+  ! CHECK: %[[asx:[0-9]*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>
   ! CHECK: %[[asc:[0-9]*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x!fir.char<1>>>>
   ! CHECK: %[[zeroc:[0-9]*]] = fir.zero_bits !fir.heap<!fir.array<?x!fir.char<1>>>
   ! CHECK: %[[shapec:[0-9]*]] = fir.shape %c0{{.*}} : (index) -> !fir.shape<1>
   ! CHECK: %[[boxc:[0-9]*]] = fir.embox %[[zeroc]](%[[shapec]]) : (!fir.heap<!fir.array<?x!fir.char<1>>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?x!fir.char<1>>>>
   ! CHECK: fir.store %[[boxc]] to %[[asc]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1>>>>>
-  ! CHECK: %[[zerox:[0-9]*]] = fir.zero_bits !fir.heap<!fir.array<?x!fir.complex<4>>>
+  ! CHECK: %[[zerox:[0-9]*]] = fir.zero_bits !fir.heap<!fir.array<?xcomplex<f32>>>
   ! CHECK: %[[shapex:[0-9]*]] = fir.shape %c0{{.*}} : (index) -> !fir.shape<1>
-  ! CHECK: %[[boxx:[0-9].*]] = fir.embox %[[zerox]](%[[shapex]]) : (!fir.heap<!fir.array<?x!fir.complex<4>>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>
-  ! CHECK: fir.store %[[boxx]] to %[[asx]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.complex<4>>>>>
+  ! CHECK: %[[boxx:[0-9].*]] = fir.embox %[[zerox]](%[[shapex]]) : (!fir.heap<!fir.array<?xcomplex<f32>>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>
+  ! CHECK: fir.store %[[boxx]] to %[[asx]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xcomplex<f32>>>>>
   asi = 3
   return
 ! CHECK-LABEL: func @_QPashapex(
diff --git a/flang/test/Lower/implicit-call-mismatch.f90 b/flang/test/Lower/implicit-call-mismatch.f90
index ca605d65f922b..e6bc75cf1f7bd 100644
--- a/flang/test/Lower/implicit-call-mismatch.f90
+++ b/flang/test/Lower/implicit-call-mismatch.f90
@@ -45,8 +45,8 @@ subroutine pass_complex_to_proc(a)
   call takes_proc(a)
 end subroutine
 ! CHECK-LABEL: func.func @_QPpass_complex_to_proc(
-! CHECK-SAME: %[[arg0:.*]]: !fir.ref<!fir.complex<4>>
-! CHECK: %[[procAddr:.*]] = fir.convert %[[arg0]] : (!fir.ref<!fir.complex<4>>) -> (() -> ())
+! CHECK-SAME: %[[arg0:.*]]: !fir.ref<complex<f32>>
+! CHECK: %[[procAddr:.*]] = fir.convert %[[arg0]] : (!fir.ref<complex<f32>>) -> (() -> ())
 ! CHECK: %[[boxProc:.*]] = fir.emboxproc %[[procAddr]] : (() -> ()) -> !fir.boxproc<() -> ()>
 ! CHECK: fir.call @_QPtakes_proc(%[[boxProc]]) {{.*}}: (!fir.boxproc<() -> ()>) -> ()
 
@@ -129,7 +129,7 @@ subroutine test_conversion_from_proc
   call pass_real_to_proc(proc)
 
   ! CHECK: %[[proc:.*]] = fir.address_of(@_QPproc) : () -> ()
-  ! CHECK: %[[convert:.*]] = fir.convert %[[proc]] : (() -> ()) -> !fir.ref<!fir.complex<4>>
+  ! CHECK: %[[convert:.*]] = fir.convert %[[proc]] : (() -> ()) -> !fir.ref<complex<f32>>
   ! CHECK: fir.call @_QPpass_complex_to_proc(%[[convert]])
   call pass_complex_to_proc(proc)
 
@@ -217,8 +217,8 @@ subroutine pass_complex_to_char_proc(a)
   call takes_char_proc(a)
 end subroutine
 ! CHECK-LABEL: func.func @_QPpass_complex_to_char_proc(
-! CHECK-SAME: %[[arg0:.*]]: !fir.ref<!fir.complex<4>>
-! CHECK: %[[procAddr:.*]] = fir.convert %[[arg0]] : (!fir.ref<!fir.complex<4>>) -> (() -> ())
+! CHECK-SAME: %[[arg0:.*]]: !fir.ref<complex<f32>>
+! CHECK: %[[procAddr:.*]] = fir.convert %[[arg0]] : (!fir.ref<complex<f32>>) -> (() -> ())
 ! CHECK: %[[boxProc:.*]] = fir.emboxproc %[[procAddr]] : (() -> ()) -> !fir.boxproc<() -> ()>
 ! CHECK: %[[charLen:.*]] = fir.undefined i64
 ! CHECK: %[[tuple:.*]] = fir.undefined tuple<!fir.boxproc<() -> ()>, i64>
@@ -328,7 +328,7 @@ subroutine test_conversion_from_char_proc
   call pass_real_to_char_proc(char_proc)
 
   ! CHECK: %[[proc:.*]] = fir.address_of(@_QPchar_proc) : (!fir.ref<!fir.char<1,8>>, index) -> !fir.boxchar<1>
-  ! CHECK: %[[convert:.*]] = fir.convert %[[proc]] : ((!fir.ref<!fir.char<1,8>>, index) -> !fir.boxchar<1>) -> !fir.ref<!fir.complex<4>>
+  ! CHECK: %[[convert:.*]] = fir.convert %[[proc]] : ((!fir.ref<!fir.char<1,8>>, index) -> !fir.boxchar<1>) -> !fir.ref<complex<f32>>
   ! CHECK: fir.call @_QPpass_complex_to_char_proc(%[[convert]])
   call pass_complex_to_char_proc(char_proc)
 
diff --git a/flang/test/Lower/math-lowering/abs.f90 b/flang/test/Lower/math-lowering/abs.f90
index 7be5ad265ef56..ead603c1d13e8 100644
--- a/flang/test/Lower/math-lowering/abs.f90
+++ b/flang/test/Lower/math-lowering/abs.f90
@@ -39,17 +39,17 @@ function test_complex4(c)
   test_complex4 = abs(c)
 end function
 ! ALL-LABEL: @_QPtest_complex4
-! ALL: {{%[A-Za-z0-9._]+}} = fir.call @cabsf({{%[A-Za-z0-9._]+}}) {{.*}}: (!fir.complex<4>) -> f32
+! ALL: {{%[A-Za-z0-9._]+}} = fir.call @cabsf({{%[A-Za-z0-9._]+}}) {{.*}}: (complex<f32>) -> f32
 
 function test_complex8(c)
   complex(8) :: c, test_complex8
   test_complex8 = abs(c)
 end function
 ! ALL-LABEL: @_QPtest_complex8
-! ALL: {{%[A-Za-z0-9._]+}} = fir.call @cabs({{%[A-Za-z0-9._]+}}) {{.*}}: (!fir.complex<8>) -> f64
+! ALL: {{%[A-Za-z0-9._]+}} = fir.call @cabs({{%[A-Za-z0-9._]+}}) {{.*}}: (complex<f64>) -> f64
 
 ! PRECISE-DAG: func.func private @fabsf(f32) -> f32 attributes {fir.bindc_name = "fabsf", fir.runtime}
 ! PRECISE-DAG: func.func private @fabs(f64) -> f64 attributes {fir.bindc_name = "fabs", fir.runtime}
 ! PRECISE-DAG: func.func private @llvm.fabs.f128(f128) -> f128 attributes {fir.bindc_name = "llvm.fabs.f128", fir.runtime}
-! PRECISE-DAG: func.func private @cabsf(!fir.complex<4>) -> f32 attributes {fir.bindc_name = "cabsf", fir.runtime}
-! PRECISE-DAG: func.func private @cabs(!fir.complex<8>) -> f64 attributes {fir.bindc_name = "cabs", fir.runtime}
+! PRECISE-DAG: func.func private @cabsf(complex<f32>) -> f32 attributes {fir.bindc_name = "cabsf", fir.runtime}
+! PRECISE-DAG: func.func private @cabs(complex<f64>) -> f64 attributes {fir.bindc_name = "cabs", fir.runtime}
diff --git a/flang/test/Lower/nsw.f90 b/flang/test/Lower/nsw.f90
new file mode 100644
index 0000000000000..84435b7133042
--- /dev/null
+++ b/flang/test/Lower/nsw.f90
@@ -0,0 +1,61 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: bbc -emit-fir -fwrapv %s -o - | FileCheck %s --check-prefix=NO-NSW
+
+! NO-NSW-NOT: overflow<nsw>
+
+subroutine subscript(a, i, j, k)
+  integer :: a(:,:,:), i, j, k
+  a(i+1, j-2, k*3) = 5
+end subroutine
+! CHECK-LABEL:   func.func @_QPsubscript(
+! CHECK:           %[[VAL_4:.*]] = arith.constant 3 : i32
+! CHECK:           %[[VAL_5:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_6:.*]] = arith.constant 1 : i32
+! CHECK:           %[[VAL_9:.*]] = fir.declare %{{.*}}a"} : (!fir.box<!fir.array<?x?x?xi32>>, !fir.dscope) -> !fir.box<!fir.array<?x?x?xi32>>
+! CHECK:           %[[VAL_10:.*]] = fir.rebox %[[VAL_9]] : (!fir.box<!fir.array<?x?x?xi32>>) -> !fir.box<!fir.array<?x?x?xi32>>
+! CHECK:           %[[VAL_11:.*]] = fir.declare %{{.*}}i"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+! CHECK:           %[[VAL_12:.*]] = fir.declare %{{.*}}j"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+! CHECK:           %[[VAL_13:.*]] = fir.declare %{{.*}}k"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+! CHECK:           %[[VAL_14:.*]] = fir.load %[[VAL_11]] : !fir.ref<i32>
+! CHECK:           %[[VAL_15:.*]] = arith.addi %[[VAL_14]], %[[VAL_6]] overflow<nsw> : i32
+! CHECK:           %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> i64
+! CHECK:           %[[VAL_17:.*]] = fir.load %[[VAL_12]] : !fir.ref<i32>
+! CHECK:           %[[VAL_18:.*]] = arith.subi %[[VAL_17]], %[[VAL_5]] overflow<nsw> : i32
+! CHECK:           %[[VAL_19:.*]] = fir.convert %[[VAL_18]] : (i32) -> i64
+! CHECK:           %[[VAL_20:.*]] = fir.load %[[VAL_13]] : !fir.ref<i32>
+! CHECK:           %[[VAL_21:.*]] = arith.muli %[[VAL_20]], %[[VAL_4]] overflow<nsw> : i32
+! CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_21]] : (i32) -> i64
+! CHECK:           %[[VAL_23:.*]] = fir.array_coor %[[VAL_10]] %[[VAL_16]], %[[VAL_19]], %[[VAL_22]] :
+
+! Test that nsw is never added to arith ops
+! on vector subscripts.
+subroutine vector_subscript_as_value(x, y, z)
+  integer :: x(100)
+  integer(8) :: y(20), z(20)
+  call bar(x(y+z))
+end subroutine
+! CHECK-LABEL:   func.func @_QPvector_subscript_as_value(
+! CHECK-NOT: overflow<nsw>
+! CHECK:           return
+
+subroutine vector_subscript_lhs(x, vector1, vector2)
+  integer(8) :: vector1(10), vector2(10)
+  real :: x(:)
+  x(vector1+vector2) = 42.
+end subroutine
+! CHECK-LABEL:   func.func @_QPvector_subscript_lhs(
+! CHECK-NOT: overflow<nsw>
+! CHECK:           return
+
+! Test that nsw is never added to arith ops
+! on arguments of bitwise comparison intrinsics.
+subroutine bitwise_comparison(a, b)
+  integer :: a, b
+  print *, bge(a+b, a-b)
+  print *, bgt(a+b, a-b)
+  print *, ble(a+b, a-b)
+  print *, blt(a+b, a-b)
+end subroutine
+! CHECK-LABEL:   func.func @_QPbitwise_comparison(
+! CHECK-NOT: overflow<nsw>
+! CHECK:           return
diff --git a/flang/test/Lower/pointer.f90 b/flang/test/Lower/pointer.f90
index 6bc548e3a392b..aeff8c71ff274 100644
--- a/flang/test/Lower/pointer.f90
+++ b/flang/test/Lower/pointer.f90
@@ -21,12 +21,12 @@ subroutine pointerTests
   ! CHECK: [[reg2:%[0-9]+]] = fir.convert [[reg1]] : (!fir.ref<none>) -> !fir.ptr<f32>
   ! CHECK: fir.has_value [[reg2]] : !fir.ptr<f32>
 
-  ! CHECK: fir.global internal @_QFpointertestsEptr3 : !fir.ptr<!fir.complex<4>>
+  ! CHECK: fir.global internal @_QFpointertestsEptr3 : !fir.ptr<complex<f32>>
   complex, pointer :: ptr3 => NULL()
   ! CHECK: %[[c0:.*]] = arith.constant 0 : index
   ! CHECK: [[reg1:%[0-9]+]] = fir.convert %[[c0:.*]] : (index) -> !fir.ref<none>
-  ! CHECK: [[reg2:%[0-9]+]] = fir.convert [[reg1]] : (!fir.ref<none>) -> !fir.ptr<!fir.complex<4>>
-  ! CHECK: fir.has_value [[reg2]] : !fir.ptr<!fir.complex<4>>
+  ! CHECK: [[reg2:%[0-9]+]] = fir.convert [[reg1]] : (!fir.ref<none>) -> !fir.ptr<complex<f32>>
+  ! CHECK: fir.has_value [[reg2]] : !fir.ptr<complex<f32>>
 
   ! CHECK: fir.global internal @_QFpointertestsEptr4 : !fir.ptr<!fir.char<1,?>>
   character(:), pointer :: ptr4 => NULL()
diff --git a/flang/test/Lower/polymorphic.f90 b/flang/test/Lower/polymorphic.f90
index 14ec8a06a964f..8c40c91bc3baa 100644
--- a/flang/test/Lower/polymorphic.f90
+++ b/flang/test/Lower/polymorphic.f90
@@ -343,7 +343,7 @@ subroutine pass_trivial_to_up()
 ! CHECK: %[[BOX_LOG:.*]] = fir.embox %{{.*}} : (!fir.ref<!fir.logical<4>>) -> !fir.class<none>
 ! CHECK: fir.call @_QMpolymorphic_testPup_input(%[[BOX_LOG]]) {{.*}} : (!fir.class<none>) -> ()
 
-! CHECK: %[[BOX_COMPLEX:.*]] = fir.embox %{{.*}} : (!fir.ref<!fir.complex<4>>) -> !fir.class<none>
+! CHECK: %[[BOX_COMPLEX:.*]] = fir.embox %{{.*}} : (!fir.ref<complex<f32>>) -> !fir.class<none>
 ! CHECK: fir.call @_QMpolymorphic_testPup_input(%[[BOX_COMPLEX]]) {{.*}} : (!fir.class<none>) -> ()
 
   subroutine up_arr_input(a)
@@ -377,7 +377,7 @@ subroutine pass_trivial_arr_to_up()
 ! CHECK: %[[BOX_LOG:.*]] = fir.embox %{{.*}}(%{{.*}}) : (!fir.ref<!fir.array<2x!fir.logical<4>>>, !fir.shape<1>) -> !fir.class<!fir.array<2xnone>>
 ! CHECK: fir.call @_QMpolymorphic_testPup_arr_input(%[[BOX_LOG]]) {{.*}} : (!fir.class<!fir.array<2xnone>>) -> ()
 
-! CHECK: %[[BOX_COMPLEX:.*]] = fir.embox %{{.*}}(%{{.*}}) : (!fir.ref<!fir.array<2x!fir.complex<4>>>, !fir.shape<1>) -> !fir.class<!fir.array<2xnone>>
+! CHECK: %[[BOX_COMPLEX:.*]] = fir.embox %{{.*}}(%{{.*}}) : (!fir.ref<!fir.array<2xcomplex<f32>>>, !fir.shape<1>) -> !fir.class<!fir.array<2xnone>>
 ! CHECK: fir.call @_QMpolymorphic_testPup_arr_input(%[[BOX_COMPLEX]]) {{.*}} : (!fir.class<!fir.array<2xnone>>) -> ()
 
   subroutine assign_polymorphic_allocatable()
@@ -578,7 +578,7 @@ subroutine test_elemental_poly_array(p)
 ! CHECK: %[[LOOP_RES:.*]] = fir.do_loop %[[IND:.*]] = %[[C0]] to %[[UB]] step %[[C1]] unordered iter_args(%[[ARG:.*]] = %[[ARRAY_LOAD_TMP]]) -> (!fir.array<5xi32>) {
 ! CHECK:   %[[COORD:.*]] = fir.coordinate_of %[[P]], %[[IND]] : (!fir.class<!fir.array<5x!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>>, index) -> !fir.ref<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>
 ! CHECK:   %[[EMBOXED:.*]] = fir.embox %[[COORD]] source_box %[[P]] : (!fir.ref<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, !fir.class<!fir.array<5x!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>>) -> !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>
-! CHECK:   %[[RES:.*]] = fir.dispatch "elemental_fct"(%[[EMBOXED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%[[EMBOXED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> i32 {pass_arg_pos = 0 : i32}
+! CHECK:   %[[RES:.*]] = fir.dispatch "elemental_fct"(%[[EMBOXED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%[[EMBOXED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> i32 proc_attrs <elemental, pure> {pass_arg_pos = 0 : i32}
 ! CHECK:   %[[ARR_UP:.*]] = fir.array_update %[[ARG]], %[[RES]], %[[IND]] : (!fir.array<5xi32>, i32, index) -> !fir.array<5xi32>
 ! CHECK:   fir.result %[[ARR_UP]] : !fir.array<5xi32>
 ! CHECK: }
@@ -609,7 +609,7 @@ subroutine test_elemental_poly_array_2d(p)
 ! CHECK:   %[[LOOP_RES0:.*]] = fir.do_loop %[[IND1:.*]] = %[[C0]] to %[[UB0]] step %[[C1]] unordered iter_args(%[[ARG0:.*]] = %[[ARG]]) -> (!fir.array<5x5xi32>) {
 ! CHECK:     %[[COORD:.*]] = fir.coordinate_of %[[P]], %[[IND1]], %[[IND0]] : (!fir.class<!fir.array<5x5x!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>>, index, index) -> !fir.ref<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>
 ! CHECK:     %[[EMBOXED:.*]] = fir.embox %[[COORD]] source_box %[[P]] : (!fir.ref<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, !fir.class<!fir.array<5x5x!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>>) -> !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>
-! CHECK:     %[[RES:.*]] = fir.dispatch "elemental_fct"(%[[EMBOXED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%[[EMBOXED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> i32 {pass_arg_pos = 0 : i32}
+! CHECK:     %[[RES:.*]] = fir.dispatch "elemental_fct"(%[[EMBOXED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%[[EMBOXED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> i32 proc_attrs <elemental, pure> {pass_arg_pos = 0 : i32}
 ! CHECK:     %[[ARR_UP:.*]] = fir.array_update %[[ARG0]], %[[RES]], %[[IND1]], %[[IND0]] : (!fir.array<5x5xi32>, i32, index, index) -> !fir.array<5x5xi32>
 ! CHECK:     fir.result %[[ARR_UP]] : !fir.array<5x5xi32>
 ! CHECK:   }
@@ -663,7 +663,7 @@ subroutine test_elemental_sub_poly_array(p)
 ! CHECK: fir.do_loop %[[IND:.*]] = %[[C0]] to %[[UB]] step %[[C1]] {
 ! CHECK:   %[[COORD:.*]] = fir.coordinate_of %[[P]], %[[IND]] : (!fir.class<!fir.array<10x!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>>, index) -> !fir.ref<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>
 ! CHECK:   %[[EMBOXED:.*]] = fir.embox %[[COORD]] source_box %[[P]] : (!fir.ref<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, !fir.class<!fir.array<10x!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>>) -> !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>
-! CHECK:   fir.dispatch "elemental_sub"(%[[EMBOXED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%[[EMBOXED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
+! CHECK:   fir.dispatch "elemental_sub"(%[[EMBOXED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%[[EMBOXED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) proc_attrs <elemental, pure> {pass_arg_pos = 0 : i32}
 ! CHECK: }
 ! CHECK: %[[C1:.*]] = arith.constant 1 : index
 ! CHECK: %[[C0:.*]] = arith.constant 0 : index
@@ -671,7 +671,7 @@ subroutine test_elemental_sub_poly_array(p)
 ! CHECK: fir.do_loop %[[IND:.*]] = %[[C0]] to %[[UB]] step %[[C1]] {
 ! CHECK:   %[[COORD:.*]] = fir.coordinate_of %[[P]], %[[IND]] : (!fir.class<!fir.array<10x!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>>, index) -> !fir.ref<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>
 ! CHECK:   %[[EMBOXED:.*]] = fir.embox %[[COORD]] source_box %[[P]] : (!fir.ref<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, !fir.class<!fir.array<10x!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>>) -> !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>
-! CHECK:   fir.dispatch "elemental_sub_pass"(%[[EMBOXED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %[[EMBOXED]] : !fir.ref<i32>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 1 : i32}
+! CHECK:   fir.dispatch "elemental_sub_pass"(%[[EMBOXED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %[[EMBOXED]] : !fir.ref<i32>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) proc_attrs <elemental, pure> {pass_arg_pos = 1 : i32}
 ! CHECK: }
 
   subroutine test_elemental_sub_array_assumed(t)
@@ -718,7 +718,7 @@ subroutine test_elemental_sub_poly_array_assumed(p)
 ! CHECK: fir.do_loop %[[IND:.*]] = %[[C0]] to %[[UB]] step %[[C1]] {
 ! CHECK:   %[[COORD:.*]] = fir.coordinate_of %[[P]], %[[IND]] : (!fir.class<!fir.array<?x!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>>, index) -> !fir.ref<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>
 ! CHECK:   %[[EMBOXED:.*]] = fir.embox %[[COORD]] source_box %[[P]] : (!fir.ref<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, !fir.class<!fir.array<?x!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>>) -> !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>
-! CHECK:   fir.dispatch "elemental_sub"(%[[EMBOXED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%[[EMBOXED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
+! CHECK:   fir.dispatch "elemental_sub"(%[[EMBOXED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%[[EMBOXED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) proc_attrs <elemental, pure> {pass_arg_pos = 0 : i32}
 ! CHECK: }
 ! CHECK: %[[C0:.*]] = arith.constant 0 : index
 ! CHECK: %[[P_DIMS:.*]]:3 = fir.box_dims %[[P]], %[[C0]] : (!fir.class<!fir.array<?x!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>>, index) -> (index, index, index)
@@ -728,7 +728,7 @@ subroutine test_elemental_sub_poly_array_assumed(p)
 ! CHECK: fir.do_loop %[[IND:.*]] = %[[C0]] to %[[UB]] step %[[C1]] {
 ! CHECK:   %[[COORD:.*]] = fir.coordinate_of %[[P]], %[[IND]] : (!fir.class<!fir.array<?x!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>>, index) -> !fir.ref<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>
 ! CHECK:   %[[EMBOXED:.*]] = fir.embox %[[COORD]] source_box %[[P]] : (!fir.ref<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>, !fir.class<!fir.array<?x!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>>) -> !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>
-! CHECK:   fir.dispatch "elemental_sub_pass"(%[[EMBOXED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %[[EMBOXED]] : !fir.ref<i32>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) {pass_arg_pos = 1 : i32}
+! CHECK:   fir.dispatch "elemental_sub_pass"(%[[EMBOXED]] : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) (%{{.*}}, %[[EMBOXED]] : !fir.ref<i32>, !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) proc_attrs <elemental, pure> {pass_arg_pos = 1 : i32}
 ! CHECK: }
 
   subroutine write_p1(dtv, unit, iotype, v_list, iostat, iomsg)
diff --git a/flang/test/Lower/sqrt.f90 b/flang/test/Lower/sqrt.f90
index 458a83b9a1469..e00d91c7abf33 100644
--- a/flang/test/Lower/sqrt.f90
+++ b/flang/test/Lower/sqrt.f90
@@ -23,14 +23,14 @@ subroutine sqrt_testd(a, b)
 ! CHECK-LABEL: sqrt_testc
 subroutine sqrt_testc(z)
   complex :: z
-! CHECK: fir.call @fir.sqrt.contract.z4.z4
+! CHECK: fir.call @fir.sqrt.contract.z32.z32
   z = sqrt(z)
 end subroutine
 
 ! CHECK-LABEL: sqrt_testcd
 subroutine sqrt_testcd(z)
   complex(kind=8) :: z
-! CHECK: fir.call @fir.sqrt.contract.z8.z8
+! CHECK: fir.call @fir.sqrt.contract.z64.z64
   z = sqrt(z)
 end subroutine
 
@@ -40,10 +40,10 @@ subroutine sqrt_testcd(z)
 ! CHECK-LABEL: @fir.sqrt.contract.f64.f64
 ! CHECK: math.sqrt %{{.*}} : f64
 
-! CHECK-LABEL: func private @fir.sqrt.contract.z4.z4
+! CHECK-LABEL: func private @fir.sqrt.contract.z32.z32
 ! CMPLX-FAST: complex.sqrt %{{.*}} : complex<f32>
 ! CMPLX-PRECISE: fir.call @csqrtf
 
-! CHECK-LABEL: @fir.sqrt.contract.z8.z8
+! CHECK-LABEL: @fir.sqrt.contract.z64.z64
 ! CMPLX-FAST: complex.sqrt %{{.*}} : complex<f64>
 ! CMPLX-PRECISE: fir.call @csqrt
diff --git a/flang/test/Lower/trigonometric-intrinsics.f90 b/flang/test/Lower/trigonometric-intrinsics.f90
index 44a6b716b3867..34cc59eaafd9b 100644
--- a/flang/test/Lower/trigonometric-intrinsics.f90
+++ b/flang/test/Lower/trigonometric-intrinsics.f90
@@ -20,14 +20,14 @@ subroutine tan_testd(a, b)
 ! CHECK-LABEL: tan_testc
 subroutine tan_testc(z)
   complex :: z
-! CHECK: fir.call @fir.tan.contract.z4.z4
+! CHECK: fir.call @fir.tan.contract.z32.z32
   z = tan(z)
 end subroutine
 
 ! CHECK-LABEL: tan_testcd
 subroutine tan_testcd(z)
   complex(kind=8) :: z
-! CHECK: fir.call @fir.tan.contract.z8.z8
+! CHECK: fir.call @fir.tan.contract.z64.z64
   z = tan(z)
 end subroutine
 
@@ -48,14 +48,14 @@ subroutine atan_testd(a, b)
 ! CHECK-LABEL: atan_testc
 subroutine atan_testc(z)
   complex :: z
-! CHECK: fir.call @fir.atan.contract.z4.z4
+! CHECK: fir.call @fir.atan.contract.z32.z32
   z = atan(z)
 end subroutine
 
 ! CHECK-LABEL: atan_testcd
 subroutine atan_testcd(z)
   complex(kind=8) :: z
-! CHECK: fir.call @fir.atan.contract.z8.z8
+! CHECK: fir.call @fir.atan.contract.z64.z64
   z = atan(z)
 end subroutine
 
@@ -76,14 +76,14 @@ subroutine cos_testd(a, b)
 ! CHECK-LABEL: cos_testc
 subroutine cos_testc(z)
   complex :: z
-! CHECK: fir.call @fir.cos.contract.z4.z4
+! CHECK: fir.call @fir.cos.contract.z32.z32
   z = cos(z)
 end subroutine
 
 ! CHECK-LABEL: cos_testcd
 subroutine cos_testcd(z)
   complex(kind=8) :: z
-! CHECK: fir.call @fir.cos.contract.z8.z8
+! CHECK: fir.call @fir.cos.contract.z64.z64
   z = cos(z)
 end subroutine
 
@@ -104,14 +104,14 @@ subroutine cosh_testd(a, b)
 ! CHECK-LABEL: cosh_testc
 subroutine cosh_testc(z)
   complex :: z
-! CHECK: fir.call @fir.cosh.contract.z4.z4
+! CHECK: fir.call @fir.cosh.contract.z32.z32
   z = cosh(z)
 end subroutine
 
 ! CHECK-LABEL: cosh_testcd
 subroutine cosh_testcd(z)
   complex(kind=8) :: z
-! CHECK: fir.call @fir.cosh.contract.z8.z8
+! CHECK: fir.call @fir.cosh.contract.z64.z64
   z = cosh(z)
 end subroutine
 
@@ -132,14 +132,14 @@ subroutine sin_testd(a, b)
 ! CHECK-LABEL: sin_testc
 subroutine sin_testc(z)
   complex :: z
-! CHECK: fir.call @fir.sin.contract.z4.z4
+! CHECK: fir.call @fir.sin.contract.z32.z32
   z = sin(z)
 end subroutine
 
 ! CHECK-LABEL: sin_testcd
 subroutine sin_testcd(z)
   complex(kind=8) :: z
-! CHECK: fir.call @fir.sin.contract.z8.z8
+! CHECK: fir.call @fir.sin.contract.z64.z64
   z = sin(z)
 end subroutine
 
@@ -160,14 +160,14 @@ subroutine sinh_testd(a, b)
 ! CHECK-LABEL: sinh_testc
 subroutine sinh_testc(z)
   complex :: z
-! CHECK: fir.call @fir.sinh.contract.z4.z4
+! CHECK: fir.call @fir.sinh.contract.z32.z32
   z = sinh(z)
 end subroutine
 
 ! CHECK-LABEL: sinh_testcd
 subroutine sinh_testcd(z)
   complex(kind=8) :: z
-! CHECK: fir.call @fir.sinh.contract.z8.z8
+! CHECK: fir.call @fir.sinh.contract.z64.z64
   z = sinh(z)
 end subroutine
 
@@ -177,11 +177,11 @@ subroutine sinh_testcd(z)
 ! CHECK-LABEL: @fir.tan.contract.f64.f64
 ! CHECK: math.tan %{{.*}} : f64
 
-! CHECK-LABEL: @fir.tan.contract.z4.z4
+! CHECK-LABEL: @fir.tan.contract.z32.z32
 ! CMPLX-FAST: complex.tan %{{.*}} : complex<f32>
 ! CMPLX-PRECISE: fir.call @ctanf
 
-! CHECK-LABEL: @fir.tan.contract.z8.z8
+! CHECK-LABEL: @fir.tan.contract.z64.z64
 ! CMPLX-FAST: complex.tan %{{.*}} : complex<f64>
 ! CMPLX-PRECISE: fir.call @ctan
 
@@ -191,10 +191,10 @@ subroutine sinh_testcd(z)
 ! CHECK-LABEL: @fir.atan.contract.f64.f64
 ! CHECK: math.atan %{{.*}} : f64
 
-! CHECK-LABEL: @fir.atan.contract.z4.z4
+! CHECK-LABEL: @fir.atan.contract.z32.z32
 ! CHECK: fir.call @catanf
 
-! CHECK-LABEL: @fir.atan.contract.z8.z8
+! CHECK-LABEL: @fir.atan.contract.z64.z64
 ! CHECK: fir.call @catan
 
 ! CHECK-LABEL: @fir.cos.contract.f32.f32
@@ -203,11 +203,11 @@ subroutine sinh_testcd(z)
 ! CHECK-LABEL: @fir.cos.contract.f64.f64
 ! CHECK: math.cos %{{.*}} : f64
 
-! CHECK-LABEL: @fir.cos.contract.z4.z4
+! CHECK-LABEL: @fir.cos.contract.z32.z32
 ! CMPLX-FAST: complex.cos %{{.*}} : complex<f32>
 ! CMPLX-PRECISE: fir.call @ccosf
 
-! CHECK-LABEL: @fir.cos.contract.z8.z8
+! CHECK-LABEL: @fir.cos.contract.z64.z64
 ! CMPLX-FAST: complex.cos %{{.*}} : complex<f64>
 ! CMPLX-PRECISE: fir.call @ccos
 
@@ -217,10 +217,10 @@ subroutine sinh_testcd(z)
 ! CHECK-LABEL: @fir.cosh.contract.f64.f64
 ! CHECK: fir.call {{.*}}cosh
 
-! CHECK-LABEL: @fir.cosh.contract.z4.z4
+! CHECK-LABEL: @fir.cosh.contract.z32.z32
 ! CHECK: fir.call @ccoshf
 
-! CHECK-LABEL: @fir.cosh.contract.z8.z8
+! CHECK-LABEL: @fir.cosh.contract.z64.z64
 ! CHECK: fir.call @ccosh
 
 ! CHECK-LABEL: @fir.sin.contract.f32.f32
@@ -229,11 +229,11 @@ subroutine sinh_testcd(z)
 ! CHECK-LABEL: @fir.sin.contract.f64.f64
 ! CHECK: math.sin %{{.*}} : f64
 
-! CHECK-LABEL: @fir.sin.contract.z4.z4
+! CHECK-LABEL: @fir.sin.contract.z32.z32
 ! CMPLX-FAST: complex.sin %{{.*}} : complex<f32>
 ! CMPLX-PRECISE: fir.call @csinf
 
-! CHECK-LABEL: @fir.sin.contract.z8.z8
+! CHECK-LABEL: @fir.sin.contract.z64.z64
 ! CMPLX-FAST: complex.sin %{{.*}} : complex<f64>
 ! CMPLX-PRECISE: fir.call @csin
 
@@ -243,8 +243,8 @@ subroutine sinh_testcd(z)
 ! CHECK-LABEL: @fir.sinh.contract.f64.f64
 ! CHECK: fir.call {{.*}}sinh
 
-! CHECK-LABEL: @fir.sinh.contract.z4.z4
+! CHECK-LABEL: @fir.sinh.contract.z32.z32
 ! CHECK: fir.call @csinhf
 
-! CHECK-LABEL: @fir.sinh.contract.z8.z8
+! CHECK-LABEL: @fir.sinh.contract.z64.z64
 ! CHECK: fir.call @csinh
diff --git a/flang/test/Lower/vector-subscript-io.f90 b/flang/test/Lower/vector-subscript-io.f90
index d298609269dae..129e3ee1206c0 100644
--- a/flang/test/Lower/vector-subscript-io.f90
+++ b/flang/test/Lower/vector-subscript-io.f90
@@ -215,7 +215,7 @@ subroutine two_vectors(x, y1, y2)
 end subroutine
 
 ! CHECK-LABEL: func @_QPtriplets_and_vector(
-! CHECK-SAME:    %[[VAL_170:.*]]: !fir.ref<!fir.array<4x4x!fir.complex<4>>>{{.*}}, %[[VAL_166:.*]]: !fir.ref<!fir.array<3xi32>>{{.*}}) {
+! CHECK-SAME:    %[[VAL_170:.*]]: !fir.ref<!fir.array<4x4xcomplex<f32>>>{{.*}}, %[[VAL_166:.*]]: !fir.ref<!fir.array<3xi32>>{{.*}}) {
 subroutine triplets_and_vector(x, y)
   integer :: y(3)
   complex :: x(4, 4)
@@ -242,8 +242,8 @@ subroutine triplets_and_vector(x, y)
 ! CHECK:   %[[VAL_165:.*]] = fir.coordinate_of %[[VAL_166]], %[[VAL_159]] : (!fir.ref<!fir.array<3xi32>>, index) -> !fir.ref<i32>
 ! CHECK:   %[[VAL_167:.*]] = fir.load %[[VAL_165]] : !fir.ref<i32>
 ! CHECK:   %[[VAL_168:.*]] = fir.convert %[[VAL_167]] : (i32) -> index
-! CHECK:   %[[VAL_169:.*]] = fir.array_coor %[[VAL_170]](%[[VAL_157]]) {{\[}}%[[VAL_158]]] %[[VAL_162]], %[[VAL_168]] : (!fir.ref<!fir.array<4x4x!fir.complex<4>>>, !fir.shape<2>, !fir.slice<2>, index, index) -> !fir.ref<!fir.complex<4>>
-! CHECK:   %[[VAL_171:.*]] = fir.convert %[[VAL_169]] : (!fir.ref<!fir.complex<4>>) -> !fir.ref<f32>
+! CHECK:   %[[VAL_169:.*]] = fir.array_coor %[[VAL_170]](%[[VAL_157]]) {{\[}}%[[VAL_158]]] %[[VAL_162]], %[[VAL_168]] : (!fir.ref<!fir.array<4x4xcomplex<f32>>>, !fir.shape<2>, !fir.slice<2>, index, index) -> !fir.ref<complex<f32>>
+! CHECK:   %[[VAL_171:.*]] = fir.convert %[[VAL_169]] : (!fir.ref<complex<f32>>) -> !fir.ref<f32>
 ! CHECK:   %[[VAL_172:.*]] = fir.call @_FortranAioInputComplex32(%[[VAL_156]], %[[VAL_171]]) {{.*}}: (!fir.ref<i8>, !fir.ref<f32>) -> i1
 ! CHECK:   %[[VAL_173:.*]] = arith.addi %[[VAL_162]], %[[VAL_153]] : index
 ! CHECK:   %[[VAL_174:.*]] = arith.subi %[[VAL_163]], %[[VAL_153]] : index
@@ -342,7 +342,7 @@ subroutine substring(x, y, i, j)
 end subroutine
 
 ! CHECK-LABEL: func @_QPcomplex_part(
-! CHECK-SAME: %[[VAL_262:.*]]: !fir.box<!fir.array<?x!fir.complex<4>>>{{.*}}, %[[VAL_253:.*]]: !fir.box<!fir.array<?xi32>>{{.*}}) {
+! CHECK-SAME: %[[VAL_262:.*]]: !fir.box<!fir.array<?xcomplex<f32>>>{{.*}}, %[[VAL_253:.*]]: !fir.box<!fir.array<?xi32>>{{.*}}) {
 subroutine complex_part(z, y)
   integer :: y(:)
   complex :: z(:)
@@ -364,7 +364,7 @@ subroutine complex_part(z, y)
 ! CHECK:   %[[VAL_258:.*]] = fir.coordinate_of %[[VAL_253]], %[[VAL_255]] : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
 ! CHECK:   %[[VAL_259:.*]] = fir.load %[[VAL_258]] : !fir.ref<i32>
 ! CHECK:   %[[VAL_260:.*]] = fir.convert %[[VAL_259]] : (i32) -> index
-! CHECK:   %[[VAL_261:.*]] = fir.array_coor %[[VAL_262]] {{\[}}%[[VAL_254]]] %[[VAL_260]] : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.slice<1>, index) -> !fir.ref<f32>
+! CHECK:   %[[VAL_261:.*]] = fir.array_coor %[[VAL_262]] {{\[}}%[[VAL_254]]] %[[VAL_260]] : (!fir.box<!fir.array<?xcomplex<f32>>>, !fir.slice<1>, index) -> !fir.ref<f32>
 ! CHECK:   %[[VAL_263:.*]] = fir.call @_FortranAioInputReal32(%[[VAL_251]], %[[VAL_261]]) {{.*}}: (!fir.ref<i8>, !fir.ref<f32>) -> i1
 ! CHECK:   %[[VAL_264:.*]] = arith.addi %[[VAL_255]], %[[VAL_248]] : index
 ! CHECK:   %[[VAL_265:.*]] = arith.subi %[[VAL_256]], %[[VAL_248]] : index
diff --git a/flang/test/Parser/OpenMP/lastprivate-clause.f90 b/flang/test/Parser/OpenMP/lastprivate-clause.f90
new file mode 100644
index 0000000000000..382f02c75e7f1
--- /dev/null
+++ b/flang/test/Parser/OpenMP/lastprivate-clause.f90
@@ -0,0 +1,54 @@
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=50 %s | FileCheck --ignore-case --check-prefix="UNPARSE" %s 
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=50 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+subroutine foo1()
+  integer :: x, i
+  x = 1
+  !$omp parallel do lastprivate(x)
+  do i = 1, 100
+    x = x + 1
+  enddo
+end
+
+!UNPARSE: SUBROUTINE foo1
+!UNPARSE:  INTEGER x, i
+!UNPARSE:   x=1_4
+!UNPARSE: !$OMP PARALLEL DO  LASTPRIVATE(x)
+!UNPARSE:  DO i=1_4,100_4
+!UNPARSE:    x=x+1_4
+!UNPARSE:  END DO
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: SubroutineStmt
+!PARSE-TREE:   Name = 'foo1'
+!PARSE-TREE: OmpLoopDirective -> llvm::omp::Directive = parallel do
+!PARSE-TREE: OmpClauseList -> OmpClause -> Lastprivate -> OmpLastprivateClause
+!PARSE-TREE:   OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: EndSubroutineStmt
+
+
+subroutine foo2()
+  integer :: x, i
+  x = 1
+  !$omp parallel do lastprivate(conditional: x)
+  do i = 1, 100
+    x = x + 1
+  enddo
+end
+
+!UNPARSE: SUBROUTINE foo2
+!UNPARSE:  INTEGER x, i
+!UNPARSE:   x=1_4
+!UNPARSE: !$OMP PARALLEL DO  LASTPRIVATE(CONDITIONAL:x)
+!UNPARSE:  DO i=1_4,100_4
+!UNPARSE:    x=x+1_4
+!UNPARSE:  END DO
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: SubroutineStmt
+!PARSE-TREE:   Name = 'foo2'
+!PARSE-TREE: OmpLoopDirective -> llvm::omp::Directive = parallel do
+!PARSE-TREE: OmpClauseList -> OmpClause -> Lastprivate -> OmpLastprivateClause
+!PARSE-TREE:   LastprivateModifier = Conditional
+!PARSE-TREE:   OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: EndSubroutineStmt
diff --git a/flang/test/Semantics/OpenMP/private03.f90 b/flang/test/Semantics/OpenMP/private03.f90
new file mode 100644
index 0000000000000..61f790fbb38b2
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/private03.f90
@@ -0,0 +1,39 @@
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! OpenMP Version 4.5
+! Variables that appear in expressions for statement function definitions
+! may not appear in private, firstprivate or lastprivate clauses.
+
+subroutine stmt_function(temp)
+
+  integer :: i, p, q, r
+  real :: c, f, s, v, t(10)
+  real, intent(in) :: temp
+
+  c(temp) = p * (temp - q) / r
+  f(temp) = q + (temp * r/p)
+  v(temp) = c(temp) + f(temp)/2 - s
+
+  p = 5
+  q = 32
+  r = 9
+
+  !ERROR: Variable 'p' in statement function expression cannot be in a PRIVATE clause
+  !$omp parallel private(p)
+  s = c(temp)
+  !$omp end parallel
+
+  !ERROR: Variable 's' in statement function expression cannot be in a FIRSTPRIVATE clause
+  !$omp parallel firstprivate(s)
+  s = s + f(temp)
+  !$omp end parallel
+
+  !ERROR: Variable 's' in statement function expression cannot be in a LASTPRIVATE clause
+  !$omp parallel do lastprivate(s, t)
+  do i = 1, 10
+  t(i) = v(temp) + i - s
+  end do
+  !$omp end parallel do
+
+  print *, t
+
+end subroutine stmt_function
diff --git a/flang/test/Semantics/OpenMP/reduction-namelist.f90 b/flang/test/Semantics/OpenMP/reduction-namelist.f90
new file mode 100644
index 0000000000000..edae3d7aef2c5
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/reduction-namelist.f90
@@ -0,0 +1,46 @@
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! This is not actually disallowed by the OpenMP standard, but it is not allowed
+! for privatisation - this seems like an oversight.
+
+module test
+  integer :: a, b, c
+  namelist /nlist1/ a, b
+end module
+
+program omp_reduction
+  use test
+
+  integer :: p(10) ,q(10)
+  namelist /nlist2/ c, d
+
+  a = 5
+  b = 10
+  c = 100
+
+  !ERROR: Variable 'd' in NAMELIST cannot be in a REDUCTION clause
+  !ERROR: Variable 'a' in NAMELIST cannot be in a REDUCTION clause
+  !$omp parallel reduction(+:d) reduction(+:a)
+  d = a + b
+  a = d
+  !$omp end parallel
+
+  call sb()
+
+  contains
+    subroutine sb()
+      namelist /nlist3/ p, q
+
+      !ERROR: Variable 'p' in NAMELIST cannot be in a REDUCTION clause
+      !ERROR: Variable 'q' in NAMELIST cannot be in a REDUCTION clause
+      !$omp parallel reduction(+:p) reduction(+:q)
+      p = c * b
+      q = p * d
+      !$omp end parallel
+
+      write(*, nlist1)
+      write(*, nlist2)
+      write(*, nlist3)
+
+    end subroutine
+
+end program omp_reduction
diff --git a/flang/test/Semantics/resolve123.f90 b/flang/test/Semantics/resolve123.f90
index 1b2c4613f2fef..8f7c3acd44311 100644
--- a/flang/test/Semantics/resolve123.f90
+++ b/flang/test/Semantics/resolve123.f90
@@ -77,3 +77,13 @@ subroutine s9()
   do concurrent(i=1:5) reduce(+:i)
   end do
 end subroutine s9
+
+subroutine s10()
+! Cannot have variable inside of a NAMELIST in a REDUCE locality spec
+  integer :: k
+  namelist /nlist1/ k
+!ERROR: NAMELIST variable 'k' not allowed in a REDUCE locality-spec
+  do concurrent(i=1:5) reduce(+:k)
+     k = k + i
+  end do
+end subroutine s10
diff --git a/flang/test/Semantics/windows.f90 b/flang/test/Semantics/windows.f90
new file mode 100644
index 0000000000000..705fb1b816895
--- /dev/null
+++ b/flang/test/Semantics/windows.f90
@@ -0,0 +1,12 @@
+! RUN: %if x86_64-registered-target %{ %python %S/test_errors.py %s %flang --target=x86_64-pc-windows-msvc -Werror %}
+! RUN: %if aarch64-registered-target %{ %python %S/test_errors.py %s %flang --target=aarch64-pc-windows-msvc -Werror %}
+
+subroutine uid
+  !WARNING: User IDs do not exist on Windows. This function will always return 1
+  i = getuid()
+end subroutine uid
+
+subroutine gid
+  !WARNING: Group IDs do not exist on Windows. This function will always return 1
+  i = getgid()
+end subroutine gid
diff --git a/flang/test/Transforms/debug-107988.fir b/flang/test/Transforms/debug-107988.fir
index 0b08cf1c0b2eb..0ba4296138f50 100644
--- a/flang/test/Transforms/debug-107988.fir
+++ b/flang/test/Transforms/debug-107988.fir
@@ -13,7 +13,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
 #loc1 = loc("test.f90":5:1)
 #loc2 = loc("test.f90":15:1)
 
-// CHECK: #[[VAR:.*]] = #llvm.di_local_variable<{{.*}}name = "._QFtestEstr"{{.*}}flags = Artificial>
+// CHECK: #[[VAR:.*]] = #llvm.di_local_variable<{{.*}}name = "._QFtestEstr{{.*}}flags = Artificial>
 // CHECK: func.func @test
 // CHECK: %[[V1:.*]]:2 = fir.unboxchar{{.*}}
 // CHECK: %[[V2:.*]] = fir.convert %[[V1]]#1 : (index) -> i64
diff --git a/flang/test/Transforms/debug-90683.fir b/flang/test/Transforms/debug-90683.fir
deleted file mode 100644
index a21332e3968a7..0000000000000
--- a/flang/test/Transforms/debug-90683.fir
+++ /dev/null
@@ -1,25 +0,0 @@
-// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s -o - | FileCheck %s
-
-// This test checks that debug information for fir.real type works ok.
-
-module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
-  func.func @_QPfn1(%arg0: !fir.ref<!fir.complex<8>> {fir.bindc_name = "a"} ) {
-    %0 = fir.declare %arg0 {uniq_name = "_QFfn1Ea"} : (!fir.ref<!fir.complex<8>>) -> !fir.ref<!fir.complex<8>>
-    %1 = fir.alloca f32 {bindc_name = "abserror", uniq_name = "_QFfn1Eabserror"}
-    %2 = fir.declare %1 {uniq_name = "_QFfn1Eabserror"} : (!fir.ref<f32>) -> !fir.ref<f32>
-    %3 = fir.load %0 : !fir.ref<!fir.complex<8>>
-    %4 = fir.extract_value %3, [0 : i32] : (!fir.complex<8>) -> !fir.real<8>
-    %5 = fir.extract_value %3, [1 : i32] : (!fir.complex<8>) -> !fir.real<8>
-    %6 = fir.call @cabs(%4, %5) : (!fir.real<8>, !fir.real<8>) -> f64
-    %7 = fir.convert %6 : (f64) -> f32
-    fir.store %7 to %2 : !fir.ref<f32>
-    return
-  } loc(#loc1)
-  func.func private @cabs(!fir.real<8>, !fir.real<8>) -> f64 attributes {fir.bindc_name = "cabs", fir.runtime}
-} loc(#loc)
-#loc1 = loc("test.f90":5:1)
-#loc = loc("test.f90":0:0)
-
-// CHECK-DAG: #[[TY:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "real", sizeInBits = 64, encoding = DW_ATE_float>
-// CHECK-DAG: #[[TY1:.*]] = #llvm.di_subroutine_type<callingConvention = DW_CC_normal, types = #[[TY]], #[[TY]], #[[TY]]>
-// CHECK-DAG: #{{.*}} = #llvm.di_subprogram<{{.*}}name = "cabs", linkageName = "cabs"{{.*}}, type = #[[TY1]]>
diff --git a/flang/test/Transforms/debug-assumed-shape-array-2.fir b/flang/test/Transforms/debug-assumed-shape-array-2.fir
new file mode 100644
index 0000000000000..212a3453d110d
--- /dev/null
+++ b/flang/test/Transforms/debug-assumed-shape-array-2.fir
@@ -0,0 +1,23 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+
+// Test assumed shape array with variable lower bound.
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func private @_QFPfn(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "n"}) attributes {} {
+    %c23_i32 = arith.constant 23 : i32
+    %c6_i32 = arith.constant 6 : i32
+    %c20_i32 = arith.constant 20 : i32
+    %0 = fir.undefined !fir.dscope
+    %1 = fircg.ext_declare %arg1 dummy_scope %0 {uniq_name = "_QFFfnEn"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32> loc(#loc1)
+    %2 = fir.load %1 : !fir.ref<i32>
+    %3 = fir.convert %2 : (i32) -> index
+    %4 = fircg.ext_declare %arg0 origin %3 dummy_scope %0 {uniq_name = "_QFFfnEb"} : (!fir.box<!fir.array<?xi32>>, index, !fir.dscope) -> !fir.box<!fir.array<?xi32>> loc(#loc2)
+    return
+  } loc(#loc3)
+}
+#loc1 = loc("test1.f90":1:1)
+#loc2 = loc("test1.f90":3:16)
+#loc3 = loc("test1.f90":4:16)
+
+// CHECK: #[[VAR:.*]] = #llvm.di_local_variable<{{.*}}name = "._QFFfnEb1"{{.*}}flags = Artificial>
+// CHECK: #llvm.di_composite_type<tag = DW_TAG_array_type{{.*}}elements = #llvm.di_subrange<count = #llvm.di_expression<[{{.*}}]>, lowerBound = #[[VAR]], stride = #llvm.di_expression<[{{.*}}]>>, dataLocation = <[DW_OP_push_object_address, DW_OP_deref]>>
diff --git a/flang/test/Transforms/debug-assumed-size-array.fir b/flang/test/Transforms/debug-assumed-size-array.fir
index 9adf711a0a6f7..892502cb64a59 100644
--- a/flang/test/Transforms/debug-assumed-size-array.fir
+++ b/flang/test/Transforms/debug-assumed-size-array.fir
@@ -16,7 +16,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
 #loc1 = loc("test.f90":3:1)
 #loc2 = loc("test.f90":4:1)
 
-// CHECK-DAG: #[[TY1:.*]] = #llvm.di_composite_type<tag = DW_TAG_array_type{{.*}}elements = #llvm.di_subrange<count = 5 : i64, lowerBound = 1 : i64>, #llvm.di_subrange<>>
+// CHECK-DAG: #[[TY1:.*]] = #llvm.di_composite_type<tag = DW_TAG_array_type{{.*}}elements = #llvm.di_subrange<count = 5 : i64>, #llvm.di_subrange<>>
 // CHECK-DAG: #[[TY2:.*]] = #llvm.di_composite_type<tag = DW_TAG_array_type{{.*}}elements = #llvm.di_subrange<lowerBound = 2 : i64>>
 // CHECK-DAG: #llvm.di_local_variable<{{.*}}name = "a1"{{.*}}type = #[[TY1]]>
 // CHECK-DAG: #llvm.di_local_variable<{{.*}}name = "a2"{{.*}}type = #[[TY2]]>
diff --git a/flang/test/Transforms/debug-complex-1.fir b/flang/test/Transforms/debug-complex-1.fir
index cc742d3b183bb..633f27af99fb1 100644
--- a/flang/test/Transforms/debug-complex-1.fir
+++ b/flang/test/Transforms/debug-complex-1.fir
@@ -4,20 +4,20 @@
 // variants are checked.
 
 module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
-  func.func @test1(%x : !fir.complex<4>) -> !fir.complex<8> {
-  %1 = fir.convert %x : (!fir.complex<4>) -> !fir.complex<8>
-  return %1 : !fir.complex<8>
+  func.func @test1(%x : complex<f32>) -> complex<f64> {
+  %1 = fir.convert %x : (complex<f32>) -> complex<f64>
+  return %1 : complex<f64>
   }loc(#loc1)
-  func.func @test2(%x : !fir.complex<4>) -> complex<f64> {
-  %1 = fir.convert %x : (!fir.complex<4>) -> complex<f64>
+  func.func @test2(%x : complex<f32>) -> complex<f64> {
+  %1 = fir.convert %x : (complex<f32>) -> complex<f64>
   return %1 : complex<f64>
   }loc(#loc2)
-  func.func @test3(%x : !fir.complex<4>) -> !fir.complex<16> {
-  %1 = fir.convert %x : (!fir.complex<4>) -> !fir.complex<16>
-  return %1 : !fir.complex<16>
+  func.func @test3(%x : complex<f32>) -> complex<f128> {
+  %1 = fir.convert %x : (complex<f32>) -> complex<f128>
+  return %1 : complex<f128>
   }loc(#loc3)
-  func.func @test4(%x : !fir.complex<4>) -> complex<f128> {
-  %1 = fir.convert %x : (!fir.complex<4>) -> complex<f128>
+  func.func @test4(%x : complex<f32>) -> complex<f128> {
+  %1 = fir.convert %x : (complex<f32>) -> complex<f128>
   return %1 : complex<f128>
   }loc(#loc4)
 }
diff --git a/flang/test/Transforms/debug-derived-type-1.fir b/flang/test/Transforms/debug-derived-type-1.fir
index 26f7017f5f5a3..cfbd361a91e72 100644
--- a/flang/test/Transforms/debug-derived-type-1.fir
+++ b/flang/test/Transforms/debug-derived-type-1.fir
@@ -8,15 +8,15 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<!llvm.ptr<272>, d
     %0 = fir.zero_bits !fir.type<_QMm_employeeTt_employee{t_person:!fir.type<_QMm_employeeTt_person{t_address:!fir.type<_QMm_employeeTt_address{house_number:i32}>,name:!fir.char<1,20>}>,hired_date:!fir.type<_QMm_employeeTt_date{year:i32,month:i32,day:i32}>,monthly_salary:f32}>
     fir.has_value %0 : !fir.type<_QMm_employeeTt_employee{t_person:!fir.type<_QMm_employeeTt_person{t_address:!fir.type<_QMm_employeeTt_address{house_number:i32}>,name:!fir.char<1,20>}>,hired_date:!fir.type<_QMm_employeeTt_date{year:i32,month:i32,day:i32}>,monthly_salary:f32}>
   } loc(#loc5)
-  fir.global @_QMt1Evar : !fir.type<_QMt1Tt_t1{age:i32,points:!fir.array<3x!fir.complex<4>>,cond:!fir.logical<1>,name:!fir.char<1,20>,ratio:f64}> {
-    %0 = fir.zero_bits !fir.type<_QMt1Tt_t1{age:i32,points:!fir.array<3x!fir.complex<4>>,cond:!fir.logical<1>,name:!fir.char<1,20>,ratio:f64}>
-    fir.has_value %0 : !fir.type<_QMt1Tt_t1{age:i32,points:!fir.array<3x!fir.complex<4>>,cond:!fir.logical<1>,name:!fir.char<1,20>,ratio:f64}>
+  fir.global @_QMt1Evar : !fir.type<_QMt1Tt_t1{age:i32,points:!fir.array<3xcomplex<f32>>,cond:!fir.logical<1>,name:!fir.char<1,20>,ratio:f64}> {
+    %0 = fir.zero_bits !fir.type<_QMt1Tt_t1{age:i32,points:!fir.array<3xcomplex<f32>>,cond:!fir.logical<1>,name:!fir.char<1,20>,ratio:f64}>
+    fir.has_value %0 : !fir.type<_QMt1Tt_t1{age:i32,points:!fir.array<3xcomplex<f32>>,cond:!fir.logical<1>,name:!fir.char<1,20>,ratio:f64}>
   } loc(#loc6)
   fir.global @_QMtest_1Exyz : !fir.type<_QMtest_1Tsometype{m_array:!fir.array<3xi32>,m_vt1:!fir.box<!fir.heap<!fir.type<_QMtest_1Tt1{name:!fir.char<1,20>,height:f32}>>>,v2:i32,m_alloc:!fir.box<!fir.heap<!fir.array<?xi32>>>,v3:i32,m_first:!fir.box<!fir.heap<!fir.char<1,?>>>,v4:i32,m_p1:!fir.box<!fir.ptr<i32>>,v5:i32,m_p2:!fir.box<!fir.ptr<i32>>,v6:i32,m_p3:!fir.box<!fir.ptr<!fir.array<?xi32>>>,v7:i32}> {
     %0 = fir.zero_bits !fir.type<_QMtest_1Tsometype{m_array:!fir.array<3xi32>,m_vt1:!fir.box<!fir.heap<!fir.type<_QMtest_1Tt1{name:!fir.char<1,20>,height:f32}>>>,v2:i32,m_alloc:!fir.box<!fir.heap<!fir.array<?xi32>>>,v3:i32,m_first:!fir.box<!fir.heap<!fir.char<1,?>>>,v4:i32,m_p1:!fir.box<!fir.ptr<i32>>,v5:i32,m_p2:!fir.box<!fir.ptr<i32>>,v6:i32,m_p3:!fir.box<!fir.ptr<!fir.array<?xi32>>>,v7:i32}>
     fir.has_value %0 : !fir.type<_QMtest_1Tsometype{m_array:!fir.array<3xi32>,m_vt1:!fir.box<!fir.heap<!fir.type<_QMtest_1Tt1{name:!fir.char<1,20>,height:f32}>>>,v2:i32,m_alloc:!fir.box<!fir.heap<!fir.array<?xi32>>>,v3:i32,m_first:!fir.box<!fir.heap<!fir.char<1,?>>>,v4:i32,m_p1:!fir.box<!fir.ptr<i32>>,v5:i32,m_p2:!fir.box<!fir.ptr<i32>>,v6:i32,m_p3:!fir.box<!fir.ptr<!fir.array<?xi32>>>,v7:i32}>
   } loc(#loc12)
-  fir.type_info @_QMt1Tt_t1 noinit nodestroy nofinal : !fir.type<_QMt1Tt_t1{age:i32,points:!fir.array<3x!fir.complex<4>>,cond:!fir.logical<1>,name:!fir.char<1,20>,ratio:f64}> loc(#loc7)
+  fir.type_info @_QMt1Tt_t1 noinit nodestroy nofinal : !fir.type<_QMt1Tt_t1{age:i32,points:!fir.array<3xcomplex<f32>>,cond:!fir.logical<1>,name:!fir.char<1,20>,ratio:f64}> loc(#loc7)
   fir.type_info @_QMm_employeeTt_address noinit nodestroy nofinal : !fir.type<_QMm_employeeTt_address{house_number:i32}> loc(#loc1)
   fir.type_info @_QMm_employeeTt_person noinit nodestroy nofinal extends !fir.type<_QMm_employeeTt_address{house_number:i32}> : !fir.type<_QMm_employeeTt_person{t_address:!fir.type<_QMm_employeeTt_address{house_number:i32}>,name:!fir.char<1,20>}> loc(#loc2)
   fir.type_info @_QMm_employeeTt_date noinit nodestroy nofinal : !fir.type<_QMm_employeeTt_date{year:i32,month:i32,day:i32}> loc(#loc3)
diff --git a/flang/test/Transforms/debug-fixed-array-type.fir b/flang/test/Transforms/debug-fixed-array-type.fir
index 1a7d8115908a0..a15975c7cc92a 100644
--- a/flang/test/Transforms/debug-fixed-array-type.fir
+++ b/flang/test/Transforms/debug-fixed-array-type.fir
@@ -31,9 +31,9 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
 
 // CHECK-DAG: #[[INT:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 32, encoding = DW_ATE_signed>
 // CHECK-DAG: #[[REAL:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "real", sizeInBits = 32, encoding = DW_ATE_float>
-// CHECK-DAG: #[[D1TY:.*]] = #llvm.di_composite_type<tag = DW_TAG_array_type{{.*}}baseType = #[[INT]], elements = #llvm.di_subrange<count = 3 : i64, lowerBound = 1 : i64>>
-// CHECK-DAG: #[[D2TY:.*]] = #llvm.di_composite_type<tag = DW_TAG_array_type{{.*}}baseType = #[[INT]], elements = #llvm.di_subrange<count = 2 : i64, lowerBound = 1 : i64>, #llvm.di_subrange<count = 5 : i64, lowerBound = 1 : i64>>
-// CHECK-DAG: #[[D3TY:.*]] = #llvm.di_composite_type<tag = DW_TAG_array_type{{.*}}baseType = #[[REAL]], elements = #llvm.di_subrange<count = 6 : i64, lowerBound = 1 : i64>, #llvm.di_subrange<count = 8 : i64, lowerBound = 1 : i64>, #llvm.di_subrange<count = 7 : i64, lowerBound = 1 : i64>>
+// CHECK-DAG: #[[D1TY:.*]] = #llvm.di_composite_type<tag = DW_TAG_array_type{{.*}}baseType = #[[INT]], elements = #llvm.di_subrange<count = 3 : i64>>
+// CHECK-DAG: #[[D2TY:.*]] = #llvm.di_composite_type<tag = DW_TAG_array_type{{.*}}baseType = #[[INT]], elements = #llvm.di_subrange<count = 2 : i64>, #llvm.di_subrange<count = 5 : i64>>
+// CHECK-DAG: #[[D3TY:.*]] = #llvm.di_composite_type<tag = DW_TAG_array_type{{.*}}baseType = #[[REAL]], elements = #llvm.di_subrange<count = 6 : i64>, #llvm.di_subrange<count = 8 : i64>, #llvm.di_subrange<count = 7 : i64>>
 // CHECK-DAG: #[[D4TY:.*]] = #llvm.di_composite_type<tag = DW_TAG_array_type, baseType = #di_basic_type, elements = #llvm.di_subrange<count = 6 : i64, lowerBound = -2 : i64>, #llvm.di_subrange<count = 7 : i64, lowerBound = 4 : i64>>
 // CHECK-DAG: #llvm.di_local_variable<{{.*}}name = "d1"{{.*}}type = #[[D1TY]]>
 // CHECK-DAG: #llvm.di_local_variable<{{.*}}name = "d2"{{.*}}type = #[[D2TY]]>
diff --git a/flang/test/Transforms/debug-fn-info.fir b/flang/test/Transforms/debug-fn-info.fir
index f23a1a27e13eb..eaf88152a21bc 100644
--- a/flang/test/Transforms/debug-fn-info.fir
+++ b/flang/test/Transforms/debug-fn-info.fir
@@ -76,7 +76,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
 // CHECK: #[[TY3:.*]] = #llvm.di_subroutine_type<callingConvention = DW_CC_normal, types = #di_null_type, #[[INT4]]>
 
 // Line numbers should match the number in corresponding loc entry.
-// CHECK: #llvm.di_subprogram<{{.*}}name = "_QQmain", linkageName = "_QQmain", file = {{.*}}, line = 15, scopeLine = 15, subprogramFlags = Definition, type = #[[TY0]]>
+// CHECK: #llvm.di_subprogram<{{.*}}name = "mn", linkageName = "_QQmain", file = {{.*}}, line = 15, scopeLine = 15, subprogramFlags = Definition, type = #[[TY0]]>
 // CHECK: #llvm.di_subprogram<{{.*}}name = "fn1", linkageName = "_QFPfn1", file = {{.*}}, line = 26, scopeLine = 26, subprogramFlags = Definition, type = #[[TY1]]>
 // CHECK: #llvm.di_subprogram<{{.*}}name = "fn2", linkageName = "_QFPfn2", file = {{.*}}, line = 43, scopeLine = 43, subprogramFlags = Definition, type = #[[TY2]]>
 // CHECK: #llvm.di_subprogram<{{.*}}name = "fn3", linkageName = "_QFPfn3", file = {{.*}}, line = 53, scopeLine = 53, subprogramFlags = Definition, type = #[[TY3]]>
diff --git a/flang/test/Transforms/debug-local-global-storage-1.fir b/flang/test/Transforms/debug-local-global-storage-1.fir
index 83a9055a6b8dc..2638464dbab0b 100644
--- a/flang/test/Transforms/debug-local-global-storage-1.fir
+++ b/flang/test/Transforms/debug-local-global-storage-1.fir
@@ -44,7 +44,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<i64, dense<64> :
 
 // CHECK-DAG: #[[CU:.*]] = #llvm.di_compile_unit<{{.*}}>
 // CHECK-DAG: #[[MOD:.*]] = #llvm.di_module<{{.*}}scope = #[[CU]]{{.*}}name = "example"{{.*}}>
-// CHECK-DAG: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "_QQmain"{{.*}}>
+// CHECK-DAG: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "test"{{.*}}>
 // CHECK-DAG: #[[MOD_SP:.*]] = #llvm.di_subprogram<{{.*}}name = "mod_sub"{{.*}}retainedNodes = {{.*}}>
 // CHECK-DAG: #llvm.di_global_variable<scope = #[[SP]], name = "arr"{{.*}}line = 22{{.*}}>
 // CHECK-DAG: #llvm.di_global_variable<scope = #[[SP]], name = "s"{{.*}}line = 23{{.*}}>
diff --git a/flang/test/Transforms/debug-local-var.fir b/flang/test/Transforms/debug-local-var.fir
index 762f5ed269efc..b7a1ff7185a63 100644
--- a/flang/test/Transforms/debug-local-var.fir
+++ b/flang/test/Transforms/debug-local-var.fir
@@ -77,7 +77,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
 // CHECK-DAG: #[[LOG1:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "logical", sizeInBits = 8, encoding = DW_ATE_boolean>
 // CHECK-DAG: #[[REAL4:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "real", sizeInBits = 32, encoding = DW_ATE_float>
 // CHECK-DAG: #[[LOG4:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "logical", sizeInBits = 32, encoding = DW_ATE_boolean>
-// CHECK-DAG: #[[MAIN:.*]] = #llvm.di_subprogram<{{.*}}name = "_QQmain"{{.*}}>
+// CHECK-DAG: #[[MAIN:.*]] = #llvm.di_subprogram<{{.*}}name = "mn"{{.*}}>
 // CHECK-DAG: #[[FN1:.*]] = #llvm.di_subprogram<{{.*}}name = "fn1"{{.*}}>
 // CHECK-DAG: #[[FN2:.*]] = #llvm.di_subprogram<{{.*}}name = "fn2"{{.*}}>
 
diff --git a/flang/test/Transforms/debug-variable-array-dim.fir b/flang/test/Transforms/debug-variable-array-dim.fir
new file mode 100644
index 0000000000000..1f401041dee57
--- /dev/null
+++ b/flang/test/Transforms/debug-variable-array-dim.fir
@@ -0,0 +1,41 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func @foo_(%arg0: !fir.ref<!fir.array<?x?xi32>> {fir.bindc_name = "a"}, %arg1: !fir.ref<i32> {fir.bindc_name = "n"}, %arg2: !fir.ref<i32> {fir.bindc_name = "m"}, %arg3: !fir.ref<i32> {fir.bindc_name = "p"}) attributes {fir.internal_name = "_QPfoo"} {
+    %c5_i32 = arith.constant 5 : i32
+    %c6_i32 = arith.constant 6 : i32
+    %c2 = arith.constant 2 : index
+    %c10_i32 = arith.constant 10 : i32
+    %c0 = arith.constant 0 : index
+    %c1 = arith.constant 1 : index
+    %0 = fir.undefined !fir.dscope
+    %1 = fircg.ext_declare %arg1 dummy_scope %0 {uniq_name = "_QFfooEn"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32> loc(#loc2)
+    %2 = fircg.ext_declare %arg2 dummy_scope %0 {uniq_name = "_QFfooEm"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32> loc(#loc3)
+    %3 = fircg.ext_declare %arg3 dummy_scope %0 {uniq_name = "_QFfooEp"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32> loc(#loc4)
+    %4 = fir.load %1 : !fir.ref<i32>
+    %5 = fir.convert %4 : (i32) -> index
+    %6 = fir.load %2 : !fir.ref<i32>
+    %7 = fir.convert %6 : (i32) -> index
+    %8 = arith.subi %7, %5 : index
+    %9 = arith.addi %8, %c1 : index
+    %10 = arith.cmpi sgt, %9, %c0 : index
+    %11 = arith.select %10, %9, %c0 : index
+    %12 = fir.load %3 : !fir.ref<i32>
+    %13 = fir.convert %12 : (i32) -> index
+    %14 = arith.cmpi sgt, %13, %c0 : index
+    %15 = arith.select %14, %13, %c0 : index
+    %16 = fircg.ext_declare %arg0(%11, %15) origin %5, %c1 dummy_scope %0 {uniq_name = "_QFfooEa"} : (!fir.ref<!fir.array<?x?xi32>>, index, index, index, index, !fir.dscope) -> !fir.ref<!fir.array<?x?xi32>> loc(#loc5)
+    return
+  } loc(#loc1)
+}
+
+#loc1 = loc("test.f90":5:1)
+#loc2 = loc("test.f90":6:11)
+#loc3 = loc("test.f90":7:11)
+#loc4 = loc("test.f90":2:8)
+#loc5 = loc("test.f90":8:11)
+
+// CHECK-DAG: #[[VAR0:.*]] = #llvm.di_local_variable<{{.*}}name = "._QFfooEa3"{{.*}}flags = Artificial>
+// CHECK-DAG: #[[VAR1:.*]] = #llvm.di_local_variable<{{.*}}name = "._QFfooEa1"{{.*}}flags = Artificial>
+// CHECK-DAG: #[[VAR2:.*]] = #llvm.di_local_variable<{{.*}}name = "._QFfooEa2"{{.*}}flags = Artificial>
+// CHECK-DAG: #llvm.di_composite_type<tag = DW_TAG_array_type{{.*}}elements = #llvm.di_subrange<count = #[[VAR1]], lowerBound = #[[VAR0]]>, #llvm.di_subrange<count = #[[VAR2]]>>
diff --git a/flang/test/Transforms/debug-variable-char-len.fir b/flang/test/Transforms/debug-variable-char-len.fir
index 598d97cee970a..9e177d22d5b10 100644
--- a/flang/test/Transforms/debug-variable-char-len.fir
+++ b/flang/test/Transforms/debug-variable-char-len.fir
@@ -22,7 +22,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
 #loc2 = loc("test.f90":17:1)
 #loc3 = loc("test.f90":15:1)
 
-// CHECK: #[[VAR:.*]] = #llvm.di_local_variable<{{.*}}name = "._QFfooEstr1"{{.*}}flags = Artificial>
+// CHECK: #[[VAR:.*]] = #llvm.di_local_variable<{{.*}}name = "._QFfooEstr1{{.*}}flags = Artificial>
 // CHECK: func.func @foo
 // CHECK: llvm.intr.dbg.value #[[VAR]]
 // CHECK: return
diff --git a/flang/test/Transforms/loop-versioning.fir b/flang/test/Transforms/loop-versioning.fir
index 0c62741656347..7528d14b3670d 100644
--- a/flang/test/Transforms/loop-versioning.fir
+++ b/flang/test/Transforms/loop-versioning.fir
@@ -1597,40 +1597,40 @@ func.func @_QPtest_real10(%arg0: !fir.box<!fir.array<?x?xf80>> {fir.bindc_name =
 // CHECK:           } else {
 // CHECK:             fir.do_loop
 
-func.func @_QPtest_complex10(%arg0: !fir.box<!fir.array<?x?x!fir.complex<10>>> {fir.bindc_name = "a"}) -> !fir.complex<10> {
+func.func @_QPtest_complex10(%arg0: !fir.box<!fir.array<?x?xcomplex<f80>>> {fir.bindc_name = "a"}) -> complex<f80> {
   %c10 = arith.constant 10 : index
   %c1 = arith.constant 1 : index
   %cst = arith.constant 0.000000e+00 : f80
-  %0 = fir.declare %arg0 {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtest_complex10Ea"} : (!fir.box<!fir.array<?x?x!fir.complex<10>>>) -> !fir.box<!fir.array<?x?x!fir.complex<10>>>
-  %1 = fir.rebox %0 : (!fir.box<!fir.array<?x?x!fir.complex<10>>>) -> !fir.box<!fir.array<?x?x!fir.complex<10>>>
+  %0 = fir.declare %arg0 {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtest_complex10Ea"} : (!fir.box<!fir.array<?x?xcomplex<f80>>>) -> !fir.box<!fir.array<?x?xcomplex<f80>>>
+  %1 = fir.rebox %0 : (!fir.box<!fir.array<?x?xcomplex<f80>>>) -> !fir.box<!fir.array<?x?xcomplex<f80>>>
   %2 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFtest_complex10Ei"}
   %3 = fir.declare %2 {uniq_name = "_QFtest_complex10Ei"} : (!fir.ref<i32>) -> !fir.ref<i32>
-  %4 = fir.alloca !fir.complex<10> {bindc_name = "res", uniq_name = "_QFtest_complex10Eres"}
-  %5 = fir.declare %4 {uniq_name = "_QFtest_complex10Eres"} : (!fir.ref<!fir.complex<10>>) -> !fir.ref<!fir.complex<10>>
+  %4 = fir.alloca complex<f80> {bindc_name = "res", uniq_name = "_QFtest_complex10Eres"}
+  %5 = fir.declare %4 {uniq_name = "_QFtest_complex10Eres"} : (!fir.ref<complex<f80>>) -> !fir.ref<complex<f80>>
   %6 = fir.address_of(@_QFtest_complex10ECxdp) : !fir.ref<i32>
   %7 = fir.declare %6 {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QFtest_complex10ECxdp"} : (!fir.ref<i32>) -> !fir.ref<i32>
-  %8 = fir.undefined !fir.complex<10>
-  %9 = fir.insert_value %8, %cst, [0 : index] : (!fir.complex<10>, f80) -> !fir.complex<10>
-  %10 = fir.insert_value %9, %cst, [1 : index] : (!fir.complex<10>, f80) -> !fir.complex<10>
-  fir.store %10 to %5 : !fir.ref<!fir.complex<10>>
+  %8 = fir.undefined complex<f80>
+  %9 = fir.insert_value %8, %cst, [0 : index] : (complex<f80>, f80) -> complex<f80>
+  %10 = fir.insert_value %9, %cst, [1 : index] : (complex<f80>, f80) -> complex<f80>
+  fir.store %10 to %5 : !fir.ref<complex<f80>>
   %11 = fir.convert %c1 : (index) -> i32
   %12:2 = fir.do_loop %arg1 = %c1 to %c10 step %c1 iter_args(%arg2 = %11) -> (index, i32) {
     fir.store %arg2 to %3 : !fir.ref<i32>
-    %14 = fir.load %5 : !fir.ref<!fir.complex<10>>
+    %14 = fir.load %5 : !fir.ref<complex<f80>>
     %15 = fir.load %3 : !fir.ref<i32>
     %16 = fir.convert %15 : (i32) -> i64
-    %17 = fir.array_coor %1 %16, %16 : (!fir.box<!fir.array<?x?x!fir.complex<10>>>, i64, i64) -> !fir.ref<!fir.complex<10>>
-    %18 = fir.load %17 : !fir.ref<!fir.complex<10>>
-    %19 = fir.addc %14, %18 {fastmath = #arith.fastmath<contract>} : !fir.complex<10>
-    fir.store %19 to %5 : !fir.ref<!fir.complex<10>>
+    %17 = fir.array_coor %1 %16, %16 : (!fir.box<!fir.array<?x?xcomplex<f80>>>, i64, i64) -> !fir.ref<complex<f80>>
+    %18 = fir.load %17 : !fir.ref<complex<f80>>
+    %19 = fir.addc %14, %18 {fastmath = #arith.fastmath<contract>} : complex<f80>
+    fir.store %19 to %5 : !fir.ref<complex<f80>>
     %20 = arith.addi %arg1, %c1 : index
     %21 = fir.load %3 : !fir.ref<i32>
     %22 = arith.addi %21, %11 : i32
     fir.result %20, %22 : index, i32
   }
   fir.store %12#1 to %3 : !fir.ref<i32>
-  %13 = fir.load %5 : !fir.ref<!fir.complex<10>>
-  return %13 : !fir.complex<10>
+  %13 = fir.load %5 : !fir.ref<complex<f80>>
+  return %13 : complex<f80>
 }
 // CHECK-LABEL:   func.func @_QPtest_complex10(
 // CHECK:           fir.if
diff --git a/flang/test/Transforms/omp-map-info-finalization.fir b/flang/test/Transforms/omp-map-info-finalization.fir
index 61af1ee308d78..fa7b65d41929b 100644
--- a/flang/test/Transforms/omp-map-info-finalization.fir
+++ b/flang/test/Transforms/omp-map-info-finalization.fir
@@ -44,24 +44,24 @@ module attributes {omp.is_target_device = false} {
 // -----
 
 module attributes {omp.is_target_device = false} {
-  func.func @test_derived_type_map_operand_and_block_addition(%arg0: !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>) {
-    %0 = hlfir.designate %arg0{"rx"}   : (!fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>) -> !fir.ref<f32>
+  func.func @test_derived_type_map_operand_and_block_addition(%arg0: !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:complex<f32>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>) {
+    %0 = hlfir.designate %arg0{"rx"}   : (!fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:complex<f32>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>) -> !fir.ref<f32>
     %1 = omp.map.info var_ptr(%0 : !fir.ref<f32>, f32) map_clauses(from) capture(ByRef) -> !fir.ref<f32> {name = "scalar_struct%rx"}
-    %2 = hlfir.designate %arg0{"ry"}   : (!fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>) -> !fir.ref<f32>
+    %2 = hlfir.designate %arg0{"ry"}   : (!fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:complex<f32>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>) -> !fir.ref<f32>
     %3 = omp.map.info var_ptr(%2 : !fir.ref<f32>, f32) map_clauses(from) capture(ByRef) -> !fir.ref<f32> {name = "scalar_struct%ry"}
-    %4 = omp.map.info var_ptr(%arg0 : !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>, !fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>) map_clauses(from) capture(ByRef) members(%1, %3 : [1], [4] : !fir.ref<f32>, !fir.ref<f32>) -> !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>> {name = "scalar_struct", partial_map = true}
-    omp.target map_entries(%4 -> %arg1 : !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>) {
+    %4 = omp.map.info var_ptr(%arg0 : !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:complex<f32>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>, !fir.type<_QFTdtype{ix:i32,rx:f32,zx:complex<f32>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>) map_clauses(from) capture(ByRef) members(%1, %3 : [1], [4] : !fir.ref<f32>, !fir.ref<f32>) -> !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:complex<f32>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>> {name = "scalar_struct", partial_map = true}
+    omp.target map_entries(%4 -> %arg1 : !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:complex<f32>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>) {
       omp.terminator
     }
     return
   }
 }
 
-// CHECK: func.func @test_derived_type_map_operand_and_block_addition(%{{.*}}: !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>) { 
+// CHECK: func.func @test_derived_type_map_operand_and_block_addition(%{{.*}}: !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:complex<f32>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>) { 
 // CHECK:   %[[MAP_MEMBER_1:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<f32>, f32) map_clauses(from) capture(ByRef) -> !fir.ref<f32> {name = "scalar_struct%rx"}
 // CHECK:   %[[MAP_MEMBER_2:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<f32>, f32) map_clauses(from) capture(ByRef) -> !fir.ref<f32> {name = "scalar_struct%ry"}
-// CHECK:   %[[MAP_PARENT:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>, !fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>) map_clauses(from) capture(ByRef) members(%[[MAP_MEMBER_1]], %[[MAP_MEMBER_2]] : [1], [4] : !fir.ref<f32>, !fir.ref<f32>) -> !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>> {name = "scalar_struct", partial_map = true}
-// CHECK:   omp.target map_entries(%[[MAP_MEMBER_1]] -> %[[ARG1:.*]], %[[MAP_MEMBER_2]] -> %[[ARG2:.*]], %[[MAP_PARENT]] -> %[[ARG3:.*]] : !fir.ref<f32>, !fir.ref<f32>, !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>) {
+// CHECK:   %[[MAP_PARENT:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:complex<f32>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>, !fir.type<_QFTdtype{ix:i32,rx:f32,zx:complex<f32>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>) map_clauses(from) capture(ByRef) members(%[[MAP_MEMBER_1]], %[[MAP_MEMBER_2]] : [1], [4] : !fir.ref<f32>, !fir.ref<f32>) -> !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:complex<f32>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>> {name = "scalar_struct", partial_map = true}
+// CHECK:   omp.target map_entries(%[[MAP_MEMBER_1]] -> %[[ARG1:.*]], %[[MAP_MEMBER_2]] -> %[[ARG2:.*]], %[[MAP_PARENT]] -> %[[ARG3:.*]] : !fir.ref<f32>, !fir.ref<f32>, !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:complex<f32>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>) {
 
 // -----
 
diff --git a/flang/test/Transforms/simplifyintrinsics.fir b/flang/test/Transforms/simplifyintrinsics.fir
index f21776e03ded8..a3930566035b3 100644
--- a/flang/test/Transforms/simplifyintrinsics.fir
+++ b/flang/test/Transforms/simplifyintrinsics.fir
@@ -247,26 +247,26 @@ module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.targ
 
 // Call to SUM with 1D COMPLEX array is not replaced.
 module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.target_triple = "native"} {
-  func.func @sum_1d_complex(%arg0: !fir.ref<!fir.array<10x!fir.complex<4>>> {fir.bindc_name = "a"}) -> !fir.complex<4> {
-    %0 = fir.alloca !fir.complex<4>
+  func.func @sum_1d_complex(%arg0: !fir.ref<!fir.array<10xcomplex<f32>>> {fir.bindc_name = "a"}) -> complex<f32> {
+    %0 = fir.alloca complex<f32>
     %c10 = arith.constant 10 : index
-    %1 = fir.alloca !fir.complex<4> {bindc_name = "sum_1d_complex", uniq_name = "_QFsum_1d_complexEsum_1d_complex"}
+    %1 = fir.alloca complex<f32> {bindc_name = "sum_1d_complex", uniq_name = "_QFsum_1d_complexEsum_1d_complex"}
     %2 = fir.shape %c10 : (index) -> !fir.shape<1>
-    %3 = fir.embox %arg0(%2) : (!fir.ref<!fir.array<10x!fir.complex<4>>>, !fir.shape<1>) -> !fir.box<!fir.array<10x!fir.complex<4>>>
+    %3 = fir.embox %arg0(%2) : (!fir.ref<!fir.array<10xcomplex<f32>>>, !fir.shape<1>) -> !fir.box<!fir.array<10xcomplex<f32>>>
     %4 = fir.absent !fir.box<i1>
     %c0 = arith.constant 0 : index
     %5 = fir.address_of(@_QQclX2E2F6973756D5F362E66393000) : !fir.ref<!fir.char<1,13>>
     %c5_i32 = arith.constant 5 : i32
-    %6 = fir.convert %0 : (!fir.ref<!fir.complex<4>>) -> !fir.ref<complex<f32>>
-    %7 = fir.convert %3 : (!fir.box<!fir.array<10x!fir.complex<4>>>) -> !fir.box<none>
+    %6 = fir.convert %0 : (!fir.ref<complex<f32>>) -> !fir.ref<complex<f32>>
+    %7 = fir.convert %3 : (!fir.box<!fir.array<10xcomplex<f32>>>) -> !fir.box<none>
     %8 = fir.convert %5 : (!fir.ref<!fir.char<1,13>>) -> !fir.ref<i8>
     %9 = fir.convert %c0 : (index) -> i32
     %10 = fir.convert %4 : (!fir.box<i1>) -> !fir.box<none>
     %11 = fir.call @_FortranACppSumComplex4(%6, %7, %8, %c5_i32, %9, %10) : (!fir.ref<complex<f32>>, !fir.box<none>, !fir.ref<i8>, i32, i32, !fir.box<none>) -> none
-    %12 = fir.load %0 : !fir.ref<!fir.complex<4>>
-    fir.store %12 to %1 : !fir.ref<!fir.complex<4>>
-    %13 = fir.load %1 : !fir.ref<!fir.complex<4>>
-    return %13 : !fir.complex<4>
+    %12 = fir.load %0 : !fir.ref<complex<f32>>
+    fir.store %12 to %1 : !fir.ref<complex<f32>>
+    %13 = fir.load %1 : !fir.ref<complex<f32>>
+    return %13 : complex<f32>
   }
   func.func private @_FortranACppSumComplex4(!fir.ref<complex<f32>>, !fir.box<none>, !fir.ref<i8>, i32, i32, !fir.box<none>) -> none attributes {fir.runtime}
   fir.global linkonce @_QQclX2E2F6973756D5F362E66393000 constant : !fir.char<1,13> {
@@ -275,7 +275,7 @@ module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.targ
   }
 }
 
-// CHECK-LABEL:   func.func @sum_1d_complex(%{{.*}}: !fir.ref<!fir.array<10x!fir.complex<4>>> {fir.bindc_name = "a"}) -> !fir.complex<4> {
+// CHECK-LABEL:   func.func @sum_1d_complex(%{{.*}}: !fir.ref<!fir.array<10xcomplex<f32>>> {fir.bindc_name = "a"}) -> complex<f32> {
 // CHECK-NOT:       fir.call @_FortranACppSumComplex4x1_simplified({{.*}})
 // CHECK:           fir.call @_FortranACppSumComplex4({{.*}}) : (!fir.ref<complex<f32>>, !fir.box<none>, !fir.ref<i8>, i32, i32, !fir.box<none>) -> none
 // CHECK-NOT:       fir.call @_FortranACppSumComplex4x1_simplified({{.*}})
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index 0a008d577cc25..3a05f5f984487 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -228,6 +228,12 @@ static llvm::cl::opt<std::string>
                          llvm::cl::desc("Override host target triple"),
                          llvm::cl::init(""));
 
+static llvm::cl::opt<bool> integerWrapAround(
+    "fwrapv",
+    llvm::cl::desc("Treat signed integer overflow as two's complement"),
+    llvm::cl::init(false));
+
+// TODO: integrate this option with the above
 static llvm::cl::opt<bool>
     setNSW("integer-overflow",
            llvm::cl::desc("add nsw flag to internal operations"),
@@ -373,6 +379,7 @@ static llvm::LogicalResult convertFortranSourceToMLIR(
   Fortran::lower::LoweringOptions loweringOptions{};
   loweringOptions.setNoPPCNativeVecElemOrder(enableNoPPCNativeVecElemOrder);
   loweringOptions.setLowerToHighLevelFIR(useHLFIR || emitHLFIR);
+  loweringOptions.setIntegerWrapAround(integerWrapAround);
   loweringOptions.setNSWOnLoopVarInc(setNSW);
   std::vector<Fortran::lower::EnvironmentDefault> envDefaults = {};
   constexpr const char *tuneCPU = "";
diff --git a/flang/unittests/Optimizer/Builder/ComplexTest.cpp b/flang/unittests/Optimizer/Builder/ComplexTest.cpp
index 17171512470ac..6472a52f25ee5 100644
--- a/flang/unittests/Optimizer/Builder/ComplexTest.cpp
+++ b/flang/unittests/Optimizer/Builder/ComplexTest.cpp
@@ -35,7 +35,7 @@ struct ComplexTest : public testing::Test {
 
     // Init commonly used types
     realTy1 = mlir::FloatType::getF32(&context);
-    complexTy1 = fir::ComplexType::get(&context, 4);
+    complexTy1 = mlir::ComplexType::get(realTy1);
     integerTy1 = mlir::IntegerType::get(&context, 32);
 
     // Create commonly used reals
@@ -52,7 +52,7 @@ struct ComplexTest : public testing::Test {
 
   // Commonly used real/complex/integer types
   mlir::FloatType realTy1;
-  fir::ComplexType complexTy1;
+  mlir::ComplexType complexTy1;
   mlir::IntegerType integerTy1;
 
   // Commonly used real numbers
@@ -64,20 +64,13 @@ struct ComplexTest : public testing::Test {
 
 TEST_F(ComplexTest, verifyTypes) {
   mlir::Value cVal1 = helper->createComplex(complexTy1, rOne, rTwo);
-  mlir::Value cVal2 = helper->createComplex(4, rOne, rTwo);
   EXPECT_TRUE(fir::isa_complex(cVal1.getType()));
-  EXPECT_TRUE(fir::isa_complex(cVal2.getType()));
   EXPECT_TRUE(fir::isa_real(helper->getComplexPartType(cVal1)));
-  EXPECT_TRUE(fir::isa_real(helper->getComplexPartType(cVal2)));
 
   mlir::Value real1 = helper->extractComplexPart(cVal1, /*isImagPart=*/false);
   mlir::Value imag1 = helper->extractComplexPart(cVal1, /*isImagPart=*/true);
-  mlir::Value real2 = helper->extractComplexPart(cVal2, /*isImagPart=*/false);
-  mlir::Value imag2 = helper->extractComplexPart(cVal2, /*isImagPart=*/true);
   EXPECT_EQ(realTy1, real1.getType());
   EXPECT_EQ(realTy1, imag1.getType());
-  EXPECT_EQ(realTy1, real2.getType());
-  EXPECT_EQ(realTy1, imag2.getType());
 
   mlir::Value cVal3 =
       helper->insertComplexPart(cVal1, rThree, /*isImagPart=*/false);
diff --git a/flang/unittests/Optimizer/Builder/FIRBuilderTest.cpp b/flang/unittests/Optimizer/Builder/FIRBuilderTest.cpp
index e5e5454ee88ad..f63afe4137683 100644
--- a/flang/unittests/Optimizer/Builder/FIRBuilderTest.cpp
+++ b/flang/unittests/Optimizer/Builder/FIRBuilderTest.cpp
@@ -585,3 +585,62 @@ TEST_F(FIRBuilderTest, genArithFastMath) {
   auto op4_fmf = op4_fmi.getFastMathFlagsAttr().getValue();
   EXPECT_EQ(op4_fmf, FMF1);
 }
+
+TEST_F(FIRBuilderTest, genArithIntegerOverflow) {
+  auto builder = getBuilder();
+  auto ctx = builder.getContext();
+  auto loc = builder.getUnknownLoc();
+
+  auto intTy = IntegerType::get(ctx, 32);
+  auto arg = builder.create<fir::UndefOp>(loc, intTy);
+
+  // Test that IntegerOverflowFlags is 'none' by default.
+  mlir::Operation *op1 = builder.create<mlir::arith::AddIOp>(loc, arg, arg);
+  auto op1_iofi =
+      mlir::dyn_cast_or_null<mlir::arith::ArithIntegerOverflowFlagsInterface>(
+          op1);
+  EXPECT_TRUE(op1_iofi);
+  auto op1_ioff = op1_iofi.getOverflowAttr().getValue();
+  EXPECT_EQ(op1_ioff, arith::IntegerOverflowFlags::none);
+
+  // Test that the builder is copied properly.
+  fir::FirOpBuilder builder_copy(builder);
+
+  arith::IntegerOverflowFlags nsw = arith::IntegerOverflowFlags::nsw;
+  builder.setIntegerOverflowFlags(nsw);
+  arith::IntegerOverflowFlags nuw = arith::IntegerOverflowFlags::nuw;
+  builder_copy.setIntegerOverflowFlags(nuw);
+
+  // Modifying IntegerOverflowFlags for the copy must not affect the original
+  // builder.
+  mlir::Operation *op2 = builder.create<mlir::arith::AddIOp>(loc, arg, arg);
+  auto op2_iofi =
+      mlir::dyn_cast_or_null<mlir::arith::ArithIntegerOverflowFlagsInterface>(
+          op2);
+  EXPECT_TRUE(op2_iofi);
+  auto op2_ioff = op2_iofi.getOverflowAttr().getValue();
+  EXPECT_EQ(op2_ioff, nsw);
+
+  // Modifying IntegerOverflowFlags for the original builder must not affect the
+  // copy.
+  mlir::Operation *op3 =
+      builder_copy.create<mlir::arith::AddIOp>(loc, arg, arg);
+  auto op3_iofi =
+      mlir::dyn_cast_or_null<mlir::arith::ArithIntegerOverflowFlagsInterface>(
+          op3);
+  EXPECT_TRUE(op3_iofi);
+  auto op3_ioff = op3_iofi.getOverflowAttr().getValue();
+  EXPECT_EQ(op3_ioff, nuw);
+
+  // Test that the builder copy inherits IntegerOverflowFlags from the original.
+  fir::FirOpBuilder builder_copy2(builder);
+
+  mlir::Operation *op4 =
+      builder_copy2.create<mlir::arith::AddIOp>(loc, arg, arg);
+  auto op4_iofi =
+      mlir::dyn_cast_or_null<mlir::arith::ArithIntegerOverflowFlagsInterface>(
+          op4);
+  EXPECT_TRUE(op4_iofi);
+  auto op4_ioff = op4_iofi.getOverflowAttr().getValue();
+  EXPECT_EQ(op4_ioff, nsw);
+}
diff --git a/flang/unittests/Optimizer/Builder/Runtime/CommandTest.cpp b/flang/unittests/Optimizer/Builder/Runtime/CommandTest.cpp
index 58a151447d5b4..8bc1e87814a98 100644
--- a/flang/unittests/Optimizer/Builder/Runtime/CommandTest.cpp
+++ b/flang/unittests/Optimizer/Builder/Runtime/CommandTest.cpp
@@ -50,4 +50,4 @@ TEST_F(RuntimeCallTest, genGetPID) {
   mlir::Value result = fir::runtime::genGetPID(*firBuilder, loc);
   checkCallOp(result.getDefiningOp(), "_FortranAGetPID", /*nbArgs=*/0,
       /*addLocArgs=*/false);
-}
\ No newline at end of file
+}
diff --git a/flang/unittests/Optimizer/Builder/Runtime/IntrinsicsTest.cpp b/flang/unittests/Optimizer/Builder/Runtime/IntrinsicsTest.cpp
new file mode 100644
index 0000000000000..1440a5fd01c2b
--- /dev/null
+++ b/flang/unittests/Optimizer/Builder/Runtime/IntrinsicsTest.cpp
@@ -0,0 +1,17 @@
+#include "flang/Optimizer/Builder/Runtime/Intrinsics.h"
+#include "RuntimeCallTestBase.h"
+#include "gtest/gtest.h"
+
+TEST_F(RuntimeCallTest, genGetGID) {
+  mlir::Location loc = firBuilder->getUnknownLoc();
+  mlir::Value result = fir::runtime::genGetGID(*firBuilder, loc);
+  checkCallOp(result.getDefiningOp(), "_FortranAGetGID", /*nbArgs=*/0,
+      /*addLocArgs=*/false);
+}
+
+TEST_F(RuntimeCallTest, genGetUID) {
+  mlir::Location loc = firBuilder->getUnknownLoc();
+  mlir::Value result = fir::runtime::genGetUID(*firBuilder, loc);
+  checkCallOp(result.getDefiningOp(), "_FortranAGetUID", /*nbArgs=*/0,
+      /*addLocArgs=*/false);
+}
diff --git a/flang/unittests/Optimizer/Builder/Runtime/RuntimeCallTestBase.h b/flang/unittests/Optimizer/Builder/Runtime/RuntimeCallTestBase.h
index 4ace359f055b0..d0ec97733e83d 100644
--- a/flang/unittests/Optimizer/Builder/Runtime/RuntimeCallTestBase.h
+++ b/flang/unittests/Optimizer/Builder/Runtime/RuntimeCallTestBase.h
@@ -47,10 +47,10 @@ struct RuntimeCallTest : public testing::Test {
     f80Ty = firBuilder->getF80Type();
     f128Ty = firBuilder->getF128Type();
 
-    c4Ty = fir::ComplexType::get(firBuilder->getContext(), 4);
-    c8Ty = fir::ComplexType::get(firBuilder->getContext(), 8);
-    c10Ty = fir::ComplexType::get(firBuilder->getContext(), 10);
-    c16Ty = fir::ComplexType::get(firBuilder->getContext(), 16);
+    c4Ty = mlir::ComplexType::get(f32Ty);
+    c8Ty = mlir::ComplexType::get(f64Ty);
+    c10Ty = mlir::ComplexType::get(f80Ty);
+    c16Ty = mlir::ComplexType::get(f128Ty);
 
     seqTy10 = fir::SequenceType::get(fir::SequenceType::Shape(1, 10), i32Ty);
     boxTy = fir::BoxType::get(mlir::NoneType::get(firBuilder->getContext()));
diff --git a/flang/unittests/Optimizer/CMakeLists.txt b/flang/unittests/Optimizer/CMakeLists.txt
index 7299e3ee0529a..c58fb226a175c 100644
--- a/flang/unittests/Optimizer/CMakeLists.txt
+++ b/flang/unittests/Optimizer/CMakeLists.txt
@@ -25,6 +25,7 @@ add_flang_unittest(FlangOptimizerTests
   Builder/Runtime/CommandTest.cpp
   Builder/Runtime/CharacterTest.cpp
   Builder/Runtime/DerivedTest.cpp
+  Builder/Runtime/IntrinsicsTest.cpp
   Builder/Runtime/NumericTest.cpp
   Builder/Runtime/RaggedTest.cpp
   Builder/Runtime/ReductionTest.cpp
diff --git a/flang/unittests/Optimizer/FIRTypesTest.cpp b/flang/unittests/Optimizer/FIRTypesTest.cpp
index 238aa6939d5df..a07c018a8afd5 100644
--- a/flang/unittests/Optimizer/FIRTypesTest.cpp
+++ b/flang/unittests/Optimizer/FIRTypesTest.cpp
@@ -229,10 +229,10 @@ TEST_F(FIRTypesTest, updateTypeForUnlimitedPolymorphic) {
   mlir::Type i32Ty = mlir::IntegerType::get(&context, 32);
   mlir::Type f32Ty = mlir::FloatType::getF32(&context);
   mlir::Type l1Ty = fir::LogicalType::get(&context, 1);
-  mlir::Type cplx4Ty = fir::ComplexType::get(&context, 4);
+  mlir::Type cplx32Ty = mlir::ComplexType::get(f32Ty);
   mlir::Type char1Ty = fir::CharacterType::get(&context, 1, 10);
   llvm::SmallVector<mlir::Type> intrinsicTypes = {
-      i32Ty, f32Ty, l1Ty, cplx4Ty, char1Ty};
+      i32Ty, f32Ty, l1Ty, cplx32Ty, char1Ty};
 
   for (mlir::Type ty : intrinsicTypes) {
     // `ty` -> none
diff --git a/flang/unittests/Optimizer/RTBuilder.cpp b/flang/unittests/Optimizer/RTBuilder.cpp
index d6cf96c4351c2..35b9f1a6d5dcb 100644
--- a/flang/unittests/Optimizer/RTBuilder.cpp
+++ b/flang/unittests/Optimizer/RTBuilder.cpp
@@ -9,6 +9,7 @@
 #include "flang/Optimizer/Builder/Runtime/RTBuilder.h"
 #include "gtest/gtest.h"
 #include "flang/Optimizer/Support/InitFIR.h"
+#include <complex>
 
 // Check that it is possible to make a difference between complex runtime
 // function using C99 complex and C++ std::complex. This is important since
@@ -30,7 +31,7 @@ TEST(RTBuilderTest, ComplexRuntimeInterface) {
   auto c99_cacosf_funcTy = mlir::cast<mlir::FunctionType>(c99_cacosf_signature);
   EXPECT_EQ(c99_cacosf_funcTy.getNumInputs(), 1u);
   EXPECT_EQ(c99_cacosf_funcTy.getNumResults(), 1u);
-  auto cplx_ty = fir::ComplexType::get(&ctx, 4);
+  auto cplx_ty = mlir::ComplexType::get(mlir::FloatType::getF32(&ctx));
   EXPECT_EQ(c99_cacosf_funcTy.getInput(0), cplx_ty);
   EXPECT_EQ(c99_cacosf_funcTy.getResult(0), cplx_ty);
 }
diff --git a/libc/config/config.json b/libc/config/config.json
index caeb3e89a84d8..2e4f878778e6e 100644
--- a/libc/config/config.json
+++ b/libc/config/config.json
@@ -22,6 +22,14 @@
       "value": false,
       "doc": "Use large table for better printf long double performance."
     },
+    "LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_DYADIC_FLOAT": {
+      "value": false,
+      "doc": "Use dyadic float for faster and smaller but less accurate printf doubles."
+    },
+    "LIBC_CONF_PRINTF_FLOAT_TO_STR_NO_SPECIALIZE_LD": {
+      "value": false,
+      "doc": "Use the same mode for double and long double in printf."
+    },
     "LIBC_CONF_PRINTF_DISABLE_FIXED_POINT": {
       "value": false,
       "doc": "Disable printing fixed point values in printf and friends."
diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 64fbe1a250c0b..72a4d9b65ce99 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -347,6 +347,13 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.wchar.wctob
 )
 
+if(LLVM_LIBC_INCLUDE_SCUDO)
+  list(APPEND TARGET_LIBC_ENTRYPOINTS
+    # malloc.h external entrypoints
+    libc.src.stdlib.mallopt
+  )
+endif()
+
 set(TARGET_LIBM_ENTRYPOINTS
     # fenv.h entrypoints
     libc.src.fenv.feclearexcept
@@ -476,9 +483,15 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.ilogb
     libc.src.math.ilogbf
     libc.src.math.ilogbl
+    libc.src.math.iscanonical
+    libc.src.math.iscanonicalf
+    libc.src.math.iscanonicall
     libc.src.math.isnan
     libc.src.math.isnanf
     libc.src.math.isnanl
+    libc.src.math.issignaling
+    libc.src.math.issignalingf
+    libc.src.math.issignalingl
     libc.src.math.ldexp
     libc.src.math.ldexpf
     libc.src.math.ldexpl
@@ -637,6 +650,8 @@ if(LIBC_TYPES_HAS_FLOAT16)
     libc.src.math.fromfpxf16
     libc.src.math.getpayloadf16
     libc.src.math.ilogbf16
+    libc.src.math.iscanonicalf16
+    libc.src.math.issignalingf16
     libc.src.math.ldexpf16
     libc.src.math.llogbf16
     libc.src.math.llrintf16
@@ -720,6 +735,8 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.fsubf128
     libc.src.math.getpayloadf128
     libc.src.math.ilogbf128
+    libc.src.math.iscanonicalf128
+    libc.src.math.issignalingf128
     libc.src.math.ldexpf128
     libc.src.math.llogbf128
     libc.src.math.llrintf128
diff --git a/libc/config/linux/aarch64/headers.txt b/libc/config/linux/aarch64/headers.txt
index ebe053af99d80..f98af5744685f 100644
--- a/libc/config/linux/aarch64/headers.txt
+++ b/libc/config/linux/aarch64/headers.txt
@@ -11,6 +11,7 @@ set(TARGET_PUBLIC_HEADERS
     libc.include.inttypes
     libc.include.limits
     libc.include.link
+    libc.include.malloc
     libc.include.math
     libc.include.pthread
     libc.include.signal
diff --git a/libc/config/linux/arm/entrypoints.txt b/libc/config/linux/arm/entrypoints.txt
index 1be9a872dd2f7..2ddb7aeefe48e 100644
--- a/libc/config/linux/arm/entrypoints.txt
+++ b/libc/config/linux/arm/entrypoints.txt
@@ -189,6 +189,13 @@ if(LLVM_LIBC_FULL_BUILD)
   )
 endif()
 
+if(LLVM_LIBC_INCLUDE_SCUDO)
+  list(APPEND TARGET_LIBC_ENTRYPOINTS
+    # malloc.h external entrypoints
+    libc.src.stdlib.mallopt
+  )
+endif()
+
 set(TARGET_LIBM_ENTRYPOINTS
     # fenv.h entrypoints
     libc.src.fenv.feclearexcept
diff --git a/libc/config/linux/arm/headers.txt b/libc/config/linux/arm/headers.txt
index 84078a947031a..6576db1f85269 100644
--- a/libc/config/linux/arm/headers.txt
+++ b/libc/config/linux/arm/headers.txt
@@ -4,6 +4,7 @@ set(TARGET_PUBLIC_HEADERS
     libc.include.fenv
     libc.include.float
     libc.include.inttypes
+    libc.include.malloc
     libc.include.math
     libc.include.search
     libc.include.setjmp
diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index ff3d821c664c5..b2e1aefccef66 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -346,6 +346,13 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.wchar.wctob
 )
 
+if(LLVM_LIBC_INCLUDE_SCUDO)
+  list(APPEND TARGET_LIBC_ENTRYPOINTS
+    # malloc.h external entrypoints
+    libc.src.stdlib.mallopt
+  )
+endif()
+
 set(TARGET_LIBM_ENTRYPOINTS
     # fenv.h entrypoints
     libc.src.fenv.feclearexcept
@@ -479,9 +486,15 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.ilogb
     libc.src.math.ilogbf
     libc.src.math.ilogbl
+    libc.src.math.iscanonical
+    libc.src.math.iscanonicalf
+    libc.src.math.iscanonicall
     libc.src.math.isnan
     libc.src.math.isnanf
     libc.src.math.isnanl
+    libc.src.math.issignaling
+    libc.src.math.issignalingf
+    libc.src.math.issignalingl
     libc.src.math.ldexp
     libc.src.math.ldexpf
     libc.src.math.ldexpl
@@ -627,6 +640,8 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.fsubf128
     libc.src.math.getpayloadf128
     libc.src.math.ilogbf128
+    libc.src.math.iscanonicalf128
+    libc.src.math.issignalingf128
     libc.src.math.ldexpf128
     libc.src.math.llogbf128
     libc.src.math.llrintf128
diff --git a/libc/config/linux/riscv/headers.txt b/libc/config/linux/riscv/headers.txt
index 0294f62bc2f7a..41c343f71998b 100644
--- a/libc/config/linux/riscv/headers.txt
+++ b/libc/config/linux/riscv/headers.txt
@@ -11,6 +11,7 @@ set(TARGET_PUBLIC_HEADERS
     libc.include.stdint
     libc.include.inttypes
     libc.include.limits
+    libc.include.malloc
     libc.include.math
     libc.include.pthread
     libc.include.sched
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index dd658af3bfb67..0daf3ccacbb27 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -346,6 +346,13 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.wchar.wctob
 )
 
+if(LLVM_LIBC_INCLUDE_SCUDO)
+  list(APPEND TARGET_LIBC_ENTRYPOINTS
+    # malloc.h external entrypoints
+    libc.src.stdlib.mallopt
+  )
+endif()
+
 set(TARGET_LIBM_ENTRYPOINTS
     # fenv.h entrypoints
     libc.src.fenv.feclearexcept
@@ -479,9 +486,15 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.ilogb
     libc.src.math.ilogbf
     libc.src.math.ilogbl
+    libc.src.math.iscanonical
+    libc.src.math.iscanonicalf
+    libc.src.math.iscanonicall
     libc.src.math.isnan
     libc.src.math.isnanf
     libc.src.math.isnanl
+    libc.src.math.issignaling
+    libc.src.math.issignalingf
+    libc.src.math.issignalingl
     libc.src.math.ldexp
     libc.src.math.ldexpf
     libc.src.math.ldexpl
@@ -637,6 +650,8 @@ if(LIBC_TYPES_HAS_FLOAT16)
     libc.src.math.fromfpxf16
     libc.src.math.getpayloadf16
     libc.src.math.ilogbf16
+    libc.src.math.iscanonicalf16
+    libc.src.math.issignalingf16
     libc.src.math.ldexpf16
     libc.src.math.llogbf16
     libc.src.math.llrintf16
@@ -717,6 +732,8 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.fsubf128
     libc.src.math.getpayloadf128
     libc.src.math.ilogbf128
+    libc.src.math.iscanonicalf128
+    libc.src.math.issignalingf128
     libc.src.math.ldexpf128
     libc.src.math.llogbf128
     libc.src.math.llrintf128
diff --git a/libc/config/linux/x86_64/headers.txt b/libc/config/linux/x86_64/headers.txt
index 881e149d9c40d..e0c04b381492d 100644
--- a/libc/config/linux/x86_64/headers.txt
+++ b/libc/config/linux/x86_64/headers.txt
@@ -13,6 +13,7 @@ set(TARGET_PUBLIC_HEADERS
     libc.include.inttypes
     libc.include.limits
     libc.include.link
+    libc.include.malloc
     libc.include.math
     libc.include.pthread
     libc.include.sched
diff --git a/libc/docs/configure.rst b/libc/docs/configure.rst
index c28d3a36ada74..867bb807d10ac 100644
--- a/libc/docs/configure.rst
+++ b/libc/docs/configure.rst
@@ -40,6 +40,8 @@ to learn about the defaults for your platform and target.
     - ``LIBC_CONF_PRINTF_DISABLE_INDEX_MODE``: Disable index mode in the printf format string.
     - ``LIBC_CONF_PRINTF_DISABLE_STRERROR``: Disable handling of %m to print strerror in printf and friends.
     - ``LIBC_CONF_PRINTF_DISABLE_WRITE_INT``: Disable handling of %n in printf format string.
+    - ``LIBC_CONF_PRINTF_FLOAT_TO_STR_NO_SPECIALIZE_LD``: Use the same mode for double and long double in printf.
+    - ``LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_DYADIC_FLOAT``: Use dyadic float for faster and smaller but less accurate printf doubles.
     - ``LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE``: Use large table for better printf long double performance.
 * **"pthread" options**
     - ``LIBC_CONF_RAW_MUTEX_DEFAULT_SPIN_COUNT``: Default number of spins before blocking if a mutex is in contention (default to 100).
diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst
index 185d2d440849a..0e757e7978bfa 100644
--- a/libc/docs/math/index.rst
+++ b/libc/docs/math/index.rst
@@ -182,6 +182,10 @@ Basic Operations
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | ilogb            | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.6.8               | F.10.3.8                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| iscanonical      | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.3.2               | N/A                        |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
+| issignaling      | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.3.8               | N/A                        |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | ldexp            | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.6.9               | F.10.3.9                   |
 +------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | llogb            | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.6.10              | F.10.3.10                  |
diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt
index 16c2ac5124c84..1f3cb59f69e96 100644
--- a/libc/include/CMakeLists.txt
+++ b/libc/include/CMakeLists.txt
@@ -146,6 +146,16 @@ add_header_macro(
     .llvm-libc-macros.limits_macros
 )
 
+add_header_macro(
+  malloc
+  ../libc/newhdrgen/yaml/malloc.yaml
+  malloc.h.def
+  malloc.h
+  DEPENDS
+    .llvm_libc_common_h
+    .llvm-libc-macros.malloc_macros
+)
+
 add_header_macro(
   math
   ../libc/newhdrgen/yaml/math.yaml
diff --git a/libc/include/llvm-libc-macros/CMakeLists.txt b/libc/include/llvm-libc-macros/CMakeLists.txt
index d28d19330e61b..2ba437c8437f2 100644
--- a/libc/include/llvm-libc-macros/CMakeLists.txt
+++ b/libc/include/llvm-libc-macros/CMakeLists.txt
@@ -109,6 +109,12 @@ add_macro_header(
     link-macros.h
 )
 
+add_macro_header(
+  malloc_macros
+  HDR
+    malloc-macros.h
+)
+
 add_macro_header(
   math_macros
   HDR
diff --git a/libc/include/llvm-libc-macros/malloc-macros.h b/libc/include/llvm-libc-macros/malloc-macros.h
new file mode 100644
index 0000000000000..65eddfc5e86d6
--- /dev/null
+++ b/libc/include/llvm-libc-macros/malloc-macros.h
@@ -0,0 +1,17 @@
+//===-- Definition of macros to be used with malloc functions -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_MACROS_MALLOC_MACROS_H
+#define LLVM_LIBC_MACROS_MALLOC_MACROS_H
+
+// Note: these values only make sense when Scudo is used as the memory
+// allocator.
+#define M_PURGE (-101)
+#define M_PURGE_ALL (-104)
+
+#endif // LLVM_LIBC_MACROS_MALLOC_MACROS_H
diff --git a/libc/include/llvm-libc-types/wchar_t.h b/libc/include/llvm-libc-types/wchar_t.h
index faa40d96efc4d..bf2633a1c244b 100644
--- a/libc/include/llvm-libc-types/wchar_t.h
+++ b/libc/include/llvm-libc-types/wchar_t.h
@@ -9,6 +9,11 @@
 #ifndef LLVM_LIBC_TYPES_WCHAR_T_H
 #define LLVM_LIBC_TYPES_WCHAR_T_H
 
+// wchar_t is a fundamental type in C++.
+#ifndef __cplusplus
+
 typedef __WCHAR_TYPE__ wchar_t;
 
+#endif
+
 #endif // LLVM_LIBC_TYPES_WCHAR_T_H
diff --git a/libc/include/malloc.h.def b/libc/include/malloc.h.def
new file mode 100644
index 0000000000000..2a2bad0f13c3e
--- /dev/null
+++ b/libc/include/malloc.h.def
@@ -0,0 +1,17 @@
+//===-- C standard library header malloc.h --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_MALLOC_H
+#define LLVM_LIBC_MALLOC_H
+
+#include "__llvm-libc-common.h"
+#include "llvm-libc-macros/malloc-macros.h"
+
+%%public_api()
+
+#endif // LLVM_LIBC_MALLOC_H
diff --git a/libc/newhdrgen/yaml/malloc.yaml b/libc/newhdrgen/yaml/malloc.yaml
new file mode 100644
index 0000000000000..8db4f3aebb9b3
--- /dev/null
+++ b/libc/newhdrgen/yaml/malloc.yaml
@@ -0,0 +1,13 @@
+header: malloc.h
+macros: []
+types: []
+enums: []
+objects: []
+functions:
+  - name: mallopt
+    standards:
+      - GNUExtensions
+    return_type: int
+    arguments:
+      - type: int
+      - type: int
diff --git a/libc/newhdrgen/yaml/sys/socket.yaml b/libc/newhdrgen/yaml/sys/socket.yaml
index accb4b14099e8..47d835fa5f4a1 100644
--- a/libc/newhdrgen/yaml/sys/socket.yaml
+++ b/libc/newhdrgen/yaml/sys/socket.yaml
@@ -49,7 +49,7 @@ functions:
     return_type: ssize_t
     arguments:
       - type: int
-      - type: const void *
+      - type: void *
       - type: size_t
       - type: int
   - name: recvfrom
@@ -58,18 +58,18 @@ functions:
     return_type: ssize_t
     arguments:
       - type: int
-      - type: const void*
+      - type: void *
       - type: size_t
       - type: int
-      - type: const struct sockaddr *
-      - type: socklen_t
+      - type: struct sockaddr *__restrict
+      - type: socklen_t *__restrict
   - name: recvmsg
     standards:
       - POSIX
     return_type: ssize_t
     arguments:
       - type: int
-      - type: const struct msghdr *
+      - type: struct msghdr *
       - type: int
   - name: send
     standards:
diff --git a/libc/spec/gnu_ext.td b/libc/spec/gnu_ext.td
index b2a2b8af104d8..64121aed9574f 100644
--- a/libc/spec/gnu_ext.td
+++ b/libc/spec/gnu_ext.td
@@ -22,6 +22,16 @@ def GnuExtensions : StandardSpec<"GNUExtensions"> {
     ]
   >;
 
+  HeaderSpec Malloc = HeaderSpec<
+      "malloc.h",
+      [], // Macros
+      [], // Types
+      [], // Enumerations
+      [
+          FunctionSpec<"mallopt", RetValSpec<IntType>, [ArgSpec<IntType>, ArgSpec<IntType>]>,
+      ]
+  >;
+
   HeaderSpec Math = HeaderSpec<
       "math.h",
       [], // Macros
@@ -291,6 +301,7 @@ def GnuExtensions : StandardSpec<"GNUExtensions"> {
   let Headers = [
     CType,
     FEnv,
+    Malloc,
     Math,
     PThread,
     Sched,
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index 7caf543748151..dadd1ef11d7d3 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -825,12 +825,24 @@ def StdC : StandardSpec<"stdc"> {
           GuardedFunctionSpec<"nanf16", RetValSpec<Float16Type>, [ArgSpec<ConstCharPtr>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"nanf128", RetValSpec<Float128Type>, [ArgSpec<ConstCharPtr>], "LIBC_TYPES_HAS_FLOAT128">,
 
+          FunctionSpec<"issignaling", RetValSpec<IntType>, [ArgSpec<DoubleType>]>,
+          FunctionSpec<"issignalingf", RetValSpec<IntType>, [ArgSpec<FloatType>]>,
+          FunctionSpec<"issignalingl", RetValSpec<IntType>, [ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"issignalingf16", RetValSpec<IntType>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
+          GuardedFunctionSpec<"issignalingf128", RetValSpec<IntType>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
+
           FunctionSpec<"canonicalize", RetValSpec<IntType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"canonicalizef", RetValSpec<IntType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
           FunctionSpec<"canonicalizel", RetValSpec<IntType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
           GuardedFunctionSpec<"canonicalizef16", RetValSpec<IntType>, [ArgSpec<Float16Type>, ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
           GuardedFunctionSpec<"canonicalizef128", RetValSpec<IntType>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
 
+          FunctionSpec<"iscanonical", RetValSpec<IntType>, [ArgSpec<DoubleType>]>,
+          FunctionSpec<"iscanonicalf", RetValSpec<IntType>, [ArgSpec<FloatType>]>,
+          FunctionSpec<"iscanonicall", RetValSpec<IntType>, [ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"iscanonicalf16", RetValSpec<IntType>, [ArgSpec<Float16Type>], "LIBC_TYPES_HAS_FLOAT16">,
+          GuardedFunctionSpec<"iscanonicalf128", RetValSpec<IntType>, [ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
+
           FunctionSpec<"dsqrtl", RetValSpec<DoubleType>, [ArgSpec<LongDoubleType>]>,
 
           FunctionSpec<"totalorder", RetValSpec<IntType>, [ArgSpec<ConstDoublePtr>, ArgSpec<ConstDoublePtr>]>,
diff --git a/libc/src/__support/FPUtil/BasicOperations.h b/libc/src/__support/FPUtil/BasicOperations.h
index 824c9bfb3947f..a69bca6acc512 100644
--- a/libc/src/__support/FPUtil/BasicOperations.h
+++ b/libc/src/__support/FPUtil/BasicOperations.h
@@ -247,6 +247,12 @@ LIBC_INLINE T fdim(T x, T y) {
   return (x > y ? x - y : 0);
 }
 
+template <typename T, cpp::enable_if_t<cpp::is_floating_point_v<T>, int> = 0>
+LIBC_INLINE int issignaling(const T &x) {
+  FPBits<T> sx(x);
+  return sx.is_signaling_nan();
+}
+
 template <typename T, cpp::enable_if_t<cpp::is_floating_point_v<T>, int> = 0>
 LIBC_INLINE int canonicalize(T &cx, const T &x) {
   FPBits<T> sx(x);
diff --git a/libc/src/__support/FPUtil/dyadic_float.h b/libc/src/__support/FPUtil/dyadic_float.h
index 165ffc7c92202..289fd01680547 100644
--- a/libc/src/__support/FPUtil/dyadic_float.h
+++ b/libc/src/__support/FPUtil/dyadic_float.h
@@ -357,7 +357,7 @@ template <size_t Bits> struct DyadicFloat {
     return as<T, /*ShouldSignalExceptions=*/false>();
   }
 
-  LIBC_INLINE explicit constexpr operator MantissaType() const {
+  LIBC_INLINE constexpr MantissaType as_mantissa_type() const {
     if (mantissa.is_zero())
       return 0;
 
diff --git a/libc/src/__support/float_to_string.h b/libc/src/__support/float_to_string.h
index c4c7a15179f95..e2e06cd0492a9 100644
--- a/libc/src/__support/float_to_string.h
+++ b/libc/src/__support/float_to_string.h
@@ -20,6 +20,7 @@
 #include "src/__support/libc_assert.h"
 #include "src/__support/macros/attributes.h"
 #include "src/__support/macros/config.h"
+#include "src/__support/sign.h"
 
 // This file has 5 compile-time flags to allow the user to configure the float
 // to string behavior. These were used to explore tradeoffs during the design
@@ -232,7 +233,7 @@ LIBC_INLINE UInt<MID_INT_SIZE> get_table_positive_df(int exponent, size_t i) {
   if (shift_amount < 0) {
     return 1;
   }
-  fputil::DyadicFloat<INT_SIZE> num(false, 0, 1);
+  fputil::DyadicFloat<INT_SIZE> num(Sign::POS, 0, 1);
   constexpr UInt<INT_SIZE> MOD_SIZE =
       (UInt<INT_SIZE>(EXP10_9)
        << (CALC_SHIFT_CONST + (IDX_SIZE > 1 ? IDX_SIZE : 0)));
@@ -242,16 +243,17 @@ LIBC_INLINE UInt<MID_INT_SIZE> get_table_positive_df(int exponent, size_t i) {
        0x89705f4136b4a597}};
 
   static const fputil::DyadicFloat<INT_SIZE> FIVE_EXP_MINUS_NINE(
-      false, -276, FIVE_EXP_MINUS_NINE_MANT);
+      Sign::POS, -276, FIVE_EXP_MINUS_NINE_MANT);
 
   if (i > 0) {
-    fputil::DyadicFloat<INT_SIZE> fives = fputil::pow_n(FIVE_EXP_MINUS_NINE, i);
+    fputil::DyadicFloat<INT_SIZE> fives =
+        fputil::pow_n(FIVE_EXP_MINUS_NINE, static_cast<uint32_t>(i));
     num = fives;
   }
   num = mul_pow_2(num, shift_amount);
 
   // Adding one is part of the formula.
-  UInt<INT_SIZE> int_num = static_cast<UInt<INT_SIZE>>(num) + 1;
+  UInt<INT_SIZE> int_num = num.as_mantissa_type() + 1;
   if (int_num > MOD_SIZE) {
     auto rem =
         int_num
@@ -339,23 +341,24 @@ LIBC_INLINE UInt<MID_INT_SIZE> get_table_negative_df(int exponent, size_t i) {
 
   int shift_amount = CALC_SHIFT_CONST - exponent;
 
-  fputil::DyadicFloat<INT_SIZE> num(false, 0, 1);
+  fputil::DyadicFloat<INT_SIZE> num(Sign::POS, 0, 1);
   constexpr UInt<INT_SIZE> MOD_SIZE =
       (UInt<INT_SIZE>(EXP10_9)
        << (CALC_SHIFT_CONST + (IDX_SIZE > 1 ? IDX_SIZE : 0)));
 
   constexpr UInt<INT_SIZE> TEN_EXP_NINE_MANT(EXP10_9);
 
-  static const fputil::DyadicFloat<INT_SIZE> TEN_EXP_NINE(false, 0,
+  static const fputil::DyadicFloat<INT_SIZE> TEN_EXP_NINE(Sign::POS, 0,
                                                           TEN_EXP_NINE_MANT);
 
   if (i > 0) {
-    fputil::DyadicFloat<INT_SIZE> tens = fputil::pow_n(TEN_EXP_NINE, i);
+    fputil::DyadicFloat<INT_SIZE> tens =
+        fputil::pow_n(TEN_EXP_NINE, static_cast<uint32_t>(i));
     num = tens;
   }
   num = mul_pow_2(num, shift_amount);
 
-  UInt<INT_SIZE> int_num = static_cast<UInt<INT_SIZE>>(num);
+  UInt<INT_SIZE> int_num = num.as_mantissa_type();
   if (int_num > MOD_SIZE) {
     auto rem =
         int_num
diff --git a/libc/src/__support/sign.h b/libc/src/__support/sign.h
index 28cfae4bab1de..4a629e4488195 100644
--- a/libc/src/__support/sign.h
+++ b/libc/src/__support/sign.h
@@ -11,6 +11,8 @@
 
 #include "src/__support/macros/attributes.h" // LIBC_INLINE, LIBC_INLINE_VAR
 
+namespace LIBC_NAMESPACE_DECL {
+
 // A type to interact with signed arithmetic types.
 struct Sign {
   LIBC_INLINE constexpr bool is_pos() const { return !is_negative; }
@@ -37,4 +39,5 @@ struct Sign {
 LIBC_INLINE_VAR constexpr Sign Sign::NEG = Sign(true);
 LIBC_INLINE_VAR constexpr Sign Sign::POS = Sign(false);
 
+} // namespace LIBC_NAMESPACE_DECL
 #endif // LLVM_LIBC_SRC___SUPPORT_SIGN_H
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index 3cba34fc24932..97e2d0d64a147 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -66,6 +66,12 @@ add_math_entrypoint_object(canonicalizel)
 add_math_entrypoint_object(canonicalizef16)
 add_math_entrypoint_object(canonicalizef128)
 
+add_math_entrypoint_object(iscanonical)
+add_math_entrypoint_object(iscanonicalf)
+add_math_entrypoint_object(iscanonicall)
+add_math_entrypoint_object(iscanonicalf16)
+add_math_entrypoint_object(iscanonicalf128)
+
 add_math_entrypoint_object(cbrt)
 add_math_entrypoint_object(cbrtf)
 
@@ -302,6 +308,12 @@ add_math_entrypoint_object(isnan)
 add_math_entrypoint_object(isnanf)
 add_math_entrypoint_object(isnanl)
 
+add_math_entrypoint_object(issignaling)
+add_math_entrypoint_object(issignalingf)
+add_math_entrypoint_object(issignalingl)
+add_math_entrypoint_object(issignalingf16)
+add_math_entrypoint_object(issignalingf128)
+
 add_math_entrypoint_object(llogb)
 add_math_entrypoint_object(llogbf)
 add_math_entrypoint_object(llogbl)
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index d0676d03420c6..93e632c38af23 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -60,6 +60,60 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.basic_operations
 )
 
+add_entrypoint_object(
+  iscanonical
+  SRCS
+    iscanonical.cpp
+  HDRS
+    ../iscanonical.h
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  iscanonicalf
+  SRCS
+    iscanonicalf.cpp
+  HDRS
+    ../iscanonicalf.h
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  iscanonicall
+  SRCS
+    iscanonicall.cpp
+  HDRS
+    ../iscanonicall.h
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  iscanonicalf16
+  SRCS
+    iscanonicalf16.cpp
+  HDRS
+    ../iscanonicalf16.h
+  COMPILE_OPTIONS
+    -O3
+  DEPENDS
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  iscanonicalf128
+  SRCS
+    iscanonicalf128.cpp
+  HDRS
+    ../iscanonicalf128.h
+  COMPILE_OPTIONS
+    -O3
+  DEPENDS
+    libc.src.__support.macros.properties.types
+)
+
 add_entrypoint_object(
   ceil
   SRCS
@@ -3377,6 +3431,60 @@ add_entrypoint_object(
     -O3
 )
 
+add_entrypoint_object(
+  issignaling
+  SRCS
+    issignaling.cpp
+  HDRS
+    ../issignaling.h
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  issignalingf
+  SRCS
+    issignalingf.cpp
+  HDRS
+    ../issignalingf.h
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  issignalingl
+  SRCS
+    issignalingl.cpp
+  HDRS
+    ../issignalingl.h
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  issignalingf16
+  SRCS
+    issignalingf16.cpp
+  HDRS
+    ../issignalingf16.h
+  COMPILE_OPTIONS
+      -O3
+  DEPENDS
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  issignalingf128
+  SRCS
+    issignalingf128.cpp
+  HDRS
+    ../issignalingf128.h
+  COMPILE_OPTIONS
+      -O3
+  DEPENDS
+    libc.src.__support.macros.properties.types
+)
+
 add_entrypoint_object(
   isnan
   SRCS
diff --git a/libc/src/math/generic/iscanonical.cpp b/libc/src/math/generic/iscanonical.cpp
new file mode 100644
index 0000000000000..ff922f3ae8ee7
--- /dev/null
+++ b/libc/src/math/generic/iscanonical.cpp
@@ -0,0 +1,22 @@
+//===-- Implementation of iscanonical function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/iscanonical.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+#undef iscanonical
+LLVM_LIBC_FUNCTION(int, iscanonical, (double x)) {
+  double temp;
+  return fputil::canonicalize(temp, x) == 0;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/iscanonicalf.cpp b/libc/src/math/generic/iscanonicalf.cpp
new file mode 100644
index 0000000000000..5c38e9b72ec58
--- /dev/null
+++ b/libc/src/math/generic/iscanonicalf.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of iscanonicalf function ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/iscanonicalf.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, iscanonicalf, (float x)) {
+  float temp;
+  return fputil::canonicalize(temp, x) == 0;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/iscanonicalf128.cpp b/libc/src/math/generic/iscanonicalf128.cpp
new file mode 100644
index 0000000000000..ff549c3194cba
--- /dev/null
+++ b/libc/src/math/generic/iscanonicalf128.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of iscanonicalf128 function ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/iscanonicalf128.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, iscanonicalf128, (float128 x)) {
+  float128 temp;
+  return fputil::canonicalize(temp, x) == 0;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/iscanonicalf16.cpp b/libc/src/math/generic/iscanonicalf16.cpp
new file mode 100644
index 0000000000000..f9b093dc8ff2d
--- /dev/null
+++ b/libc/src/math/generic/iscanonicalf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of iscanonicalf16 function -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/iscanonicalf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, iscanonicalf16, (float16 x)) {
+  float16 temp;
+  return fputil::canonicalize(temp, x) == 0;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/iscanonicall.cpp b/libc/src/math/generic/iscanonicall.cpp
new file mode 100644
index 0000000000000..34e00359912b7
--- /dev/null
+++ b/libc/src/math/generic/iscanonicall.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of iscanonicall function ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/iscanonicall.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, iscanonicall, (long double x)) {
+  long double temp;
+  return fputil::canonicalize(temp, x) == 0;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/issignaling.cpp b/libc/src/math/generic/issignaling.cpp
new file mode 100644
index 0000000000000..0687a32817b29
--- /dev/null
+++ b/libc/src/math/generic/issignaling.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of issignaling function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/issignaling.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, issignaling, (double x)) {
+  return fputil::issignaling(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/issignalingf.cpp b/libc/src/math/generic/issignalingf.cpp
new file mode 100644
index 0000000000000..410bf7b4d4278
--- /dev/null
+++ b/libc/src/math/generic/issignalingf.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of issignalingf function ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/issignalingf.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, issignalingf, (float x)) {
+  return fputil::issignaling(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/issignalingf128.cpp b/libc/src/math/generic/issignalingf128.cpp
new file mode 100644
index 0000000000000..29612ccf677f6
--- /dev/null
+++ b/libc/src/math/generic/issignalingf128.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of issignalingf128 function ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/issignalingf128.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, issignalingf128, (float128 x)) {
+  return fputil::issignaling(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/issignalingf16.cpp b/libc/src/math/generic/issignalingf16.cpp
new file mode 100644
index 0000000000000..976b8e2b689b3
--- /dev/null
+++ b/libc/src/math/generic/issignalingf16.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of issignalingf16 function -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/issignalingf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, issignalingf16, (float16 x)) {
+  return fputil::issignaling(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/issignalingl.cpp b/libc/src/math/generic/issignalingl.cpp
new file mode 100644
index 0000000000000..a3959bcae2357
--- /dev/null
+++ b/libc/src/math/generic/issignalingl.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of issignalingl function ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/issignalingl.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, issignalingl, (long double x)) {
+  return fputil::issignaling(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/iscanonical.h b/libc/src/math/iscanonical.h
new file mode 100644
index 0000000000000..14b2d17c8bbe9
--- /dev/null
+++ b/libc/src/math/iscanonical.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for iscanonical -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ISCANONICAL_H
+#define LLVM_LIBC_SRC_MATH_ISCANONICAL_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int iscanonical(double x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_ISCANONICAL_H
diff --git a/libc/src/math/iscanonicalf.h b/libc/src/math/iscanonicalf.h
new file mode 100644
index 0000000000000..fdb48bc2767be
--- /dev/null
+++ b/libc/src/math/iscanonicalf.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for iscanonicalf ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ISCANONICALF_H
+#define LLVM_LIBC_SRC_MATH_ISCANONICALF_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int iscanonicalf(float x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_ISCANONICALF_H
diff --git a/libc/src/math/iscanonicalf128.h b/libc/src/math/iscanonicalf128.h
new file mode 100644
index 0000000000000..9ba021c86ec14
--- /dev/null
+++ b/libc/src/math/iscanonicalf128.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for iscanonicalf128 ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ISCANONICALF128_H
+#define LLVM_LIBC_SRC_MATH_ISCANONICALF128_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int iscanonicalf128(float128 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_ISCANONICALF128_H
diff --git a/libc/src/math/iscanonicalf16.h b/libc/src/math/iscanonicalf16.h
new file mode 100644
index 0000000000000..565d3eae35c49
--- /dev/null
+++ b/libc/src/math/iscanonicalf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for iscanonicalf16 ----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ISCANONICALF16_H
+#define LLVM_LIBC_SRC_MATH_ISCANONICALF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int iscanonicalf16(float16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_ISCANONICALF16_H
diff --git a/libc/src/math/iscanonicall.h b/libc/src/math/iscanonicall.h
new file mode 100644
index 0000000000000..edda6e334f1ab
--- /dev/null
+++ b/libc/src/math/iscanonicall.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for iscanonicall ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ISCANONICALL_H
+#define LLVM_LIBC_SRC_MATH_ISCANONICALL_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int iscanonicall(long double x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_ISCANONICALL_H
diff --git a/libc/src/math/issignaling.h b/libc/src/math/issignaling.h
new file mode 100644
index 0000000000000..093fd7d48770d
--- /dev/null
+++ b/libc/src/math/issignaling.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for issignaling -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ISSIGNALING_H
+#define LLVM_LIBC_SRC_MATH_ISSIGNALING_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int issignaling(double x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_ISSIGNALING_H
diff --git a/libc/src/math/issignalingf.h b/libc/src/math/issignalingf.h
new file mode 100644
index 0000000000000..97522600916cc
--- /dev/null
+++ b/libc/src/math/issignalingf.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for issignalingf ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ISSIGNALINGF_H
+#define LLVM_LIBC_SRC_MATH_ISSIGNALINGF_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int issignalingf(float x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_ISSIGNALINGF_H
diff --git a/libc/src/math/issignalingf128.h b/libc/src/math/issignalingf128.h
new file mode 100644
index 0000000000000..7d1bfdec646de
--- /dev/null
+++ b/libc/src/math/issignalingf128.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for issignalingf128 ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ISSIGNALINGF128_H
+#define LLVM_LIBC_SRC_MATH_ISSIGNALINGF128_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int issignalingf128(float128 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_ISSIGNALINGF128_H
diff --git a/libc/src/math/issignalingf16.h b/libc/src/math/issignalingf16.h
new file mode 100644
index 0000000000000..2b38525e1a3e0
--- /dev/null
+++ b/libc/src/math/issignalingf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for issignalingf16 ----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ISSIGNALINGF16_H
+#define LLVM_LIBC_SRC_MATH_ISSIGNALINGF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int issignalingf16(float16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_ISSIGNALINGF16_H
diff --git a/libc/src/math/issignalingl.h b/libc/src/math/issignalingl.h
new file mode 100644
index 0000000000000..edc039416d1f4
--- /dev/null
+++ b/libc/src/math/issignalingl.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for issignalingl ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ISSIGNALINGL_H
+#define LLVM_LIBC_SRC_MATH_ISSIGNALINGL_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int issignalingl(long double x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_ISSIGNALINGL_H
diff --git a/libc/src/stdio/printf_core/CMakeLists.txt b/libc/src/stdio/printf_core/CMakeLists.txt
index 542327ad5a49a..8172fda1e866e 100644
--- a/libc/src/stdio/printf_core/CMakeLists.txt
+++ b/libc/src/stdio/printf_core/CMakeLists.txt
@@ -10,6 +10,12 @@ endif()
 if(LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE)
   list(APPEND printf_config_copts "-DLIBC_COPT_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE")
 endif()
+if(LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_DYADIC_FLOAT)
+  list(APPEND printf_config_copts "-DLIBC_COPT_FLOAT_TO_STR_USE_DYADIC_FLOAT")
+endif()
+if(LIBC_CONF_PRINTF_FLOAT_TO_STR_NO_SPECIALIZE_LD)
+  list(APPEND printf_config_copts "-DLIBC_COPT_FLOAT_TO_STR_NO_SPECIALIZE_LD")
+endif()
 if(LIBC_CONF_PRINTF_DISABLE_FIXED_POINT)
   list(APPEND printf_config_copts "-DLIBC_COPT_PRINTF_DISABLE_FIXED_POINT")
 endif()
diff --git a/libc/src/stdlib/CMakeLists.txt b/libc/src/stdlib/CMakeLists.txt
index 1b5b2cb155264..d997cd41e630f 100644
--- a/libc/src/stdlib/CMakeLists.txt
+++ b/libc/src/stdlib/CMakeLists.txt
@@ -384,6 +384,11 @@ if(NOT LIBC_TARGET_OS_IS_GPU)
       DEPENDS
         ${SCUDO_DEPS}
     )
+    add_entrypoint_external(
+      mallopt
+      DEPENDS
+        ${SCUDO_DEPS}
+    )
   else()
     # Only use freelist malloc for baremetal targets.
     add_entrypoint_object(
diff --git a/libc/src/sys/socket/linux/recv.cpp b/libc/src/sys/socket/linux/recv.cpp
index 55a766aec0e77..5e9f2d3233fcf 100644
--- a/libc/src/sys/socket/linux/recv.cpp
+++ b/libc/src/sys/socket/linux/recv.cpp
@@ -8,6 +8,9 @@
 
 #include "src/sys/socket/recv.h"
 
+#include <linux/net.h>   // For SYS_SOCKET socketcall number.
+#include <sys/syscall.h> // For syscall numbers.
+
 #include "hdr/types/socklen_t.h"
 #include "hdr/types/ssize_t.h"
 #include "hdr/types/struct_sockaddr.h"
@@ -15,26 +18,23 @@
 #include "src/__support/common.h"
 #include "src/__support/macros/sanitizer.h"
 #include "src/errno/libc_errno.h"
-#include <linux/net.h>   // For SYS_SOCKET socketcall number.
-#include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(ssize_t, recv,
-                   (int sockfd, const void *buf, size_t len, int flags)) {
+                   (int sockfd, void *buf, size_t len, int flags)) {
 #ifdef SYS_recv
   ssize_t ret =
-      LIBC_NAMESPACE::syscall_impl<int>(SYS_recv, sockfd, buf, len, flags);
+      LIBC_NAMESPACE::syscall_impl<ssize_t>(SYS_recv, sockfd, buf, len, flags);
 #elif defined(SYS_recvfrom)
-  ssize_t ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_recvfrom, sockfd,
-                                                  reinterpret_cast<long>(buf),
-                                                  len, flags, nullptr, 0);
+  ssize_t ret = LIBC_NAMESPACE::syscall_impl<ssize_t>(
+      SYS_recvfrom, sockfd, buf, len, flags, nullptr, nullptr);
 #elif defined(SYS_socketcall)
   unsigned long sockcall_args[4] = {
       static_cast<unsigned long>(sockfd), reinterpret_cast<unsigned long>(buf),
       static_cast<unsigned long>(len), static_cast<unsigned long>(flags)};
-  ssize_t ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_socketcall, SYS_RECV,
-                                                  sockcall_args);
+  ssize_t ret = LIBC_NAMESPACE::syscall_impl<ssize_t>(SYS_socketcall, SYS_RECV,
+                                                      sockcall_args);
 #else
 #error "socket and socketcall syscalls unavailable for this platform."
 #endif
diff --git a/libc/src/sys/socket/linux/recvfrom.cpp b/libc/src/sys/socket/linux/recvfrom.cpp
index 990e58da3c1b6..a0f8278cd5deb 100644
--- a/libc/src/sys/socket/linux/recvfrom.cpp
+++ b/libc/src/sys/socket/linux/recvfrom.cpp
@@ -8,6 +8,9 @@
 
 #include "src/sys/socket/recvfrom.h"
 
+#include <linux/net.h>   // For SYS_SOCKET socketcall number.
+#include <sys/syscall.h> // For syscall numbers.
+
 #include "hdr/types/socklen_t.h"
 #include "hdr/types/ssize_t.h"
 #include "hdr/types/struct_sockaddr.h"
@@ -15,19 +18,16 @@
 #include "src/__support/common.h"
 #include "src/__support/macros/sanitizer.h"
 #include "src/errno/libc_errno.h"
-#include <linux/net.h>   // For SYS_SOCKET socketcall number.
-#include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(ssize_t, recvfrom,
-                   (int sockfd, const void *buf, size_t len, int flags,
-                    const struct sockaddr *dest_addr, socklen_t addrlen)) {
+                   (int sockfd, void *buf, size_t len, int flags,
+                    struct sockaddr *__restrict dest_addr,
+                    socklen_t *__restrict addrlen)) {
 #ifdef SYS_recvfrom
-
-  ssize_t ret = LIBC_NAMESPACE::syscall_impl<int>(
-      SYS_recvfrom, sockfd, reinterpret_cast<long>(buf), len, flags,
-      reinterpret_cast<long>(dest_addr), addrlen);
+  ssize_t ret = LIBC_NAMESPACE::syscall_impl<ssize_t>(
+      SYS_recvfrom, sockfd, buf, len, flags, dest_addr, addrlen);
 #elif defined(SYS_socketcall)
   unsigned long sockcall_args[6] = {static_cast<unsigned long>(sockfd),
                                     reinterpret_cast<unsigned long>(buf),
@@ -35,8 +35,8 @@ LLVM_LIBC_FUNCTION(ssize_t, recvfrom,
                                     static_cast<unsigned long>(flags),
                                     reinterpret_cast<unsigned long>(dest_addr),
                                     static_cast<unsigned long>(addrlen)};
-  ssize_t ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_socketcall, SYS_RECVFROM,
-                                                  sockcall_args);
+  ssize_t ret = LIBC_NAMESPACE::syscall_impl<ssize_t>(
+      SYS_socketcall, SYS_RECVFROM, sockcall_args);
 #else
 #error "socket and socketcall syscalls unavailable for this platform."
 #endif
@@ -46,6 +46,7 @@ LLVM_LIBC_FUNCTION(ssize_t, recvfrom,
   }
 
   MSAN_UNPOISON(buf, ret);
+  MSAN_UNPOISON(addrlen, sizeof(socklen_t));
 
   return ret;
 }
diff --git a/libc/src/sys/socket/linux/recvmsg.cpp b/libc/src/sys/socket/linux/recvmsg.cpp
index f44e5800d817f..60bbc84877b85 100644
--- a/libc/src/sys/socket/linux/recvmsg.cpp
+++ b/libc/src/sys/socket/linux/recvmsg.cpp
@@ -8,28 +8,29 @@
 
 #include "src/sys/socket/recvmsg.h"
 
+#include <linux/net.h>   // For SYS_SOCKET socketcall number.
+#include <sys/syscall.h> // For syscall numbers.
+
 #include "hdr/types/ssize_t.h"
 #include "hdr/types/struct_msghdr.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 #include "src/__support/macros/sanitizer.h"
 #include "src/errno/libc_errno.h"
-#include <linux/net.h>   // For SYS_SOCKET socketcall number.
-#include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(ssize_t, recvmsg,
-                   (int sockfd, const struct msghdr *msg, int flags)) {
+                   (int sockfd, struct msghdr *msg, int flags)) {
 #ifdef SYS_recvmsg
-  ssize_t ret = LIBC_NAMESPACE::syscall_impl<int>(
-      SYS_recvmsg, sockfd, reinterpret_cast<long>(msg), flags);
+  ssize_t ret =
+      LIBC_NAMESPACE::syscall_impl<ssize_t>(SYS_recvmsg, sockfd, msg, flags);
 #elif defined(SYS_socketcall)
   unsigned long sockcall_args[3] = {static_cast<unsigned long>(sockfd),
                                     reinterpret_cast<unsigned long>(msg),
                                     static_cast<unsigned long>(flags)};
-  ssize_t ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_socketcall, SYS_RECVMSG,
-                                                  sockcall_args);
+  ssize_t ret = LIBC_NAMESPACE::syscall_impl<ssize_t>(
+      SYS_socketcall, SYS_RECVMSG, sockcall_args);
 #else
 #error "socket and socketcall syscalls unavailable for this platform."
 #endif
diff --git a/libc/src/sys/socket/linux/send.cpp b/libc/src/sys/socket/linux/send.cpp
index 0d29b6fd35b90..cb3b4d5a9ece7 100644
--- a/libc/src/sys/socket/linux/send.cpp
+++ b/libc/src/sys/socket/linux/send.cpp
@@ -8,14 +8,15 @@
 
 #include "src/sys/socket/send.h"
 
+#include <linux/net.h>   // For SYS_SOCKET socketcall number.
+#include <sys/syscall.h> // For syscall numbers.
+
 #include "hdr/types/socklen_t.h"
 #include "hdr/types/ssize_t.h"
 #include "hdr/types/struct_sockaddr.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 #include "src/errno/libc_errno.h"
-#include <linux/net.h>   // For SYS_SOCKET socketcall number.
-#include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -23,16 +24,16 @@ LLVM_LIBC_FUNCTION(ssize_t, send,
                    (int sockfd, const void *buf, size_t len, int flags)) {
 #ifdef SYS_send
   ssize_t ret =
-      LIBC_NAMESPACE::syscall_impl<int>(SYS_send, sockfd, buf, len, flags);
+      LIBC_NAMESPACE::syscall_impl<ssize_t>(SYS_send, sockfd, buf, len, flags);
 #elif defined(SYS_sendto)
-  ssize_t ret = LIBC_NAMESPACE::syscall_impl<int>(
-      SYS_sendto, sockfd, reinterpret_cast<long>(buf), len, flags, nullptr, 0);
+  ssize_t ret = LIBC_NAMESPACE::syscall_impl<ssize_t>(SYS_sendto, sockfd, buf,
+                                                      len, flags, nullptr, 0);
 #elif defined(SYS_socketcall)
   unsigned long sockcall_args[4] = {
       static_cast<unsigned long>(sockfd), reinterpret_cast<unsigned long>(buf),
       static_cast<unsigned long>(len), static_cast<unsigned long>(flags)};
-  ssize_t ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_socketcall, SYS_SEND,
-                                                  sockcall_args);
+  ssize_t ret = LIBC_NAMESPACE::syscall_impl<ssize_t>(SYS_socketcall, SYS_SEND,
+                                                      sockcall_args);
 #else
 #error "socket and socketcall syscalls unavailable for this platform."
 #endif
diff --git a/libc/src/sys/socket/linux/sendmsg.cpp b/libc/src/sys/socket/linux/sendmsg.cpp
index ba2b37768cc24..b4d9c9deda028 100644
--- a/libc/src/sys/socket/linux/sendmsg.cpp
+++ b/libc/src/sys/socket/linux/sendmsg.cpp
@@ -8,27 +8,28 @@
 
 #include "src/sys/socket/sendmsg.h"
 
+#include <linux/net.h>   // For SYS_SOCKET socketcall number.
+#include <sys/syscall.h> // For syscall numbers.
+
 #include "hdr/types/ssize_t.h"
 #include "hdr/types/struct_msghdr.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 #include "src/errno/libc_errno.h"
-#include <linux/net.h>   // For SYS_SOCKET socketcall number.
-#include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(ssize_t, sendmsg,
                    (int sockfd, const struct msghdr *msg, int flags)) {
 #ifdef SYS_sendmsg
-  ssize_t ret = LIBC_NAMESPACE::syscall_impl<int>(
-      SYS_sendmsg, sockfd, reinterpret_cast<long>(msg), flags);
+  ssize_t ret =
+      LIBC_NAMESPACE::syscall_impl<ssize_t>(SYS_sendmsg, sockfd, msg, flags);
 #elif defined(SYS_socketcall)
   unsigned long sockcall_args[3] = {static_cast<unsigned long>(sockfd),
                                     reinterpret_cast<unsigned long>(msg),
                                     static_cast<unsigned long>(flags)};
-  ssize_t ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_socketcall, SYS_SENDMSG,
-                                                  sockcall_args);
+  ssize_t ret = LIBC_NAMESPACE::syscall_impl<ssize_t>(
+      SYS_socketcall, SYS_SENDMSG, sockcall_args);
 #else
 #error "socket and socketcall syscalls unavailable for this platform."
 #endif
diff --git a/libc/src/sys/socket/linux/sendto.cpp b/libc/src/sys/socket/linux/sendto.cpp
index f5c3ffe8954da..2fada192b0865 100644
--- a/libc/src/sys/socket/linux/sendto.cpp
+++ b/libc/src/sys/socket/linux/sendto.cpp
@@ -8,14 +8,15 @@
 
 #include "src/sys/socket/sendto.h"
 
+#include <linux/net.h>   // For SYS_SOCKET socketcall number.
+#include <sys/syscall.h> // For syscall numbers.
+
 #include "hdr/types/socklen_t.h"
 #include "hdr/types/ssize_t.h"
 #include "hdr/types/struct_sockaddr.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
 #include "src/errno/libc_errno.h"
-#include <linux/net.h>   // For SYS_SOCKET socketcall number.
-#include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -23,10 +24,8 @@ LLVM_LIBC_FUNCTION(ssize_t, sendto,
                    (int sockfd, const void *buf, size_t len, int flags,
                     const struct sockaddr *dest_addr, socklen_t addrlen)) {
 #ifdef SYS_sendto
-
-  ssize_t ret = LIBC_NAMESPACE::syscall_impl<int>(
-      SYS_sendto, sockfd, reinterpret_cast<long>(buf), len, flags,
-      reinterpret_cast<long>(dest_addr), addrlen);
+  ssize_t ret = LIBC_NAMESPACE::syscall_impl<ssize_t>(
+      SYS_sendto, sockfd, buf, len, flags, dest_addr, addrlen);
 #elif defined(SYS_socketcall)
   unsigned long sockcall_args[6] = {static_cast<unsigned long>(sockfd),
                                     reinterpret_cast<unsigned long>(buf),
@@ -34,8 +33,8 @@ LLVM_LIBC_FUNCTION(ssize_t, sendto,
                                     static_cast<unsigned long>(flags),
                                     reinterpret_cast<unsigned long>(dest_addr),
                                     static_cast<unsigned long>(addrlen)};
-  ssize_t ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_socketcall, SYS_SENDTO,
-                                                  sockcall_args);
+  ssize_t ret = LIBC_NAMESPACE::syscall_impl<ssize_t>(
+      SYS_socketcall, SYS_SENDTO, sockcall_args);
 #else
 #error "socket and socketcall syscalls unavailable for this platform."
 #endif
diff --git a/libc/src/sys/socket/recv.h b/libc/src/sys/socket/recv.h
index 093e8ca9b1230..ea64daa1183fe 100644
--- a/libc/src/sys/socket/recv.h
+++ b/libc/src/sys/socket/recv.h
@@ -14,7 +14,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
-ssize_t recv(int sockfd, const void *buf, size_t len, int flags);
+ssize_t recv(int sockfd, void *buf, size_t len, int flags);
 
 } // namespace LIBC_NAMESPACE_DECL
 
diff --git a/libc/src/sys/socket/recvfrom.h b/libc/src/sys/socket/recvfrom.h
index ee8d52157b705..14869802e7256 100644
--- a/libc/src/sys/socket/recvfrom.h
+++ b/libc/src/sys/socket/recvfrom.h
@@ -17,8 +17,9 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
-ssize_t recvfrom(int sockfd, const void *buf, size_t len, int flags,
-                 const struct sockaddr *address, socklen_t addrlen);
+ssize_t recvfrom(int sockfd, void *buf, size_t len, int flags,
+                 struct sockaddr *__restrict address,
+                 socklen_t *__restrict addrlen);
 
 } // namespace LIBC_NAMESPACE_DECL
 
diff --git a/libc/src/sys/socket/recvmsg.h b/libc/src/sys/socket/recvmsg.h
index a506fb0ae68fa..6230136f439b1 100644
--- a/libc/src/sys/socket/recvmsg.h
+++ b/libc/src/sys/socket/recvmsg.h
@@ -15,7 +15,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
-ssize_t recvmsg(int sockfd, const struct msghdr *msg, int flags);
+ssize_t recvmsg(int sockfd, struct msghdr *msg, int flags);
 
 } // namespace LIBC_NAMESPACE_DECL
 
diff --git a/libc/test/UnitTest/FPMatcher.h b/libc/test/UnitTest/FPMatcher.h
index 43752a4942ad5..3cf0b04952ec0 100644
--- a/libc/test/UnitTest/FPMatcher.h
+++ b/libc/test/UnitTest/FPMatcher.h
@@ -21,6 +21,8 @@
 
 #include "hdr/math_macros.h"
 
+using LIBC_NAMESPACE::Sign;
+
 namespace LIBC_NAMESPACE_DECL {
 namespace testing {
 
diff --git a/libc/test/src/__support/FPUtil/dyadic_float_test.cpp b/libc/test/src/__support/FPUtil/dyadic_float_test.cpp
index 3b1f9deb64ac8..709e1df06d21a 100644
--- a/libc/test/src/__support/FPUtil/dyadic_float_test.cpp
+++ b/libc/test/src/__support/FPUtil/dyadic_float_test.cpp
@@ -16,6 +16,7 @@
 using Float128 = LIBC_NAMESPACE::fputil::DyadicFloat<128>;
 using Float192 = LIBC_NAMESPACE::fputil::DyadicFloat<192>;
 using Float256 = LIBC_NAMESPACE::fputil::DyadicFloat<256>;
+using LIBC_NAMESPACE::Sign;
 
 TEST(LlvmLibcDyadicFloatTest, BasicConversions) {
   Float128 x(Sign::POS, /*exponent*/ 0,
diff --git a/libc/test/src/__support/FPUtil/fpbits_test.cpp b/libc/test/src/__support/FPUtil/fpbits_test.cpp
index df50d8546f34f..bcab0286480c3 100644
--- a/libc/test/src/__support/FPUtil/fpbits_test.cpp
+++ b/libc/test/src/__support/FPUtil/fpbits_test.cpp
@@ -13,6 +13,7 @@
 #include "src/__support/sign.h" // Sign
 #include "test/UnitTest/Test.h"
 
+using LIBC_NAMESPACE::Sign;
 using LIBC_NAMESPACE::fputil::FPBits;
 using LIBC_NAMESPACE::fputil::FPType;
 using LIBC_NAMESPACE::fputil::internal::FPRep;
diff --git a/libc/test/src/math/FDimTest.h b/libc/test/src/math/FDimTest.h
index 20c63e7d1ab59..03ff26ab23798 100644
--- a/libc/test/src/math/FDimTest.h
+++ b/libc/test/src/math/FDimTest.h
@@ -13,6 +13,8 @@
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
 
+using LIBC_NAMESPACE::Sign;
+
 template <typename T>
 class FDimTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 public:
diff --git a/libc/test/src/math/HypotTest.h b/libc/test/src/math/HypotTest.h
index 58b5338318245..fd0c1b394b8f7 100644
--- a/libc/test/src/math/HypotTest.h
+++ b/libc/test/src/math/HypotTest.h
@@ -17,6 +17,8 @@
 
 #include "hdr/math_macros.h"
 
+using LIBC_NAMESPACE::Sign;
+
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
 template <typename T>
diff --git a/libc/test/src/math/ILogbTest.h b/libc/test/src/math/ILogbTest.h
index ccb92eb552301..5d805ef7a1e22 100644
--- a/libc/test/src/math/ILogbTest.h
+++ b/libc/test/src/math/ILogbTest.h
@@ -16,6 +16,8 @@
 #include "test/UnitTest/FEnvSafeTest.h"
 #include "test/UnitTest/Test.h"
 
+using LIBC_NAMESPACE::Sign;
+
 class LlvmLibcILogbTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 public:
   template <typename T> struct ILogbFunc {
diff --git a/libc/test/src/math/LdExpTest.h b/libc/test/src/math/LdExpTest.h
index 34466a526d60f..08dce0d4d4789 100644
--- a/libc/test/src/math/LdExpTest.h
+++ b/libc/test/src/math/LdExpTest.h
@@ -19,6 +19,8 @@
 #include "hdr/math_macros.h"
 #include <stdint.h>
 
+using LIBC_NAMESPACE::Sign;
+
 template <typename T>
 class LdExpTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
   using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
diff --git a/libc/test/src/math/NextAfterTest.h b/libc/test/src/math/NextAfterTest.h
index b3b03f763992a..d97c264cbd806 100644
--- a/libc/test/src/math/NextAfterTest.h
+++ b/libc/test/src/math/NextAfterTest.h
@@ -18,6 +18,8 @@
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
 
+using LIBC_NAMESPACE::Sign;
+
 template <typename T>
 class NextAfterTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
   using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
diff --git a/libc/test/src/math/RIntTest.h b/libc/test/src/math/RIntTest.h
index d31bf743f1a37..c9d8ebcbc9e23 100644
--- a/libc/test/src/math/RIntTest.h
+++ b/libc/test/src/math/RIntTest.h
@@ -21,6 +21,7 @@
 #include "hdr/math_macros.h"
 
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+using LIBC_NAMESPACE::Sign;
 
 static constexpr int ROUNDING_MODES[4] = {FE_UPWARD, FE_DOWNWARD, FE_TOWARDZERO,
                                           FE_TONEAREST};
diff --git a/libc/test/src/math/RemQuoTest.h b/libc/test/src/math/RemQuoTest.h
index a35a6eee8c2e0..e664000fd18d1 100644
--- a/libc/test/src/math/RemQuoTest.h
+++ b/libc/test/src/math/RemQuoTest.h
@@ -18,6 +18,7 @@
 #include "utils/MPFRWrapper/MPFRUtils.h"
 
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+using LIBC_NAMESPACE::Sign;
 
 template <typename T>
 class RemQuoTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
diff --git a/libc/test/src/math/RoundToIntegerTest.h b/libc/test/src/math/RoundToIntegerTest.h
index d3e557a847953..910ff968c1a03 100644
--- a/libc/test/src/math/RoundToIntegerTest.h
+++ b/libc/test/src/math/RoundToIntegerTest.h
@@ -22,6 +22,7 @@
 #include <errno.h>
 
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+using LIBC_NAMESPACE::Sign;
 
 static constexpr int ROUNDING_MODES[4] = {FE_UPWARD, FE_DOWNWARD, FE_TOWARDZERO,
                                           FE_TONEAREST};
diff --git a/libc/test/src/math/atanhf_test.cpp b/libc/test/src/math/atanhf_test.cpp
index b0505e4c1182a..488a9a942ce91 100644
--- a/libc/test/src/math/atanhf_test.cpp
+++ b/libc/test/src/math/atanhf_test.cpp
@@ -18,6 +18,7 @@
 #include <stdint.h>
 
 using LlvmLibcAtanhfTest = LIBC_NAMESPACE::testing::FPTest<float>;
+using LIBC_NAMESPACE::Sign;
 
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
diff --git a/libc/test/src/math/smoke/AddTest.h b/libc/test/src/math/smoke/AddTest.h
index f06a0868a520f..66b188f4fa7b3 100644
--- a/libc/test/src/math/smoke/AddTest.h
+++ b/libc/test/src/math/smoke/AddTest.h
@@ -17,6 +17,8 @@
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
 
+using LIBC_NAMESPACE::Sign;
+
 template <typename OutType, typename InType>
 class AddTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 
diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt
index 9f9203c491d04..a70b65f23f4b7 100644
--- a/libc/test/src/math/smoke/CMakeLists.txt
+++ b/libc/test/src/math/smoke/CMakeLists.txt
@@ -327,6 +327,66 @@ add_fp_unittest(
     libc.src.__support.integer_literals
 )
 
+add_fp_unittest(
+  iscanonical_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    iscanonical_test.cpp
+  HDRS
+    IsCanonicalTest.h
+  DEPENDS
+    libc.src.math.iscanonical
+)
+
+add_fp_unittest(
+  iscanonicalf_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    iscanonicalf_test.cpp
+  HDRS
+    IsCanonicalTest.h
+  DEPENDS
+    libc.src.math.iscanonicalf
+)
+
+add_fp_unittest(
+  iscanonicall_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    iscanonicall_test.cpp
+  HDRS
+    IsCanonicalTest.h
+  DEPENDS
+    libc.src.math.iscanonicall
+)
+
+add_fp_unittest(
+  iscanonicalf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    iscanonicalf16_test.cpp
+  HDRS
+    IsCanonicalTest.h
+  DEPENDS
+    libc.src.math.iscanonicalf16
+)
+
+add_fp_unittest(
+  iscanonicalf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    iscanonicalf128_test.cpp
+  HDRS
+    IsCanonicalTest.h
+  DEPENDS
+    libc.src.math.iscanonicalf128
+)
+
 add_fp_unittest(
   ceil_test
   SUITE
@@ -1597,6 +1657,66 @@ add_fp_unittest(
     libc.src.__support.FPUtil.manipulation_functions
 )
 
+add_fp_unittest(
+  issignaling_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    issignaling_test.cpp
+  HDRS
+    IsSignalingTest.h
+  DEPENDS
+    libc.src.math.issignaling
+)
+
+add_fp_unittest(
+  issignalingf_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    issignalingf_test.cpp
+  HDRS
+    IsSignalingTest.h
+  DEPENDS
+    libc.src.math.issignalingf
+)
+
+add_fp_unittest(
+  issignalingl_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    issignalingl_test.cpp
+  HDRS
+    IsSignalingTest.h
+  DEPENDS
+    libc.src.math.issignalingl
+)
+
+add_fp_unittest(
+  issignalingf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    issignalingf16_test.cpp
+  HDRS
+    IsSignalingTest.h
+  DEPENDS
+    libc.src.math.issignalingf16
+)
+
+add_fp_unittest(
+  issignalingf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    issignalingf128_test.cpp
+  HDRS
+    IsSignalingTest.h
+  DEPENDS
+    libc.src.math.issignalingf128
+)
+
 add_fp_unittest(
   llogb_test
   SUITE
diff --git a/libc/test/src/math/smoke/DivTest.h b/libc/test/src/math/smoke/DivTest.h
index 60e7a8adc9eba..d8074797449e9 100644
--- a/libc/test/src/math/smoke/DivTest.h
+++ b/libc/test/src/math/smoke/DivTest.h
@@ -16,6 +16,8 @@
 #include "test/UnitTest/RoundingModeUtils.h"
 #include "test/UnitTest/Test.h"
 
+using LIBC_NAMESPACE::Sign;
+
 template <typename OutType, typename InType>
 class DivTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 
diff --git a/libc/test/src/math/smoke/FDimTest.h b/libc/test/src/math/smoke/FDimTest.h
index cc0ce3bc10c46..56cc70c985ba2 100644
--- a/libc/test/src/math/smoke/FDimTest.h
+++ b/libc/test/src/math/smoke/FDimTest.h
@@ -12,6 +12,8 @@
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
 
+using LIBC_NAMESPACE::Sign;
+
 template <typename T>
 class FDimTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 public:
diff --git a/libc/test/src/math/smoke/GetPayloadTest.h b/libc/test/src/math/smoke/GetPayloadTest.h
index d904571269e13..1b1bf4f5d56ff 100644
--- a/libc/test/src/math/smoke/GetPayloadTest.h
+++ b/libc/test/src/math/smoke/GetPayloadTest.h
@@ -13,6 +13,8 @@
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
 
+using LIBC_NAMESPACE::Sign;
+
 template <typename T>
 class GetPayloadTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 
diff --git a/libc/test/src/math/smoke/ILogbTest.h b/libc/test/src/math/smoke/ILogbTest.h
index 988f71f54bf0d..605698e1d3452 100644
--- a/libc/test/src/math/smoke/ILogbTest.h
+++ b/libc/test/src/math/smoke/ILogbTest.h
@@ -15,6 +15,8 @@
 #include "test/UnitTest/FEnvSafeTest.h"
 #include "test/UnitTest/Test.h"
 
+using LIBC_NAMESPACE::Sign;
+
 template <typename OutType, typename InType>
 class LlvmLibcILogbTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
   using FPBits = LIBC_NAMESPACE::fputil::FPBits<InType>;
diff --git a/libc/test/src/math/smoke/IsCanonicalTest.h b/libc/test/src/math/smoke/IsCanonicalTest.h
new file mode 100644
index 0000000000000..8b6554b97f035
--- /dev/null
+++ b/libc/test/src/math/smoke/IsCanonicalTest.h
@@ -0,0 +1,58 @@
+//===-- Utility class to test different flavors of iscanonical --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_ISCANONICALTEST_H
+#define LLVM_LIBC_TEST_SRC_MATH_SMOKE_ISCANONICALTEST_H
+
+#include "test/UnitTest/FEnvSafeTest.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+template <typename T>
+class IsCanonicalTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
+
+  DECLARE_SPECIAL_CONSTANTS(T)
+
+public:
+  typedef int (*IsCanonicalFunc)(T);
+
+  void testSpecialNumbers(IsCanonicalFunc func) {
+    EXPECT_EQ(func(aNaN), 1);
+    EXPECT_EQ(func(neg_aNaN), 1);
+    EXPECT_EQ(func(sNaN), 0);
+    EXPECT_EQ(func(neg_sNaN), 0);
+    EXPECT_EQ(func(inf), 1);
+    EXPECT_EQ(func(neg_inf), 1);
+    EXPECT_EQ(func(min_normal), 1);
+    EXPECT_EQ(func(max_normal), 1);
+    EXPECT_EQ(func(neg_max_normal), 1);
+    EXPECT_EQ(func(min_denormal), 1);
+    EXPECT_EQ(func(neg_min_denormal), 1);
+    EXPECT_EQ(func(max_denormal), 1);
+    EXPECT_EQ(func(zero), 1);
+    EXPECT_EQ(func(neg_zero), 1);
+  }
+
+  void testRoundedNumbers(IsCanonicalFunc func) {
+    EXPECT_EQ(func(T(1.0)), 1);
+    EXPECT_EQ(func(T(-1.0)), 1);
+    EXPECT_EQ(func(T(10.0)), 1);
+    EXPECT_EQ(func(T(-10.0)), 1);
+    EXPECT_EQ(func(T(1234.0)), 1);
+    EXPECT_EQ(func(T(-1234.0)), 1);
+  }
+};
+
+#define LIST_ISCANONICAL_TESTS(T, func)                                        \
+  using LlvmLibcIsCanonicalTest = IsCanonicalTest<T>;                          \
+  TEST_F(LlvmLibcIsCanonicalTest, SpecialNumbers) {                            \
+    testSpecialNumbers(&func);                                                 \
+  }                                                                            \
+  TEST_F(LlvmLibcIsCanonicalTest, RoundedNubmers) { testRoundedNumbers(&func); }
+
+#endif // LLVM_LIBC_TEST_SRC_MATH_SMOKE_ISCANONICALTEST_H
diff --git a/libc/test/src/math/smoke/IsSignalingTest.h b/libc/test/src/math/smoke/IsSignalingTest.h
new file mode 100644
index 0000000000000..f52642ad27bea
--- /dev/null
+++ b/libc/test/src/math/smoke/IsSignalingTest.h
@@ -0,0 +1,60 @@
+//===-- Utility class to test different flavors of issignaling --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_ISSIGNALINGTEST_H
+#define LLVM_LIBC_TEST_SRC_MATH_SMOKE_ISSIGNALINGTEST_H
+
+#include "test/UnitTest/FEnvSafeTest.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+#include "hdr/math_macros.h"
+
+template <typename T>
+class IsSignalingTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
+
+  DECLARE_SPECIAL_CONSTANTS(T)
+
+public:
+  typedef int (*IsSignalingFunc)(T);
+
+  void testSpecialNumbers(IsSignalingFunc func) {
+    EXPECT_EQ(func(aNaN), 0);
+    EXPECT_EQ(func(neg_aNaN), 0);
+    EXPECT_EQ(func(sNaN), 1);
+    EXPECT_EQ(func(neg_sNaN), 1);
+    EXPECT_EQ(func(inf), 0);
+    EXPECT_EQ(func(neg_inf), 0);
+    EXPECT_EQ(func(min_normal), 0);
+    EXPECT_EQ(func(max_normal), 0);
+    EXPECT_EQ(func(neg_max_normal), 0);
+    EXPECT_EQ(func(min_denormal), 0);
+    EXPECT_EQ(func(neg_min_denormal), 0);
+    EXPECT_EQ(func(max_denormal), 0);
+    EXPECT_EQ(func(zero), 0);
+    EXPECT_EQ(func(neg_zero), 0);
+  }
+
+  void testRoundedNumbers(IsSignalingFunc func) {
+    EXPECT_EQ(func(T(1.0)), 0);
+    EXPECT_EQ(func(T(-1.0)), 0);
+    EXPECT_EQ(func(T(10.0)), 0);
+    EXPECT_EQ(func(T(-10.0)), 0);
+    EXPECT_EQ(func(T(1234.0)), 0);
+    EXPECT_EQ(func(T(-1234.0)), 0);
+  }
+};
+
+#define LIST_ISSIGNALING_TESTS(T, func)                                        \
+  using LlvmLibcIsSignalingTest = IsSignalingTest<T>;                          \
+  TEST_F(LlvmLibcIsSignalingTest, SpecialNumbers) {                            \
+    testSpecialNumbers(&func);                                                 \
+  }                                                                            \
+  TEST_F(LlvmLibcIsSignalingTest, RoundedNubmers) { testRoundedNumbers(&func); }
+
+#endif // LLVM_LIBC_TEST_SRC_MATH_SMOKE_ISSIGNALINGTEST_H
diff --git a/libc/test/src/math/smoke/LdExpTest.h b/libc/test/src/math/smoke/LdExpTest.h
index 7739bd71e5e7a..42e4c6a3a5184 100644
--- a/libc/test/src/math/smoke/LdExpTest.h
+++ b/libc/test/src/math/smoke/LdExpTest.h
@@ -18,6 +18,8 @@
 
 #include <stdint.h>
 
+using LIBC_NAMESPACE::Sign;
+
 template <typename T, typename U = int>
 class LdExpTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
   using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
diff --git a/libc/test/src/math/smoke/MulTest.h b/libc/test/src/math/smoke/MulTest.h
index c409122397b1d..cf7f41abf5500 100644
--- a/libc/test/src/math/smoke/MulTest.h
+++ b/libc/test/src/math/smoke/MulTest.h
@@ -16,6 +16,8 @@
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
 
+using LIBC_NAMESPACE::Sign;
+
 template <typename OutType, typename InType>
 class MulTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 
diff --git a/libc/test/src/math/smoke/NearbyIntTest.h b/libc/test/src/math/smoke/NearbyIntTest.h
index 0051ff9447a7e..092700e2eef03 100644
--- a/libc/test/src/math/smoke/NearbyIntTest.h
+++ b/libc/test/src/math/smoke/NearbyIntTest.h
@@ -15,6 +15,8 @@
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
 
+using LIBC_NAMESPACE::Sign;
+
 static constexpr int ROUNDING_MODES[4] = {FE_UPWARD, FE_DOWNWARD, FE_TOWARDZERO,
                                           FE_TONEAREST};
 
diff --git a/libc/test/src/math/smoke/NextAfterTest.h b/libc/test/src/math/smoke/NextAfterTest.h
index 6278f899d8a8b..e62832e6712f3 100644
--- a/libc/test/src/math/smoke/NextAfterTest.h
+++ b/libc/test/src/math/smoke/NextAfterTest.h
@@ -18,6 +18,8 @@
 
 #include "hdr/fenv_macros.h"
 
+using LIBC_NAMESPACE::Sign;
+
 // TODO: Strengthen errno,exception checks and remove these assert macros
 // after new matchers/test fixtures are added
 #define ASSERT_FP_EQ_WITH_EXCEPTION(result, expected, expected_exception)      \
diff --git a/libc/test/src/math/smoke/NextTowardTest.h b/libc/test/src/math/smoke/NextTowardTest.h
index 61528f71305db..a4cd5d0cc8c8f 100644
--- a/libc/test/src/math/smoke/NextTowardTest.h
+++ b/libc/test/src/math/smoke/NextTowardTest.h
@@ -18,6 +18,8 @@
 
 #include "hdr/fenv_macros.h"
 
+using LIBC_NAMESPACE::Sign;
+
 // TODO: Strengthen errno,exception checks and remove these assert macros
 // after new matchers/test fixtures are added
 #define ASSERT_FP_EQ_WITH_EXCEPTION(result, expected, expected_exception)      \
diff --git a/libc/test/src/math/smoke/RIntTest.h b/libc/test/src/math/smoke/RIntTest.h
index fb2c89c4980b4..7f14aeb674d68 100644
--- a/libc/test/src/math/smoke/RIntTest.h
+++ b/libc/test/src/math/smoke/RIntTest.h
@@ -18,6 +18,8 @@
 #include "hdr/fenv_macros.h"
 #include "hdr/math_macros.h"
 
+using LIBC_NAMESPACE::Sign;
+
 static constexpr int ROUNDING_MODES[4] = {FE_UPWARD, FE_DOWNWARD, FE_TOWARDZERO,
                                           FE_TONEAREST};
 
diff --git a/libc/test/src/math/smoke/RemQuoTest.h b/libc/test/src/math/smoke/RemQuoTest.h
index e9263263dfb24..2fdb594d4f8af 100644
--- a/libc/test/src/math/smoke/RemQuoTest.h
+++ b/libc/test/src/math/smoke/RemQuoTest.h
@@ -14,6 +14,8 @@
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
 
+using LIBC_NAMESPACE::Sign;
+
 template <typename T>
 class RemQuoTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
   using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
diff --git a/libc/test/src/math/smoke/SetPayloadSigTest.h b/libc/test/src/math/smoke/SetPayloadSigTest.h
index 60913c60b481c..f480479618a08 100644
--- a/libc/test/src/math/smoke/SetPayloadSigTest.h
+++ b/libc/test/src/math/smoke/SetPayloadSigTest.h
@@ -13,6 +13,8 @@
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
 
+using LIBC_NAMESPACE::Sign;
+
 template <typename T>
 class SetPayloadSigTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 
diff --git a/libc/test/src/math/smoke/SetPayloadTest.h b/libc/test/src/math/smoke/SetPayloadTest.h
index 128d609f0632a..9ede5678fef1b 100644
--- a/libc/test/src/math/smoke/SetPayloadTest.h
+++ b/libc/test/src/math/smoke/SetPayloadTest.h
@@ -13,6 +13,8 @@
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
 
+using LIBC_NAMESPACE::Sign;
+
 template <typename T>
 class SetPayloadTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 
diff --git a/libc/test/src/math/smoke/SubTest.h b/libc/test/src/math/smoke/SubTest.h
index 8793b9f157f72..ca952009c87c6 100644
--- a/libc/test/src/math/smoke/SubTest.h
+++ b/libc/test/src/math/smoke/SubTest.h
@@ -16,6 +16,8 @@
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
 
+using LIBC_NAMESPACE::Sign;
+
 template <typename OutType, typename InType>
 class SubTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 
diff --git a/libc/test/src/math/smoke/TotalOrderMagTest.h b/libc/test/src/math/smoke/TotalOrderMagTest.h
index 3d7f24fc74d10..0a13fd2922e4c 100644
--- a/libc/test/src/math/smoke/TotalOrderMagTest.h
+++ b/libc/test/src/math/smoke/TotalOrderMagTest.h
@@ -13,6 +13,8 @@
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
 
+using LIBC_NAMESPACE::Sign;
+
 template <typename T>
 class TotalOrderMagTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 
diff --git a/libc/test/src/math/smoke/TotalOrderTest.h b/libc/test/src/math/smoke/TotalOrderTest.h
index 4d4257d089daf..e426eb35016b9 100644
--- a/libc/test/src/math/smoke/TotalOrderTest.h
+++ b/libc/test/src/math/smoke/TotalOrderTest.h
@@ -13,6 +13,8 @@
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
 
+using LIBC_NAMESPACE::Sign;
+
 template <typename T>
 class TotalOrderTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 
diff --git a/libc/test/src/math/smoke/atanhf_test.cpp b/libc/test/src/math/smoke/atanhf_test.cpp
index 2d2acfeeab4e5..fa07b62e9f43f 100644
--- a/libc/test/src/math/smoke/atanhf_test.cpp
+++ b/libc/test/src/math/smoke/atanhf_test.cpp
@@ -16,6 +16,8 @@
 #include <errno.h>
 #include <stdint.h>
 
+using LIBC_NAMESPACE::Sign;
+
 using LlvmLibcAtanhfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 TEST_F(LlvmLibcAtanhfTest, SpecialNumbers) {
diff --git a/libc/test/src/math/smoke/iscanonical_test.cpp b/libc/test/src/math/smoke/iscanonical_test.cpp
new file mode 100644
index 0000000000000..8e8b3e4308f88
--- /dev/null
+++ b/libc/test/src/math/smoke/iscanonical_test.cpp
@@ -0,0 +1,17 @@
+//===-- Unittests for iscanonical -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "IsCanonicalTest.h"
+
+// The testing framework might include math.h and iscanonical macro definition
+// in overlay mode.
+#undef iscanonical
+
+#include "src/math/iscanonical.h"
+
+LIST_ISCANONICAL_TESTS(double, LIBC_NAMESPACE::iscanonical)
diff --git a/libc/test/src/math/smoke/iscanonicalf128_test.cpp b/libc/test/src/math/smoke/iscanonicalf128_test.cpp
new file mode 100644
index 0000000000000..0176ae2f223da
--- /dev/null
+++ b/libc/test/src/math/smoke/iscanonicalf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for iscanonicalf128 -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "IsCanonicalTest.h"
+
+#include "src/math/iscanonicalf128.h"
+
+LIST_ISCANONICAL_TESTS(float128, LIBC_NAMESPACE::iscanonicalf128)
diff --git a/libc/test/src/math/smoke/iscanonicalf16_test.cpp b/libc/test/src/math/smoke/iscanonicalf16_test.cpp
new file mode 100644
index 0000000000000..d17d79f86139e
--- /dev/null
+++ b/libc/test/src/math/smoke/iscanonicalf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for iscanonicalf16 --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "IsCanonicalTest.h"
+
+#include "src/math/iscanonicalf16.h"
+
+LIST_ISCANONICAL_TESTS(float16, LIBC_NAMESPACE::iscanonicalf16)
diff --git a/libc/test/src/math/smoke/iscanonicalf_test.cpp b/libc/test/src/math/smoke/iscanonicalf_test.cpp
new file mode 100644
index 0000000000000..eba8688f61f94
--- /dev/null
+++ b/libc/test/src/math/smoke/iscanonicalf_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for iscanonicalf ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "IsCanonicalTest.h"
+
+#include "src/math/iscanonicalf.h"
+
+LIST_ISCANONICAL_TESTS(float, LIBC_NAMESPACE::iscanonicalf)
diff --git a/libc/test/src/math/smoke/iscanonicall_test.cpp b/libc/test/src/math/smoke/iscanonicall_test.cpp
new file mode 100644
index 0000000000000..d180bcb7443e5
--- /dev/null
+++ b/libc/test/src/math/smoke/iscanonicall_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for iscanonicall ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "IsCanonicalTest.h"
+
+#include "src/math/iscanonicall.h"
+
+LIST_ISCANONICAL_TESTS(long double, LIBC_NAMESPACE::iscanonicall)
diff --git a/libc/test/src/math/smoke/issignaling_test.cpp b/libc/test/src/math/smoke/issignaling_test.cpp
new file mode 100644
index 0000000000000..4292cbbb852b4
--- /dev/null
+++ b/libc/test/src/math/smoke/issignaling_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for issignaling -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "IsSignalingTest.h"
+
+#include "src/math/issignaling.h"
+
+LIST_ISSIGNALING_TESTS(double, LIBC_NAMESPACE::issignaling)
diff --git a/libc/test/src/math/smoke/issignalingf128_test.cpp b/libc/test/src/math/smoke/issignalingf128_test.cpp
new file mode 100644
index 0000000000000..929e32a8facdd
--- /dev/null
+++ b/libc/test/src/math/smoke/issignalingf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for issignalingf128 -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "IsSignalingTest.h"
+
+#include "src/math/issignalingf128.h"
+
+LIST_ISSIGNALING_TESTS(float128, LIBC_NAMESPACE::issignalingf128)
diff --git a/libc/test/src/math/smoke/issignalingf16_test.cpp b/libc/test/src/math/smoke/issignalingf16_test.cpp
new file mode 100644
index 0000000000000..fc2a65a329183
--- /dev/null
+++ b/libc/test/src/math/smoke/issignalingf16_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for issignalingf16 --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "IsSignalingTest.h"
+
+#include "src/math/issignalingf16.h"
+
+LIST_ISSIGNALING_TESTS(float16, LIBC_NAMESPACE::issignalingf16)
diff --git a/libc/test/src/math/smoke/issignalingf_test.cpp b/libc/test/src/math/smoke/issignalingf_test.cpp
new file mode 100644
index 0000000000000..0ceb9a80c0310
--- /dev/null
+++ b/libc/test/src/math/smoke/issignalingf_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for issignalingf ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "IsSignalingTest.h"
+
+#include "src/math/issignalingf.h"
+
+LIST_ISSIGNALING_TESTS(float, LIBC_NAMESPACE::issignalingf)
diff --git a/libc/test/src/math/smoke/issignalingl_test.cpp b/libc/test/src/math/smoke/issignalingl_test.cpp
new file mode 100644
index 0000000000000..049769009ba7c
--- /dev/null
+++ b/libc/test/src/math/smoke/issignalingl_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for issignalingl ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "IsSignalingTest.h"
+
+#include "src/math/issignalingl.h"
+
+LIST_ISSIGNALING_TESTS(long double, LIBC_NAMESPACE::issignalingl)
diff --git a/libc/test/src/stdio/CMakeLists.txt b/libc/test/src/stdio/CMakeLists.txt
index e970e4cf01405..ec94f5aaaf9b6 100644
--- a/libc/test/src/stdio/CMakeLists.txt
+++ b/libc/test/src/stdio/CMakeLists.txt
@@ -116,6 +116,9 @@ add_libc_test(
 if(LIBC_CONF_PRINTF_DISABLE_FLOAT)
   list(APPEND sprintf_test_copts "-DLIBC_COPT_PRINTF_DISABLE_FLOAT")
 endif()
+if(LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_DYADIC_FLOAT)
+  list(APPEND sprintf_test_copts "-DLIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION")
+endif()
 if(LIBC_CONF_PRINTF_DISABLE_INDEX_MODE)
   list(APPEND sprintf_test_copts "-DLIBC_COPT_PRINTF_DISABLE_INDEX_MODE")
 endif()
diff --git a/libc/test/src/stdio/sprintf_test.cpp b/libc/test/src/stdio/sprintf_test.cpp
index 09a175104d2fe..e8303ff3416b2 100644
--- a/libc/test/src/stdio/sprintf_test.cpp
+++ b/libc/test/src/stdio/sprintf_test.cpp
@@ -56,8 +56,7 @@ TEST(LlvmLibcSPrintfTest, SimpleNoConv) {
 
   written =
       LIBC_NAMESPACE::sprintf(buff, "A simple string with no conversions.");
-  EXPECT_EQ(written, 36);
-  ASSERT_STREQ(buff, "A simple string with no conversions.");
+  ASSERT_STREQ_LEN(written, buff, "A simple string with no conversions.");
 }
 
 TEST(LlvmLibcSPrintfTest, PercentConv) {
@@ -65,16 +64,13 @@ TEST(LlvmLibcSPrintfTest, PercentConv) {
   int written;
 
   written = LIBC_NAMESPACE::sprintf(buff, "%%");
-  EXPECT_EQ(written, 1);
-  ASSERT_STREQ(buff, "%");
+  ASSERT_STREQ_LEN(written, buff, "%");
 
   written = LIBC_NAMESPACE::sprintf(buff, "abc %% def");
-  EXPECT_EQ(written, 9);
-  ASSERT_STREQ(buff, "abc % def");
+  ASSERT_STREQ_LEN(written, buff, "abc % def");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%%%%%%");
-  EXPECT_EQ(written, 3);
-  ASSERT_STREQ(buff, "%%%");
+  ASSERT_STREQ_LEN(written, buff, "%%%");
 }
 
 TEST(LlvmLibcSPrintfTest, CharConv) {
@@ -82,16 +78,13 @@ TEST(LlvmLibcSPrintfTest, CharConv) {
   int written;
 
   written = LIBC_NAMESPACE::sprintf(buff, "%c", 'a');
-  EXPECT_EQ(written, 1);
-  ASSERT_STREQ(buff, "a");
+  ASSERT_STREQ_LEN(written, buff, "a");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%3c %-3c", '1', '2');
-  EXPECT_EQ(written, 7);
-  ASSERT_STREQ(buff, "  1 2  ");
+  ASSERT_STREQ_LEN(written, buff, "  1 2  ");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%*c", 2, '3');
-  EXPECT_EQ(written, 2);
-  ASSERT_STREQ(buff, " 3");
+  ASSERT_STREQ_LEN(written, buff, " 3");
 }
 
 TEST(LlvmLibcSPrintfTest, StringConv) {
@@ -99,27 +92,22 @@ TEST(LlvmLibcSPrintfTest, StringConv) {
   int written;
 
   written = LIBC_NAMESPACE::sprintf(buff, "%s", "abcDEF123");
-  EXPECT_EQ(written, 9);
-  ASSERT_STREQ(buff, "abcDEF123");
+  ASSERT_STREQ_LEN(written, buff, "abcDEF123");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%10s %-10s", "centered", "title");
-  EXPECT_EQ(written, 21);
-  ASSERT_STREQ(buff, "  centered title     ");
+  ASSERT_STREQ_LEN(written, buff, "  centered title     ");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%-5.4s%-4.4s", "words can describe",
                                     "soups most delicious");
-  EXPECT_EQ(written, 9);
-  ASSERT_STREQ(buff, "word soup");
+  ASSERT_STREQ_LEN(written, buff, "word soup");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%*s %.*s %*.*s", 10, "beginning", 2,
                                     "isn't", 12, 10, "important. Ever.");
-  EXPECT_EQ(written, 26);
-  ASSERT_STREQ(buff, " beginning is   important.");
+  ASSERT_STREQ_LEN(written, buff, " beginning is   important.");
 
 #ifndef LIBC_COPT_PRINTF_NO_NULLPTR_CHECKS
   written = LIBC_NAMESPACE::sprintf(buff, "%s", nullptr);
-  EXPECT_EQ(written, 6);
-  ASSERT_STREQ(buff, "(null)");
+  ASSERT_STREQ_LEN(written, buff, "(null)");
 #endif // LIBC_COPT_PRINTF_NO_NULLPTR_CHECKS
 }
 
@@ -130,73 +118,57 @@ TEST(LlvmLibcSPrintfTest, IntConv) {
   // Basic Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%d", 123);
-  EXPECT_EQ(written, 3);
-  ASSERT_STREQ(buff, "123");
+  ASSERT_STREQ_LEN(written, buff, "123");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%i", -456);
-  EXPECT_EQ(written, 4);
-  ASSERT_STREQ(buff, "-456");
+  ASSERT_STREQ_LEN(written, buff, "-456");
 
   // Length Modifier Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%hhu", 257); // 0x101
-  EXPECT_EQ(written, 1);
-  ASSERT_STREQ(buff, "1");
+  ASSERT_STREQ_LEN(written, buff, "1");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%llu", 18446744073709551615ull);
-  EXPECT_EQ(written, 20);
-  ASSERT_STREQ(buff, "18446744073709551615"); // ull max
+  ASSERT_STREQ_LEN(written, buff, "18446744073709551615"); // ull max
 
   written = LIBC_NAMESPACE::sprintf(buff, "%u", ~0);
   if (sizeof(int) == 4) {
-    EXPECT_EQ(written, 10);
-    ASSERT_STREQ(buff, "4294967295");
+    ASSERT_STREQ_LEN(written, buff, "4294967295");
   }
 
   written = LIBC_NAMESPACE::sprintf(buff, "%tu", ~ptrdiff_t(0));
   if (sizeof(ptrdiff_t) == 8) {
-    EXPECT_EQ(written, 20);
-    ASSERT_STREQ(buff, "18446744073709551615");
+    ASSERT_STREQ_LEN(written, buff, "18446744073709551615");
   } else if (sizeof(ptrdiff_t) == 4) {
-    EXPECT_EQ(written, 10);
-    ASSERT_STREQ(buff, "4294967295");
+    ASSERT_STREQ_LEN(written, buff, "4294967295");
   }
 
   written = LIBC_NAMESPACE::sprintf(buff, "%lld", -9223372036854775807ll - 1ll);
-  EXPECT_EQ(written, 20);
-  ASSERT_STREQ(buff, "-9223372036854775808"); // ll min
+  ASSERT_STREQ_LEN(written, buff, "-9223372036854775808"); // ll min
 
   written = LIBC_NAMESPACE::sprintf(buff, "%w3d", 5807);
-  EXPECT_EQ(written, 1);
-  ASSERT_STREQ(buff, "7");
+  ASSERT_STREQ_LEN(written, buff, "7");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%w3d", 1);
-  EXPECT_EQ(written, 1);
-  ASSERT_STREQ(buff, "1");
+  ASSERT_STREQ_LEN(written, buff, "1");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%w64d", 9223372036854775807ll);
-  EXPECT_EQ(written, 19);
-  ASSERT_STREQ(buff, "9223372036854775807");
+  ASSERT_STREQ_LEN(written, buff, "9223372036854775807");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%w-1d", 5807);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "%w-1d");
+  ASSERT_STREQ_LEN(written, buff, "%w-1d");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%w0d", 5807);
-  EXPECT_EQ(written, 4);
-  ASSERT_STREQ(buff, "%w0d");
+  ASSERT_STREQ_LEN(written, buff, "%w0d");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%w999d", 9223372036854775807ll);
-  EXPECT_EQ(written, 19);
-  ASSERT_STREQ(buff, "9223372036854775807");
+  ASSERT_STREQ_LEN(written, buff, "9223372036854775807");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%winvalid%w1d", 5807, 5807);
-  EXPECT_EQ(written, 10);
-  ASSERT_STREQ(buff, "%winvalid1");
+  ASSERT_STREQ_LEN(written, buff, "%winvalid1");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%w-1d%w1d", 5807, 5807);
-  EXPECT_EQ(written, 6);
-  ASSERT_STREQ(buff, "%w-1d1");
+  ASSERT_STREQ_LEN(written, buff, "%w-1d1");
 
   char format[64];
   char uintmax[128];
@@ -208,157 +180,122 @@ TEST(LlvmLibcSPrintfTest, IntConv) {
   ASSERT_STREQ(buff, uintmax);
 
   written = LIBC_NAMESPACE::sprintf(buff, "%w64u", 18446744073709551615ull);
-  EXPECT_EQ(written, 20);
-  ASSERT_STREQ(buff, "18446744073709551615"); // ull max
+  ASSERT_STREQ_LEN(written, buff, "18446744073709551615"); // ull max
 
   written =
       LIBC_NAMESPACE::sprintf(buff, "%w64d", -9223372036854775807ll - 1ll);
-  EXPECT_EQ(written, 20);
-  ASSERT_STREQ(buff, "-9223372036854775808"); // ll min
+  ASSERT_STREQ_LEN(written, buff, "-9223372036854775808"); // ll min
 
   written = LIBC_NAMESPACE::sprintf(buff, "%wf3d", 5807);
-  EXPECT_EQ(written, 1);
-  ASSERT_STREQ(buff, "7");
+  ASSERT_STREQ_LEN(written, buff, "7");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%wf3d", 1);
-  EXPECT_EQ(written, 1);
-  ASSERT_STREQ(buff, "1");
+  ASSERT_STREQ_LEN(written, buff, "1");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%wf64u", 18446744073709551615ull);
-  EXPECT_EQ(written, 20);
-  ASSERT_STREQ(buff, "18446744073709551615"); // ull max
+  ASSERT_STREQ_LEN(written, buff, "18446744073709551615"); // ull max
 
   written =
       LIBC_NAMESPACE::sprintf(buff, "%wf64d", -9223372036854775807ll - 1ll);
-  EXPECT_EQ(written, 20);
-  ASSERT_STREQ(buff, "-9223372036854775808"); // ll min
+  ASSERT_STREQ_LEN(written, buff, "-9223372036854775808"); // ll min
 
   written = LIBC_NAMESPACE::sprintf(buff, "%wf0d", 5807);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "%wf0d");
+  ASSERT_STREQ_LEN(written, buff, "%wf0d");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%wf-1d", 5807);
-  EXPECT_EQ(written, 6);
-  ASSERT_STREQ(buff, "%wf-1d");
+  ASSERT_STREQ_LEN(written, buff, "%wf-1d");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%wfinvalid%wf1d", 5807, 5807);
-  EXPECT_EQ(written, 11);
-  ASSERT_STREQ(buff, "%wfinvalid1");
+  ASSERT_STREQ_LEN(written, buff, "%wfinvalid1");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%wf-1d%wf1d", 5807, 5807);
-  EXPECT_EQ(written, 7);
-  ASSERT_STREQ(buff, "%wf-1d1");
+  ASSERT_STREQ_LEN(written, buff, "%wf-1d1");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%wf999d", 9223372036854775807ll);
-  EXPECT_EQ(written, 19);
-  ASSERT_STREQ(buff, "9223372036854775807");
+  ASSERT_STREQ_LEN(written, buff, "9223372036854775807");
 
   // Min Width Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%4d", 789);
-  EXPECT_EQ(written, 4);
-  ASSERT_STREQ(buff, " 789");
+  ASSERT_STREQ_LEN(written, buff, " 789");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%2d", 987);
-  EXPECT_EQ(written, 3);
-  ASSERT_STREQ(buff, "987");
+  ASSERT_STREQ_LEN(written, buff, "987");
 
   // Precision Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%d", 0);
-  EXPECT_EQ(written, 1);
-  ASSERT_STREQ(buff, "0");
+  ASSERT_STREQ_LEN(written, buff, "0");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.0d", 0);
-  EXPECT_EQ(written, 0);
-  ASSERT_STREQ(buff, "");
+  ASSERT_STREQ_LEN(written, buff, "");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.5d", 654);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "00654");
+  ASSERT_STREQ_LEN(written, buff, "00654");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.5d", -321);
-  EXPECT_EQ(written, 6);
-  ASSERT_STREQ(buff, "-00321");
+  ASSERT_STREQ_LEN(written, buff, "-00321");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.2d", 135);
-  EXPECT_EQ(written, 3);
-  ASSERT_STREQ(buff, "135");
+  ASSERT_STREQ_LEN(written, buff, "135");
 
   // Flag Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.5d", -321);
-  EXPECT_EQ(written, 6);
-  ASSERT_STREQ(buff, "-00321");
+  ASSERT_STREQ_LEN(written, buff, "-00321");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%-5d", 246);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "246  ");
+  ASSERT_STREQ_LEN(written, buff, "246  ");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%-5d", -147);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "-147 ");
+  ASSERT_STREQ_LEN(written, buff, "-147 ");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%+d", 258);
-  EXPECT_EQ(written, 4);
-  ASSERT_STREQ(buff, "+258");
+  ASSERT_STREQ_LEN(written, buff, "+258");
 
   written = LIBC_NAMESPACE::sprintf(buff, "% d", 369);
-  EXPECT_EQ(written, 4);
-  ASSERT_STREQ(buff, " 369");
+  ASSERT_STREQ_LEN(written, buff, " 369");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%05d", 470);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "00470");
+  ASSERT_STREQ_LEN(written, buff, "00470");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%05d", -581);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "-0581");
+  ASSERT_STREQ_LEN(written, buff, "-0581");
 
   // Combined Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%+ u", 692);
-  EXPECT_EQ(written, 3);
-  ASSERT_STREQ(buff, "692");
+  ASSERT_STREQ_LEN(written, buff, "692");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%+ -05d", 703);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "+703 ");
+  ASSERT_STREQ_LEN(written, buff, "+703 ");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%7.5d", 814);
-  EXPECT_EQ(written, 7);
-  ASSERT_STREQ(buff, "  00814");
+  ASSERT_STREQ_LEN(written, buff, "  00814");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%7.5d", -925);
-  EXPECT_EQ(written, 7);
-  ASSERT_STREQ(buff, " -00925");
+  ASSERT_STREQ_LEN(written, buff, " -00925");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%7.5d", 159);
-  EXPECT_EQ(written, 7);
-  ASSERT_STREQ(buff, "  00159");
+  ASSERT_STREQ_LEN(written, buff, "  00159");
 
   written = LIBC_NAMESPACE::sprintf(buff, "% -7.5d", 260);
-  EXPECT_EQ(written, 7);
-  ASSERT_STREQ(buff, " 00260 ");
+  ASSERT_STREQ_LEN(written, buff, " 00260 ");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%5.4d", 10000);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "10000");
+  ASSERT_STREQ_LEN(written, buff, "10000");
 
   // Multiple Conversion Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%10d %-10d", 456, -789);
-  EXPECT_EQ(written, 21);
-  ASSERT_STREQ(buff, "       456 -789      ");
+  ASSERT_STREQ_LEN(written, buff, "       456 -789      ");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%-5.4d%+.4u", 75, 25);
-  EXPECT_EQ(written, 9);
-  ASSERT_STREQ(buff, "0075 0025");
+  ASSERT_STREQ_LEN(written, buff, "0075 0025");
 
   written = LIBC_NAMESPACE::sprintf(buff, "% 05hhi %+-0.5llu %-+ 06.3zd",
                                     256 + 127, 68719476736ll, size_t(2));
-  EXPECT_EQ(written, 24);
-  ASSERT_STREQ(buff, " 0127 68719476736 +002  ");
+  ASSERT_STREQ_LEN(written, buff, " 0127 68719476736 +002  ");
 }
 
 TEST(LlvmLibcSPrintfTest, HexConv) {
@@ -368,130 +305,102 @@ TEST(LlvmLibcSPrintfTest, HexConv) {
   // Basic Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%x", 0x123a);
-  EXPECT_EQ(written, 4);
-  ASSERT_STREQ(buff, "123a");
+  ASSERT_STREQ_LEN(written, buff, "123a");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%X", 0x456b);
-  EXPECT_EQ(written, 4);
-  ASSERT_STREQ(buff, "456B");
+  ASSERT_STREQ_LEN(written, buff, "456B");
 
   // Length Modifier Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%hhx", 0x10001);
-  EXPECT_EQ(written, 1);
-  ASSERT_STREQ(buff, "1");
+  ASSERT_STREQ_LEN(written, buff, "1");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%llx", 0xffffffffffffffffull);
-  EXPECT_EQ(written, 16);
-  ASSERT_STREQ(buff, "ffffffffffffffff"); // ull max
+  ASSERT_STREQ_LEN(written, buff, "ffffffffffffffff"); // ull max
 
   written = LIBC_NAMESPACE::sprintf(buff, "%tX", ~ptrdiff_t(0));
   if (sizeof(ptrdiff_t) == 8) {
-    EXPECT_EQ(written, 16);
-    ASSERT_STREQ(buff, "FFFFFFFFFFFFFFFF");
+    ASSERT_STREQ_LEN(written, buff, "FFFFFFFFFFFFFFFF");
   } else if (sizeof(ptrdiff_t) == 4) {
-    EXPECT_EQ(written, 8);
-    ASSERT_STREQ(buff, "FFFFFFFF");
+    ASSERT_STREQ_LEN(written, buff, "FFFFFFFF");
   }
 
   // Min Width Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%4x", 0x789);
-  EXPECT_EQ(written, 4);
-  ASSERT_STREQ(buff, " 789");
+  ASSERT_STREQ_LEN(written, buff, " 789");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%2X", 0x987);
-  EXPECT_EQ(written, 3);
-  ASSERT_STREQ(buff, "987");
+  ASSERT_STREQ_LEN(written, buff, "987");
 
   // Precision Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%x", 0);
-  EXPECT_EQ(written, 1);
-  ASSERT_STREQ(buff, "0");
+  ASSERT_STREQ_LEN(written, buff, "0");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.0x", 0);
-  EXPECT_EQ(written, 0);
-  ASSERT_STREQ(buff, "");
+  ASSERT_STREQ_LEN(written, buff, "");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.5x", 0x1F3);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "001f3");
+  ASSERT_STREQ_LEN(written, buff, "001f3");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.2x", 0x135);
-  EXPECT_EQ(written, 3);
-  ASSERT_STREQ(buff, "135");
+  ASSERT_STREQ_LEN(written, buff, "135");
 
   // Flag Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%-5x", 0x246);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "246  ");
+  ASSERT_STREQ_LEN(written, buff, "246  ");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%#x", 0xd3f);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "0xd3f");
+  ASSERT_STREQ_LEN(written, buff, "0xd3f");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%#x", 0);
-  EXPECT_EQ(written, 1);
-  ASSERT_STREQ(buff, "0");
+  ASSERT_STREQ_LEN(written, buff, "0");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%#X", 0xE40);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "0XE40");
+  ASSERT_STREQ_LEN(written, buff, "0XE40");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%05x", 0x470);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "00470");
+  ASSERT_STREQ_LEN(written, buff, "00470");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%0#6x", 0x8c3);
-  EXPECT_EQ(written, 6);
-  ASSERT_STREQ(buff, "0x08c3");
+  ASSERT_STREQ_LEN(written, buff, "0x08c3");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%-#6x", 0x5f0);
-  EXPECT_EQ(written, 6);
-  ASSERT_STREQ(buff, "0x5f0 ");
+  ASSERT_STREQ_LEN(written, buff, "0x5f0 ");
 
   // Combined Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%#-07x", 0x703);
-  EXPECT_EQ(written, 7);
-  ASSERT_STREQ(buff, "0x703  ");
+  ASSERT_STREQ_LEN(written, buff, "0x703  ");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%7.5x", 0x814);
-  EXPECT_EQ(written, 7);
-  ASSERT_STREQ(buff, "  00814");
+  ASSERT_STREQ_LEN(written, buff, "  00814");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%#9.5X", 0x9d4);
-  EXPECT_EQ(written, 9);
-  ASSERT_STREQ(buff, "  0X009D4");
+  ASSERT_STREQ_LEN(written, buff, "  0X009D4");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%#.x", 0);
-  EXPECT_EQ(written, 0);
-  ASSERT_STREQ(buff, "");
+  ASSERT_STREQ_LEN(written, buff, "");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%-7.5x", 0x260);
-  EXPECT_EQ(written, 7);
-  ASSERT_STREQ(buff, "00260  ");
+  ASSERT_STREQ_LEN(written, buff, "00260  ");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%5.4x", 0x10000);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "10000");
+  ASSERT_STREQ_LEN(written, buff, "10000");
 
   // Multiple Conversion Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%10X %-#10x", 0x45b, 0x789);
-  EXPECT_EQ(written, 21);
-  ASSERT_STREQ(buff, "       45B 0x789     ");
+  ASSERT_STREQ_LEN(written, buff, "       45B 0x789     ");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%-5.4x%#.4x", 0x75, 0x25);
-  EXPECT_EQ(written, 11);
-  ASSERT_STREQ(buff, "0075 0x0025");
+  ASSERT_STREQ_LEN(written, buff, "0075 0x0025");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%04hhX %#.5llx %-6.3zX", 256 + 0x7f,
                                     0x1000000000ll, size_t(2));
-  EXPECT_EQ(written, 24);
-  ASSERT_STREQ(buff, "007F 0x1000000000 002   ");
+  ASSERT_STREQ_LEN(written, buff, "007F 0x1000000000 002   ");
 }
 
 TEST(LlvmLibcSPrintfTest, BinConv) {
@@ -501,110 +410,86 @@ TEST(LlvmLibcSPrintfTest, BinConv) {
   // Basic Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%b", 42);
-  EXPECT_EQ(written, 6);
-  ASSERT_STREQ(buff, "101010");
+  ASSERT_STREQ_LEN(written, buff, "101010");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%B", 12081991);
-  EXPECT_EQ(written, 24);
-  ASSERT_STREQ(buff, "101110000101101101000111");
+  ASSERT_STREQ_LEN(written, buff, "101110000101101101000111");
 
   // Min Width Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%10b", 0b101010);
-  EXPECT_EQ(written, 10);
-  ASSERT_STREQ(buff, "    101010");
+  ASSERT_STREQ_LEN(written, buff, "    101010");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%2B", 0b101010);
-  EXPECT_EQ(written, 6);
-  ASSERT_STREQ(buff, "101010");
+  ASSERT_STREQ_LEN(written, buff, "101010");
 
   // Precision Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%b", 0);
-  EXPECT_EQ(written, 1);
-  ASSERT_STREQ(buff, "0");
+  ASSERT_STREQ_LEN(written, buff, "0");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.0b", 0);
-  EXPECT_EQ(written, 0);
-  ASSERT_STREQ(buff, "");
+  ASSERT_STREQ_LEN(written, buff, "");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.5b", 0b111);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "00111");
+  ASSERT_STREQ_LEN(written, buff, "00111");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.2b", 0b111);
-  EXPECT_EQ(written, 3);
-  ASSERT_STREQ(buff, "111");
+  ASSERT_STREQ_LEN(written, buff, "111");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%3b", 0b111);
-  EXPECT_EQ(written, 3);
-  ASSERT_STREQ(buff, "111");
+  ASSERT_STREQ_LEN(written, buff, "111");
 
   // Flag Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%-5b", 0b111);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "111  ");
+  ASSERT_STREQ_LEN(written, buff, "111  ");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%#b", 0b111);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "0b111");
+  ASSERT_STREQ_LEN(written, buff, "0b111");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%#b", 0);
-  EXPECT_EQ(written, 1);
-  ASSERT_STREQ(buff, "0");
+  ASSERT_STREQ_LEN(written, buff, "0");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%#B", 0b111);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "0B111");
+  ASSERT_STREQ_LEN(written, buff, "0B111");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%05b", 0b111);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "00111");
+  ASSERT_STREQ_LEN(written, buff, "00111");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%0#6b", 0b111);
-  EXPECT_EQ(written, 6);
-  ASSERT_STREQ(buff, "0b0111");
+  ASSERT_STREQ_LEN(written, buff, "0b0111");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%-#6b", 0b111);
-  EXPECT_EQ(written, 6);
-  ASSERT_STREQ(buff, "0b111 ");
+  ASSERT_STREQ_LEN(written, buff, "0b111 ");
 
   // Combined Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%#-07b", 0b111);
-  EXPECT_EQ(written, 7);
-  ASSERT_STREQ(buff, "0b111  ");
+  ASSERT_STREQ_LEN(written, buff, "0b111  ");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%7.5b", 0b111);
-  EXPECT_EQ(written, 7);
-  ASSERT_STREQ(buff, "  00111");
+  ASSERT_STREQ_LEN(written, buff, "  00111");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%#9.5B", 0b111);
-  EXPECT_EQ(written, 9);
-  ASSERT_STREQ(buff, "  0B00111");
+  ASSERT_STREQ_LEN(written, buff, "  0B00111");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%#.b", 0);
-  EXPECT_EQ(written, 0);
-  ASSERT_STREQ(buff, "");
+  ASSERT_STREQ_LEN(written, buff, "");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%-7.5b", 0b111);
-  EXPECT_EQ(written, 7);
-  ASSERT_STREQ(buff, "00111  ");
+  ASSERT_STREQ_LEN(written, buff, "00111  ");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%5.4b", 0b1111);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, " 1111");
+  ASSERT_STREQ_LEN(written, buff, " 1111");
 
   // Multiple Conversion Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%10B %-#10b", 0b101, 0b110);
-  EXPECT_EQ(written, 21);
-  ASSERT_STREQ(buff, "       101 0b110     ");
+  ASSERT_STREQ_LEN(written, buff, "       101 0b110     ");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%-5.4b%#.4b", 0b101, 0b110);
-  EXPECT_EQ(written, 11);
-  ASSERT_STREQ(buff, "0101 0b0110");
+  ASSERT_STREQ_LEN(written, buff, "0101 0b0110");
 }
 
 TEST(LlvmLibcSPrintfTest, PointerConv) {
@@ -612,17 +497,14 @@ TEST(LlvmLibcSPrintfTest, PointerConv) {
   int written;
 
   written = LIBC_NAMESPACE::sprintf(buff, "%p", nullptr);
-  EXPECT_EQ(written, 9);
-  ASSERT_STREQ(buff, "(nullptr)");
+  ASSERT_STREQ_LEN(written, buff, "(nullptr)");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%p", 0x1a2b3c4d);
-  EXPECT_EQ(written, 10);
-  ASSERT_STREQ(buff, "0x1a2b3c4d");
+  ASSERT_STREQ_LEN(written, buff, "0x1a2b3c4d");
 
   if constexpr (sizeof(void *) > 4) {
     written = LIBC_NAMESPACE::sprintf(buff, "%p", 0x1a2b3c4d5e6f7081);
-    EXPECT_EQ(written, 18);
-    ASSERT_STREQ(buff, "0x1a2b3c4d5e6f7081");
+    ASSERT_STREQ_LEN(written, buff, "0x1a2b3c4d5e6f7081");
   }
 
   written = LIBC_NAMESPACE::sprintf(buff, "%p", &written);
@@ -631,42 +513,35 @@ TEST(LlvmLibcSPrintfTest, PointerConv) {
   // Width tests:
 
   written = LIBC_NAMESPACE::sprintf(buff, "%20p", nullptr);
-  EXPECT_EQ(written, 20);
-  ASSERT_STREQ(buff, "           (nullptr)");
+  ASSERT_STREQ_LEN(written, buff, "           (nullptr)");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%20p", 0x1a2b3c4d);
-  EXPECT_EQ(written, 20);
-  ASSERT_STREQ(buff, "          0x1a2b3c4d");
+  ASSERT_STREQ_LEN(written, buff, "          0x1a2b3c4d");
 
   // Flag tests:
 
   written = LIBC_NAMESPACE::sprintf(buff, "%-20p", nullptr);
-  EXPECT_EQ(written, 20);
-  ASSERT_STREQ(buff, "(nullptr)           ");
+  ASSERT_STREQ_LEN(written, buff, "(nullptr)           ");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%-20p", 0x1a2b3c4d);
-  EXPECT_EQ(written, 20);
-  ASSERT_STREQ(buff, "0x1a2b3c4d          ");
+  ASSERT_STREQ_LEN(written, buff, "0x1a2b3c4d          ");
 
   // Using the 0 flag is technically undefined, but here we're following the
   // convention of matching the behavior of %#x.
   written = LIBC_NAMESPACE::sprintf(buff, "%020p", 0x1a2b3c4d);
-  EXPECT_EQ(written, 20);
-  ASSERT_STREQ(buff, "0x00000000001a2b3c4d");
+  ASSERT_STREQ_LEN(written, buff, "0x00000000001a2b3c4d");
 
   // Precision tests:
   // These are all undefined behavior. The precision option is undefined for %p.
 
   // Precision specifies the number of characters for a string conversion.
   written = LIBC_NAMESPACE::sprintf(buff, "%.5p", nullptr);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "(null");
+  ASSERT_STREQ_LEN(written, buff, "(null");
 
   // Precision specifies the number of digits to be written for %x conversions,
   // and the "0x" doesn't count as part of the digits.
   written = LIBC_NAMESPACE::sprintf(buff, "%.20p", 0x1a2b3c4d);
-  EXPECT_EQ(written, 22);
-  ASSERT_STREQ(buff, "0x0000000000001a2b3c4d");
+  ASSERT_STREQ_LEN(written, buff, "0x0000000000001a2b3c4d");
 }
 
 TEST(LlvmLibcSPrintfTest, OctConv) {
@@ -676,126 +551,99 @@ TEST(LlvmLibcSPrintfTest, OctConv) {
   // Basic Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%o", 01234);
-  EXPECT_EQ(written, 4);
-  ASSERT_STREQ(buff, "1234");
+  ASSERT_STREQ_LEN(written, buff, "1234");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%o", 04567);
-  EXPECT_EQ(written, 4);
-  ASSERT_STREQ(buff, "4567");
+  ASSERT_STREQ_LEN(written, buff, "4567");
 
   // Length Modifier Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%hho", 0401);
-  EXPECT_EQ(written, 1);
-  ASSERT_STREQ(buff, "1");
+  ASSERT_STREQ_LEN(written, buff, "1");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%llo", 01777777777777777777777ull);
-  EXPECT_EQ(written, 22);
-  ASSERT_STREQ(buff, "1777777777777777777777"); // ull max
+  ASSERT_STREQ_LEN(written, buff, "1777777777777777777777"); // ull max
 
   written = LIBC_NAMESPACE::sprintf(buff, "%to", ~ptrdiff_t(0));
   if (sizeof(ptrdiff_t) == 8) {
-    EXPECT_EQ(written, 22);
-    ASSERT_STREQ(buff, "1777777777777777777777");
+    ASSERT_STREQ_LEN(written, buff, "1777777777777777777777");
   } else if (sizeof(ptrdiff_t) == 4) {
-    EXPECT_EQ(written, 11);
-    ASSERT_STREQ(buff, "37777777777");
+    ASSERT_STREQ_LEN(written, buff, "37777777777");
   }
 
   // Min Width Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%4o", 0701);
-  EXPECT_EQ(written, 4);
-  ASSERT_STREQ(buff, " 701");
+  ASSERT_STREQ_LEN(written, buff, " 701");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%2o", 0107);
-  EXPECT_EQ(written, 3);
-  ASSERT_STREQ(buff, "107");
+  ASSERT_STREQ_LEN(written, buff, "107");
 
   // Precision Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%o", 0);
-  EXPECT_EQ(written, 1);
-  ASSERT_STREQ(buff, "0");
+  ASSERT_STREQ_LEN(written, buff, "0");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.0o", 0);
-  EXPECT_EQ(written, 0);
-  ASSERT_STREQ(buff, "");
+  ASSERT_STREQ_LEN(written, buff, "");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.5o", 0153);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "00153");
+  ASSERT_STREQ_LEN(written, buff, "00153");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.2o", 0135);
-  EXPECT_EQ(written, 3);
-  ASSERT_STREQ(buff, "135");
+  ASSERT_STREQ_LEN(written, buff, "135");
 
   // Flag Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%-5o", 0246);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "246  ");
+  ASSERT_STREQ_LEN(written, buff, "246  ");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%#o", 0234);
-  EXPECT_EQ(written, 4);
-  ASSERT_STREQ(buff, "0234");
+  ASSERT_STREQ_LEN(written, buff, "0234");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%#o", 0);
-  EXPECT_EQ(written, 1);
-  ASSERT_STREQ(buff, "0");
+  ASSERT_STREQ_LEN(written, buff, "0");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%05o", 0470);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "00470");
+  ASSERT_STREQ_LEN(written, buff, "00470");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%0#6o", 0753);
-  EXPECT_EQ(written, 6);
-  ASSERT_STREQ(buff, "000753");
+  ASSERT_STREQ_LEN(written, buff, "000753");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%-#6o", 0642);
-  EXPECT_EQ(written, 6);
-  ASSERT_STREQ(buff, "0642  ");
+  ASSERT_STREQ_LEN(written, buff, "0642  ");
 
   // Combined Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%#-07o", 0703);
-  EXPECT_EQ(written, 7);
-  ASSERT_STREQ(buff, "0703   ");
+  ASSERT_STREQ_LEN(written, buff, "0703   ");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%#.o", 0);
-  EXPECT_EQ(written, 1);
-  ASSERT_STREQ(buff, "0");
+  ASSERT_STREQ_LEN(written, buff, "0");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%7.5o", 0314);
-  EXPECT_EQ(written, 7);
-  ASSERT_STREQ(buff, "  00314");
+  ASSERT_STREQ_LEN(written, buff, "  00314");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%#9.5o", 0234);
-  EXPECT_EQ(written, 9);
-  ASSERT_STREQ(buff, "    00234");
+  ASSERT_STREQ_LEN(written, buff, "    00234");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%-7.5o", 0260);
-  EXPECT_EQ(written, 7);
-  ASSERT_STREQ(buff, "00260  ");
+  ASSERT_STREQ_LEN(written, buff, "00260  ");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%5.4o", 010000);
-  EXPECT_EQ(written, 5);
-  ASSERT_STREQ(buff, "10000");
+  ASSERT_STREQ_LEN(written, buff, "10000");
 
   // Multiple Conversion Tests.
 
   written = LIBC_NAMESPACE::sprintf(buff, "%10o %-#10o", 0456, 0123);
-  EXPECT_EQ(written, 21);
-  ASSERT_STREQ(buff, "       456 0123      ");
+  ASSERT_STREQ_LEN(written, buff, "       456 0123      ");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%-5.4o%#.4o", 075, 025);
-  EXPECT_EQ(written, 9);
-  ASSERT_STREQ(buff, "0075 0025");
+  ASSERT_STREQ_LEN(written, buff, "0075 0025");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%04hho %#.5llo %-6.3zo", 256 + 077,
                                     01000000000000ll, size_t(2));
-  EXPECT_EQ(written, 26);
-  ASSERT_STREQ(buff, "0077 01000000000000 002   ");
+  ASSERT_STREQ_LEN(written, buff, "0077 01000000000000 002   ");
 }
 
 #ifndef LIBC_COPT_PRINTF_DISABLE_FLOAT
@@ -1168,7 +1016,7 @@ TEST(LlvmLibcSPrintfTest, FloatHexExpConv) {
 }
 
 TEST(LlvmLibcSPrintfTest, FloatDecimalConv) {
-  char buff[1000];
+  char buff[1500];
   int written;
 
   ForceRoundingMode r(RoundingMode::Nearest);
@@ -1179,9 +1027,6 @@ TEST(LlvmLibcSPrintfTest, FloatDecimalConv) {
   long double ld_nan =
       LIBC_NAMESPACE::fputil::FPBits<long double>::quiet_nan().get_val();
 
-  char big_buff[10000]; // Used for long doubles and other extremely wide
-                        // numbers.
-
   written = LIBC_NAMESPACE::sprintf(buff, "%f", 1.0);
   ASSERT_STREQ_LEN(written, buff, "1.000000");
 
@@ -1197,6 +1042,7 @@ TEST(LlvmLibcSPrintfTest, FloatDecimalConv) {
   written = LIBC_NAMESPACE::sprintf(buff, "%f", 1.5);
   ASSERT_STREQ_LEN(written, buff, "1.500000");
 
+#ifndef LIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION
   written = LIBC_NAMESPACE::sprintf(buff, "%f", 1e300);
   ASSERT_STREQ_LEN(
       written, buff,
@@ -1205,6 +1051,7 @@ TEST(LlvmLibcSPrintfTest, FloatDecimalConv) {
       "218478670698284838720092657580373783023379478809005936895323497079994508"
       "111903896764088007465274278014249457925878882005684283811566947219638686"
       "5459400540160.000000");
+#endif // LIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION
 
   written = LIBC_NAMESPACE::sprintf(buff, "%f", 0.1);
   ASSERT_STREQ_LEN(written, buff, "0.100000");
@@ -1350,6 +1197,7 @@ TEST(LlvmLibcSPrintfTest, FloatDecimalConv) {
   written = LIBC_NAMESPACE::sprintf(buff, "%.9f", 1.9999999999999514);
   ASSERT_STREQ_LEN(written, buff, "2.000000000");
 
+#ifndef LIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION
   // The number continues after the literal because floating point numbers can't
   // represent every value. The printed value is the closest value a double can
   // represent, rounded to the requested precision.
@@ -1361,14 +1209,15 @@ TEST(LlvmLibcSPrintfTest, FloatDecimalConv) {
       "131959884853339045938639911360973972585316399767392273697826861241937664"
       "824105639342441431495119762431744054912109728706985341609159156917030486"
       "5110665559768676757812");
+#endif
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.36f", 9.9e-77);
   ASSERT_STREQ_LEN(written, buff, "0.000000000000000000000000000000000000");
 
-  written =
-      LIBC_NAMESPACE::sprintf(big_buff, "%.1071f", 2.0226568751604562E-314);
+#ifndef LIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION
+  written = LIBC_NAMESPACE::sprintf(buff, "%.1071f", 2.0226568751604562E-314);
   ASSERT_STREQ_LEN(
-      written, big_buff,
+      written, buff,
       "0."
       "000000000000000000000000000000000000000000000000000000000000000000000000"
       "000000000000000000000000000000000000000000000000000000000000000000000000"
@@ -1385,6 +1234,7 @@ TEST(LlvmLibcSPrintfTest, FloatDecimalConv) {
       "242480201291872969114441104973910102402751449901108484914924879541248714"
       "939096548775588293353689592872854495101242645279589976452453829724479805"
       "750016448075109469332839157162950982637994457036256790161132812");
+#endif
 
   // If no precision is specified it defaults to 6 for %f.
   written = LIBC_NAMESPACE::sprintf(buff, "%f", 2325885.4901960781);
@@ -1442,26 +1292,6 @@ TEST(LlvmLibcSPrintfTest, FloatDecimalConv) {
       "000000000000000000000000000000000000000000000000000000000000000000000000"
       "000000000000000000000000000000000004940656");
 
-  /*
-    written = LIBC_NAMESPACE::sprintf(buff, "%.1La", 0.1L);
-  #if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80)
-    ASSERT_STREQ_LEN(written, buff, "0xc.dp-7");
-  #elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64)
-    ASSERT_STREQ_LEN(written, buff, "0x1.ap-4");
-  #elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128)
-    ASSERT_STREQ_LEN(written, buff, "0x1.ap-4");
-  #endif
-
-    written = LIBC_NAMESPACE::sprintf(buff, "%.1La",
-  0xf.fffffffffffffffp16380L); #if
-  defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) ASSERT_STREQ_LEN(written, buff,
-  "0x1.0p+16384"); #elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64)
-    ASSERT_STREQ_LEN(written, buff, "inf");
-  #elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128)
-    ASSERT_STREQ_LEN(written, buff, "0x2.0p+16383");
-  #endif
-  */
-
   // Rounding Mode Tests.
 
   if (ForceRoundingMode r(RoundingMode::Nearest); r.success) {
@@ -1706,10 +1536,12 @@ TEST(LlvmLibcSPrintfTest, FloatDecimalLongDoubleConv) {
 
 #if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80)
 
+#ifndef LIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION
   written = LIBC_NAMESPACE::sprintf(buff, "%Lf", 1e100L);
   ASSERT_STREQ_LEN(written, buff,
                    "99999999999999999996693535322073426194986990198284960792713"
                    "91541752018669482644324418977840117055488.000000");
+#endif
 
   written = LIBC_NAMESPACE::sprintf(buff, "%Lf", 0xd.96ed1192687859ap-24L);
   ASSERT_STREQ_LEN(written, buff, "0.000001");
@@ -1717,6 +1549,7 @@ TEST(LlvmLibcSPrintfTest, FloatDecimalLongDoubleConv) {
   written = LIBC_NAMESPACE::sprintf(buff, "%Lf", 10000000000000000.25L);
   ASSERT_STREQ_LEN(written, buff, "10000000000000000.250000");
 
+#ifndef LIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION
   written = LIBC_NAMESPACE::sprintf(buff, "%.510Lf", 0x8p-503L);
   ASSERT_STREQ_LEN(
       written, buff,
@@ -1729,6 +1562,7 @@ TEST(LlvmLibcSPrintfTest, FloatDecimalLongDoubleConv) {
       "948791366338993705251636497892270212003524508209121908744820211960149463"
       "721109340307985507678283651836204093399373959982767701148986816406250000"
       "000000");
+#endif
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.500Lf", -4327677766926336.0L);
   ASSERT_STREQ_LEN(
@@ -1744,6 +1578,7 @@ TEST(LlvmLibcSPrintfTest, FloatDecimalLongDoubleConv) {
 
   char big_buff[10000]; // Used for extremely wide numbers.
 
+#ifndef LIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION
   written = LIBC_NAMESPACE::sprintf(big_buff, "%Lf", 1e1000L);
   ASSERT_STREQ_LEN(
       written, big_buff,
@@ -1909,10 +1744,12 @@ TEST(LlvmLibcSPrintfTest, FloatDecimalLongDoubleConv) {
       "542411931794417513706468964386151771884986701034153254238591108962471088"
       "538580868883777725864856414593426212108664758848926003176234596076950884"
       "9149662444156604419552086811989770240.000000");
+#endif
 
   written = LIBC_NAMESPACE::sprintf(big_buff, "%.10Lf", 1e-10L);
   ASSERT_STREQ_LEN(written, big_buff, "0.0000000001");
 
+#ifndef LIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION
   written = LIBC_NAMESPACE::sprintf(big_buff, "%.7500Lf", 1e-4900L);
   ASSERT_STREQ_LEN(
       written, big_buff,
@@ -2022,6 +1859,7 @@ TEST(LlvmLibcSPrintfTest, FloatDecimalLongDoubleConv) {
       "570449525088342437216896462077260223998756027453411520977536701491759878"
       "422771447006016890777855573925295187921971811871399320142563330377888532"
       "179817332113");
+#endif
 #endif // LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80
 }
 
@@ -2193,12 +2031,14 @@ TEST(LlvmLibcSPrintfTest, FloatExponentConv) {
   written = LIBC_NAMESPACE::sprintf(buff, "%.0e", 9.059E+200);
   ASSERT_STREQ_LEN(written, buff, "9e+200");
 
+#ifndef LIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION
   written = LIBC_NAMESPACE::sprintf(buff, "%.166e", 1.131959884853339E-72);
   ASSERT_STREQ_LEN(written, buff,
                    "1."
                    "13195988485333904593863991136097397258531639976739227369782"
                    "68612419376648241056393424414314951197624317440549121097287"
                    "069853416091591569170304865110665559768676757812e-72");
+#endif
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.0e", 9.5);
   ASSERT_STREQ_LEN(written, buff, "1e+01");
@@ -2217,6 +2057,7 @@ TEST(LlvmLibcSPrintfTest, FloatExponentConv) {
 
   // Subnormal Precision Tests
 
+#ifndef LIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION
   written = LIBC_NAMESPACE::sprintf(buff, "%.310e", 0x1.0p-1022);
   ASSERT_STREQ_LEN(
       written, buff,
@@ -2226,10 +2067,12 @@ TEST(LlvmLibcSPrintfTest, FloatExponentConv) {
       "385377718821130777993532002330479610147442583636071921565046942503734208"
       "375250806650616658158948720491179968591639648500635908770118304874799780"
       "8877537499494515804516e-308");
+#endif
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.30e", 0x1.0p-1022);
   ASSERT_STREQ_LEN(written, buff, "2.225073858507201383090232717332e-308");
 
+#ifndef LIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION
   written = LIBC_NAMESPACE::sprintf(buff, "%.310e", 0x1.0p-1023);
   ASSERT_STREQ_LEN(
       written, buff,
@@ -2239,6 +2082,7 @@ TEST(LlvmLibcSPrintfTest, FloatExponentConv) {
       "192688859410565388996766001165239805073721291818035960782523471251867104"
       "187625403325308329079474360245589984295819824250317954385059152437399890"
       "4438768749747257902258e-308");
+#endif
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.6e", 9.99999e-310);
   ASSERT_STREQ_LEN(written, buff, "9.999990e-310");
@@ -2264,26 +2108,6 @@ TEST(LlvmLibcSPrintfTest, FloatExponentConv) {
   written = LIBC_NAMESPACE::sprintf(buff, "%.10e", 0x1.0p-1074);
   ASSERT_STREQ_LEN(written, buff, "4.9406564584e-324");
 
-  /*
-    written = LIBC_NAMESPACE::sprintf(buff, "%.1La", 0.1L);
-  #if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80)
-    ASSERT_STREQ_LEN(written, buff, "0xc.dp-7");
-  #elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64)
-    ASSERT_STREQ_LEN(written, buff, "0x1.ap-4");
-  #elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128)
-    ASSERT_STREQ_LEN(written, buff, "0x1.ap-4");
-  #endif
-
-    written = LIBC_NAMESPACE::sprintf(buff, "%.1La",
-  0xf.fffffffffffffffp16380L); #if
-  defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) ASSERT_STREQ_LEN(written, buff,
-  "0x1.0p+16384"); #elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64)
-    ASSERT_STREQ_LEN(written, buff, "inf");
-  #elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128)
-    ASSERT_STREQ_LEN(written, buff, "0x2.0p+16383");
-  #endif
-  */
-
   // Rounding Mode Tests.
 
   if (ForceRoundingMode r(RoundingMode::Nearest); r.success) {
@@ -2521,121 +2345,31 @@ TEST(LlvmLibcSPrintfTest, FloatExponentLongDoubleConv) {
   ForceRoundingMode r(RoundingMode::Nearest);
   // Length Modifier Tests.
 
-#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80)
   written = LIBC_NAMESPACE::sprintf(buff, "%.9Le", 1000000000500000000.1L);
   ASSERT_STREQ_LEN(written, buff, "1.000000001e+18");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.9Le", 1000000000500000000.0L);
   ASSERT_STREQ_LEN(written, buff, "1.000000000e+18");
 
+  written = LIBC_NAMESPACE::sprintf(buff, "%Le", 1e100L);
+  ASSERT_STREQ_LEN(written, buff, "1.000000e+100");
+
+  written = LIBC_NAMESPACE::sprintf(buff, "%Le", 1.0L);
+  ASSERT_STREQ_LEN(written, buff, "1.000000e+00");
+
+#if !defined(LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE)
   written = LIBC_NAMESPACE::sprintf(buff, "%Le", 0xf.fffffffffffffffp+16380L);
   ASSERT_STREQ_LEN(written, buff, "1.189731e+4932");
-#endif
-
-  // TODO: Fix long doubles (needs bigger table or alternate algorithm.)
-  // Currently the table values are generated, which is very slow.
-  /*
-  written = LIBC_NAMESPACE::sprintf(buff, "%Lf", 1e100L);
-  ASSERT_STREQ_LEN(written, buff,
-                   "99999999999999999996693535322073426194986990198284960792713"
-                   "91541752018669482644324418977840117055488.000000");
 
-  written = LIBC_NAMESPACE::sprintf(buff, "%Lf", 1.0L);
-  ASSERT_STREQ_LEN(written, buff, "1.000000");
+  written = LIBC_NAMESPACE::sprintf(buff, "%Le", 1e1000L);
+  ASSERT_STREQ_LEN(written, buff, "1.000000e+1000");
 
-  char big_buff[10000];
-  written = LIBC_NAMESPACE::sprintf(big_buff, "%Lf", 1e1000L);
-  ASSERT_STREQ_LEN(
-      written, big_buff,
-      "999999999999999999973107317669562353428234857594552594925899449376328728"
-      "202461036775511405481186963193066642191664822065529414252060696836533522"
-      "387143501724276282079456797058697369889056407118642873669166717313763499"
-      "277025985141177344925615052465165938514140943010597323750202561187880136"
-      "174810574553749194614479541820148407958204853833697063267336294787191005"
-      "628217462261955103745349844675732989944229689277833828743730290177882029"
-      "042613704915899149603539993716885598351951895974316347947147507970269673"
-      "097709017164643598452451201499004104341931127294141495501309305995449742"
-      "273419524803597130450457553871345958049837885085168840317195672271085085"
-      "950520957945970913451088104971436093671776829538796532762184174216651692"
-      "640931965387852083906784898823494867055070322768919156031682291829761007"
-      "101483799978382119231551218582499361996919560548090784230386907125151658"
-      "086767207295524036170321059257942621398084478974000973622199163292708506"
-      "2431457550909271560663602154947063707982236377366647567795879936."
-      "000000");
+  written = LIBC_NAMESPACE::sprintf(buff, "%Le", 1e4900L);
+  ASSERT_STREQ_LEN(written, buff, "1.000000e+4900");
 
-  written = LIBC_NAMESPACE::sprintf(big_buff, "%Lf", 1e4900L);
-  ASSERT_STREQ_LEN(
-      written, big_buff,
-      "100000000000000000002708312230690349833224052504078834346502930111959028"
-      "517260692666637048230414374897655201843766090626319971729765251179632020"
-      "313912652522792711197087872698264530532442630109549129842736280196919130"
-      "242615101228133188193853826983121366159061148351354364472807590931218045"
-      "387490935930967150336231085015126034696883068553581691802388371635128003"
-      "615577299166097675723780877126495909902479233742826339471026068806070433"
-      "075629449530819183550315434973800271862658869400009022028602967197463980"
-      "126881829804282202449930132940824361207087494829502385835258094836304011"
-      "876250359661206802659650567866176246063987902366800491980400341950657151"
-      "370854446585517805253310195469184699955519312761482572080479702840420595"
-      "377369017651259376039167277822106875560385309101650382998482652792335482"
-      "865443482342801545877390859444282105890147577937366066315975231014810320"
-      "888482059656248277607763361589359794524314002443575149260630989130103550"
-      "443177966380769341050735632338583912575890190136462629316287947355057647"
-      "111088565611192544631519843618778618820046304429723908484879583579178075"
-      "456701368334212923379389029311286386996015804122917416008806233549005183"
-      "152461084266176543129004016414959261473645240454289630182591200574019087"
-      "358223489767381636349719510715487188747217311279465814538495924567014916"
-      "238565628036285599497236493491668884212847699052761266207598941300449276"
-      "447201387520841811835583254242213093566548778954711633721122784159793843"
-      "766802019309395771984693609426401362800013936338891483689127845928572536"
-      "790651156184721483511507878883282891696900630100211914227950790472211403"
-      "392549466062537498185758854079775888444518306635752468713312357556380082"
-      "275500658967283696421824354930077523691855699312544373220921962817907078"
-      "445538421941800259027487429330768616490865438859612697367766323925013940"
-      "918384858952407145253573823848733994146335416209309233074165707437420756"
-      "438833918763109580759409985573826485055208965115587885226774453455112406"
-      "581351429640282227888764449360534584421929291565334894907337572527922691"
-      "473242328379737396430908523008687037407295838014450772162091496534584696"
-      "605157436893236842602956298545594095307060870397506421786236892553632163"
-      "491468601982681381011940409602294892199042638682530687578982576819839451"
-      "907594697546439533559153604700750696252355362322662219852740143212566818"
-      "745528402265116534684566273868361460640280523251242059850044328669692159"
-      "629900374576027104298177006629276014371540945261309319363704125592775129"
-      "543526908667388673739382491147471395192495459318806593271282662311169392"
-      "196897003517840025298267505925987901751541005546610016067658227181318892"
-      "914686508281007582655667597441346214499847364272258631922040641860333431"
-      "409838623713258383681350233064164940590695888300919626215847587544298023"
-      "636416943680102708406086295669759876682046839368574433996997648445207805"
-      "615784339667691231286807666753972942872019850432610318031627872612657513"
-      "588188267160616660825719678199868371370527508463011236193719286066916786"
-      "169956541349011494927225747024994619057884118692213564790598702879596058"
-      "672338334720925179141906809470606964896245458600635183723159228561689808"
-      "246141482736625197373238197777325580142168245885279594913851700941789475"
-      "252421784152262567254611571822468808675893407728003047921107885664474662"
-      "930921581384003950729114103689170603748380178682003976896397305836815761"
-      "717676338115866650889936516794601457549097578905329423919798362140648664"
-      "569177147076571576101649257502509463877402424847669830852345415301684820"
-      "395813946416649808062227494112874521812750160935760825922220707178083076"
-      "380203450993589198835885505461509442443773367592842795410339065860781804"
-      "024975272228687688301824830333940416256885455008512598774611538878683158"
-      "183931461086893832255176926531299425504132104728730288984598001187854507"
-      "900417184206801359847651992484444933900133130832052346600926424167009902"
-      "829803553087005800387704758687923428053612864451456596148162238935900033"
-      "917094683141205188616000211702577553792389670853917118547527592495253773"
-      "028135298405566315903922235989614934474805789300370437580494193066066314"
-      "056627605207631392651010580925826419831250810981343093764403877594495896"
-      "516881097415880926429607388979497471571321217205535961262051641426436441"
-      "668989765107456413733909427384182109285933511623871034309722437967253289"
-      "084018145083721513211807496392673952789642893241520398827805325610653506"
-      "029060153153064455898648607959013571280930834475689835845791849456112104"
-      "462337569019001580859906425911782967213265389744605395555069797947978230"
-      "708108432086217134763779632408473684293543722127232658767439906910370146"
-      "716836295909075482355827087389127370874842532825987593970846704144140471"
-      "956027276735614286138656432085771988513977140957180090146798065497158947"
-      "229765733489703157617307078835099906185890777007500964162371428641176460"
-      "739074789794941408428328217107759915202650066155868439585510978709442590"
-      "231934194956788626761834746430104077432547436359522462253411168467463134"
-      "24896.000000");
-*/
+  written = LIBC_NAMESPACE::sprintf(buff, "%Le", 1.2345678e4900L);
+  ASSERT_STREQ_LEN(written, buff, "1.234568e+4900");
+#endif
 }
 
 TEST(LlvmLibcSPrintfTest, FloatAutoConv) {
@@ -2834,6 +2568,7 @@ TEST(LlvmLibcSPrintfTest, FloatAutoConv) {
   ASSERT_STREQ_LEN(written, buff, "1e+01");
 
   // Subnormal Precision Tests
+#ifndef LIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION
   written = LIBC_NAMESPACE::sprintf(buff, "%.310g", 0x1.0p-1022);
   ASSERT_STREQ_LEN(
       written, buff,
@@ -2843,10 +2578,12 @@ TEST(LlvmLibcSPrintfTest, FloatAutoConv) {
       "385377718821130777993532002330479610147442583636071921565046942503734208"
       "375250806650616658158948720491179968591639648500635908770118304874799780"
       "887753749949451580452e-308");
+#endif
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.30g", 0x1.0p-1022);
   ASSERT_STREQ_LEN(written, buff, "2.22507385850720138309023271733e-308");
 
+#ifndef LIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION
   written = LIBC_NAMESPACE::sprintf(buff, "%.310g", 0x1.0p-1023);
   ASSERT_STREQ_LEN(
       written, buff,
@@ -2856,6 +2593,7 @@ TEST(LlvmLibcSPrintfTest, FloatAutoConv) {
       "192688859410565388996766001165239805073721291818035960782523471251867104"
       "187625403325308329079474360245589984295819824250317954385059152437399890"
       "443876874974725790226e-308");
+#endif
 
   written = LIBC_NAMESPACE::sprintf(buff, "%.7g", 9.99999e-310);
   ASSERT_STREQ_LEN(written, buff, "9.99999e-310");
@@ -2884,38 +2622,6 @@ TEST(LlvmLibcSPrintfTest, FloatAutoConv) {
   written = LIBC_NAMESPACE::sprintf(buff, "%.10g", 0x1.0p-1074);
   ASSERT_STREQ_LEN(written, buff, "4.940656458e-324");
 
-#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80)
-
-  written = LIBC_NAMESPACE::sprintf(buff, "%.60Lg", 0xa.aaaaaaaaaaaaaabp-7L);
-  ASSERT_STREQ_LEN(
-      written, buff,
-      "0.0833333333333333333355920878593448009041821933351457118988037");
-
-#endif // LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80
-
-  // Long double precision tests.
-  // These are currently commented out because they require long double support
-  // that isn't ready yet.
-  /*
-    written = LIBC_NAMESPACE::sprintf(buff, "%.1La", 0.1L);
-  #if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80)
-    ASSERT_STREQ_LEN(written, buff, "0xc.dp-7");
-  #elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64)
-    ASSERT_STREQ_LEN(written, buff, "0x1.ap-4");
-  #elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128)
-    ASSERT_STREQ_LEN(written, buff, "0x1.ap-4");
-  #endif
-
-    written = LIBC_NAMESPACE::sprintf(buff, "%.1La",
-  0xf.fffffffffffffffp16380L); #if
-  defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) ASSERT_STREQ_LEN(written, buff,
-  "0x1.0p+16384"); #elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64)
-    ASSERT_STREQ_LEN(written, buff, "inf");
-  #elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128)
-    ASSERT_STREQ_LEN(written, buff, "0x2.0p+16383");
-  #endif
-  */
-
   // Rounding Mode Tests.
 
   if (ForceRoundingMode r(RoundingMode::Nearest); r.success) {
@@ -3157,6 +2863,7 @@ TEST(LlvmLibcSPrintfTest, FloatAutoLongDoubleConv) {
 
   // Length Modifier Tests.
 
+// TODO: Tests for other long double types
 #if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80)
 
   written = LIBC_NAMESPACE::sprintf(buff, "%Lg", 0xf.fffffffffffffffp+16380L);
@@ -3165,142 +2872,33 @@ TEST(LlvmLibcSPrintfTest, FloatAutoLongDoubleConv) {
   written = LIBC_NAMESPACE::sprintf(buff, "%Lg", 0xa.aaaaaaaaaaaaaabp-7L);
   ASSERT_STREQ_LEN(written, buff, "0.0833333");
 
+#ifndef LIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION
+  written = LIBC_NAMESPACE::sprintf(buff, "%.60Lg", 0xa.aaaaaaaaaaaaaabp-7L);
+  ASSERT_STREQ_LEN(
+      written, buff,
+      "0.0833333333333333333355920878593448009041821933351457118988037");
+#endif
+
+#endif // LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80
+
   written = LIBC_NAMESPACE::sprintf(buff, "%Lg", 9.99999999999e-100L);
   ASSERT_STREQ_LEN(written, buff, "1e-99");
 
-#endif // LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80
+  written = LIBC_NAMESPACE::sprintf(buff, "%Lg", 1e100L);
+  ASSERT_STREQ_LEN(written, buff, "1e+100");
 
-  // TODO: Uncomment the below tests after long double support is added
-  /*
-  written = LIBC_NAMESPACE::sprintf(buff, "%Lf", 1e100L);
-  ASSERT_STREQ_LEN(written, buff,
-                   "99999999999999999996693535322073426194986990198284960792713"
-                   "91541752018669482644324418977840117055488.000000");
+  written = LIBC_NAMESPACE::sprintf(buff, "%Lg", 1.0L);
+  ASSERT_STREQ_LEN(written, buff, "1");
 
-  written = LIBC_NAMESPACE::sprintf(buff, "%Lf", 1.0L);
-  ASSERT_STREQ_LEN(written, buff, "1.000000");
+  written = LIBC_NAMESPACE::sprintf(buff, "%Lg", 0.1L);
+  ASSERT_STREQ_LEN(written, buff, "0.1");
 
   char big_buff[10000];
-  written = LIBC_NAMESPACE::sprintf(big_buff, "%Lf", 1e1000L);
-  ASSERT_STREQ_LEN(
-      written, big_buff,
-      "999999999999999999973107317669562353428234857594552594925899449376328728"
-      "202461036775511405481186963193066642191664822065529414252060696836533522"
-      "387143501724276282079456797058697369889056407118642873669166717313763499"
-      "277025985141177344925615052465165938514140943010597323750202561187880136"
-      "174810574553749194614479541820148407958204853833697063267336294787191005"
-      "628217462261955103745349844675732989944229689277833828743730290177882029"
-      "042613704915899149603539993716885598351951895974316347947147507970269673"
-      "097709017164643598452451201499004104341931127294141495501309305995449742"
-      "273419524803597130450457553871345958049837885085168840317195672271085085"
-      "950520957945970913451088104971436093671776829538796532762184174216651692"
-      "640931965387852083906784898823494867055070322768919156031682291829761007"
-      "101483799978382119231551218582499361996919560548090784230386907125151658"
-      "086767207295524036170321059257942621398084478974000973622199163292708506"
-      "2431457550909271560663602154947063707982236377366647567795879936."
-      "000000");
+  written = LIBC_NAMESPACE::sprintf(big_buff, "%Lg", 1e1000L);
+  ASSERT_STREQ_LEN(written, big_buff, "1e+1000");
 
-  written = LIBC_NAMESPACE::sprintf(big_buff, "%Lf", 1e4900L);
-  ASSERT_STREQ_LEN(
-      written, big_buff,
-      "100000000000000000002708312230690349833224052504078834346502930111959028"
-      "517260692666637048230414374897655201843766090626319971729765251179632020"
-      "313912652522792711197087872698264530532442630109549129842736280196919130"
-      "242615101228133188193853826983121366159061148351354364472807590931218045"
-      "387490935930967150336231085015126034696883068553581691802388371635128003"
-      "615577299166097675723780877126495909902479233742826339471026068806070433"
-      "075629449530819183550315434973800271862658869400009022028602967197463980"
-      "126881829804282202449930132940824361207087494829502385835258094836304011"
-      "876250359661206802659650567866176246063987902366800491980400341950657151"
-      "370854446585517805253310195469184699955519312761482572080479702840420595"
-      "377369017651259376039167277822106875560385309101650382998482652792335482"
-      "865443482342801545877390859444282105890147577937366066315975231014810320"
-      "888482059656248277607763361589359794524314002443575149260630989130103550"
-      "443177966380769341050735632338583912575890190136462629316287947355057647"
-      "111088565611192544631519843618778618820046304429723908484879583579178075"
-      "456701368334212923379389029311286386996015804122917416008806233549005183"
-      "152461084266176543129004016414959261473645240454289630182591200574019087"
-      "358223489767381636349719510715487188747217311279465814538495924567014916"
-      "238565628036285599497236493491668884212847699052761266207598941300449276"
-      "447201387520841811835583254242213093566548778954711633721122784159793843"
-      "766802019309395771984693609426401362800013936338891483689127845928572536"
-      "790651156184721483511507878883282891696900630100211914227950790472211403"
-      "392549466062537498185758854079775888444518306635752468713312357556380082"
-      "275500658967283696421824354930077523691855699312544373220921962817907078"
-      "445538421941800259027487429330768616490865438859612697367766323925013940"
-      "918384858952407145253573823848733994146335416209309233074165707437420756"
-      "438833918763109580759409985573826485055208965115587885226774453455112406"
-      "581351429640282227888764449360534584421929291565334894907337572527922691"
-      "473242328379737396430908523008687037407295838014450772162091496534584696"
-      "605157436893236842602956298545594095307060870397506421786236892553632163"
-      "491468601982681381011940409602294892199042638682530687578982576819839451"
-      "907594697546439533559153604700750696252355362322662219852740143212566818"
-      "745528402265116534684566273868361460640280523251242059850044328669692159"
-      "629900374576027104298177006629276014371540945261309319363704125592775129"
-      "543526908667388673739382491147471395192495459318806593271282662311169392"
-      "196897003517840025298267505925987901751541005546610016067658227181318892"
-      "914686508281007582655667597441346214499847364272258631922040641860333431"
-      "409838623713258383681350233064164940590695888300919626215847587544298023"
-      "636416943680102708406086295669759876682046839368574433996997648445207805"
-      "615784339667691231286807666753972942872019850432610318031627872612657513"
-      "588188267160616660825719678199868371370527508463011236193719286066916786"
-      "169956541349011494927225747024994619057884118692213564790598702879596058"
-      "672338334720925179141906809470606964896245458600635183723159228561689808"
-      "246141482736625197373238197777325580142168245885279594913851700941789475"
-      "252421784152262567254611571822468808675893407728003047921107885664474662"
-      "930921581384003950729114103689170603748380178682003976896397305836815761"
-      "717676338115866650889936516794601457549097578905329423919798362140648664"
-      "569177147076571576101649257502509463877402424847669830852345415301684820"
-      "395813946416649808062227494112874521812750160935760825922220707178083076"
-      "380203450993589198835885505461509442443773367592842795410339065860781804"
-      "024975272228687688301824830333940416256885455008512598774611538878683158"
-      "183931461086893832255176926531299425504132104728730288984598001187854507"
-      "900417184206801359847651992484444933900133130832052346600926424167009902"
-      "829803553087005800387704758687923428053612864451456596148162238935900033"
-      "917094683141205188616000211702577553792389670853917118547527592495253773"
-      "028135298405566315903922235989614934474805789300370437580494193066066314"
-      "056627605207631392651010580925826419831250810981343093764403877594495896"
-      "516881097415880926429607388979497471571321217205535961262051641426436441"
-      "668989765107456413733909427384182109285933511623871034309722437967253289"
-      "084018145083721513211807496392673952789642893241520398827805325610653506"
-      "029060153153064455898648607959013571280930834475689835845791849456112104"
-      "462337569019001580859906425911782967213265389744605395555069797947978230"
-      "708108432086217134763779632408473684293543722127232658767439906910370146"
-      "716836295909075482355827087389127370874842532825987593970846704144140471"
-      "956027276735614286138656432085771988513977140957180090146798065497158947"
-      "229765733489703157617307078835099906185890777007500964162371428641176460"
-      "739074789794941408428328217107759915202650066155868439585510978709442590"
-      "231934194956788626761834746430104077432547436359522462253411168467463134"
-      "24896.000000");
-*/
-  /*
-    written = LIBC_NAMESPACE::sprintf(buff, "%La", 0.1L);
-  #if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80)
-    ASSERT_STREQ_LEN(written, buff, "0xc.ccccccccccccccdp-7");
-  #elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64)
-    ASSERT_STREQ_LEN(written, buff, "0x1.999999999999ap-4");
-  #elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128)
-    ASSERT_STREQ_LEN(written, buff, "0x1.999999999999999999999999999ap-4");
-  #endif
-
-    written = LIBC_NAMESPACE::sprintf(buff, "%La", 1.0e1000L);
-  #if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80)
-    ASSERT_STREQ_LEN(written, buff, "0xf.38db1f9dd3dac05p+3318");
-  #elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64)
-    ASSERT_STREQ_LEN(written, buff, "inf");
-  #elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128)
-    ASSERT_STREQ_LEN(written, buff, "0x1.e71b63f3ba7b580af1a52d2a7379p+3321");
-  #endif
-
-    written = LIBC_NAMESPACE::sprintf(buff, "%La", 1.0e-1000L);
-  #if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80)
-    ASSERT_STREQ_LEN(written, buff, "0x8.68a9188a89e1467p-3325");
-  #elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64)
-    ASSERT_STREQ_LEN(written, buff, "0x0p+0");
-  #elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128)
-    ASSERT_STREQ_LEN(written, buff, "0x1.0d152311513c28ce202627c06ec2p-3322");
-  #endif
-  */
+  written = LIBC_NAMESPACE::sprintf(big_buff, "%Lg", 1e4900L);
+  ASSERT_STREQ_LEN(written, big_buff, "1e+4900");
 }
 
 #endif // LIBC_COPT_PRINTF_DISABLE_FLOAT
@@ -3710,27 +3308,23 @@ TEST(LlvmLibcSPrintfTest, WriteIntConv) {
 
   test_val = -1;
   written = LIBC_NAMESPACE::sprintf(buff, "12345%n67890", &test_val);
-  EXPECT_EQ(written, 10);
   EXPECT_EQ(test_val, 5);
-  ASSERT_STREQ(buff, "1234567890");
+  ASSERT_STREQ_LEN(written, buff, "1234567890");
 
   test_val = -1;
   written = LIBC_NAMESPACE::sprintf(buff, "%n", &test_val);
-  EXPECT_EQ(written, 0);
   EXPECT_EQ(test_val, 0);
-  ASSERT_STREQ(buff, "");
+  ASSERT_STREQ_LEN(written, buff, "");
 
   test_val = 0x100;
   written = LIBC_NAMESPACE::sprintf(buff, "ABC%hhnDEF", &test_val);
-  EXPECT_EQ(written, 6);
   EXPECT_EQ(test_val, 0x103);
-  ASSERT_STREQ(buff, "ABCDEF");
+  ASSERT_STREQ_LEN(written, buff, "ABCDEF");
 
   test_val = -1;
   written = LIBC_NAMESPACE::sprintf(buff, "%s%n", "87654321", &test_val);
-  EXPECT_EQ(written, 8);
   EXPECT_EQ(test_val, 8);
-  ASSERT_STREQ(buff, "87654321");
+  ASSERT_STREQ_LEN(written, buff, "87654321");
 
 #ifndef LIBC_COPT_PRINTF_NO_NULLPTR_CHECKS
   written = LIBC_NAMESPACE::sprintf(buff, "abc123%n", nullptr);
@@ -3745,23 +3339,20 @@ TEST(LlvmLibcSPrintfTest, IndexModeParsing) {
   int written;
 
   written = LIBC_NAMESPACE::sprintf(buff, "%1$s", "abcDEF123");
-  EXPECT_EQ(written, 9);
-  ASSERT_STREQ(buff, "abcDEF123");
+  ASSERT_STREQ_LEN(written, buff, "abcDEF123");
 
   written = LIBC_NAMESPACE::sprintf(buff, "%1$s %%", "abcDEF123");
-  EXPECT_EQ(written, 11);
-  ASSERT_STREQ(buff, "abcDEF123 %");
+  ASSERT_STREQ_LEN(written, buff, "abcDEF123 %");
 
   written =
       LIBC_NAMESPACE::sprintf(buff, "%3$s %1$s %2$s", "is", "hard", "ordering");
-  EXPECT_EQ(written, 16);
-  ASSERT_STREQ(buff, "ordering is hard");
+  ASSERT_STREQ_LEN(written, buff, "ordering is hard");
 
   written = LIBC_NAMESPACE::sprintf(
       buff, "%10$s %9$s %8$c %7$s %6$s, %6$s %5$s %4$-*1$s %3$.*11$s %2$s. %%",
       6, "pain", "alphabetical", "such", "is", "this", "do", 'u', "would",
       "why", 1);
-  EXPECT_EQ(written, 45);
-  ASSERT_STREQ(buff, "why would u do this, this is such   a pain. %");
+  ASSERT_STREQ_LEN(written, buff,
+                   "why would u do this, this is such   a pain. %");
 }
 #endif // LIBC_COPT_PRINTF_DISABLE_INDEX_MODE
diff --git a/libc/test/src/stdlib/CMakeLists.txt b/libc/test/src/stdlib/CMakeLists.txt
index db90d9a4741eb..c68627a668741 100644
--- a/libc/test/src/stdlib/CMakeLists.txt
+++ b/libc/test/src/stdlib/CMakeLists.txt
@@ -168,13 +168,17 @@ add_libc_test(
     .strtol_test_support
 )
 
+if(LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_DYADIC_FLOAT)
+  list(APPEND strfrom_test_copts "-DLIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION")
+endif()
+
 add_header_library(
   strfrom_test_support
   HDRS
-  StrfromTest.h
+    StrfromTest.h
   DEPENDS
-  libc.src.__support.CPP.type_traits
-  libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.CPP.type_traits
+    libc.src.__support.FPUtil.fp_bits
 )
 
 add_libc_test(
@@ -186,6 +190,8 @@ add_libc_test(
   DEPENDS
     .strfrom_test_support
     libc.src.stdlib.strfromf
+  COMPILE_OPTIONS
+    ${strfrom_test_copts}
 )
 
 add_libc_test(
@@ -197,6 +203,8 @@ add_libc_test(
   DEPENDS
     .strfrom_test_support
     libc.src.stdlib.strfromd
+  COMPILE_OPTIONS
+    ${strfrom_test_copts}
 )
 
 add_libc_test(
@@ -208,6 +216,8 @@ add_libc_test(
   DEPENDS
     .strfrom_test_support
     libc.src.stdlib.strfroml
+  COMPILE_OPTIONS
+    ${strfrom_test_copts}
 )
 
 add_libc_test(
diff --git a/libc/test/src/stdlib/StrfromTest.h b/libc/test/src/stdlib/StrfromTest.h
index 0db507ef0716e..5209472886f78 100644
--- a/libc/test/src/stdlib/StrfromTest.h
+++ b/libc/test/src/stdlib/StrfromTest.h
@@ -156,6 +156,9 @@ class StrfromTest : public LIBC_NAMESPACE::testing::Test {
     written = func(buff, 99, "%f", 1.5);
     ASSERT_STREQ_LEN(written, buff, "1.500000");
 
+// Dyadic float is only accurate to ~50 digits, so skip this 300 digit test.
+// TODO: Create way to test just the first ~50 digits of a number.
+#ifndef LIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION
     written = func(buff, 499, "%f", 1e300);
     ASSERT_STREQ_LEN(written, buff,
                      "100000000000000005250476025520442024870446858110815915491"
@@ -167,6 +170,7 @@ class StrfromTest : public LIBC_NAMESPACE::testing::Test {
                      "111903896764088007465274278014249457925878882005684283811"
                      "566947219638686"
                      "5459400540160.000000");
+#endif // DLIBC_COPT_FLOAT_TO_STR_REDUCED_PRECISION
 
     written = func(buff, 99, "%f", 0.1);
     ASSERT_STREQ_LEN(written, buff, "0.100000");
diff --git a/libc/utils/HdrGen/PublicAPICommand.cpp b/libc/utils/HdrGen/PublicAPICommand.cpp
index 2a7f63eb25c72..5c46c82a23853 100644
--- a/libc/utils/HdrGen/PublicAPICommand.cpp
+++ b/libc/utils/HdrGen/PublicAPICommand.cpp
@@ -109,7 +109,7 @@ static AttributeMap collectAttributeMacros(const SpecMap &Spec,
 static void emitAttributeMacroDecls(const AttributeMap &MacroAttr,
                                     llvm::raw_ostream &OS) {
   for (auto &[Macro, Attr] : MacroAttr) {
-    std::vector<llvm::Record *> Instances =
+    std::vector<const llvm::Record *> Instances =
         Attr->getValueAsListOfDefs("Instances");
     llvm::SmallVector<std::pair<AttributeStyle, const llvm::Record *>> Styles;
     std::transform(Instances.begin(), Instances.end(),
@@ -190,7 +190,7 @@ static void emitAttributeMacroDecls(const AttributeMap &MacroAttr,
 
 static void emitAttributeMacroForFunction(const llvm::Record *FunctionSpec,
                                           llvm::raw_ostream &OS) {
-  std::vector<llvm::Record *> Attributes =
+  std::vector<const llvm::Record *> Attributes =
       FunctionSpec->getValueAsListOfDefs("Attributes");
   llvm::interleave(
       Attributes.begin(), Attributes.end(),
diff --git a/libc/utils/LibcTableGenUtil/APIIndexer.cpp b/libc/utils/LibcTableGenUtil/APIIndexer.cpp
index 0a77584071f99..cf66d0a7aef6e 100644
--- a/libc/utils/LibcTableGenUtil/APIIndexer.cpp
+++ b/libc/utils/LibcTableGenUtil/APIIndexer.cpp
@@ -81,22 +81,22 @@ std::string APIIndexer::getTypeAsString(const llvm::Record *TypeRecord) {
 
 void APIIndexer::indexStandardSpecDef(const llvm::Record *StandardSpec) {
   auto HeaderSpecList = StandardSpec->getValueAsListOfDefs("Headers");
-  for (llvm::Record *HeaderSpec : HeaderSpecList) {
+  for (const llvm::Record *HeaderSpec : HeaderSpecList) {
     llvm::StringRef Header = HeaderSpec->getValueAsString("Name");
     if (!StdHeader.has_value() || Header == StdHeader) {
       PublicHeaders.emplace(Header);
       auto MacroSpecList = HeaderSpec->getValueAsListOfDefs("Macros");
       // TODO: Trigger a fatal error on duplicate specs.
-      for (llvm::Record *MacroSpec : MacroSpecList)
+      for (const llvm::Record *MacroSpec : MacroSpecList)
         MacroSpecMap[std::string(MacroSpec->getValueAsString("Name"))] =
             MacroSpec;
 
       auto TypeSpecList = HeaderSpec->getValueAsListOfDefs("Types");
-      for (llvm::Record *TypeSpec : TypeSpecList)
+      for (const llvm::Record *TypeSpec : TypeSpecList)
         TypeSpecMap[std::string(TypeSpec->getValueAsString("Name"))] = TypeSpec;
 
       auto FunctionSpecList = HeaderSpec->getValueAsListOfDefs("Functions");
-      for (llvm::Record *FunctionSpec : FunctionSpecList) {
+      for (const llvm::Record *FunctionSpec : FunctionSpecList) {
         auto FunctionName = std::string(FunctionSpec->getValueAsString("Name"));
         FunctionSpecMap[FunctionName] = FunctionSpec;
         FunctionToHeaderMap[FunctionName] = std::string(Header);
@@ -104,13 +104,13 @@ void APIIndexer::indexStandardSpecDef(const llvm::Record *StandardSpec) {
 
       auto EnumerationSpecList =
           HeaderSpec->getValueAsListOfDefs("Enumerations");
-      for (llvm::Record *EnumerationSpec : EnumerationSpecList) {
+      for (const llvm::Record *EnumerationSpec : EnumerationSpecList) {
         EnumerationSpecMap[std::string(
             EnumerationSpec->getValueAsString("Name"))] = EnumerationSpec;
       }
 
       auto ObjectSpecList = HeaderSpec->getValueAsListOfDefs("Objects");
-      for (llvm::Record *ObjectSpec : ObjectSpecList) {
+      for (const llvm::Record *ObjectSpec : ObjectSpecList) {
         auto ObjectName = std::string(ObjectSpec->getValueAsString("Name"));
         ObjectSpecMap[ObjectName] = ObjectSpec;
         ObjectToHeaderMap[ObjectName] = std::string(Header);
@@ -124,7 +124,7 @@ void APIIndexer::indexPublicAPIDef(const llvm::Record *PublicAPI) {
   // requested is from an included standard. Such a check is done while
   // generating the API.
   auto MacroDefList = PublicAPI->getValueAsListOfDefs("Macros");
-  for (llvm::Record *MacroDef : MacroDefList)
+  for (const llvm::Record *MacroDef : MacroDefList)
     MacroDefsMap[std::string(MacroDef->getValueAsString("Name"))] = MacroDef;
 
   auto TypeList = PublicAPI->getValueAsListOfStrings("Types");
diff --git a/libcxx/cmake/caches/hexagon-linux-runtimes.cmake b/libcxx/cmake/caches/hexagon-linux-runtimes.cmake
new file mode 100644
index 0000000000000..7f309de9a14e6
--- /dev/null
+++ b/libcxx/cmake/caches/hexagon-linux-runtimes.cmake
@@ -0,0 +1,15 @@
+
+set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR OFF CACHE BOOL "")
+set(LLVM_ENABLE_RUNTIMES libcxx;libcxxabi;libunwind;compiler-rt CACHE STRING "")
+set(LIBCXX_INCLUDE_BENCHMARKS OFF CACHE BOOL "")
+set(LIBCXX_HAS_MUSL_LIBC ON CACHE BOOL "")
+set(LIBCXX_INCLUDE_TESTS OFF CACHE BOOL "")
+set(LIBCXXABI_INCLUDE_TESTS OFF CACHE BOOL "")
+set(LIBUNWIND_INCLUDE_TESTS OFF CACHE BOOL "")
+set(LIBCXX_CXX_ABI libcxxabi CACHE STRING "")
+set(LIBCXXABI_USE_LLVM_UNWINDER ON CACHE BOOL "")
+set(LIBCXXABI_ENABLE_SHARED ON CACHE BOOL "")
+
+set(LIBCXX_USE_COMPILER_RT ON CACHE BOOL "")
+set(LIBCXXABI_USE_COMPILER_RT ON CACHE BOOL "")
+set(LIBUNWIND_USE_COMPILER_RT ON CACHE BOOL "")
diff --git a/libcxx/docs/Status/Cxx20Issues.csv b/libcxx/docs/Status/Cxx20Issues.csv
index 8e9fc1398bc09..b0fa5346e873d 100644
--- a/libcxx/docs/Status/Cxx20Issues.csv
+++ b/libcxx/docs/Status/Cxx20Issues.csv
@@ -239,7 +239,7 @@
 "`LWG3314 <https://wg21.link/LWG3314>`__","Is stream insertion behavior locale dependent when ``Period::type``\  is ``micro``\ ?","2020-02 (Prague)","|Complete|","16.0",""
 "`LWG3315 <https://wg21.link/LWG3315>`__","LWG3315: Correct Allocator Default Behavior","2020-02 (Prague)","|Complete|","",""
 "`LWG3316 <https://wg21.link/LWG3316>`__","Correctly define epoch for ``utc_clock``\  / ``utc_timepoint``\ ","2020-02 (Prague)","","",""
-"`LWG3317 <https://wg21.link/LWG3317>`__","Incorrect ``operator<<``\  for floating-point durations","2020-02 (Prague)","","",""
+"`LWG3317 <https://wg21.link/LWG3317>`__","Incorrect ``operator<<``\  for floating-point durations","2020-02 (Prague)","|Complete|","16.0",""
 "`LWG3318 <https://wg21.link/LWG3318>`__","Clarify whether clocks can represent time before their epoch","2020-02 (Prague)","","",""
 "`LWG3319 <https://wg21.link/LWG3319>`__","Properly reference specification of IANA time zone database","2020-02 (Prague)","|Nothing To Do|","",""
 "`LWG3320 <https://wg21.link/LWG3320>`__","``span::cbegin/cend``\  methods produce different results than ``std::[ranges::]cbegin/cend``\ ","2020-02 (Prague)","|Complete|","",""
diff --git a/libcxx/docs/Status/Cxx23Issues.csv b/libcxx/docs/Status/Cxx23Issues.csv
index 1c8bb057b0966..63e4176ecba1d 100644
--- a/libcxx/docs/Status/Cxx23Issues.csv
+++ b/libcxx/docs/Status/Cxx23Issues.csv
@@ -296,7 +296,7 @@
 "`LWG3862 <https://wg21.link/LWG3862>`__","``basic_const_iterator``'s ``common_type`` specialization is underconstrained","2023-02 (Issaquah)","","",""
 "`LWG3865 <https://wg21.link/LWG3865>`__","Sorting a range of ``pairs``","2023-02 (Issaquah)","|Complete|","17.0",""
 "`LWG3869 <https://wg21.link/LWG3869>`__","Deprecate ``std::errc`` constants related to UNIX STREAMS","2023-02 (Issaquah)","|Complete|","19.0",""
-"`LWG3870 <https://wg21.link/LWG3870>`__","Remove ``voidify``","2023-02 (Issaquah)","","",""
+"`LWG3870 <https://wg21.link/LWG3870>`__","Remove ``voidify``","2023-02 (Issaquah)","|Complete|","20.0",""
 "`LWG3871 <https://wg21.link/LWG3871>`__","Adjust note about ``terminate``","2023-02 (Issaquah)","","",""
 "`LWG3872 <https://wg21.link/LWG3872>`__","``basic_const_iterator`` should have custom ``iter_move``","2023-02 (Issaquah)","","",""
 "`LWG3875 <https://wg21.link/LWG3875>`__","``std::ranges::repeat_view<T, IntegerClass>::iterator`` may be ill-formed","2023-02 (Issaquah)","|Complete|","17.0",""
diff --git a/libcxx/docs/Status/Cxx23Papers.csv b/libcxx/docs/Status/Cxx23Papers.csv
index 6dde153ee8aab..e1d7bb8f37863 100644
--- a/libcxx/docs/Status/Cxx23Papers.csv
+++ b/libcxx/docs/Status/Cxx23Papers.csv
@@ -96,7 +96,7 @@
 "`P1202R5 <https://wg21.link/P1202R5>`__","Asymmetric Fences","2022-11 (Kona)","","",""
 "`P1264R2 <https://wg21.link/P1264R2>`__","Revising the wording of ``stream`` input operations","2022-11 (Kona)","|Complete|","9.0",""
 "`P1478R8 <https://wg21.link/P1478R8>`__","``Byte-wise`` ``atomic`` ``memcpy``","2022-11 (Kona)","","",""
-"`P2167R3 <https://wg21.link/P2167R3>`__","Improved Proposed Wording for LWG 2114","2022-11 (Kona)","","",""
+"`P2167R3 <https://wg21.link/P2167R3>`__","Improved Proposed Wording for LWG 2114","2022-11 (Kona)","|Complete|","20.0","The `[cmp.alg] <https://eel.is/c++draft/cmp.alg>`__ part is implemented as a DR against C++20. MSVC STL does the same. Other parts are Nothing To Do."
 "`P2396R1 <https://wg21.link/P2396R1>`__","Concurrency TS 2 fixes ","2022-11 (Kona)","","",""
 "`P2505R5 <https://wg21.link/P2505R5>`__","Monadic Functions for ``std::expected``","2022-11 (Kona)","|Complete|","17.0",""
 "`P2539R4 <https://wg21.link/P2539R4>`__","Should the output of ``std::print`` to a terminal be synchronized with the underlying stream?","2022-11 (Kona)","|Complete|","18.0",""
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 8a63280053340..9bd1b41b8bfac 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -560,7 +560,6 @@ set(files
   __memory/unique_temporary_buffer.h
   __memory/uses_allocator.h
   __memory/uses_allocator_construction.h
-  __memory/voidify.h
   __memory_resource/memory_resource.h
   __memory_resource/monotonic_buffer_resource.h
   __memory_resource/polymorphic_allocator.h
diff --git a/libcxx/include/__compare/compare_partial_order_fallback.h b/libcxx/include/__compare/compare_partial_order_fallback.h
index e0efa3ccb88db..80f2aca661faa 100644
--- a/libcxx/include/__compare/compare_partial_order_fallback.h
+++ b/libcxx/include/__compare/compare_partial_order_fallback.h
@@ -11,6 +11,7 @@
 
 #include <__compare/ordering.h>
 #include <__compare/partial_order.h>
+#include <__concepts/boolean_testable.h>
 #include <__config>
 #include <__type_traits/decay.h>
 #include <__type_traits/is_same.h>
@@ -37,18 +38,16 @@ struct __fn {
   }
 
   template <class _Tp, class _Up>
-    requires is_same_v<decay_t<_Tp>, decay_t<_Up>>
-  _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) noexcept(noexcept(
-      std::forward<_Tp>(__t) == std::forward<_Up>(__u)  ? partial_ordering::equivalent
-      : std::forward<_Tp>(__t) < std::forward<_Up>(__u) ? partial_ordering::less
-      : std::forward<_Up>(__u) < std::forward<_Tp>(__t)
-          ? partial_ordering::greater
-          : partial_ordering::unordered))
-      -> decltype(std::forward<_Tp>(__t) == std::forward<_Up>(__u)  ? partial_ordering::equivalent
-                  : std::forward<_Tp>(__t) < std::forward<_Up>(__u) ? partial_ordering::less
-                  : std::forward<_Up>(__u) < std::forward<_Tp>(__t)
-                      ? partial_ordering::greater
-                      : partial_ordering::unordered) {
+    requires is_same_v<decay_t<_Tp>, decay_t<_Up>> && requires(_Tp&& __t, _Up&& __u) {
+      { std::forward<_Tp>(__t) == std::forward<_Up>(__u) } -> __boolean_testable;
+      { std::forward<_Tp>(__t) < std::forward<_Up>(__u) } -> __boolean_testable;
+      { std::forward<_Up>(__u) < std::forward<_Tp>(__t) } -> __boolean_testable;
+    }
+  _LIBCPP_HIDE_FROM_ABI static constexpr partial_ordering __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) noexcept(
+      noexcept(std::forward<_Tp>(__t) == std::forward<_Up>(__u)  ? partial_ordering::equivalent
+               : std::forward<_Tp>(__t) < std::forward<_Up>(__u) ? partial_ordering::less
+               : std::forward<_Up>(__u) < std::forward<_Tp>(__t) ? partial_ordering::greater
+                                                                 : partial_ordering::unordered)) {
     return std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? partial_ordering::equivalent
          : std::forward<_Tp>(__t) < std::forward<_Up>(__u)  ? partial_ordering::less
          : std::forward<_Up>(__u) < std::forward<_Tp>(__t)
diff --git a/libcxx/include/__compare/compare_strong_order_fallback.h b/libcxx/include/__compare/compare_strong_order_fallback.h
index a94d517ed30fc..c41a90c5afa82 100644
--- a/libcxx/include/__compare/compare_strong_order_fallback.h
+++ b/libcxx/include/__compare/compare_strong_order_fallback.h
@@ -11,6 +11,7 @@
 
 #include <__compare/ordering.h>
 #include <__compare/strong_order.h>
+#include <__concepts/boolean_testable.h>
 #include <__config>
 #include <__type_traits/decay.h>
 #include <__type_traits/is_same.h>
@@ -37,16 +38,14 @@ struct __fn {
   }
 
   template <class _Tp, class _Up>
-    requires is_same_v<decay_t<_Tp>, decay_t<_Up>>
-  _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) noexcept(noexcept(
-      std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? strong_ordering::equal
-      : std::forward<_Tp>(__t) < std::forward<_Up>(__u)
-          ? strong_ordering::less
-          : strong_ordering::greater))
-      -> decltype(std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? strong_ordering::equal
-                  : std::forward<_Tp>(__t) < std::forward<_Up>(__u)
-                      ? strong_ordering::less
-                      : strong_ordering::greater) {
+    requires is_same_v<decay_t<_Tp>, decay_t<_Up>> && requires(_Tp&& __t, _Up&& __u) {
+      { std::forward<_Tp>(__t) == std::forward<_Up>(__u) } -> __boolean_testable;
+      { std::forward<_Tp>(__t) < std::forward<_Up>(__u) } -> __boolean_testable;
+    }
+  _LIBCPP_HIDE_FROM_ABI static constexpr strong_ordering __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) noexcept(
+      noexcept(std::forward<_Tp>(__t) == std::forward<_Up>(__u)  ? strong_ordering::equal
+               : std::forward<_Tp>(__t) < std::forward<_Up>(__u) ? strong_ordering::less
+                                                                 : strong_ordering::greater)) {
     return std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? strong_ordering::equal
          : std::forward<_Tp>(__t) < std::forward<_Up>(__u)
              ? strong_ordering::less
diff --git a/libcxx/include/__compare/compare_weak_order_fallback.h b/libcxx/include/__compare/compare_weak_order_fallback.h
index 062b7b582cd7e..26689fbd9f445 100644
--- a/libcxx/include/__compare/compare_weak_order_fallback.h
+++ b/libcxx/include/__compare/compare_weak_order_fallback.h
@@ -11,6 +11,7 @@
 
 #include <__compare/ordering.h>
 #include <__compare/weak_order.h>
+#include <__concepts/boolean_testable.h>
 #include <__config>
 #include <__type_traits/decay.h>
 #include <__type_traits/is_same.h>
@@ -37,16 +38,15 @@ struct __fn {
   }
 
   template <class _Tp, class _Up>
-    requires is_same_v<decay_t<_Tp>, decay_t<_Up>>
-  _LIBCPP_HIDE_FROM_ABI static constexpr auto __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) noexcept(noexcept(
+    requires is_same_v<decay_t<_Tp>, decay_t<_Up>> && requires(_Tp&& __t, _Up&& __u) {
+      { std::forward<_Tp>(__t) == std::forward<_Up>(__u) } -> __boolean_testable;
+      { std::forward<_Tp>(__t) < std::forward<_Up>(__u) } -> __boolean_testable;
+    }
+  _LIBCPP_HIDE_FROM_ABI static constexpr weak_ordering __go(_Tp&& __t, _Up&& __u, __priority_tag<0>) noexcept(noexcept(
       std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? weak_ordering::equivalent
       : std::forward<_Tp>(__t) < std::forward<_Up>(__u)
           ? weak_ordering::less
-          : weak_ordering::greater))
-      -> decltype(std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? weak_ordering::equivalent
-                  : std::forward<_Tp>(__t) < std::forward<_Up>(__u)
-                      ? weak_ordering::less
-                      : weak_ordering::greater) {
+          : weak_ordering::greater)) {
     return std::forward<_Tp>(__t) == std::forward<_Up>(__u) ? weak_ordering::equivalent
          : std::forward<_Tp>(__t) < std::forward<_Up>(__u)
              ? weak_ordering::less
diff --git a/libcxx/include/__memory/array_cookie.h b/libcxx/include/__memory/array_cookie.h
index 34eec64320610..10b29c9dcc78e 100644
--- a/libcxx/include/__memory/array_cookie.h
+++ b/libcxx/include/__memory/array_cookie.h
@@ -24,7 +24,7 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 // Trait representing whether a type requires an array cookie at the start of its allocation when
-// allocated as `new T[n]` and deallocated as `delete array`.
+// allocated as `new T[n]` and deallocated as `delete[] array`.
 //
 // Under the Itanium C++ ABI [1], we know that an array cookie is available unless `T` is trivially
 // destructible and the call to `operator delete[]` is not a sized operator delete. Under ABIs other
diff --git a/libcxx/include/__memory/compressed_pair.h b/libcxx/include/__memory/compressed_pair.h
index 6454b2b7ab070..9d44775bdb487 100644
--- a/libcxx/include/__memory/compressed_pair.h
+++ b/libcxx/include/__memory/compressed_pair.h
@@ -56,7 +56,7 @@ template <class _ToPad,
           bool _Empty = ((is_empty<_ToPad>::value && !__libcpp_is_final<_ToPad>::value) ||
                          is_reference<_ToPad>::value || sizeof(_ToPad) == __datasizeof_v<_ToPad>)>
 class __compressed_pair_padding {
-  char __padding_[sizeof(_ToPad) - __datasizeof_v<_ToPad>];
+  char __padding_[sizeof(_ToPad) - __datasizeof_v<_ToPad>] = {};
 };
 
 template <class _ToPad>
diff --git a/libcxx/include/__memory/construct_at.h b/libcxx/include/__memory/construct_at.h
index eb02132480064..d8c97467f54b9 100644
--- a/libcxx/include/__memory/construct_at.h
+++ b/libcxx/include/__memory/construct_at.h
@@ -14,7 +14,6 @@
 #include <__config>
 #include <__iterator/access.h>
 #include <__memory/addressof.h>
-#include <__memory/voidify.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_array.h>
 #include <__utility/declval.h>
@@ -38,7 +37,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 template <class _Tp, class... _Args, class = decltype(::new(std::declval<void*>()) _Tp(std::declval<_Args>()...))>
 _LIBCPP_HIDE_FROM_ABI constexpr _Tp* construct_at(_Tp* __location, _Args&&... __args) {
   _LIBCPP_ASSERT_NON_NULL(__location != nullptr, "null pointer given to construct_at");
-  return ::new (std::__voidify(*__location)) _Tp(std::forward<_Args>(__args)...);
+  return ::new (static_cast<void*>(__location)) _Tp(std::forward<_Args>(__args)...);
 }
 
 #endif
@@ -49,7 +48,7 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp* __construct_at(_Tp* __l
   return std::construct_at(__location, std::forward<_Args>(__args)...);
 #else
   return _LIBCPP_ASSERT_NON_NULL(__location != nullptr, "null pointer given to construct_at"),
-         ::new (std::__voidify(*__location)) _Tp(std::forward<_Args>(__args)...);
+         ::new (static_cast<void*>(__location)) _Tp(std::forward<_Args>(__args)...);
 #endif
 }
 
diff --git a/libcxx/include/__memory/shared_ptr.h b/libcxx/include/__memory/shared_ptr.h
index 70964e6122d5a..20c1b69f45ae6 100644
--- a/libcxx/include/__memory/shared_ptr.h
+++ b/libcxx/include/__memory/shared_ptr.h
@@ -248,33 +248,35 @@ struct __for_overwrite_tag {};
 
 template <class _Tp, class _Alloc>
 struct __shared_ptr_emplace : __shared_weak_count {
+  using __value_type = __remove_cv_t<_Tp>;
+
   template <class... _Args,
             class _Allocator                                                                         = _Alloc,
             __enable_if_t<is_same<typename _Allocator::value_type, __for_overwrite_tag>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI explicit __shared_ptr_emplace(_Alloc __a, _Args&&...) : __storage_(std::move(__a)) {
     static_assert(
         sizeof...(_Args) == 0, "No argument should be provided to the control block when using _for_overwrite");
-    ::new ((void*)__get_elem()) _Tp;
+    ::new (static_cast<void*>(__get_elem())) __value_type;
   }
 
   template <class... _Args,
             class _Allocator                                                                          = _Alloc,
             __enable_if_t<!is_same<typename _Allocator::value_type, __for_overwrite_tag>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI explicit __shared_ptr_emplace(_Alloc __a, _Args&&... __args) : __storage_(std::move(__a)) {
-    using _TpAlloc = typename __allocator_traits_rebind<_Alloc, __remove_cv_t<_Tp> >::type;
+    using _TpAlloc = typename __allocator_traits_rebind<_Alloc, __value_type>::type;
     _TpAlloc __tmp(*__get_alloc());
     allocator_traits<_TpAlloc>::construct(__tmp, __get_elem(), std::forward<_Args>(__args)...);
   }
 
   _LIBCPP_HIDE_FROM_ABI _Alloc* __get_alloc() _NOEXCEPT { return __storage_.__get_alloc(); }
 
-  _LIBCPP_HIDE_FROM_ABI _Tp* __get_elem() _NOEXCEPT { return __storage_.__get_elem(); }
+  _LIBCPP_HIDE_FROM_ABI __value_type* __get_elem() _NOEXCEPT { return __storage_.__get_elem(); }
 
 private:
   template <class _Allocator                                                                         = _Alloc,
             __enable_if_t<is_same<typename _Allocator::value_type, __for_overwrite_tag>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI void __on_zero_shared_impl() _NOEXCEPT {
-    __get_elem()->~_Tp();
+    __get_elem()->~__value_type();
   }
 
   template <class _Allocator                                                                          = _Alloc,
@@ -300,7 +302,7 @@ struct __shared_ptr_emplace : __shared_weak_count {
   // through `std::allocate_shared` and `std::make_shared`.
   struct _Storage {
     struct _Data {
-      _LIBCPP_COMPRESSED_PAIR(_Alloc, __alloc_, _Tp, __elem_);
+      _LIBCPP_COMPRESSED_PAIR(_Alloc, __alloc_, __value_type, __elem_);
     };
 
     _ALIGNAS_TYPE(_Data) char __buffer_[sizeof(_Data)];
@@ -312,7 +314,7 @@ struct __shared_ptr_emplace : __shared_weak_count {
       return std::addressof(reinterpret_cast<_Data*>(__buffer_)->__alloc_);
     }
 
-    _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_CFI _Tp* __get_elem() _NOEXCEPT {
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_CFI __value_type* __get_elem() _NOEXCEPT {
       return std::addressof(reinterpret_cast<_Data*>(__buffer_)->__elem_);
     }
   };
diff --git a/libcxx/include/__memory/uninitialized_algorithms.h b/libcxx/include/__memory/uninitialized_algorithms.h
index 8ff87e28b3bb5..dd72f3c10cf15 100644
--- a/libcxx/include/__memory/uninitialized_algorithms.h
+++ b/libcxx/include/__memory/uninitialized_algorithms.h
@@ -21,7 +21,6 @@
 #include <__memory/allocator_traits.h>
 #include <__memory/construct_at.h>
 #include <__memory/pointer_traits.h>
-#include <__memory/voidify.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/extent.h>
 #include <__type_traits/is_array.h>
@@ -64,7 +63,7 @@ inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator> __uninitiali
   try {
 #endif
     for (; __ifirst != __ilast && !__stop_copying(__idx); ++__ifirst, (void)++__idx)
-      ::new (std::__voidify(*__idx)) _ValueType(*__ifirst);
+      ::new (static_cast<void*>(std::addressof(*__idx))) _ValueType(*__ifirst);
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
   } catch (...) {
     std::__destroy(__ofirst, __idx);
@@ -94,7 +93,7 @@ __uninitialized_copy_n(_InputIterator __ifirst, _Size __n, _ForwardIterator __of
   try {
 #endif
     for (; __n > 0 && !__stop_copying(__idx); ++__ifirst, (void)++__idx, (void)--__n)
-      ::new (std::__voidify(*__idx)) _ValueType(*__ifirst);
+      ::new (static_cast<void*>(std::addressof(*__idx))) _ValueType(*__ifirst);
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
   } catch (...) {
     std::__destroy(__ofirst, __idx);
@@ -124,7 +123,7 @@ __uninitialized_fill(_ForwardIterator __first, _Sentinel __last, const _Tp& __x)
   try {
 #endif
     for (; __idx != __last; ++__idx)
-      ::new (std::__voidify(*__idx)) _ValueType(__x);
+      ::new (static_cast<void*>(std::addressof(*__idx))) _ValueType(__x);
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
   } catch (...) {
     std::__destroy(__first, __idx);
@@ -152,7 +151,7 @@ __uninitialized_fill_n(_ForwardIterator __first, _Size __n, const _Tp& __x) {
   try {
 #endif
     for (; __n > 0; ++__idx, (void)--__n)
-      ::new (std::__voidify(*__idx)) _ValueType(__x);
+      ::new (static_cast<void*>(std::addressof(*__idx))) _ValueType(__x);
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
   } catch (...) {
     std::__destroy(__first, __idx);
@@ -182,7 +181,7 @@ __uninitialized_default_construct(_ForwardIterator __first, _Sentinel __last) {
   try {
 #  endif
     for (; __idx != __last; ++__idx)
-      ::new (std::__voidify(*__idx)) _ValueType;
+      ::new (static_cast<void*>(std::addressof(*__idx))) _ValueType;
 #  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
   } catch (...) {
     std::__destroy(__first, __idx);
@@ -208,7 +207,7 @@ inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator __uninitialized_default_construct_
   try {
 #  endif
     for (; __n > 0; ++__idx, (void)--__n)
-      ::new (std::__voidify(*__idx)) _ValueType;
+      ::new (static_cast<void*>(std::addressof(*__idx))) _ValueType;
 #  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
   } catch (...) {
     std::__destroy(__first, __idx);
@@ -235,7 +234,7 @@ __uninitialized_value_construct(_ForwardIterator __first, _Sentinel __last) {
   try {
 #  endif
     for (; __idx != __last; ++__idx)
-      ::new (std::__voidify(*__idx)) _ValueType();
+      ::new (static_cast<void*>(std::addressof(*__idx))) _ValueType();
 #  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
   } catch (...) {
     std::__destroy(__first, __idx);
@@ -261,7 +260,7 @@ inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator __uninitialized_value_construct_n(
   try {
 #  endif
     for (; __n > 0; ++__idx, (void)--__n)
-      ::new (std::__voidify(*__idx)) _ValueType();
+      ::new (static_cast<void*>(std::addressof(*__idx))) _ValueType();
 #  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
   } catch (...) {
     std::__destroy(__first, __idx);
@@ -297,7 +296,7 @@ inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator> __uninitiali
   try {
 #  endif
     for (; __ifirst != __ilast && !__stop_moving(__idx); ++__idx, (void)++__ifirst) {
-      ::new (std::__voidify(*__idx)) _ValueType(__iter_move(__ifirst));
+      ::new (static_cast<void*>(std::addressof(*__idx))) _ValueType(__iter_move(__ifirst));
     }
 #  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
   } catch (...) {
@@ -335,7 +334,7 @@ inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator> __uninitiali
   try {
 #  endif
     for (; __n > 0 && !__stop_moving(__idx); ++__idx, (void)++__ifirst, --__n)
-      ::new (std::__voidify(*__idx)) _ValueType(__iter_move(__ifirst));
+      ::new (static_cast<void*>(std::addressof(*__idx))) _ValueType(__iter_move(__ifirst));
 #  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
   } catch (...) {
     std::__destroy(__ofirst, __idx);
diff --git a/libcxx/include/__memory/unique_ptr.h b/libcxx/include/__memory/unique_ptr.h
index 11215dc111e36..6e42ef1eaa1a3 100644
--- a/libcxx/include/__memory/unique_ptr.h
+++ b/libcxx/include/__memory/unique_ptr.h
@@ -101,6 +101,12 @@ struct _LIBCPP_TEMPLATE_VIS default_delete<_Tp[]> {
   }
 };
 
+template <class _Deleter>
+struct __is_default_deleter : false_type {};
+
+template <class _Tp>
+struct __is_default_deleter<default_delete<_Tp> > : true_type {};
+
 template <class _Deleter>
 struct __unique_ptr_deleter_sfinae {
   static_assert(!is_reference<_Deleter>::value, "incorrect specialization");
@@ -307,11 +313,16 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr {
 // 1. When an array cookie (see [1]) exists at the beginning of the array allocation, we are
 //    able to reuse that cookie to extract the size of the array and perform bounds checking.
 //    An array cookie is a size inserted at the beginning of the allocation by the compiler.
-//    That size is inserted implicitly when doing `new T[n]` in some cases, and its purpose
-//    is to allow the runtime to destroy the `n` array elements when doing `delete array`.
+//    That size is inserted implicitly when doing `new T[n]` in some cases (as of writing this
+//    exactly when the array elements are not trivially destructible), and its main purpose is
+//    to allow the runtime to destroy the `n` array elements when doing `delete[] array`.
 //    When we are able to use array cookies, we reuse information already available in the
 //    current runtime, so bounds checking does not require changing libc++'s ABI.
 //
+//    However, note that we cannot assume the presence of an array cookie when a custom deleter
+//    is used, because the unique_ptr could have been created from an allocation that wasn't
+//    obtained via `new T[n]` (since it may not be deleted with `delete[] arr`).
+//
 // 2. When the "bounded unique_ptr" ABI configuration (controlled by `_LIBCPP_ABI_BOUNDED_UNIQUE_PTR`)
 //    is enabled, we store the size of the allocation (when it is known) so we can check it when
 //    indexing into the `unique_ptr`. That changes the layout of `std::unique_ptr<T[]>`, which is
@@ -328,7 +339,7 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr {
 //    try to fall back to using an array cookie when available.
 //
 //    Finally, note that when this ABI configuration is enabled, we have no choice but to always
-//    make space for a size to be stored in the unique_ptr. Indeed, while we might want to avoid
+//    make space for the size to be stored in the unique_ptr. Indeed, while we might want to avoid
 //    storing the size when an array cookie is available, knowing whether an array cookie is available
 //    requires the type stored in the unique_ptr to be complete, while unique_ptr can normally
 //    accommodate incomplete types.
@@ -339,7 +350,9 @@ struct __unique_ptr_array_bounds_stateless {
   __unique_ptr_array_bounds_stateless() = default;
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __unique_ptr_array_bounds_stateless(size_t) {}
 
-  template <class _Tp, __enable_if_t<__has_array_cookie<_Tp>::value, int> = 0>
+  template <class _Deleter,
+            class _Tp,
+            __enable_if_t<__is_default_deleter<_Deleter>::value && __has_array_cookie<_Tp>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool __in_bounds(_Tp* __ptr, size_t __index) const {
     // In constant expressions, we can't check the array cookie so we just pretend that the index
     // is in-bounds. The compiler catches invalid accesses anyway.
@@ -349,7 +362,9 @@ struct __unique_ptr_array_bounds_stateless {
     return __index < __cookie;
   }
 
-  template <class _Tp, __enable_if_t<!__has_array_cookie<_Tp>::value, int> = 0>
+  template <class _Deleter,
+            class _Tp,
+            __enable_if_t<!__is_default_deleter<_Deleter>::value || !__has_array_cookie<_Tp>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool __in_bounds(_Tp*, size_t) const {
     return true; // If we don't have an array cookie, we assume the access is in-bounds
   }
@@ -365,7 +380,9 @@ struct __unique_ptr_array_bounds_stored {
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __unique_ptr_array_bounds_stored(size_t __size) : __size_(__size) {}
 
   // Use the array cookie if there's one
-  template <class _Tp, __enable_if_t<__has_array_cookie<_Tp>::value, int> = 0>
+  template <class _Deleter,
+            class _Tp,
+            __enable_if_t<__is_default_deleter<_Deleter>::value && __has_array_cookie<_Tp>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool __in_bounds(_Tp* __ptr, size_t __index) const {
     if (__libcpp_is_constant_evaluated())
       return true;
@@ -374,7 +391,9 @@ struct __unique_ptr_array_bounds_stored {
   }
 
   // Otherwise, fall back on the stored size (if any)
-  template <class _Tp, __enable_if_t<!__has_array_cookie<_Tp>::value, int> = 0>
+  template <class _Deleter,
+            class _Tp,
+            __enable_if_t<!__is_default_deleter<_Deleter>::value || !__has_array_cookie<_Tp>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool __in_bounds(_Tp*, size_t __index) const {
     return __index < __size_;
   }
@@ -562,7 +581,7 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp>
   }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 __add_lvalue_reference_t<_Tp> operator[](size_t __i) const {
-    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__checker_.__in_bounds(std::__to_address(__ptr_), __i),
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__checker_.__in_bounds<deleter_type>(std::__to_address(__ptr_), __i),
                                         "unique_ptr<T[]>::operator[](index): index out of range");
     return __ptr_[__i];
   }
diff --git a/libcxx/include/__memory/voidify.h b/libcxx/include/__memory/voidify.h
deleted file mode 100644
index dbd083bd8c1e9..0000000000000
--- a/libcxx/include/__memory/voidify.h
+++ /dev/null
@@ -1,30 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _LIBCPP___MEMORY_VOIDIFY_H
-#define _LIBCPP___MEMORY_VOIDIFY_H
-
-#include <__config>
-#include <__memory/addressof.h>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-_LIBCPP_BEGIN_NAMESPACE_STD
-
-template <typename _Tp>
-_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void* __voidify(_Tp& __from) {
-  // Cast away cv-qualifiers to allow modifying elements of a range through const iterators.
-  return const_cast<void*>(static_cast<const volatile void*>(std::addressof(__from)));
-}
-
-_LIBCPP_END_NAMESPACE_STD
-
-#endif // _LIBCPP___MEMORY_VOIDIFY_H
diff --git a/libcxx/include/__ranges/chunk_by_view.h b/libcxx/include/__ranges/chunk_by_view.h
index 00014d9f10ae8..e4df589bc286f 100644
--- a/libcxx/include/__ranges/chunk_by_view.h
+++ b/libcxx/include/__ranges/chunk_by_view.h
@@ -215,7 +215,7 @@ struct __fn {
     requires constructible_from<decay_t<_Pred>, _Pred>
   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Pred&& __pred) const
       noexcept(is_nothrow_constructible_v<decay_t<_Pred>, _Pred>) {
-    return __range_adaptor_closure_t(std::__bind_back(*this, std::forward<_Pred>(__pred)));
+    return __pipeable(std::__bind_back(*this, std::forward<_Pred>(__pred)));
   }
 };
 } // namespace __chunk_by
diff --git a/libcxx/include/__ranges/drop_view.h b/libcxx/include/__ranges/drop_view.h
index 853e22a402cad..bd66371f4ed21 100644
--- a/libcxx/include/__ranges/drop_view.h
+++ b/libcxx/include/__ranges/drop_view.h
@@ -307,7 +307,7 @@ struct __fn {
     requires constructible_from<decay_t<_Np>, _Np>
   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Np&& __n) const
       noexcept(is_nothrow_constructible_v<decay_t<_Np>, _Np>) {
-    return __range_adaptor_closure_t(std::__bind_back(*this, std::forward<_Np>(__n)));
+    return __pipeable(std::__bind_back(*this, std::forward<_Np>(__n)));
   }
 };
 
diff --git a/libcxx/include/__ranges/drop_while_view.h b/libcxx/include/__ranges/drop_while_view.h
index 92f48bd0ecfba..6413ff52bc0e0 100644
--- a/libcxx/include/__ranges/drop_while_view.h
+++ b/libcxx/include/__ranges/drop_while_view.h
@@ -115,7 +115,7 @@ struct __fn {
     requires constructible_from<decay_t<_Pred>, _Pred>
   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Pred&& __pred) const
       noexcept(is_nothrow_constructible_v<decay_t<_Pred>, _Pred>) {
-    return __range_adaptor_closure_t(std::__bind_back(*this, std::forward<_Pred>(__pred)));
+    return __pipeable(std::__bind_back(*this, std::forward<_Pred>(__pred)));
   }
 };
 
diff --git a/libcxx/include/__ranges/filter_view.h b/libcxx/include/__ranges/filter_view.h
index 5b938dd4c16e1..22f67b2d967d5 100644
--- a/libcxx/include/__ranges/filter_view.h
+++ b/libcxx/include/__ranges/filter_view.h
@@ -239,7 +239,7 @@ struct __fn {
     requires constructible_from<decay_t<_Pred>, _Pred>
   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Pred&& __pred) const
       noexcept(is_nothrow_constructible_v<decay_t<_Pred>, _Pred>) {
-    return __range_adaptor_closure_t(std::__bind_back(*this, std::forward<_Pred>(__pred)));
+    return __pipeable(std::__bind_back(*this, std::forward<_Pred>(__pred)));
   }
 };
 } // namespace __filter
diff --git a/libcxx/include/__ranges/lazy_split_view.h b/libcxx/include/__ranges/lazy_split_view.h
index db031fe5f8b49..0dcbc134a21f2 100644
--- a/libcxx/include/__ranges/lazy_split_view.h
+++ b/libcxx/include/__ranges/lazy_split_view.h
@@ -420,7 +420,7 @@ struct __fn {
     requires constructible_from<decay_t<_Pattern>, _Pattern>
   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Pattern&& __pattern) const
       noexcept(is_nothrow_constructible_v<decay_t<_Pattern>, _Pattern>) {
-    return __range_adaptor_closure_t(std::__bind_back(*this, std::forward<_Pattern>(__pattern)));
+    return __pipeable(std::__bind_back(*this, std::forward<_Pattern>(__pattern)));
   }
 };
 } // namespace __lazy_split_view
diff --git a/libcxx/include/__ranges/range_adaptor.h b/libcxx/include/__ranges/range_adaptor.h
index b20165319bc35..d944a83406ba7 100644
--- a/libcxx/include/__ranges/range_adaptor.h
+++ b/libcxx/include/__ranges/range_adaptor.h
@@ -47,15 +47,15 @@ namespace ranges {
 // - `f1 | f2` is an adaptor closure `g` such that `g(x)` is equivalent to `f2(f1(x))`
 template <class _Tp>
   requires is_class_v<_Tp> && same_as<_Tp, remove_cv_t<_Tp>>
-struct __range_adaptor_closure;
+struct __range_adaptor_closure {};
 
 // Type that wraps an arbitrary function object and makes it into a range adaptor closure,
 // i.e. something that can be called via the `x | f` notation.
 template <class _Fn>
-struct __range_adaptor_closure_t : _Fn, __range_adaptor_closure<__range_adaptor_closure_t<_Fn>> {
-  _LIBCPP_HIDE_FROM_ABI constexpr explicit __range_adaptor_closure_t(_Fn&& __f) : _Fn(std::move(__f)) {}
+struct __pipeable : _Fn, __range_adaptor_closure<__pipeable<_Fn>> {
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __pipeable(_Fn&& __f) : _Fn(std::move(__f)) {}
 };
-_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(__range_adaptor_closure_t);
+_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(__pipeable);
 
 template <class _Tp>
 _Tp __derived_from_range_adaptor_closure(__range_adaptor_closure<_Tp>*);
@@ -79,13 +79,9 @@ template <_RangeAdaptorClosure _Closure, _RangeAdaptorClosure _OtherClosure>
 [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator|(_Closure&& __c1, _OtherClosure&& __c2) noexcept(
     is_nothrow_constructible_v<decay_t<_Closure>, _Closure> &&
     is_nothrow_constructible_v<decay_t<_OtherClosure>, _OtherClosure>) {
-  return __range_adaptor_closure_t(std::__compose(std::forward<_OtherClosure>(__c2), std::forward<_Closure>(__c1)));
+  return __pipeable(std::__compose(std::forward<_OtherClosure>(__c2), std::forward<_Closure>(__c1)));
 }
 
-template <class _Tp>
-  requires is_class_v<_Tp> && same_as<_Tp, remove_cv_t<_Tp>>
-struct __range_adaptor_closure {};
-
 #  if _LIBCPP_STD_VER >= 23
 template <class _Tp>
   requires is_class_v<_Tp> && same_as<_Tp, remove_cv_t<_Tp>>
diff --git a/libcxx/include/__ranges/split_view.h b/libcxx/include/__ranges/split_view.h
index ce3606aedfefb..7527281905ffe 100644
--- a/libcxx/include/__ranges/split_view.h
+++ b/libcxx/include/__ranges/split_view.h
@@ -211,7 +211,7 @@ struct __fn {
     requires constructible_from<decay_t<_Pattern>, _Pattern>
   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Pattern&& __pattern) const
       noexcept(is_nothrow_constructible_v<decay_t<_Pattern>, _Pattern>) {
-    return __range_adaptor_closure_t(std::__bind_back(*this, std::forward<_Pattern>(__pattern)));
+    return __pipeable(std::__bind_back(*this, std::forward<_Pattern>(__pattern)));
   }
 };
 } // namespace __split_view
diff --git a/libcxx/include/__ranges/take_view.h b/libcxx/include/__ranges/take_view.h
index 27ca8155a69b1..8e2d354b58a55 100644
--- a/libcxx/include/__ranges/take_view.h
+++ b/libcxx/include/__ranges/take_view.h
@@ -347,7 +347,7 @@ struct __fn {
     requires constructible_from<decay_t<_Np>, _Np>
   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Np&& __n) const
       noexcept(is_nothrow_constructible_v<decay_t<_Np>, _Np>) {
-    return __range_adaptor_closure_t(std::__bind_back(*this, std::forward<_Np>(__n)));
+    return __pipeable(std::__bind_back(*this, std::forward<_Np>(__n)));
   }
 };
 
diff --git a/libcxx/include/__ranges/take_while_view.h b/libcxx/include/__ranges/take_while_view.h
index 77ea9f7bb8131..b7cb0aef58f11 100644
--- a/libcxx/include/__ranges/take_while_view.h
+++ b/libcxx/include/__ranges/take_while_view.h
@@ -149,7 +149,7 @@ struct __fn {
     requires constructible_from<decay_t<_Pred>, _Pred>
   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Pred&& __pred) const
       noexcept(is_nothrow_constructible_v<decay_t<_Pred>, _Pred>) {
-    return __range_adaptor_closure_t(std::__bind_back(*this, std::forward<_Pred>(__pred)));
+    return __pipeable(std::__bind_back(*this, std::forward<_Pred>(__pred)));
   }
 };
 
diff --git a/libcxx/include/__ranges/to.h b/libcxx/include/__ranges/to.h
index e0abe6290b8f7..85fc580b53c9b 100644
--- a/libcxx/include/__ranges/to.h
+++ b/libcxx/include/__ranges/to.h
@@ -214,7 +214,7 @@ template <class _Container, class... _Args>
     }
   { return ranges::to<_Container>(std::forward<_Range>(__range), std::forward<_Tail>(__tail)...); };
 
-  return __range_adaptor_closure_t(std::__bind_back(__to_func, std::forward<_Args>(__args)...));
+  return __pipeable(std::__bind_back(__to_func, std::forward<_Args>(__args)...));
 }
 
 // Range adaptor closure object 2 -- wrapping the `ranges::to` version where `_Container` is a template template
@@ -233,7 +233,7 @@ template <template <class...> class _Container, class... _Args>
   };
   // clang-format on
 
-  return __range_adaptor_closure_t(std::__bind_back(__to_func, std::forward<_Args>(__args)...));
+  return __pipeable(std::__bind_back(__to_func, std::forward<_Args>(__args)...));
 }
 
 } // namespace ranges
diff --git a/libcxx/include/__ranges/transform_view.h b/libcxx/include/__ranges/transform_view.h
index 42e9a131f73b1..0ce1bfe883500 100644
--- a/libcxx/include/__ranges/transform_view.h
+++ b/libcxx/include/__ranges/transform_view.h
@@ -398,7 +398,7 @@ struct __fn {
     requires constructible_from<decay_t<_Fn>, _Fn>
   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Fn&& __f) const
       noexcept(is_nothrow_constructible_v<decay_t<_Fn>, _Fn>) {
-    return __range_adaptor_closure_t(std::__bind_back(*this, std::forward<_Fn>(__f)));
+    return __pipeable(std::__bind_back(*this, std::forward<_Fn>(__f)));
   }
 };
 } // namespace __transform
diff --git a/libcxx/include/algorithm b/libcxx/include/algorithm
index 17d63ce0cf1c0..14355d63f2901 100644
--- a/libcxx/include/algorithm
+++ b/libcxx/include/algorithm
@@ -1237,9 +1237,9 @@ template <class InputIterator1, class InputIterator2>
     mismatch(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2);
 
 template <class InputIterator1, class InputIterator2>
-    constexpr pair<InputIterator1, InputIterator2>   // constexpr in C++20
+    constexpr pair<InputIterator1, InputIterator2>
     mismatch(InputIterator1 first1, InputIterator1 last1,
-             InputIterator2 first2, InputIterator2 last2); // **C++14**
+             InputIterator2 first2, InputIterator2 last2);              // since C++14, constexpr in C++20
 
 template <class InputIterator1, class InputIterator2, class BinaryPredicate>
     constexpr pair<InputIterator1, InputIterator2>   // constexpr in C++20
@@ -1247,19 +1247,19 @@ template <class InputIterator1, class InputIterator2, class BinaryPredicate>
              InputIterator2 first2, BinaryPredicate pred);
 
 template <class InputIterator1, class InputIterator2, class BinaryPredicate>
-    constexpr pair<InputIterator1, InputIterator2>   // constexpr in C++20
+    constexpr pair<InputIterator1, InputIterator2>
     mismatch(InputIterator1 first1, InputIterator1 last1,
              InputIterator2 first2, InputIterator2 last2,
-             BinaryPredicate pred); // **C++14**
+             BinaryPredicate pred);                                     // since C++14, constexpr in C++20
 
 template <class InputIterator1, class InputIterator2>
     constexpr bool      // constexpr in C++20
     equal(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2);
 
 template <class InputIterator1, class InputIterator2>
-    constexpr bool      // constexpr in C++20
+    constexpr bool
     equal(InputIterator1 first1, InputIterator1 last1,
-          InputIterator2 first2, InputIterator2 last2); // **C++14**
+          InputIterator2 first2, InputIterator2 last2);                 // since C++14, constexpr in C++20
 
 template <class InputIterator1, class InputIterator2, class BinaryPredicate>
     constexpr bool      // constexpr in C++20
@@ -1267,10 +1267,10 @@ template <class InputIterator1, class InputIterator2, class BinaryPredicate>
           InputIterator2 first2, BinaryPredicate pred);
 
 template <class InputIterator1, class InputIterator2, class BinaryPredicate>
-    constexpr bool      // constexpr in C++20
+    constexpr bool
     equal(InputIterator1 first1, InputIterator1 last1,
           InputIterator2 first2, InputIterator2 last2,
-          BinaryPredicate pred); // **C++14**
+          BinaryPredicate pred);                                        // since C++14, constexpr in C++20
 
 template<class ForwardIterator1, class ForwardIterator2>
     constexpr bool      // constexpr in C++20
@@ -1278,9 +1278,9 @@ template<class ForwardIterator1, class ForwardIterator2>
                    ForwardIterator2 first2);
 
 template<class ForwardIterator1, class ForwardIterator2>
-    constexpr bool      // constexpr in C++20
+    constexpr bool
     is_permutation(ForwardIterator1 first1, ForwardIterator1 last1,
-                   ForwardIterator2 first2, ForwardIterator2 last2); // **C++14**
+                   ForwardIterator2 first2, ForwardIterator2 last2);    // since C++14, constexpr in C++20
 
 template<class ForwardIterator1, class ForwardIterator2, class BinaryPredicate>
     constexpr bool      // constexpr in C++20
@@ -1288,10 +1288,10 @@ template<class ForwardIterator1, class ForwardIterator2, class BinaryPredicate>
                    ForwardIterator2 first2, BinaryPredicate pred);
 
 template<class ForwardIterator1, class ForwardIterator2, class BinaryPredicate>
-    constexpr bool      // constexpr in C++20
+    constexpr bool
     is_permutation(ForwardIterator1 first1, ForwardIterator1 last1,
                    ForwardIterator2 first2, ForwardIterator2 last2,
-                   BinaryPredicate pred);  // **C++14**
+                   BinaryPredicate pred);                               // since C++14, constexpr in C++20
 
 template <class ForwardIterator1, class ForwardIterator2>
     constexpr ForwardIterator1      // constexpr in C++20
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index 881c0ca6c3669..dee9b0b88b794 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -1528,7 +1528,6 @@ module std [system] {
     }
     module uses_allocator                     { header "__memory/uses_allocator.h" }
     module uses_allocator_construction        { header "__memory/uses_allocator_construction.h" }
-    module voidify                            { header "__memory/voidify.h" }
 
     header "memory"
     export *
diff --git a/libcxx/include/optional b/libcxx/include/optional
index 7578833685ec1..4e44ef990f5d2 100644
--- a/libcxx/include/optional
+++ b/libcxx/include/optional
@@ -287,7 +287,7 @@ struct __optional_destruct_base<_Tp, false> {
   static_assert(is_object_v<value_type>, "instantiation of optional with a non-object type is undefined behavior");
   union {
     char __null_state_;
-    value_type __val_;
+    remove_cv_t<value_type> __val_;
   };
   bool __engaged_;
 
@@ -323,7 +323,7 @@ struct __optional_destruct_base<_Tp, true> {
   static_assert(is_object_v<value_type>, "instantiation of optional with a non-object type is undefined behavior");
   union {
     char __null_state_;
-    value_type __val_;
+    remove_cv_t<value_type> __val_;
   };
   bool __engaged_;
 
@@ -377,7 +377,7 @@ struct __optional_storage_base : __optional_destruct_base<_Tp> {
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __assign_from(_That&& __opt) {
     if (this->__engaged_ == __opt.has_value()) {
       if (this->__engaged_)
-        this->__val_ = std::forward<_That>(__opt).__get();
+        static_cast<_Tp&>(this->__val_) = std::forward<_That>(__opt).__get();
     } else {
       if (this->__engaged_)
         this->reset();
diff --git a/libcxx/include/string b/libcxx/include/string
index fdb189016bfba..685dbcbd4418e 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -941,7 +941,7 @@ private:
       __set_short_size(__size);
     } else {
       auto __capacity   = __recommend(__size) + 1;
-      auto __allocation = __alloc_traits::allocate(__alloc(), __capacity);
+      auto __allocation = __alloc_traits::allocate(__alloc_, __capacity);
       __begin_lifetime(__allocation, __capacity);
       __set_long_cap(__capacity);
       __set_long_pointer(__allocation);
@@ -1006,7 +1006,7 @@ public:
   }
 
   _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS basic_string(const basic_string& __str)
-      : __alloc_(__alloc_traits::select_on_container_copy_construction(__str.__alloc())) {
+      : __alloc_(__alloc_traits::select_on_container_copy_construction(__str.__alloc_)) {
     if (!__str.__is_long()) {
       __rep_ = __str.__rep_;
       __annotate_new(__get_short_size());
@@ -1048,7 +1048,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(basic_string&& __str, const allocator_type& __a)
       : __alloc_(__a) {
-    if (__str.__is_long() && __a != __str.__alloc()) // copy, not move
+    if (__str.__is_long() && __a != __str.__alloc_) // copy, not move
       __init(std::__to_address(__str.__get_long_pointer()), __str.__get_long_size());
     else {
       if (__libcpp_is_constant_evaluated())
@@ -1107,7 +1107,7 @@ public:
       __throw_out_of_range();
 
     auto __len = std::min<size_type>(__n, __str.size() - __pos);
-    if (__alloc_traits::is_always_equal::value || __alloc == __str.__alloc()) {
+    if (__alloc_traits::is_always_equal::value || __alloc == __str.__alloc_) {
       __move_assign(std::move(__str), __pos, __len);
     } else {
       // Perform a copy because the allocators are not compatible.
@@ -1212,7 +1212,7 @@ public:
   inline _LIBCPP_CONSTEXPR_SINCE_CXX20 ~basic_string() {
     __annotate_delete();
     if (__is_long())
-      __alloc_traits::deallocate(__alloc(), __get_long_pointer(), __get_long_cap());
+      __alloc_traits::deallocate(__alloc_, __get_long_pointer(), __get_long_cap());
   }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 operator __self_view() const _NOEXCEPT {
@@ -1289,7 +1289,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type length() const _NOEXCEPT { return size(); }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type max_size() const _NOEXCEPT {
-    size_type __m = __alloc_traits::max_size(__alloc());
+    size_type __m = __alloc_traits::max_size(__alloc_);
     if (__m <= std::numeric_limits<size_type>::max() / 2) {
       return __m - __alignment;
     } else {
@@ -1409,7 +1409,7 @@ public:
   template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
   _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
   append(_InputIterator __first, _InputIterator __last) {
-    const basic_string __temp(__first, __last, __alloc());
+    const basic_string __temp(__first, __last, __alloc_);
     append(__temp.data(), __temp.size());
     return *this;
   }
@@ -1465,7 +1465,7 @@ public:
 #if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI constexpr void __move_assign(basic_string&& __str, size_type __pos, size_type __len) {
     // Pilfer the allocation from __str.
-    _LIBCPP_ASSERT_INTERNAL(__alloc() == __str.__alloc(), "__move_assign called with wrong allocator");
+    _LIBCPP_ASSERT_INTERNAL(__alloc_ == __str.__alloc_, "__move_assign called with wrong allocator");
     size_type __old_sz = __str.size();
     if (!__str.__is_long())
       __str.__annotate_delete();
@@ -1571,7 +1571,7 @@ public:
       return __insert_with_size(__position, ranges::begin(__range), ranges::end(__range), __n);
 
     } else {
-      basic_string __temp(from_range, std::forward<_Range>(__range), __alloc());
+      basic_string __temp(from_range, std::forward<_Range>(__range), __alloc_);
       return insert(__position, __temp.data(), __temp.data() + __temp.size());
     }
   }
@@ -1666,7 +1666,7 @@ public:
   template <_ContainerCompatibleRange<_CharT> _Range>
   _LIBCPP_HIDE_FROM_ABI constexpr basic_string&
   replace_with_range(const_iterator __i1, const_iterator __i2, _Range&& __range) {
-    basic_string __temp(from_range, std::forward<_Range>(__range), __alloc());
+    basic_string __temp(from_range, std::forward<_Range>(__range), __alloc_);
     return replace(__i1, __i2, __temp);
   }
 #endif
@@ -1713,7 +1713,7 @@ public:
 #endif
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 allocator_type get_allocator() const _NOEXCEPT {
-    return __alloc();
+    return __alloc_;
   }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
@@ -1949,9 +1949,6 @@ private:
   _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator
   __insert_with_size(const_iterator __pos, _Iterator __first, _Sentinel __last, size_type __n);
 
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 allocator_type& __alloc() _NOEXCEPT { return __alloc_; }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR const allocator_type& __alloc() const _NOEXCEPT { return __alloc_; }
-
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS void
   __set_short_size(size_type __s) _NOEXCEPT {
     _LIBCPP_ASSERT_INTERNAL(__s < __min_cap, "__s should never be greater than or equal to the short string capacity");
@@ -2152,20 +2149,20 @@ private:
   }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __copy_assign_alloc(const basic_string& __str, true_type) {
-    if (__alloc() == __str.__alloc())
-      __alloc() = __str.__alloc();
+    if (__alloc_ == __str.__alloc_)
+      __alloc_ = __str.__alloc_;
     else {
       if (!__str.__is_long()) {
         __clear_and_shrink();
-        __alloc() = __str.__alloc();
+        __alloc_ = __str.__alloc_;
       } else {
         __annotate_delete();
-        allocator_type __a = __str.__alloc();
+        allocator_type __a = __str.__alloc_;
         auto __allocation  = std::__allocate_at_least(__a, __str.__get_long_cap());
         __begin_lifetime(__allocation.ptr, __allocation.count);
         if (__is_long())
-          __alloc_traits::deallocate(__alloc(), __get_long_pointer(), __get_long_cap());
-        __alloc() = std::move(__a);
+          __alloc_traits::deallocate(__alloc_, __get_long_pointer(), __get_long_cap());
+        __alloc_ = std::move(__a);
         __set_long_pointer(__allocation.ptr);
         __set_long_cap(__allocation.count);
         __set_long_size(__str.size());
@@ -2198,7 +2195,7 @@ private:
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign_alloc(basic_string& __c, true_type)
       _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
-    __alloc() = std::move(__c.__alloc());
+    __alloc_ = std::move(__c.__alloc_);
   }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __move_assign_alloc(basic_string&, false_type) _NOEXCEPT {}
@@ -2321,7 +2318,7 @@ basic_string<_CharT, _Traits, _Allocator>::__init(const value_type* __s, size_ty
     __set_short_size(__sz);
     __p = __get_short_pointer();
   } else {
-    auto __allocation = std::__allocate_at_least(__alloc(), __recommend(__reserve) + 1);
+    auto __allocation = std::__allocate_at_least(__alloc_, __recommend(__reserve) + 1);
     __p               = __allocation.ptr;
     __begin_lifetime(__p, __allocation.count);
     __set_long_pointer(__p);
@@ -2345,7 +2342,7 @@ basic_string<_CharT, _Traits, _Allocator>::__init(const value_type* __s, size_ty
     __set_short_size(__sz);
     __p = __get_short_pointer();
   } else {
-    auto __allocation = std::__allocate_at_least(__alloc(), __recommend(__sz) + 1);
+    auto __allocation = std::__allocate_at_least(__alloc_, __recommend(__sz) + 1);
     __p               = __allocation.ptr;
     __begin_lifetime(__p, __allocation.count);
     __set_long_pointer(__p);
@@ -2370,7 +2367,7 @@ basic_string<_CharT, _Traits, _Allocator>::__init_copy_ctor_external(const value
   } else {
     if (__sz > max_size())
       __throw_length_error();
-    auto __allocation = std::__allocate_at_least(__alloc(), __recommend(__sz) + 1);
+    auto __allocation = std::__allocate_at_least(__alloc_, __recommend(__sz) + 1);
     __p               = __allocation.ptr;
     __begin_lifetime(__p, __allocation.count);
     __set_long_pointer(__p);
@@ -2393,7 +2390,7 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::__
     __set_short_size(__n);
     __p = __get_short_pointer();
   } else {
-    auto __allocation = std::__allocate_at_least(__alloc(), __recommend(__n) + 1);
+    auto __allocation = std::__allocate_at_least(__alloc_, __recommend(__n) + 1);
     __p               = __allocation.ptr;
     __begin_lifetime(__p, __allocation.count);
     __set_long_pointer(__p);
@@ -2428,7 +2425,7 @@ basic_string<_CharT, _Traits, _Allocator>::__init_with_sentinel(_InputIterator _
   } catch (...) {
     __annotate_delete();
     if (__is_long())
-      __alloc_traits::deallocate(__alloc(), __get_long_pointer(), __get_long_cap());
+      __alloc_traits::deallocate(__alloc_, __get_long_pointer(), __get_long_cap());
     throw;
   }
 #endif // _LIBCPP_HAS_NO_EXCEPTIONS
@@ -2458,7 +2455,7 @@ basic_string<_CharT, _Traits, _Allocator>::__init_with_size(_InputIterator __fir
     __p = __get_short_pointer();
 
   } else {
-    auto __allocation = std::__allocate_at_least(__alloc(), __recommend(__sz) + 1);
+    auto __allocation = std::__allocate_at_least(__alloc_, __recommend(__sz) + 1);
     __p               = __allocation.ptr;
     __begin_lifetime(__p, __allocation.count);
     __set_long_pointer(__p);
@@ -2474,7 +2471,7 @@ basic_string<_CharT, _Traits, _Allocator>::__init_with_size(_InputIterator __fir
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
   } catch (...) {
     if (__is_long())
-      __alloc_traits::deallocate(__alloc(), __get_long_pointer(), __get_long_cap());
+      __alloc_traits::deallocate(__alloc_, __get_long_pointer(), __get_long_cap());
     throw;
   }
 #endif // _LIBCPP_HAS_NO_EXCEPTIONS
@@ -2497,7 +2494,7 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::__
   size_type __cap =
       __old_cap < __ms / 2 - __alignment ? __recommend(std::max(__old_cap + __delta_cap, 2 * __old_cap)) : __ms - 1;
   __annotate_delete();
-  auto __allocation = std::__allocate_at_least(__alloc(), __cap + 1);
+  auto __allocation = std::__allocate_at_least(__alloc_, __cap + 1);
   pointer __p       = __allocation.ptr;
   __begin_lifetime(__p, __allocation.count);
   if (__n_copy != 0)
@@ -2509,7 +2506,7 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::__
     traits_type::copy(
         std::__to_address(__p) + __n_copy + __n_add, std::__to_address(__old_p) + __n_copy + __n_del, __sec_cp_sz);
   if (__old_cap + 1 != __min_cap)
-    __alloc_traits::deallocate(__alloc(), __old_p, __old_cap + 1);
+    __alloc_traits::deallocate(__alloc_, __old_p, __old_cap + 1);
   __set_long_pointer(__p);
   __set_long_cap(__allocation.count);
   __old_sz = __n_copy + __n_add + __sec_cp_sz;
@@ -2540,7 +2537,7 @@ _LIBCPP_DEPRECATED_("use __grow_by_without_replace") basic_string<_CharT, _Trait
   size_type __cap =
       __old_cap < __ms / 2 - __alignment ? __recommend(std::max(__old_cap + __delta_cap, 2 * __old_cap)) : __ms - 1;
   __annotate_delete();
-  auto __allocation = std::__allocate_at_least(__alloc(), __cap + 1);
+  auto __allocation = std::__allocate_at_least(__alloc_, __cap + 1);
   pointer __p       = __allocation.ptr;
   __begin_lifetime(__p, __allocation.count);
   if (__n_copy != 0)
@@ -2550,7 +2547,7 @@ _LIBCPP_DEPRECATED_("use __grow_by_without_replace") basic_string<_CharT, _Trait
     traits_type::copy(
         std::__to_address(__p) + __n_copy + __n_add, std::__to_address(__old_p) + __n_copy + __n_del, __sec_cp_sz);
   if (__old_cap + 1 != __min_cap)
-    __alloc_traits::deallocate(__alloc(), __old_p, __old_cap + 1);
+    __alloc_traits::deallocate(__alloc_, __old_p, __old_cap + 1);
   __set_long_pointer(__p);
   __set_long_cap(__allocation.count);
 }
@@ -2685,7 +2682,7 @@ basic_string<_CharT, _Traits, _Allocator>::operator=(const basic_string& __str)
 template <class _CharT, class _Traits, class _Allocator>
 inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::__move_assign(
     basic_string& __str, false_type) noexcept(__alloc_traits::is_always_equal::value) {
-  if (__alloc() != __str.__alloc())
+  if (__alloc_ != __str.__alloc_)
     assign(__str);
   else
     __move_assign(__str, true_type());
@@ -2702,7 +2699,7 @@ basic_string<_CharT, _Traits, _Allocator>::__move_assign(basic_string& __str, tr
 {
   __annotate_delete();
   if (__is_long()) {
-    __alloc_traits::deallocate(__alloc(), __get_long_pointer(), __get_long_cap());
+    __alloc_traits::deallocate(__alloc_, __get_long_pointer(), __get_long_cap());
 #  if _LIBCPP_STD_VER <= 14
     if (!is_nothrow_move_assignable<allocator_type>::value) {
       __set_short_size(0);
@@ -2752,7 +2749,7 @@ template <class _CharT, class _Traits, class _Allocator>
 template <class _InputIterator, class _Sentinel>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
 basic_string<_CharT, _Traits, _Allocator>::__assign_with_sentinel(_InputIterator __first, _Sentinel __last) {
-  const basic_string __temp(__init_with_sentinel_tag(), std::move(__first), std::move(__last), __alloc());
+  const basic_string __temp(__init_with_sentinel_tag(), std::move(__first), std::move(__last), __alloc_);
   assign(__temp.data(), __temp.size());
 }
 
@@ -2940,7 +2937,7 @@ basic_string<_CharT, _Traits, _Allocator>::append(_ForwardIterator __first, _For
       traits_type::assign(*__end, value_type());
       __set_size(__sz + __n);
     } else {
-      const basic_string __temp(__first, __last, __alloc());
+      const basic_string __temp(__first, __last, __alloc_);
       append(__temp.data(), __temp.size());
     }
   }
@@ -3038,7 +3035,7 @@ template <class _CharT, class _Traits, class _Allocator>
 template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> >
 _LIBCPP_CONSTEXPR_SINCE_CXX20 typename basic_string<_CharT, _Traits, _Allocator>::iterator
 basic_string<_CharT, _Traits, _Allocator>::insert(const_iterator __pos, _InputIterator __first, _InputIterator __last) {
-  const basic_string __temp(__first, __last, __alloc());
+  const basic_string __temp(__first, __last, __alloc_);
   return insert(__pos, __temp.data(), __temp.data() + __temp.size());
 }
 
@@ -3063,7 +3060,7 @@ basic_string<_CharT, _Traits, _Allocator>::__insert_with_size(
   if (__string_is_trivial_iterator<_Iterator>::value && !__addr_in_range(*__first)) {
     return __insert_from_safe_copy(__n, __ip, __first, __last);
   } else {
-    const basic_string __temp(__init_with_sentinel_tag(), __first, __last, __alloc());
+    const basic_string __temp(__init_with_sentinel_tag(), __first, __last, __alloc_);
     return __insert_from_safe_copy(__n, __ip, __temp.begin(), __temp.end());
   }
 }
@@ -3200,7 +3197,7 @@ template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_Inp
 _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
 basic_string<_CharT, _Traits, _Allocator>::replace(
     const_iterator __i1, const_iterator __i2, _InputIterator __j1, _InputIterator __j2) {
-  const basic_string __temp(__j1, __j2, __alloc());
+  const basic_string __temp(__j1, __j2, __alloc_);
   return replace(__i1, __i2, __temp);
 }
 
@@ -3372,7 +3369,7 @@ basic_string<_CharT, _Traits, _Allocator>::__shrink_or_extend(size_type __target
     if (__target_capacity > __cap) {
       // Extend
       // - called from reserve should propagate the exception thrown.
-      auto __allocation = std::__allocate_at_least(__alloc(), __target_capacity + 1);
+      auto __allocation = std::__allocate_at_least(__alloc_, __target_capacity + 1);
       __new_data        = __allocation.ptr;
       __target_capacity = __allocation.count - 1;
     } else {
@@ -3382,13 +3379,13 @@ basic_string<_CharT, _Traits, _Allocator>::__shrink_or_extend(size_type __target
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
       try {
 #endif // _LIBCPP_HAS_NO_EXCEPTIONS
-        auto __allocation = std::__allocate_at_least(__alloc(), __target_capacity + 1);
+        auto __allocation = std::__allocate_at_least(__alloc_, __target_capacity + 1);
 
         // The Standard mandates shrink_to_fit() does not increase the capacity.
         // With equal capacity keep the existing buffer. This avoids extra work
         // due to swapping the elements.
         if (__allocation.count - 1 > __target_capacity) {
-          __alloc_traits::deallocate(__alloc(), __allocation.ptr, __allocation.count);
+          __alloc_traits::deallocate(__alloc_, __allocation.ptr, __allocation.count);
           __annotate_new(__sz); // Undoes the __annotate_delete()
           return;
         }
@@ -3407,7 +3404,7 @@ basic_string<_CharT, _Traits, _Allocator>::__shrink_or_extend(size_type __target
   }
   traits_type::copy(std::__to_address(__new_data), std::__to_address(__p), size() + 1);
   if (__was_long)
-    __alloc_traits::deallocate(__alloc(), __p, __cap + 1);
+    __alloc_traits::deallocate(__alloc_, __p, __cap + 1);
   if (__now_long) {
     __set_long_cap(__target_capacity + 1);
     __set_long_size(__sz);
@@ -3454,14 +3451,14 @@ inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocat
 {
   _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(
       __alloc_traits::propagate_on_container_swap::value || __alloc_traits::is_always_equal::value ||
-          __alloc() == __str.__alloc(),
+          __alloc_ == __str.__alloc_,
       "swapping non-equal allocators");
   if (!__is_long())
     __annotate_delete();
   if (this != std::addressof(__str) && !__str.__is_long())
     __str.__annotate_delete();
   std::swap(__rep_, __str.__rep_);
-  std::__swap_allocator(__alloc(), __str.__alloc());
+  std::__swap_allocator(__alloc_, __str.__alloc_);
   if (!__is_long())
     __annotate_new(__get_short_size());
   if (this != std::addressof(__str) && !__str.__is_long())
@@ -3814,7 +3811,7 @@ inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocat
   clear();
   if (__is_long()) {
     __annotate_delete();
-    __alloc_traits::deallocate(__alloc(), __get_long_pointer(), capacity() + 1);
+    __alloc_traits::deallocate(__alloc_, __get_long_pointer(), capacity() + 1);
     __rep_ = __rep();
   }
 }
diff --git a/libcxx/lib/abi/CHANGELOG.TXT b/libcxx/lib/abi/CHANGELOG.TXT
index 68c9d980a016e..6911694b75d8a 100644
--- a/libcxx/lib/abi/CHANGELOG.TXT
+++ b/libcxx/lib/abi/CHANGELOG.TXT
@@ -12,6 +12,47 @@ To generate a summary, re-generate the new ABI list using the
 
 New entries should be added directly below the "Version" header.
 
+------------
+Version 20.0
+------------
+
+* [libc++] Stop trying to avoid exporting some typeinfo names
+
+  This patch removes the explicit list of symbols to avoid exporting
+  from the library, which at this point doesn't cover much anyways.
+  That makes us export some new symbols on Appl platforms, namely the
+  typeinfo names for some implementation-detail types. While we
+  technically wouldn't have to export those (I don't think any user
+  can get their hands on those typeinfo names), that makes the library
+  more consistent on all platforms.
+
+  <arch>-apple-darwin
+  -------------------
+  Symbol added: __ZTSNSt3__114__codecvt_utf8IDsEE
+  Symbol added: __ZTSNSt3__114__codecvt_utf8IwEE
+  Symbol added: __ZTSNSt3__114__shared_countE
+  Symbol added: __ZTSNSt3__114codecvt_bynameIDiDu11__mbstate_tEE
+  Symbol added: __ZTSNSt3__114codecvt_bynameIDic11__mbstate_tEE
+  Symbol added: __ZTSNSt3__114codecvt_bynameIDsDu11__mbstate_tEE
+  Symbol added: __ZTSNSt3__114codecvt_bynameIDsc11__mbstate_tEE
+  Symbol added: __ZTSNSt3__114codecvt_bynameIcc11__mbstate_tEE
+  Symbol added: __ZTSNSt3__114codecvt_bynameIwc11__mbstate_tEE
+  Symbol added: __ZTSNSt3__115__codecvt_utf16IDiLb0EEE
+  Symbol added: __ZTSNSt3__115__codecvt_utf16IDiLb1EEE
+  Symbol added: __ZTSNSt3__115__codecvt_utf16IDsLb0EEE
+  Symbol added: __ZTSNSt3__115__codecvt_utf16IDsLb1EEE
+  Symbol added: __ZTSNSt3__115__codecvt_utf16IwLb0EEE
+  Symbol added: __ZTSNSt3__115__codecvt_utf16IwLb1EEE
+  Symbol added: __ZTSNSt3__116__narrow_to_utf8ILm16EEE
+  Symbol added: __ZTSNSt3__116__narrow_to_utf8ILm32EEE
+  Symbol added: __ZTSNSt3__117__assoc_sub_stateE
+  Symbol added: __ZTSNSt3__117__widen_from_utf8ILm16EEE
+  Symbol added: __ZTSNSt3__117__widen_from_utf8ILm32EEE
+  Symbol added: __ZTSNSt3__119__shared_weak_countE
+  Symbol added: __ZTSNSt3__120__codecvt_utf8_utf16IDiEE
+  Symbol added: __ZTSNSt3__120__codecvt_utf8_utf16IDsEE
+  Symbol added: __ZTSNSt3__120__codecvt_utf8_utf16IwEE
+
 ------------
 Version 19.0
 ------------
diff --git a/libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
index 32acae46e292d..db77e1d0ac30b 100644
--- a/libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -2238,12 +2238,28 @@
 {'is_defined': True, 'name': '__ZTSNSt3__113basic_istreamIwNS_11char_traitsIwEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__113basic_ostreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__113basic_ostreamIwNS_11char_traitsIwEEEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__114__codecvt_utf8IDiEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__114__codecvt_utf8IDsEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__114__codecvt_utf8IwEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__114__shared_countE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__114basic_ifstreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__114basic_iostreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__114basic_ofstreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__114codecvt_bynameIDiDu11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__114codecvt_bynameIDic11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__114codecvt_bynameIDsDu11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__114codecvt_bynameIDsc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__114codecvt_bynameIcc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__114codecvt_bynameIwc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__114collate_bynameIcEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__114collate_bynameIwEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__114error_categoryE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__115__codecvt_utf16IDiLb0EEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__115__codecvt_utf16IDiLb1EEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__115__codecvt_utf16IDsLb0EEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__115__codecvt_utf16IDsLb1EEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__115__codecvt_utf16IwLb0EEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__115__codecvt_utf16IwLb1EEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__115basic_streambufIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__115basic_streambufIwNS_11char_traitsIwEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__115basic_stringbufIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
@@ -2255,15 +2271,24 @@
 {'is_defined': True, 'name': '__ZTSNSt3__115time_get_bynameIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__115time_put_bynameIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__115time_put_bynameIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__116__narrow_to_utf8ILm16EEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__116__narrow_to_utf8ILm32EEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__117__assoc_sub_stateE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__117__widen_from_utf8ILm16EEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__117__widen_from_utf8ILm32EEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__117bad_function_callE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__117moneypunct_bynameIcLb0EEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__117moneypunct_bynameIcLb1EEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__117moneypunct_bynameIwLb0EEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__117moneypunct_bynameIwLb1EEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__119__shared_weak_countE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__119bad_expected_accessIvEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__120__codecvt_utf8_utf16IDiEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__120__codecvt_utf8_utf16IDsEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__120__codecvt_utf8_utf16IwEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__13pmr15memory_resourceE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__13pmr25monotonic_buffer_resourceE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__13pmr26synchronized_pool_resourceE', 'size': 0, 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
index 7b5d77499d55f..62716f5c415f0 100644
--- a/libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -2264,13 +2264,29 @@
 {'is_defined': True, 'name': '__ZTSNSt3__113basic_istreamIwNS_11char_traitsIwEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__113basic_ostreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__113basic_ostreamIwNS_11char_traitsIwEEEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__114__codecvt_utf8IDiEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__114__codecvt_utf8IDsEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__114__codecvt_utf8IwEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__114__shared_countE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__113messages_baseE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__114basic_ifstreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__114basic_iostreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__114basic_ofstreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__114codecvt_bynameIDiDu11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__114codecvt_bynameIDic11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__114codecvt_bynameIDsDu11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__114codecvt_bynameIDsc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__114codecvt_bynameIcc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__114codecvt_bynameIwc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__114collate_bynameIcEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__114collate_bynameIwEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__114error_categoryE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__115__codecvt_utf16IDiLb0EEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__115__codecvt_utf16IDiLb1EEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__115__codecvt_utf16IDsLb0EEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__115__codecvt_utf16IDsLb1EEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__115__codecvt_utf16IwLb0EEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__115__codecvt_utf16IwLb1EEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__115basic_streambufIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__115basic_streambufIwNS_11char_traitsIwEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__115basic_stringbufIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
@@ -2282,15 +2298,24 @@
 {'is_defined': True, 'name': '__ZTSNSt3__115time_get_bynameIwNS_19istreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__115time_put_bynameIcNS_19ostreambuf_iteratorIcNS_11char_traitsIcEEEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__115time_put_bynameIwNS_19ostreambuf_iteratorIwNS_11char_traitsIwEEEEEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__116__narrow_to_utf8ILm16EEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__116__narrow_to_utf8ILm32EEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__117__assoc_sub_stateE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__117__widen_from_utf8ILm16EEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__117__widen_from_utf8ILm32EEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__117bad_function_callE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__117moneypunct_bynameIcLb0EEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__117moneypunct_bynameIcLb1EEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__117moneypunct_bynameIwLb0EEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__117moneypunct_bynameIwLb1EEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__119__shared_weak_countE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__119bad_expected_accessIvEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__120__codecvt_utf8_utf16IDiEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__120__codecvt_utf8_utf16IDsEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__120__codecvt_utf8_utf16IwEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__13pmr15memory_resourceE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__13pmr25monotonic_buffer_resourceE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__13pmr26__null_memory_resource_impE', 'size': 0, 'type': 'OBJECT'}
diff --git a/libcxx/lib/libc++unexp.exp b/libcxx/lib/libc++unexp.exp
deleted file mode 100644
index 9507fc57f1841..0000000000000
--- a/libcxx/lib/libc++unexp.exp
+++ /dev/null
@@ -1,19 +0,0 @@
-# all guard variables
-__ZGVNSt3__*
-# all vtables
-# __ZTV*
-# all VTT
-# __ZTT*
-# all non-virtual thunks
-# __ZTh*
-# all virtual thunks
-# __ZTv*
-# typeinfo for std::__1::__types
-#    There are no std::__types
-# __ZTINSt3__1[0-9][0-9]*__*
-# typeinfo name for std::__1::__types
-__ZTSNSt3__1[0-9][0-9]*__*
-# anything using __hidden_allocator
-*__hidden_allocator*
-# anything using __sso_allocator
-*__sso_allocator*
diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt
index 48c5111a0acbf..806e341ba3b72 100644
--- a/libcxx/src/CMakeLists.txt
+++ b/libcxx/src/CMakeLists.txt
@@ -219,7 +219,6 @@ if (LIBCXX_ENABLE_SHARED)
 
     # TODO: These exports controls should not be tied to whether we re-export libc++abi symbols
     target_link_libraries(cxx_shared PRIVATE
-      "-Wl,-unexported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/../lib/libc++unexp.exp"
       "-Wl,-force_symbols_not_weak_list,${CMAKE_CURRENT_SOURCE_DIR}/../lib/notweak.exp"
       "-Wl,-force_symbols_weak_list,${CMAKE_CURRENT_SOURCE_DIR}/../lib/weak.exp")
   endif()
diff --git a/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp b/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp
index 2c8534977febc..9d4b7039402a4 100644
--- a/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp
+++ b/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp
@@ -232,9 +232,7 @@ void u32string_test() {
 
 void tuple_test() {
   std::tuple<int, int, int> test0(2, 3, 4);
-  ComparePrettyPrintToChars(
-      test0,
-      "std::tuple containing = {[1] = 2, [2] = 3, [3] = 4}");
+  ComparePrettyPrintToChars(test0, "std::tuple containing = {[0] = 2, [1] = 3, [2] = 4}");
 
   std::tuple<> test1;
   ComparePrettyPrintToChars(
diff --git a/libcxx/test/libcxx/type_traits/is_specialization.verify.cpp b/libcxx/test/libcxx/type_traits/is_specialization.verify.cpp
index 3593c2e095db9..a2ea8efcbe5de 100644
--- a/libcxx/test/libcxx/type_traits/is_specialization.verify.cpp
+++ b/libcxx/test/libcxx/type_traits/is_specialization.verify.cpp
@@ -17,5 +17,5 @@
 #include <array>
 #include <utility>
 
-// expected-error-re@*:* {{{{could not match _Size against 'type-parameter-0-0'|different template parameters}}}}
+// expected-error-re@*:* {{{{must be an expression|different template parameters}}}}
 static_assert(!std::__is_specialization_v<std::pair<int, std::size_t>, std::array>);
diff --git a/libcxx/test/std/atomics/atomics.lockfree/is_always_lock_free.cpp b/libcxx/test/std/atomics/atomics.lockfree/is_always_lock_free.pass.cpp
similarity index 94%
rename from libcxx/test/std/atomics/atomics.lockfree/is_always_lock_free.cpp
rename to libcxx/test/std/atomics/atomics.lockfree/is_always_lock_free.pass.cpp
index db17221e515d3..1ebe31375079b 100644
--- a/libcxx/test/std/atomics/atomics.lockfree/is_always_lock_free.cpp
+++ b/libcxx/test/std/atomics/atomics.lockfree/is_always_lock_free.pass.cpp
@@ -5,8 +5,9 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
+
 // UNSUPPORTED: c++03, c++11, c++14
+// XFAIL: LIBCXX-PICOLIBC-FIXME
 
 // <atomic>
 //
@@ -15,6 +16,10 @@
 //
 // static constexpr bool is_always_lock_free;
 
+// Ignore diagnostic about vector types changing the ABI on some targets, since
+// that is irrelevant for this test.
+// ADDITIONAL_COMPILE_FLAGS: -Wno-psabi
+
 #include <atomic>
 #include <cassert>
 #include <concepts>
@@ -27,7 +32,8 @@ template <typename T>
 void check_always_lock_free(std::atomic<T> const& a) {
   using InfoT = LockFreeStatusInfo<T>;
 
-  constexpr std::same_as<const bool> decltype(auto) is_always_lock_free = std::atomic<T>::is_always_lock_free;
+  constexpr auto is_always_lock_free = std::atomic<T>::is_always_lock_free;
+  ASSERT_SAME_TYPE(decltype(is_always_lock_free), bool const);
 
   // If we know the status of T for sure, validate the exact result of the function.
   if constexpr (InfoT::status_known) {
@@ -45,7 +51,8 @@ void check_always_lock_free(std::atomic<T> const& a) {
 
   // In all cases, also sanity-check it based on the implication always-lock-free => lock-free.
   if (is_always_lock_free) {
-    std::same_as<bool> decltype(auto) is_lock_free = a.is_lock_free();
+    auto is_lock_free = a.is_lock_free();
+    ASSERT_SAME_TYPE(decltype(is_always_lock_free), bool const);
     assert(is_lock_free);
   }
   ASSERT_NOEXCEPT(a.is_lock_free());
diff --git a/libcxx/test/std/language.support/cmp/cmp.alg/compare_partial_order_fallback.pass.cpp b/libcxx/test/std/language.support/cmp/cmp.alg/compare_partial_order_fallback.pass.cpp
index a057a34136e0f..f3fdd2ecb04ce 100644
--- a/libcxx/test/std/language.support/cmp/cmp.alg/compare_partial_order_fallback.pass.cpp
+++ b/libcxx/test/std/language.support/cmp/cmp.alg/compare_partial_order_fallback.pass.cpp
@@ -267,6 +267,44 @@ namespace N2 {
         friend bool operator<(const VC2&, VC2&);
         friend bool operator<(VC2&, const VC2&);
     };
+
+    enum class comparison_result_kind : bool {
+      convertible_bool,
+      boolean_testable,
+    };
+
+    template <comparison_result_kind K>
+    struct comparison_result {
+      bool value;
+
+      constexpr operator bool() const noexcept { return value; }
+
+      constexpr auto operator!() const noexcept {
+        if constexpr (K == comparison_result_kind::boolean_testable) {
+          return comparison_result{!value};
+        }
+      }
+    };
+
+    template <comparison_result_kind EqKind, comparison_result_kind LeKind>
+    struct boolean_tested_type {
+      friend constexpr comparison_result<EqKind> operator==(boolean_tested_type, boolean_tested_type) noexcept {
+        return comparison_result<EqKind>{true};
+      }
+
+      friend constexpr comparison_result<LeKind> operator<(boolean_tested_type, boolean_tested_type) noexcept {
+        return comparison_result<LeKind>{false};
+      }
+    };
+
+    using test_only_convertible =
+        boolean_tested_type<comparison_result_kind::convertible_bool, comparison_result_kind::convertible_bool>;
+    using test_eq_boolean_testable =
+        boolean_tested_type<comparison_result_kind::boolean_testable, comparison_result_kind::convertible_bool>;
+    using test_le_boolean_testable =
+        boolean_tested_type<comparison_result_kind::convertible_bool, comparison_result_kind::boolean_testable>;
+    using test_boolean_testable =
+        boolean_tested_type<comparison_result_kind::boolean_testable, comparison_result_kind::boolean_testable>;
 }
 
 constexpr bool test_2()
@@ -306,6 +344,21 @@ constexpr bool test_2()
         assert( has_partial_order(cvc, vc));
         assert(!has_partial_order(vc, cvc));
     }
+    {
+      // P2167R3 as modified by the intent of LWG3465:
+      //   All of decltype(e == f), decltype(e < f), and decltype(f < e) need to be well-formed and boolean-testable.
+      N2::test_only_convertible tc;
+      N2::test_eq_boolean_testable teq;
+      N2::test_le_boolean_testable tle;
+      N2::test_boolean_testable tbt;
+
+      assert(!has_partial_order(tc, tc));
+      assert(!has_partial_order(teq, teq));
+      assert(!has_partial_order(tle, tle));
+      assert(has_partial_order(tbt, tbt));
+
+      assert(std::compare_partial_order_fallback(tbt, tbt) == std::partial_ordering::equivalent);
+    }
     return true;
 }
 
diff --git a/libcxx/test/std/language.support/cmp/cmp.alg/compare_strong_order_fallback.pass.cpp b/libcxx/test/std/language.support/cmp/cmp.alg/compare_strong_order_fallback.pass.cpp
index 2ad05cb1ec6dd..e1e7bc37b1d79 100644
--- a/libcxx/test/std/language.support/cmp/cmp.alg/compare_strong_order_fallback.pass.cpp
+++ b/libcxx/test/std/language.support/cmp/cmp.alg/compare_strong_order_fallback.pass.cpp
@@ -473,6 +473,44 @@ namespace N2 {
         friend bool operator<(const VC2&, VC2&);
         friend bool operator<(VC2&, const VC2&);
     };
+
+    enum class comparison_result_kind : bool {
+      convertible_bool,
+      boolean_testable,
+    };
+
+    template <comparison_result_kind K>
+    struct comparison_result {
+      bool value;
+
+      constexpr operator bool() const noexcept { return value; }
+
+      constexpr auto operator!() const noexcept {
+        if constexpr (K == comparison_result_kind::boolean_testable) {
+          return comparison_result{!value};
+        }
+      }
+    };
+
+    template <comparison_result_kind EqKind, comparison_result_kind LeKind>
+    struct boolean_tested_type {
+      friend constexpr comparison_result<EqKind> operator==(boolean_tested_type, boolean_tested_type) noexcept {
+        return comparison_result<EqKind>{true};
+      }
+
+      friend constexpr comparison_result<LeKind> operator<(boolean_tested_type, boolean_tested_type) noexcept {
+        return comparison_result<LeKind>{false};
+      }
+    };
+
+    using test_only_convertible =
+        boolean_tested_type<comparison_result_kind::convertible_bool, comparison_result_kind::convertible_bool>;
+    using test_eq_boolean_testable =
+        boolean_tested_type<comparison_result_kind::boolean_testable, comparison_result_kind::convertible_bool>;
+    using test_le_boolean_testable =
+        boolean_tested_type<comparison_result_kind::convertible_bool, comparison_result_kind::boolean_testable>;
+    using test_boolean_testable =
+        boolean_tested_type<comparison_result_kind::boolean_testable, comparison_result_kind::boolean_testable>;
 }
 
 constexpr bool test_2()
@@ -506,6 +544,20 @@ constexpr bool test_2()
         assert( has_strong_order(cvc, vc));
         assert(!has_strong_order(vc, cvc));
     }
+    {
+      // P2167R3: Both decltype(e == f) and decltype(e < f) need to be well-formed and boolean-testable.
+      N2::test_only_convertible tc;
+      N2::test_eq_boolean_testable teq;
+      N2::test_le_boolean_testable tle;
+      N2::test_boolean_testable tbt;
+
+      assert(!has_strong_order(tc, tc));
+      assert(!has_strong_order(teq, teq));
+      assert(!has_strong_order(tle, tle));
+      assert(has_strong_order(tbt, tbt));
+
+      assert(std::compare_strong_order_fallback(tbt, tbt) == std::strong_ordering::equal);
+    }
     return true;
 }
 
@@ -515,13 +567,17 @@ int main(int, char**)
     test_1_2();
     test_1_3<float>();
     test_1_3<double>();
-    // test_1_3<long double>();  // UNIMPLEMENTED
+#ifdef TEST_LONG_DOUBLE_IS_DOUBLE
+    test_1_3<long double>(); // UNIMPLEMENTED when long double is a distinct type
+#endif
     test_1_4();
     test_2();
 
     static_assert(test_1_3<float>());
     static_assert(test_1_3<double>());
-    // static_assert(test_1_3<long double>());  // UNIMPLEMENTED
+#ifdef TEST_LONG_DOUBLE_IS_DOUBLE
+    static_assert(test_1_3<long double>()); // UNIMPLEMENTED when long double is a distinct type
+#endif
     static_assert(test_1_4());
     static_assert(test_2());
 
diff --git a/libcxx/test/std/language.support/cmp/cmp.alg/compare_weak_order_fallback.pass.cpp b/libcxx/test/std/language.support/cmp/cmp.alg/compare_weak_order_fallback.pass.cpp
index 65674b4006fc8..07b5fcd0a6d17 100644
--- a/libcxx/test/std/language.support/cmp/cmp.alg/compare_weak_order_fallback.pass.cpp
+++ b/libcxx/test/std/language.support/cmp/cmp.alg/compare_weak_order_fallback.pass.cpp
@@ -520,6 +520,44 @@ namespace N2 {
         friend bool operator<(const VC2&, VC2&);
         friend bool operator<(VC2&, const VC2&);
     };
+
+    enum class comparison_result_kind : bool {
+      convertible_bool,
+      boolean_testable,
+    };
+
+    template <comparison_result_kind K>
+    struct comparison_result {
+      bool value;
+
+      constexpr operator bool() const noexcept { return value; }
+
+      constexpr auto operator!() const noexcept {
+        if constexpr (K == comparison_result_kind::boolean_testable) {
+          return comparison_result{!value};
+        }
+      }
+    };
+
+    template <comparison_result_kind EqKind, comparison_result_kind LeKind>
+    struct boolean_tested_type {
+      friend constexpr comparison_result<EqKind> operator==(boolean_tested_type, boolean_tested_type) noexcept {
+        return comparison_result<EqKind>{true};
+      }
+
+      friend constexpr comparison_result<LeKind> operator<(boolean_tested_type, boolean_tested_type) noexcept {
+        return comparison_result<LeKind>{false};
+      }
+    };
+
+    using test_only_convertible =
+        boolean_tested_type<comparison_result_kind::convertible_bool, comparison_result_kind::convertible_bool>;
+    using test_eq_boolean_testable =
+        boolean_tested_type<comparison_result_kind::boolean_testable, comparison_result_kind::convertible_bool>;
+    using test_le_boolean_testable =
+        boolean_tested_type<comparison_result_kind::convertible_bool, comparison_result_kind::boolean_testable>;
+    using test_boolean_testable =
+        boolean_tested_type<comparison_result_kind::boolean_testable, comparison_result_kind::boolean_testable>;
 }
 
 constexpr bool test_2()
@@ -553,6 +591,20 @@ constexpr bool test_2()
         assert( has_weak_order(cvc, vc));
         assert(!has_weak_order(vc, cvc));
     }
+    {
+      // P2167R3: Both decltype(e == f) and decltype(e < f) need to be well-formed and boolean-testable.
+      N2::test_only_convertible tc;
+      N2::test_eq_boolean_testable teq;
+      N2::test_le_boolean_testable tle;
+      N2::test_boolean_testable tbt;
+
+      assert(!has_weak_order(tc, tc));
+      assert(!has_weak_order(teq, teq));
+      assert(!has_weak_order(tle, tle));
+      assert(has_weak_order(tbt, tbt));
+
+      assert(std::compare_weak_order_fallback(tbt, tbt) == std::weak_ordering::equivalent);
+    }
     return true;
 }
 
diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp
index 9a3e7c9bac1e8..9b6f34a7e8a3c 100644
--- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp
+++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp
@@ -8,7 +8,6 @@
 //
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // XFAIL: availability-synchronization_library-missing
 
 // <condition_variable>
diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp
index 271c67273b272..de4816eb641dd 100644
--- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp
+++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp
@@ -8,7 +8,6 @@
 //
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // XFAIL: availability-synchronization_library-missing
 
 // <condition_variable>
diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp
index 631ac889b936f..c123e6b5659cc 100644
--- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp
+++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11, c++14, c++17
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
+
 // XFAIL: availability-synchronization_library-missing
 
 // <condition_variable>
diff --git a/libcxx/test/std/thread/thread.jthread/assign.move.pass.cpp b/libcxx/test/std/thread/thread.jthread/assign.move.pass.cpp
index d7e50fedd468f..b714cc58cbd3b 100644
--- a/libcxx/test/std/thread/thread.jthread/assign.move.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/assign.move.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 // ADDITIONAL_COMPILE_FLAGS(gcc-style-warnings): -Wno-self-move
diff --git a/libcxx/test/std/thread/thread.jthread/cons.default.pass.cpp b/libcxx/test/std/thread/thread.jthread/cons.default.pass.cpp
index 88585eaef599e..6dcc11cfbbf7c 100644
--- a/libcxx/test/std/thread/thread.jthread/cons.default.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/cons.default.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.jthread/cons.func.token.pass.cpp b/libcxx/test/std/thread/thread.jthread/cons.func.token.pass.cpp
index 5236b84b1fd96..726d70430ed76 100644
--- a/libcxx/test/std/thread/thread.jthread/cons.func.token.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/cons.func.token.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.jthread/cons.move.pass.cpp b/libcxx/test/std/thread/thread.jthread/cons.move.pass.cpp
index c3c04467703c9..4a3e21c7c0134 100644
--- a/libcxx/test/std/thread/thread.jthread/cons.move.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/cons.move.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.jthread/copy.delete.compile.pass.cpp b/libcxx/test/std/thread/thread.jthread/copy.delete.compile.pass.cpp
index e1c34afca9d47..26d457982b3f5 100644
--- a/libcxx/test/std/thread/thread.jthread/copy.delete.compile.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/copy.delete.compile.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.jthread/detach.pass.cpp b/libcxx/test/std/thread/thread.jthread/detach.pass.cpp
index 54fd5fd93bed6..eeab0e78120fa 100644
--- a/libcxx/test/std/thread/thread.jthread/detach.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/detach.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.jthread/dtor.pass.cpp b/libcxx/test/std/thread/thread.jthread/dtor.pass.cpp
index 35be0f6c0dd82..0dde648584cac 100644
--- a/libcxx/test/std/thread/thread.jthread/dtor.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/dtor.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.jthread/get_id.pass.cpp b/libcxx/test/std/thread/thread.jthread/get_id.pass.cpp
index b3a2beff9f416..b95929cd2ea26 100644
--- a/libcxx/test/std/thread/thread.jthread/get_id.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/get_id.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.jthread/get_stop_source.pass.cpp b/libcxx/test/std/thread/thread.jthread/get_stop_source.pass.cpp
index 8f35db297b749..5256bac1b2e0d 100644
--- a/libcxx/test/std/thread/thread.jthread/get_stop_source.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/get_stop_source.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.jthread/get_stop_token.pass.cpp b/libcxx/test/std/thread/thread.jthread/get_stop_token.pass.cpp
index 070761e0a3ab8..06b4b9842b2ea 100644
--- a/libcxx/test/std/thread/thread.jthread/get_stop_token.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/get_stop_token.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.jthread/hardware_concurrency.pass.cpp b/libcxx/test/std/thread/thread.jthread/hardware_concurrency.pass.cpp
index 669d72f1fd5f5..f3051fd5af47e 100644
--- a/libcxx/test/std/thread/thread.jthread/hardware_concurrency.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/hardware_concurrency.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.jthread/join.deadlock.pass.cpp b/libcxx/test/std/thread/thread.jthread/join.deadlock.pass.cpp
index 8e2f1e5f5d9d4..21fbacde36cb9 100644
--- a/libcxx/test/std/thread/thread.jthread/join.deadlock.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/join.deadlock.pass.cpp
@@ -15,7 +15,6 @@
 
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: no-exceptions
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.jthread/join.pass.cpp b/libcxx/test/std/thread/thread.jthread/join.pass.cpp
index 2bafd86338247..8435b4567e68e 100644
--- a/libcxx/test/std/thread/thread.jthread/join.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/join.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.jthread/joinable.pass.cpp b/libcxx/test/std/thread/thread.jthread/joinable.pass.cpp
index 3a88100d934db..575a132e9a098 100644
--- a/libcxx/test/std/thread/thread.jthread/joinable.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/joinable.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.jthread/nodiscard.verify.cpp b/libcxx/test/std/thread/thread.jthread/nodiscard.verify.cpp
index d0fbef733860c..35fcc03968c0f 100644
--- a/libcxx/test/std/thread/thread.jthread/nodiscard.verify.cpp
+++ b/libcxx/test/std/thread/thread.jthread/nodiscard.verify.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.jthread/request_stop.pass.cpp b/libcxx/test/std/thread/thread.jthread/request_stop.pass.cpp
index ccbea9f145e50..b53f4f3903a20 100644
--- a/libcxx/test/std/thread/thread.jthread/request_stop.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/request_stop.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.jthread/swap.free.pass.cpp b/libcxx/test/std/thread/thread.jthread/swap.free.pass.cpp
index 01c8ccd659687..7d9b4a3aec1af 100644
--- a/libcxx/test/std/thread/thread.jthread/swap.free.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/swap.free.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.jthread/swap.member.pass.cpp b/libcxx/test/std/thread/thread.jthread/swap.member.pass.cpp
index 8ae17f435aa31..9a14ecf4fb7d0 100644
--- a/libcxx/test/std/thread/thread.jthread/swap.member.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/swap.member.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.jthread/type.compile.pass.cpp b/libcxx/test/std/thread/thread.jthread/type.compile.pass.cpp
index 496f793896757..ac5c1a90b6901 100644
--- a/libcxx/test/std/thread/thread.jthread/type.compile.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/type.compile.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/nostopstate/cons.default.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/nostopstate/cons.default.pass.cpp
index 73693fc30b580..400fb356c6380 100644
--- a/libcxx/test/std/thread/thread.stoptoken/nostopstate/cons.default.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/nostopstate/cons.default.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 // struct nostopstate_t {
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopcallback/cons.const.token.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stopcallback/cons.const.token.pass.cpp
index 3d1f61d0b43a3..9fdfdb8337be6 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopcallback/cons.const.token.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopcallback/cons.const.token.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopcallback/cons.rvalue.token.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stopcallback/cons.rvalue.token.pass.cpp
index f78643def5ce3..49c97dbc4507b 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopcallback/cons.rvalue.token.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopcallback/cons.rvalue.token.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopcallback/copy.move.compile.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stopcallback/copy.move.compile.pass.cpp
index 8f414ad000a6a..2e66b64664055 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopcallback/copy.move.compile.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopcallback/copy.move.compile.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopcallback/ctad.compile.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stopcallback/ctad.compile.pass.cpp
index bd4104ca4744d..9c1d4fe3a8e2f 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopcallback/ctad.compile.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopcallback/ctad.compile.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopcallback/dtor.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stopcallback/dtor.pass.cpp
index eac3d688a1d9f..8539058d8d418 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopcallback/dtor.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopcallback/dtor.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopcallback/typedef.compile.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stopcallback/typedef.compile.pass.cpp
index b45dcc49dd568..d60bad9b501db 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopcallback/typedef.compile.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopcallback/typedef.compile.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 // using callback_type = _Callback;
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopsource/assign.copy.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stopsource/assign.copy.pass.cpp
index 628c5251bdc79..0684899b3bab1 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopsource/assign.copy.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopsource/assign.copy.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopsource/cons.copy.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stopsource/cons.copy.pass.cpp
index 1347b1d3d0689..b130e4135c2ba 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopsource/cons.copy.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopsource/cons.copy.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopsource/cons.default.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stopsource/cons.default.pass.cpp
index 1deafc433cb44..1dba0b645bdab 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopsource/cons.default.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopsource/cons.default.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopsource/cons.move.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stopsource/cons.move.pass.cpp
index 36477ba0693c6..3eb73ce2e92fd 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopsource/cons.move.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopsource/cons.move.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopsource/cons.nostopstate.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stopsource/cons.nostopstate.pass.cpp
index 97f2dfb0abcd3..13067a562da02 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopsource/cons.nostopstate.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopsource/cons.nostopstate.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopsource/equals.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stopsource/equals.pass.cpp
index ec22bbc88c1f0..4aa3d530577b6 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopsource/equals.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopsource/equals.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopsource/get_token.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stopsource/get_token.pass.cpp
index ccab810278dbd..8cd500523f5e0 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopsource/get_token.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopsource/get_token.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopsource/move.copy.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stopsource/move.copy.pass.cpp
index d49c0c48828de..09b479611b7fe 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopsource/move.copy.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopsource/move.copy.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopsource/nodiscard.verify.cpp b/libcxx/test/std/thread/thread.stoptoken/stopsource/nodiscard.verify.cpp
index 5c22511008bfd..4616f1c56999c 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopsource/nodiscard.verify.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopsource/nodiscard.verify.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopsource/request_stop.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stopsource/request_stop.pass.cpp
index 97431b771a3d9..8f91bea2506a6 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopsource/request_stop.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopsource/request_stop.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopsource/stop_possible.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stopsource/stop_possible.pass.cpp
index 3d7980b0fc7bd..da4992ff0dee7 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopsource/stop_possible.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopsource/stop_possible.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopsource/stop_requested.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stopsource/stop_requested.pass.cpp
index f8ff441a75a2f..a33a035aa12f8 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopsource/stop_requested.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopsource/stop_requested.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopsource/swap.free.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stopsource/swap.free.pass.cpp
index 48303207e3bff..097dabf49f214 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopsource/swap.free.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopsource/swap.free.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopsource/swap.member.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stopsource/swap.member.pass.cpp
index bab19147b86b4..7c724aced2cbc 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopsource/swap.member.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopsource/swap.member.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stoptoken/assign.copy.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stoptoken/assign.copy.pass.cpp
index 41bdd2c905993..753cf51c5b514 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stoptoken/assign.copy.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stoptoken/assign.copy.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stoptoken/assign.move.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stoptoken/assign.move.pass.cpp
index b9c3aa41d735c..2a94bebb62aa8 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stoptoken/assign.move.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stoptoken/assign.move.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stoptoken/cons.copy.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stoptoken/cons.copy.pass.cpp
index 9507b43f8d7dc..8794f704c8720 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stoptoken/cons.copy.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stoptoken/cons.copy.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stoptoken/cons.default.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stoptoken/cons.default.pass.cpp
index 4b069c2908cfa..389725be9cdeb 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stoptoken/cons.default.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stoptoken/cons.default.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stoptoken/cons.move.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stoptoken/cons.move.pass.cpp
index 621493187ad8c..11cedc0cf7cae 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stoptoken/cons.move.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stoptoken/cons.move.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stoptoken/equals.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stoptoken/equals.pass.cpp
index 4de9b81f71868..4009988981459 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stoptoken/equals.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stoptoken/equals.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stoptoken/nodiscard.verify.cpp b/libcxx/test/std/thread/thread.stoptoken/stoptoken/nodiscard.verify.cpp
index 89ab39d11edc8..b62ecdc8551d0 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stoptoken/nodiscard.verify.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stoptoken/nodiscard.verify.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stoptoken/stop_possible.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stoptoken/stop_possible.pass.cpp
index fd73f5e516678..daa7f9dee4708 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stoptoken/stop_possible.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stoptoken/stop_possible.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stoptoken/stop_requested.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stoptoken/stop_requested.pass.cpp
index 4dbc3137ef209..acf986e978b61 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stoptoken/stop_requested.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stoptoken/stop_requested.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stoptoken/swap.free.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stoptoken/swap.free.pass.cpp
index e910c7bd9d2ab..90e95b11277cc 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stoptoken/swap.free.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stoptoken/swap.free.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/thread/thread.stoptoken/stoptoken/swap.member.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stoptoken/swap.member.pass.cpp
index af8cb2d9bc020..9819c7db2e7be 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stoptoken/swap.member.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stoptoken/swap.member.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// UNSUPPORTED: libcpp-has-no-experimental-stop_token
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 // XFAIL: availability-synchronization_library-missing
 
diff --git a/libcxx/test/std/utilities/any/any.class/any.cons/default.pass.cpp b/libcxx/test/std/utilities/any/any.class/any.cons/default.pass.cpp
index a20d22c1073fa..82c2e46bfa1ef 100644
--- a/libcxx/test/std/utilities/any/any.class/any.cons/default.pass.cpp
+++ b/libcxx/test/std/utilities/any/any.class/any.cons/default.pass.cpp
@@ -32,7 +32,7 @@ int main(int, char**)
         struct TestConstexpr : public std::any {
           constexpr TestConstexpr() : std::any() {}
         };
-        static TEST_CONSTINIT std::any a;
+        TEST_CONSTINIT static std::any a;
         (void)a;
     }
     {
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/specialized.construct/construct_at.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/specialized.construct/construct_at.pass.cpp
index 13442df9db3ae..272441ebedc2f 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/specialized.construct/construct_at.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/specialized.construct/construct_at.pass.cpp
@@ -80,21 +80,6 @@ constexpr bool test()
         a.deallocate(p, 2);
     }
 
-    {
-        std::allocator<Counted> a;
-        Counted const* p = a.allocate(2);
-        int count = 0;
-        std::construct_at(p, count);
-        assert(count == 1);
-        std::construct_at(p+1, count);
-        assert(count == 2);
-        (p+1)->~Counted();
-        assert(count == 1);
-        p->~Counted();
-        assert(count == 0);
-        a.deallocate(const_cast<Counted*>(p), 2);
-    }
-
     return true;
 }
 
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/specialized.construct/ranges_construct_at.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/specialized.construct/ranges_construct_at.pass.cpp
index 396fed7cc3e49..f66bf0fd64777 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/specialized.construct/ranges_construct_at.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/specialized.construct/ranges_construct_at.pass.cpp
@@ -99,16 +99,6 @@ constexpr bool test() {
     alloc.deallocate(out, 2);
   }
 
-  // Works with const pointers.
-  {
-    int x = 1;
-    const int* ptr = &x;
-
-    const int* result = std::ranges::construct_at(ptr, 42);
-    assert(result == ptr);
-    assert(x == 42);
-  }
-
   return true;
 }
 
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.construct.default/ranges_uninitialized_default_construct.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.construct.default/ranges_uninitialized_default_construct.pass.cpp
index 4581f1c909e38..ef969190c6314 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.construct.default/ranges_uninitialized_default_construct.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.construct.default/ranges_uninitialized_default_construct.pass.cpp
@@ -163,30 +163,5 @@ int main(int, char**) {
   }
 #endif  // TEST_HAS_NO_EXCEPTIONS
 
-  // Works with const iterators, (iter, sentinel) overload.
-  {
-    constexpr int N = 5;
-    Buffer<Counted, N> buf;
-
-    std::ranges::uninitialized_default_construct(buf.cbegin(), buf.cend());
-    assert(Counted::current_objects == N);
-    assert(Counted::total_objects == N);
-    std::destroy(buf.begin(), buf.end());
-    Counted::reset();
-  }
-
-  // Works with const iterators, (range) overload.
-  {
-    constexpr int N = 5;
-    Buffer<Counted, N> buf;
-    auto range = std::ranges::subrange(buf.cbegin(), buf.cend());
-
-    std::ranges::uninitialized_default_construct(range);
-    assert(Counted::current_objects == N);
-    assert(Counted::total_objects == N);
-    std::destroy(buf.begin(), buf.end());
-    Counted::reset();
-  }
-
   return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.construct.default/ranges_uninitialized_default_construct_n.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.construct.default/ranges_uninitialized_default_construct_n.pass.cpp
index 9bebe4b52a8cc..40fbf22695909 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.construct.default/ranges_uninitialized_default_construct_n.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.construct.default/ranges_uninitialized_default_construct_n.pass.cpp
@@ -75,17 +75,5 @@ int main(int, char**) {
   }
 #endif  // TEST_HAS_NO_EXCEPTIONS
 
-  // Works with const iterators.
-  {
-    constexpr int N = 5;
-    Buffer<Counted, N> buf;
-
-    std::ranges::uninitialized_default_construct_n(buf.cbegin(), N);
-    assert(Counted::current_objects == N);
-    assert(Counted::total_objects == N);
-    std::destroy(buf.begin(), buf.end());
-    Counted::reset();
-  }
-
   return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.construct.value/ranges_uninitialized_value_construct.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.construct.value/ranges_uninitialized_value_construct.pass.cpp
index ad74b82dce1f2..6bab25ca38475 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.construct.value/ranges_uninitialized_value_construct.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.construct.value/ranges_uninitialized_value_construct.pass.cpp
@@ -183,30 +183,5 @@ int main(int, char**) {
   }
 #endif // TEST_HAS_NO_EXCEPTIONS
 
-  // Works with const iterators, (iter, sentinel) overload.
-  {
-    constexpr int N = 5;
-    Buffer<Counted, N> buf;
-
-    std::ranges::uninitialized_value_construct(buf.cbegin(), buf.cend());
-    assert(Counted::current_objects == N);
-    assert(Counted::total_objects == N);
-    std::destroy(buf.begin(), buf.end());
-    Counted::reset();
-  }
-
-  // Works with const iterators, (range) overload.
-  {
-    constexpr int N = 5;
-    Buffer<Counted, N> buf;
-
-    auto range = std::ranges::subrange(buf.cbegin(), buf.cend());
-    std::ranges::uninitialized_value_construct(range);
-    assert(Counted::current_objects == N);
-    assert(Counted::total_objects == N);
-    std::destroy(buf.begin(), buf.end());
-    Counted::reset();
-  }
-
   return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.construct.value/ranges_uninitialized_value_construct_n.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.construct.value/ranges_uninitialized_value_construct_n.pass.cpp
index 8f315ce0076d4..4742aefcdb5ad 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.construct.value/ranges_uninitialized_value_construct_n.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.construct.value/ranges_uninitialized_value_construct_n.pass.cpp
@@ -94,17 +94,5 @@ int main(int, char**) {
   }
 #endif // TEST_HAS_NO_EXCEPTIONS
 
-  // Works with const iterators.
-  {
-    constexpr int N = 5;
-    Buffer<Counted, N> buf;
-
-    std::ranges::uninitialized_value_construct_n(buf.cbegin(), N);
-    assert(Counted::current_objects == N);
-    assert(Counted::total_objects == N);
-    std::destroy(buf.begin(), buf.end());
-    Counted::reset();
-  }
-
   return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/ranges_uninitialized_copy.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/ranges_uninitialized_copy.pass.cpp
index 92dc380728e24..52ba70b009bab 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/ranges_uninitialized_copy.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/ranges_uninitialized_copy.pass.cpp
@@ -278,39 +278,6 @@ int main(int, char**) {
   Counted::reset();
 #endif // TEST_HAS_NO_EXCEPTIONS
 
-  // Works with const iterators, (iter, sentinel) overload.
-  {
-    constexpr int N = 5;
-    Counted in[N] = {Counted(1), Counted(2), Counted(3), Counted(4), Counted(5)};
-    Buffer<Counted, N> out;
-    Counted::reset();
-
-    std::ranges::uninitialized_copy(in, in + N, out.cbegin(), out.cend());
-    assert(Counted::current_objects == N);
-    assert(Counted::total_objects == N);
-    assert(std::equal(in, in + N, out.begin(), out.end()));
-
-    std::destroy(out.begin(), out.end());
-  }
-  Counted::reset();
-
-  // Works with const iterators, (range) overload.
-  {
-    constexpr int N = 5;
-    Counted in[N] = {Counted(1), Counted(2), Counted(3), Counted(4), Counted(5)};
-    Buffer<Counted, N> out;
-    Counted::reset();
-
-    std::ranges::subrange out_range(out.cbegin(), out.cend());
-    std::ranges::uninitialized_copy(in, out_range);
-    assert(Counted::current_objects == N);
-    assert(Counted::total_objects == N);
-    assert(std::equal(in, in + N, out.begin(), out.end()));
-
-    std::destroy(out.begin(), out.end());
-  }
-  Counted::reset();
-
   // Conversions, (iter, sentinel) overload.
   {
     constexpr int N = 3;
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/ranges_uninitialized_copy_n.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/ranges_uninitialized_copy_n.pass.cpp
index 80082eb3b98e6..84fba1aa79293 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/ranges_uninitialized_copy_n.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/ranges_uninitialized_copy_n.pass.cpp
@@ -104,22 +104,6 @@ int main(int, char**) {
 
 #endif // TEST_HAS_NO_EXCEPTIONS
 
-  // Works with const iterators.
-  {
-    constexpr int N = 5;
-    Counted in[N] = {Counted(1), Counted(2), Counted(3), Counted(4), Counted(5)};
-    Buffer<Counted, N> out;
-    Counted::reset();
-
-    std::ranges::uninitialized_copy_n(in, N, out.cbegin(), out.cend());
-    assert(Counted::current_objects == N);
-    assert(Counted::total_objects == N);
-    assert(std::equal(in, in + N, out.begin(), out.end()));
-
-    std::destroy(out.begin(), out.end());
-  }
-  Counted::reset();
-
   // Conversions.
   {
     constexpr int N = 3;
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.fill.n/ranges_uninitialized_fill_n.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.fill.n/ranges_uninitialized_fill_n.pass.cpp
index 0e8846e8c7c8f..0b35c1114d87c 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.fill.n/ranges_uninitialized_fill_n.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.fill.n/ranges_uninitialized_fill_n.pass.cpp
@@ -101,19 +101,5 @@ int main(int, char**) {
   }
 #endif // TEST_HAS_NO_EXCEPTIONS
 
-  // Works with const iterators.
-  {
-    constexpr int N = 5;
-    Buffer<Counted, N> buf;
-
-    std::ranges::uninitialized_fill_n(buf.cbegin(), N, x);
-    assert(Counted::current_objects == N);
-    assert(Counted::total_objects == N);
-    assert(std::all_of(buf.begin(), buf.end(), pred));
-
-    std::destroy(buf.begin(), buf.end());
-    Counted::reset();
-  }
-
   return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.fill/ranges_uninitialized_fill.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.fill/ranges_uninitialized_fill.pass.cpp
index 482515ec9483d..6f75fc13a1c39 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.fill/ranges_uninitialized_fill.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.fill/ranges_uninitialized_fill.pass.cpp
@@ -198,34 +198,5 @@ int main(int, char**) {
   }
 #endif // TEST_HAS_NO_EXCEPTIONS
 
-  // Works with const iterators, (iter, sentinel) overload.
-  {
-    constexpr int N = 5;
-    Buffer<Counted, N> buf;
-
-    std::ranges::uninitialized_fill(buf.cbegin(), buf.cend(), x);
-    assert(Counted::current_objects == N);
-    assert(Counted::total_objects == N);
-    assert(std::all_of(buf.begin(), buf.end(), pred));
-
-    std::destroy(buf.begin(), buf.end());
-    Counted::reset();
-  }
-
-  // Works with const iterators, (range) overload.
-  {
-    constexpr int N = 5;
-    Buffer<Counted, N> buf;
-
-    auto range = std::ranges::subrange(buf.cbegin(), buf.cend());
-    std::ranges::uninitialized_fill(range, x);
-    assert(Counted::current_objects == N);
-    assert(Counted::total_objects == N);
-    assert(std::all_of(buf.begin(), buf.end(), pred));
-
-    std::destroy(buf.begin(), buf.end());
-    Counted::reset();
-  }
-
   return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/ranges_uninitialized_move.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/ranges_uninitialized_move.pass.cpp
index 56dd25c66e199..c6b38b4fea864 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/ranges_uninitialized_move.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/ranges_uninitialized_move.pass.cpp
@@ -282,39 +282,6 @@ int main(int, char**) {
   Counted::reset();
 #endif // TEST_HAS_NO_EXCEPTIONS
 
-  // Works with const iterators, (iter, sentinel) overload.
-  {
-    constexpr int N = 5;
-    Counted in[N] = {Counted(1), Counted(2), Counted(3), Counted(4), Counted(5)};
-    Buffer<Counted, N> out;
-    Counted::reset();
-
-    std::ranges::uninitialized_move(in, in + N, out.cbegin(), out.cend());
-    assert(Counted::current_objects == N);
-    assert(Counted::total_objects == N);
-    assert(std::equal(in, in + N, out.begin(), out.end()));
-
-    std::destroy(out.begin(), out.end());
-  }
-  Counted::reset();
-
-  // Works with const iterators, (range) overload.
-  {
-    constexpr int N = 5;
-    Counted in[N] = {Counted(1), Counted(2), Counted(3), Counted(4), Counted(5)};
-    Buffer<Counted, N> out;
-    Counted::reset();
-
-    std::ranges::subrange out_range (out.cbegin(), out.cend());
-    std::ranges::uninitialized_move(in, out_range);
-    assert(Counted::current_objects == N);
-    assert(Counted::total_objects == N);
-    assert(std::equal(in, in + N, out.begin(), out.end()));
-
-    std::destroy(out.begin(), out.end());
-  }
-  Counted::reset();
-
   // Conversions, (iter, sentinel) overload.
   {
     constexpr int N = 3;
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/ranges_uninitialized_move_n.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/ranges_uninitialized_move_n.pass.cpp
index 162b4a48537ff..cac2acc5932b3 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/ranges_uninitialized_move_n.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/ranges_uninitialized_move_n.pass.cpp
@@ -105,22 +105,6 @@ int main(int, char**) {
 
 #endif // TEST_HAS_NO_EXCEPTIONS
 
-  // Works with const iterators.
-  {
-    constexpr int N = 5;
-    Counted in[N] = {Counted(1), Counted(2), Counted(3), Counted(4), Counted(5)};
-    Buffer<Counted, N> out;
-    Counted::reset();
-
-    std::ranges::uninitialized_move_n(in, N, out.cbegin(), out.cend());
-    assert(Counted::current_objects == N);
-    assert(Counted::total_objects == N);
-    assert(std::equal(in, in + N, out.begin(), out.end()));
-
-    std::destroy(out.begin(), out.end());
-  }
-  Counted::reset();
-
   // Conversions.
   {
     constexpr int N = 3;
diff --git a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/assert.subscript.pass.cpp b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/assert.subscript.pass.cpp
index 1eaf2d5900356..bb4ac981600f9 100644
--- a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/assert.subscript.pass.cpp
+++ b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/assert.subscript.pass.cpp
@@ -48,31 +48,24 @@ struct MyDeleter {
 
 template <class WithCookie, class NoCookie>
 void test() {
-  // For types with an array cookie, we can always detect OOB accesses.
+  // For types with an array cookie, we can always detect OOB accesses. Note that reliance on an array
+  // cookie is limited to the default deleter, since a unique_ptr with a custom deleter may not have
+  // been allocated with `new T[n]`.
   {
-    // Check with the default deleter
     {
-      {
-        std::unique_ptr<WithCookie[]> ptr(new WithCookie[5]);
-        TEST_LIBCPP_ASSERT_FAILURE(ptr[6], "unique_ptr<T[]>::operator[](index): index out of range");
-      }
-      {
-        std::unique_ptr<WithCookie[]> ptr = std::make_unique<WithCookie[]>(5);
-        TEST_LIBCPP_ASSERT_FAILURE(ptr[6], "unique_ptr<T[]>::operator[](index): index out of range");
-      }
-#if TEST_STD_VER >= 20
-      {
-        std::unique_ptr<WithCookie[]> ptr = std::make_unique_for_overwrite<WithCookie[]>(5);
-        TEST_LIBCPP_ASSERT_FAILURE(ptr[6] = WithCookie(), "unique_ptr<T[]>::operator[](index): index out of range");
-      }
-#endif
+      std::unique_ptr<WithCookie[]> ptr(new WithCookie[5]);
+      TEST_LIBCPP_ASSERT_FAILURE(ptr[6], "unique_ptr<T[]>::operator[](index): index out of range");
     }
-
-    // Check with a custom deleter
     {
-      std::unique_ptr<WithCookie[], MyDeleter> ptr(new WithCookie[5]);
+      std::unique_ptr<WithCookie[]> ptr = std::make_unique<WithCookie[]>(5);
       TEST_LIBCPP_ASSERT_FAILURE(ptr[6], "unique_ptr<T[]>::operator[](index): index out of range");
     }
+#if TEST_STD_VER >= 20
+    {
+      std::unique_ptr<WithCookie[]> ptr = std::make_unique_for_overwrite<WithCookie[]>(5);
+      TEST_LIBCPP_ASSERT_FAILURE(ptr[6] = WithCookie(), "unique_ptr<T[]>::operator[](index): index out of range");
+    }
+#endif
   }
 
   // For types that don't have an array cookie, things are a bit more complicated. We can detect OOB accesses
@@ -97,14 +90,9 @@ void test() {
 #endif
 
   // Make sure that we carry the bounds information properly through conversions, assignments, etc.
-  // These tests are mostly relevant when the ABI setting is enabled (with a stateful bounds-checker),
-  // but we still run them for types with an array cookie either way.
+  // These tests are only relevant when the ABI setting is enabled (with a stateful bounds-checker).
 #if defined(_LIBCPP_ABI_BOUNDED_UNIQUE_PTR)
-  using Types = types::type_list<NoCookie, WithCookie>;
-#else
-  using Types = types::type_list<WithCookie>;
-#endif
-  types::for_each(Types(), []<class T> {
+  types::for_each(types::type_list<NoCookie, WithCookie>(), []<class T> {
     // Bounds carried through move construction
     {
       std::unique_ptr<T[]> ptr = std::make_unique<T[]>(5);
@@ -135,6 +123,7 @@ void test() {
       TEST_LIBCPP_ASSERT_FAILURE(other[6], "unique_ptr<T[]>::operator[](index): index out of range");
     }
   });
+#endif
 }
 
 template <std::size_t Size>
diff --git a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/op_subscript.runtime.pass.cpp b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/op_subscript.runtime.pass.cpp
index ebfad8ec724e5..477667274f6ec 100644
--- a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/op_subscript.runtime.pass.cpp
+++ b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/op_subscript.runtime.pass.cpp
@@ -46,6 +46,11 @@ struct WithNonTrivialDtor {
 template <class T>
 struct CustomDeleter : std::default_delete<T> {};
 
+struct NoopDeleter {
+  template <class T>
+  TEST_CONSTEXPR_CXX23 void operator()(T*) const {}
+};
+
 TEST_CONSTEXPR_CXX23 bool test() {
   // Basic test
   {
@@ -112,12 +117,33 @@ TEST_CONSTEXPR_CXX23 bool test() {
         WithNonTrivialDtor<16>,
         WithNonTrivialDtor<256>>;
     types::for_each(TrickyCookieTypes(), []<class T> {
-      types::for_each(types::type_list<std::default_delete<T[]>, CustomDeleter<T[]>>(), []<class Deleter> {
-        std::unique_ptr<T[], Deleter> p(new T[3]);
+      // Array allocated with `new T[n]`, default deleter
+      {
+        std::unique_ptr<T[], std::default_delete<T[]>> p(new T[3]);
+        assert(p[0] == T());
+        assert(p[1] == T());
+        assert(p[2] == T());
+      }
+
+      // Array allocated with `new T[n]`, custom deleter
+      {
+        std::unique_ptr<T[], CustomDeleter<T[]>> p(new T[3]);
+        assert(p[0] == T());
+        assert(p[1] == T());
+        assert(p[2] == T());
+      }
+
+      // Array not allocated with `new T[n]`, custom deleter
+      //
+      // This test aims to ensure that the implementation doesn't try to use an array cookie
+      // when there is none.
+      {
+        T array[50] = {};
+        std::unique_ptr<T[], NoopDeleter> p(&array[0]);
         assert(p[0] == T());
         assert(p[1] == T());
         assert(p[2] == T());
-      });
+      }
     });
   }
 #endif // C++20
diff --git a/libcxx/test/support/test_macros.h b/libcxx/test/support/test_macros.h
index 3aa818af1d269..ae171b3f223ab 100644
--- a/libcxx/test/support/test_macros.h
+++ b/libcxx/test/support/test_macros.h
@@ -221,9 +221,11 @@
 #endif
 
 #if TEST_STD_VER > 17
-#define TEST_CONSTINIT constinit
+#  define TEST_CONSTINIT constinit
+#elif __has_cpp_attribute(clang::require_constant_initialization)
+#  define TEST_CONSTINIT [[clang::require_constant_initialization]]
 #else
-#define TEST_CONSTINIT
+#  define TEST_CONSTINIT
 #endif
 
 #if TEST_STD_VER < 11
diff --git a/libcxx/utils/gdb/libcxx/printers.py b/libcxx/utils/gdb/libcxx/printers.py
index 49087f94c0626..8e74b93e7b121 100644
--- a/libcxx/utils/gdb/libcxx/printers.py
+++ b/libcxx/utils/gdb/libcxx/printers.py
@@ -173,7 +173,7 @@ def __next__(self):
             field_name = next(self.child_iter)
             child = self.val["__base_"][field_name]["__value_"]
             self.count += 1
-            return ("[%d]" % self.count, child)
+            return ("[%d]" % (self.count - 1), child)
 
         next = __next__  # Needed for GDB built against Python 2.7.
 
@@ -331,7 +331,7 @@ def __next__(self):
             if self.offset >= self.bits_per_word:
                 self.item += 1
                 self.offset = 0
-            return ("[%d]" % self.count, outbit)
+            return ("[%d]" % (self.count - 1), outbit)
 
         next = __next__  # Needed for GDB built against Python 2.7.
 
@@ -352,7 +352,7 @@ def __next__(self):
                 raise StopIteration
             entry = self.item.dereference()
             self.item += 1
-            return ("[%d]" % self.count, entry)
+            return ("[%d]" % (self.count - 1), entry)
 
         next = __next__  # Needed for GDB built against Python 2.7.
 
diff --git a/libcxx/utils/libcxx/test/params.py b/libcxx/utils/libcxx/test/params.py
index 13c7297fd7304..6c2713ac92fdb 100644
--- a/libcxx/utils/libcxx/test/params.py
+++ b/libcxx/utils/libcxx/test/params.py
@@ -360,7 +360,6 @@ def getSuitableClangTidy(cfg):
         if experimental
         else [
             AddFeature("libcpp-has-no-incomplete-pstl"),
-            AddFeature("libcpp-has-no-experimental-stop_token"),
             AddFeature("libcpp-has-no-experimental-tzdb"),
             AddFeature("libcpp-has-no-experimental-syncstream"),
         ],
diff --git a/libunwind/test/configs/cmake-bridge.cfg.in b/libunwind/test/configs/cmake-bridge.cfg.in
index bd7122bbea32c..f627f401f9f7b 100644
--- a/libunwind/test/configs/cmake-bridge.cfg.in
+++ b/libunwind/test/configs/cmake-bridge.cfg.in
@@ -31,4 +31,4 @@ if not @LIBUNWIND_ENABLE_THREADS@:
 # Add substitutions for bootstrapping the test suite configuration
 config.substitutions.append(('%{install-prefix}', '@LIBUNWIND_TESTING_INSTALL_PREFIX@'))
 config.substitutions.append(('%{include}', '@LIBUNWIND_TESTING_INSTALL_PREFIX@/include'))
-config.substitutions.append(('%{lib}', '@LIBUNWIND_TESTING_INSTALL_PREFIX@/lib'))
+config.substitutions.append(('%{lib}', '@LIBUNWIND_TESTING_INSTALL_PREFIX@/@LIBUNWIND_INSTALL_LIBRARY_DIR@'))
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index dc757bf7dd1e7..85a58a3677181 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -2628,7 +2628,9 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
     createECExportThunks();
 
   // Resolve remaining undefined symbols and warn about imported locals.
-  ctx.symtab.resolveRemainingUndefines();
+  while (ctx.symtab.resolveRemainingUndefines())
+    run();
+
   if (errorCount())
     return;
 
diff --git a/lld/COFF/SymbolTable.cpp b/lld/COFF/SymbolTable.cpp
index 0ef58910151cf..3bd471389daa1 100644
--- a/lld/COFF/SymbolTable.cpp
+++ b/lld/COFF/SymbolTable.cpp
@@ -479,10 +479,11 @@ void SymbolTable::reportUnresolvable() {
                        /* localImports */ nullptr, true);
 }
 
-void SymbolTable::resolveRemainingUndefines() {
+bool SymbolTable::resolveRemainingUndefines() {
   llvm::TimeTraceScope timeScope("Resolve remaining undefined symbols");
   SmallPtrSet<Symbol *, 8> undefs;
   DenseMap<Symbol *, Symbol *> localImports;
+  bool foundLazy = false;
 
   for (auto &i : symMap) {
     Symbol *sym = i.second;
@@ -510,6 +511,11 @@ void SymbolTable::resolveRemainingUndefines() {
           undef->resolveWeakAlias();
         }
       }
+      if (imp && imp->isLazy()) {
+        forceLazy(imp);
+        foundLazy = true;
+        continue;
+      }
       if (imp && isa<Defined>(imp)) {
         auto *d = cast<Defined>(imp);
         replaceSymbol<DefinedLocalImport>(sym, ctx, name, d);
@@ -537,6 +543,7 @@ void SymbolTable::resolveRemainingUndefines() {
   reportProblemSymbols(
       ctx, undefs,
       ctx.config.warnLocallyDefinedImported ? &localImports : nullptr, false);
+  return foundLazy;
 }
 
 std::pair<Symbol *, bool> SymbolTable::insert(StringRef name) {
diff --git a/lld/COFF/SymbolTable.h b/lld/COFF/SymbolTable.h
index e3f674b8098f8..dab03afde3f98 100644
--- a/lld/COFF/SymbolTable.h
+++ b/lld/COFF/SymbolTable.h
@@ -57,7 +57,10 @@ class SymbolTable {
   // Try to resolve any undefined symbols and update the symbol table
   // accordingly, then print an error message for any remaining undefined
   // symbols and warn about imported local symbols.
-  void resolveRemainingUndefines();
+  // Returns whether more files might need to be linked in to resolve lazy
+  // symbols, in which case the caller is expected to call the function again
+  // after linking those files.
+  bool resolveRemainingUndefines();
 
   // Load lazy objects that are needed for MinGW automatic import and for
   // doing stdcall fixups.
diff --git a/lld/ELF/AArch64ErrataFix.cpp b/lld/ELF/AArch64ErrataFix.cpp
index 7a1477ebb79ad..01873cbc1e5c1 100644
--- a/lld/ELF/AArch64ErrataFix.cpp
+++ b/lld/ELF/AArch64ErrataFix.cpp
@@ -374,9 +374,9 @@ class elf::Patch843419Section final : public SyntheticSection {
 public:
   Patch843419Section(InputSection *p, uint64_t off);
 
-  void writeTo(uint8_t *buf) override;
+  void writeTo(Ctx &, uint8_t *buf) override;
 
-  size_t getSize() const override { return 8; }
+  size_t getSize(Ctx &) const override { return 8; }
 
   uint64_t getLDSTAddr() const;
 
@@ -399,7 +399,7 @@ Patch843419Section::Patch843419Section(InputSection *p, uint64_t off)
   this->parent = p->getParent();
   patchSym = addSyntheticLocal(
       saver().save("__CortexA53843419_" + utohexstr(getLDSTAddr())), STT_FUNC,
-      0, getSize(), *this);
+      0, getSize(ctx), *this);
   addSyntheticLocal(saver().save("$x"), STT_NOTYPE, 0, 0, *this);
 }
 
@@ -407,7 +407,7 @@ uint64_t Patch843419Section::getLDSTAddr() const {
   return patchee->getVA(patcheeOffset);
 }
 
-void Patch843419Section::writeTo(uint8_t *buf) {
+void Patch843419Section::writeTo(Ctx &ctx, uint8_t *buf) {
   // Copy the instruction that we will be replacing with a branch in the
   // patchee Section.
   write32le(buf, read32le(patchee->content().begin() + patcheeOffset));
diff --git a/lld/ELF/ARMErrataFix.cpp b/lld/ELF/ARMErrataFix.cpp
index 7068344a73b95..03feabe6d676f 100644
--- a/lld/ELF/ARMErrataFix.cpp
+++ b/lld/ELF/ARMErrataFix.cpp
@@ -72,9 +72,9 @@ class elf::Patch657417Section final : public SyntheticSection {
 public:
   Patch657417Section(InputSection *p, uint64_t off, uint32_t instr, bool isARM);
 
-  void writeTo(uint8_t *buf) override;
+  void writeTo(Ctx &, uint8_t *buf) override;
 
-  size_t getSize() const override { return 4; }
+  size_t getSize(Ctx &) const override { return 4; }
 
   // Get the virtual address of the branch instruction at patcheeOffset.
   uint64_t getBranchAddr() const;
@@ -141,7 +141,7 @@ Patch657417Section::Patch657417Section(InputSection *p, uint64_t off,
   parent = p->getParent();
   patchSym = addSyntheticLocal(
       saver().save("__CortexA8657417_" + utohexstr(getBranchAddr())), STT_FUNC,
-      isARM ? 0 : 1, getSize(), *this);
+      isARM ? 0 : 1, getSize(ctx), *this);
   addSyntheticLocal(saver().save(isARM ? "$a" : "$t"), STT_NOTYPE, 0, 0, *this);
 }
 
@@ -174,7 +174,7 @@ static uint64_t getThumbDestAddr(uint64_t sourceAddr, uint32_t instr) {
   return sourceAddr + offset + 4;
 }
 
-void Patch657417Section::writeTo(uint8_t *buf) {
+void Patch657417Section::writeTo(Ctx &ctx, uint8_t *buf) {
   // The base instruction of the patch is always a 32-bit unconditional branch.
   if (isARM)
     write32le(buf, 0xea000000);
diff --git a/lld/ELF/Arch/ARM.cpp b/lld/ELF/Arch/ARM.cpp
index 77bdd656dd8cd..d22ac540c92a1 100644
--- a/lld/ELF/Arch/ARM.cpp
+++ b/lld/ELF/Arch/ARM.cpp
@@ -1380,7 +1380,7 @@ void ArmCmseSGSection::addSGVeneer(Symbol *acleSeSym, Symbol *sym) {
   sgVeneers.emplace_back(ss);
 }
 
-void ArmCmseSGSection::writeTo(uint8_t *buf) {
+void ArmCmseSGSection::writeTo(Ctx &ctx, uint8_t *buf) {
   for (ArmCmseSGVeneer *s : sgVeneers) {
     uint8_t *p = buf + s->offset;
     write16(p + 0, 0xe97f); // SG
@@ -1397,14 +1397,14 @@ void ArmCmseSGSection::addMappingSymbol() {
   addSyntheticLocal("$t", STT_NOTYPE, /*off=*/0, /*size=*/0, *this);
 }
 
-size_t ArmCmseSGSection::getSize() const {
+size_t ArmCmseSGSection::getSize(Ctx &) const {
   if (sgVeneers.empty())
     return (impLibMaxAddr ? impLibMaxAddr - getVA() : 0) + newEntries * entsize;
 
   return entries.size() * entsize;
 }
 
-void ArmCmseSGSection::finalizeContents() {
+void ArmCmseSGSection::finalizeContents(Ctx &) {
   if (sgVeneers.empty())
     return;
 
@@ -1469,10 +1469,10 @@ template <typename ELFT> void elf::writeARMCmseImportLib() {
   for (auto &[osec, isec] : osIsPairs) {
     osec->sectionIndex = ++idx;
     osec->recordSection(isec);
-    osec->finalizeInputSections();
+    osec->finalizeInputSections(ctx);
     osec->shName = shstrtab->addString(osec->name);
-    osec->size = isec->getSize();
-    isec->finalizeContents();
+    osec->size = isec->getSize(ctx);
+    isec->finalizeContents(ctx);
     osec->offset = alignToPowerOf2(off, osec->addralign);
     off = osec->offset + osec->size;
   }
@@ -1525,7 +1525,7 @@ template <typename ELFT> void elf::writeARMCmseImportLib() {
   {
     parallel::TaskGroup tg;
     for (auto &[osec, _] : osIsPairs)
-      osec->template writeTo<ELFT>(buf + osec->offset, tg);
+      osec->template writeTo<ELFT>(ctx, buf + osec->offset, tg);
   }
 
   if (auto e = buffer->commit())
diff --git a/lld/ELF/Arch/RISCV.cpp b/lld/ELF/Arch/RISCV.cpp
index f776ac8ede1bd..f13000fcadec2 100644
--- a/lld/ELF/Arch/RISCV.cpp
+++ b/lld/ELF/Arch/RISCV.cpp
@@ -1047,8 +1047,8 @@ class RISCVAttributesSection final : public SyntheticSection {
   RISCVAttributesSection()
       : SyntheticSection(0, SHT_RISCV_ATTRIBUTES, 1, ".riscv.attributes") {}
 
-  size_t getSize() const override { return size; }
-  void writeTo(uint8_t *buf) override;
+  size_t getSize(Ctx &) const override { return size; }
+  void writeTo(Ctx &, uint8_t *buf) override;
 
   static constexpr StringRef vendor = "riscv";
   DenseMap<unsigned, unsigned> intAttr;
@@ -1276,8 +1276,8 @@ mergeAttributesSection(const SmallVector<InputSectionBase *, 0> &sections) {
   return &merged;
 }
 
-void RISCVAttributesSection::writeTo(uint8_t *buf) {
-  const size_t size = getSize();
+void RISCVAttributesSection::writeTo(Ctx &ctx, uint8_t *buf) {
+  const size_t size = getSize(ctx);
   uint8_t *const end = buf + size;
   *buf = ELFAttrs::Format_Version;
   write32(buf + 1, size - 1);
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 94cd060b697d2..373505a9e965b 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -3194,7 +3194,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
     // sectionBases.
     for (SectionCommand *cmd : ctx.script->sectionCommands)
       if (auto *osd = dyn_cast<OutputDesc>(cmd))
-        osd->osec.finalizeInputSections(ctx.script);
+        osd->osec.finalizeInputSections(ctx);
   }
 
   // Two input sections with different output sections should not be folded.
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index 7265ed56e957f..3f5d2fbb5cbae 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -102,7 +102,7 @@ static ELFKind getELFKind(MemoryBufferRef mb, StringRef archiveName) {
 // For ARM only, to set the EF_ARM_ABI_FLOAT_SOFT or EF_ARM_ABI_FLOAT_HARD
 // flag in the ELF Header we need to look at Tag_ABI_VFP_args to find out how
 // the input objects have been compiled.
-static void updateARMVFPArgs(const ARMAttributeParser &attributes,
+static void updateARMVFPArgs(Ctx &ctx, const ARMAttributeParser &attributes,
                              const InputFile *f) {
   std::optional<unsigned> attr =
       attributes.getAttributeValue(ARMBuildAttrs::ABI_VFP_args);
@@ -150,7 +150,8 @@ static void updateARMVFPArgs(const ARMAttributeParser &attributes,
 // at compile time. We follow the convention that if at least one input object
 // is compiled with an architecture that supports these features then lld is
 // permitted to use them.
-static void updateSupportedARMFeatures(const ARMAttributeParser &attributes) {
+static void updateSupportedARMFeatures(Ctx &ctx,
+                                       const ARMAttributeParser &attributes) {
   std::optional<unsigned> attr =
       attributes.getAttributeValue(ARMBuildAttrs::CPU_arch);
   if (!attr)
@@ -264,7 +265,7 @@ std::optional<MemoryBufferRef> elf::readFile(StringRef path) {
 // All input object files must be for the same architecture
 // (e.g. it does not make sense to link x86 object files with
 // MIPS object files.) This function checks for that error.
-static bool isCompatible(InputFile *file) {
+static bool isCompatible(Ctx &ctx, InputFile *file) {
   if (!file->isElf() && !isa<BitcodeFile>(file))
     return true;
 
@@ -297,7 +298,7 @@ static bool isCompatible(InputFile *file) {
 }
 
 template <class ELFT> static void doParseFile(Ctx &ctx, InputFile *file) {
-  if (!isCompatible(file))
+  if (!isCompatible(ctx, file))
     return;
 
   // Lazy object file
@@ -418,7 +419,8 @@ StringRef InputFile::getNameForScript() const {
 // the various ways that a library can be specified to LLD. This ELF extension
 // is a form of autolinking and is called `dependent libraries`. It is currently
 // unique to LLVM and lld.
-static void addDependentLibrary(StringRef specifier, const InputFile *f) {
+static void addDependentLibrary(Ctx &ctx, StringRef specifier,
+                                const InputFile *f) {
   if (!ctx.arg.dependentLibraries)
     return;
   if (std::optional<std::string> s = searchLibraryBaseName(specifier))
@@ -611,7 +613,7 @@ template <class ELFT> void ObjFile<ELFT>::parse(bool ignoreComdats) {
       } else {
         for (const char *d = data.begin(), *e = data.end(); d < e;) {
           StringRef s(d);
-          addDependentLibrary(s, this);
+          addDependentLibrary(ctx, s, this);
           d += s.size() + 1;
         }
       }
@@ -631,8 +633,8 @@ template <class ELFT> void ObjFile<ELFT>::parse(bool ignoreComdats) {
         InputSection isec(*this, sec, name);
         warn(toString(&isec) + ": " + llvm::toString(std::move(e)));
       } else {
-        updateSupportedARMFeatures(attributes);
-        updateARMVFPArgs(attributes, this);
+        updateSupportedARMFeatures(ctx, attributes);
+        updateARMVFPArgs(ctx, attributes, this);
 
         // FIXME: Retain the first attribute section we see. The eglibc ARM
         // dynamic loaders require the presence of an attribute section for
@@ -1704,7 +1706,7 @@ BitcodeFile::BitcodeFile(MemoryBufferRef mb, StringRef archiveName,
 
   std::string path = mb.getBufferIdentifier().str();
   if (ctx.arg.thinLTOIndexOnly)
-    path = replaceThinLTOSuffix(mb.getBufferIdentifier());
+    path = replaceThinLTOSuffix(ctx, mb.getBufferIdentifier());
 
   // ThinLTO assumes that all MemoryBufferRefs given to it have a unique
   // name. If two archives define two members with the same name, this
@@ -1738,9 +1740,10 @@ static uint8_t mapVisibility(GlobalValue::VisibilityTypes gvVisibility) {
   llvm_unreachable("unknown visibility");
 }
 
-static void
-createBitcodeSymbol(Symbol *&sym, const std::vector<bool> &keptComdats,
-                    const lto::InputFile::Symbol &objSym, BitcodeFile &f) {
+static void createBitcodeSymbol(Ctx &ctx, Symbol *&sym,
+                                const std::vector<bool> &keptComdats,
+                                const lto::InputFile::Symbol &objSym,
+                                BitcodeFile &f) {
   uint8_t binding = objSym.isWeak() ? STB_WEAK : STB_GLOBAL;
   uint8_t type = objSym.isTLS() ? STT_TLS : STT_NOTYPE;
   uint8_t visibility = mapVisibility(objSym.getVisibility());
@@ -1791,13 +1794,13 @@ void BitcodeFile::parse() {
   // ObjFile<ELFT>::initializeSymbols.
   for (auto [i, irSym] : llvm::enumerate(obj->symbols()))
     if (!irSym.isUndefined())
-      createBitcodeSymbol(symbols[i], keptComdats, irSym, *this);
+      createBitcodeSymbol(ctx, symbols[i], keptComdats, irSym, *this);
   for (auto [i, irSym] : llvm::enumerate(obj->symbols()))
     if (irSym.isUndefined())
-      createBitcodeSymbol(symbols[i], keptComdats, irSym, *this);
+      createBitcodeSymbol(ctx, symbols[i], keptComdats, irSym, *this);
 
   for (auto l : obj->getDependentLibraries())
-    addDependentLibrary(l, this);
+    addDependentLibrary(ctx, l, this);
 }
 
 void BitcodeFile::parseLazy() {
@@ -1917,7 +1920,7 @@ bool InputFile::shouldExtractForCommon(StringRef name) const {
   return isNonCommonDef(mb, name, archiveName);
 }
 
-std::string elf::replaceThinLTOSuffix(StringRef path) {
+std::string elf::replaceThinLTOSuffix(Ctx &ctx, StringRef path) {
   auto [suffix, repl] = ctx.arg.thinLTOObjectSuffixReplace;
   if (path.consume_back(suffix))
     return (path + repl).str();
diff --git a/lld/ELF/InputFiles.h b/lld/ELF/InputFiles.h
index 730a4d8855c6b..fc5ecbc2f5d22 100644
--- a/lld/ELF/InputFiles.h
+++ b/lld/ELF/InputFiles.h
@@ -382,7 +382,7 @@ InputFile *createInternalFile(StringRef name);
 ELFFileBase *createObjFile(MemoryBufferRef mb, StringRef archiveName = "",
                            bool lazy = false);
 
-std::string replaceThinLTOSuffix(StringRef path);
+std::string replaceThinLTOSuffix(Ctx &, StringRef path);
 
 } // namespace elf
 } // namespace lld
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index 54a214e01b089..7c8da8944e502 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -101,7 +101,7 @@ InputSectionBase::InputSectionBase(ObjFile<ELFT> &file,
 
 size_t InputSectionBase::getSize() const {
   if (auto *s = dyn_cast<SyntheticSection>(this))
-    return s->getSize();
+    return s->getSize(ctx);
   return size - bytesDropped;
 }
 
diff --git a/lld/ELF/InputSection.h b/lld/ELF/InputSection.h
index 53dbaf0784b91..c368ed4dc2e9c 100644
--- a/lld/ELF/InputSection.h
+++ b/lld/ELF/InputSection.h
@@ -478,13 +478,13 @@ class SyntheticSection : public InputSection {
                      InputSectionBase::Synthetic) {}
 
   virtual ~SyntheticSection() = default;
-  virtual size_t getSize() const = 0;
+  virtual size_t getSize(Ctx &) const = 0;
   virtual bool updateAllocSize() { return false; }
   // If the section has the SHF_ALLOC flag and the size may be changed if
   // thunks are added, update the section size.
   virtual bool isNeeded() const { return true; }
-  virtual void finalizeContents() {}
-  virtual void writeTo(uint8_t *buf) = 0;
+  virtual void finalizeContents(Ctx &) {}
+  virtual void writeTo(Ctx &, uint8_t *buf) = 0;
 
   static bool classof(const SectionBase *sec) {
     return sec->kind() == InputSectionBase::Synthetic;
diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp
index 6b4b0716b9ccb..4df1b0c289eef 100644
--- a/lld/ELF/LTO.cpp
+++ b/lld/ELF/LTO.cpp
@@ -292,7 +292,7 @@ static void thinLTOCreateEmptyIndexFiles(Ctx &ctx) {
     if (linkedBitCodeFiles.contains(f->getName()))
       continue;
     std::string path =
-        replaceThinLTOSuffix(getThinLTOOutputFile(ctx, f->obj->getName()));
+        replaceThinLTOSuffix(ctx, getThinLTOOutputFile(ctx, f->obj->getName()));
     std::unique_ptr<raw_fd_ostream> os = openFile(path + ".thinlto.bc");
     if (!os)
       continue;
diff --git a/lld/ELF/LinkerScript.cpp b/lld/ELF/LinkerScript.cpp
index cce584ae4d867..2852c90fb57ef 100644
--- a/lld/ELF/LinkerScript.cpp
+++ b/lld/ELF/LinkerScript.cpp
@@ -1442,7 +1442,7 @@ void LinkerScript::allocateHeaders(SmallVector<PhdrEntry *, 0> &phdrs) {
         return cmd.hasPhdrs || cmd.hasFilehdr;
       });
   bool paged = !ctx.arg.omagic && !ctx.arg.nmagic;
-  uint64_t headerSize = getHeaderSize();
+  uint64_t headerSize = getHeaderSize(ctx);
 
   uint64_t base = 0;
   // If SECTIONS is present and the linkerscript is not explicit about program
@@ -1491,7 +1491,7 @@ LinkerScript::assignAddresses() {
     dot = ctx.target->getImageBase();
     ctx.out.elfHeader->addr = dot;
     ctx.out.programHeaders->addr = dot + ctx.out.elfHeader->size;
-    dot += getHeaderSize();
+    dot += getHeaderSize(ctx);
   }
 
   OutputSection *changedOsec = nullptr;
diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp
index b775eed9af2bf..3468420b3bb0b 100644
--- a/lld/ELF/OutputSections.cpp
+++ b/lld/ELF/OutputSections.cpp
@@ -78,7 +78,7 @@ OutputSection::OutputSection(StringRef name, uint32_t type, uint64_t flags)
 //
 // NOTE: clang since rL252300 emits SHT_X86_64_UNWIND .eh_frame sections. Allow
 // them to be merged into SHT_PROGBITS .eh_frame (GNU as .cfi_*).
-static bool canMergeToProgbits(unsigned type) {
+static bool canMergeToProgbits(Ctx &ctx, unsigned type) {
   return type == SHT_NOBITS || type == SHT_PROGBITS || type == SHT_INIT_ARRAY ||
          type == SHT_PREINIT_ARRAY || type == SHT_FINI_ARRAY ||
          type == SHT_NOTE ||
@@ -105,7 +105,7 @@ void OutputSection::recordSection(InputSectionBase *isec) {
 // Update fields (type, flags, alignment, etc) according to the InputSection
 // isec. Also check whether the InputSection flags and type are consistent with
 // other InputSections.
-void OutputSection::commitSection(InputSection *isec) {
+void OutputSection::commitSection(Ctx &ctx, InputSection *isec) {
   if (LLVM_UNLIKELY(type != isec->type)) {
     if (!hasInputSections && !typeIsSet) {
       type = isec->type;
@@ -120,8 +120,8 @@ void OutputSection::commitSection(InputSection *isec) {
         name = saver().save(".crel" + name);
       }
     } else {
-      if (typeIsSet || !canMergeToProgbits(type) ||
-          !canMergeToProgbits(isec->type)) {
+      if (typeIsSet || !canMergeToProgbits(ctx, type) ||
+          !canMergeToProgbits(ctx, isec->type)) {
         // The (NOLOAD) changes the section type to SHT_NOBITS, the intention is
         // that the contents at that address is provided by some other means.
         // Some projects (e.g.
@@ -172,7 +172,7 @@ void OutputSection::commitSection(InputSection *isec) {
     entsize = 0;
 }
 
-static MergeSyntheticSection *createMergeSynthetic(StringRef name,
+static MergeSyntheticSection *createMergeSynthetic(Ctx &ctx, StringRef name,
                                                    uint32_t type,
                                                    uint64_t flags,
                                                    uint32_t addralign) {
@@ -188,7 +188,8 @@ static MergeSyntheticSection *createMergeSynthetic(StringRef name,
 // new synthetic sections at the location of the first input section
 // that it replaces. It then finalizes each synthetic section in order
 // to compute an output offset for each piece of each input section.
-void OutputSection::finalizeInputSections(LinkerScript *script) {
+void OutputSection::finalizeInputSections(Ctx &ctx) {
+  auto *script = ctx.script;
   std::vector<MergeSyntheticSection *> mergeSections;
   for (SectionCommand *cmd : commands) {
     auto *isd = dyn_cast<InputSectionDescription>(cmd);
@@ -222,8 +223,8 @@ void OutputSection::finalizeInputSections(LinkerScript *script) {
                (sec->addralign == ms->addralign || !(sec->flags & SHF_STRINGS));
       });
       if (i == mergeSections.end()) {
-        MergeSyntheticSection *syn =
-            createMergeSynthetic(s->name, ms->type, ms->flags, ms->addralign);
+        MergeSyntheticSection *syn = createMergeSynthetic(
+            ctx, s->name, ms->type, ms->flags, ms->addralign);
         mergeSections.push_back(syn);
         i = std::prev(mergeSections.end());
         syn->entsize = ms->entsize;
@@ -243,10 +244,10 @@ void OutputSection::finalizeInputSections(LinkerScript *script) {
 
     // Some input sections may be removed from the list after ICF.
     for (InputSection *s : isd->sections)
-      commitSection(s);
+      commitSection(ctx, s);
   }
   for (auto *ms : mergeSections)
-    ms->finalizeContents();
+    ms->finalizeContents(ctx);
 }
 
 static void sortByOrder(MutableArrayRef<InputSection *> in,
@@ -260,7 +261,7 @@ static void sortByOrder(MutableArrayRef<InputSection *> in,
     in[i] = v[i].second;
 }
 
-uint64_t elf::getHeaderSize() {
+uint64_t elf::getHeaderSize(Ctx &ctx) {
   if (ctx.arg.oFormatBinary)
     return 0;
   return ctx.out.elfHeader->size + ctx.out.programHeaders->size;
@@ -273,7 +274,7 @@ void OutputSection::sort(llvm::function_ref<int(InputSectionBase *s)> order) {
       sortByOrder(isd->sections, order);
 }
 
-static void nopInstrFill(uint8_t *buf, size_t size) {
+static void nopInstrFill(Ctx &ctx, uint8_t *buf, size_t size) {
   if (size == 0)
     return;
   unsigned i = 0;
@@ -341,7 +342,7 @@ static SmallVector<uint8_t, 0> deflateShard(ArrayRef<uint8_t> in, int level,
 //
 // * (if --compress-debug-sections is specified) non-empty .debug_* sections
 // * (if --compress-sections is specified) matched sections
-template <class ELFT> void OutputSection::maybeCompress() {
+template <class ELFT> void OutputSection::maybeCompress(Ctx &ctx) {
   using Elf_Chdr = typename ELFT::Chdr;
   (void)sizeof(Elf_Chdr);
 
@@ -367,7 +368,7 @@ template <class ELFT> void OutputSection::maybeCompress() {
   // Write uncompressed data to a temporary zero-initialized buffer.
   {
     parallel::TaskGroup tg;
-    writeTo<ELFT>(buf.get(), tg);
+    writeTo<ELFT>(ctx, buf.get(), tg);
   }
   // The generic ABI specifies "The sh_size and sh_addralign fields of the
   // section header for a compressed section reflect the requirements of the
@@ -468,7 +469,7 @@ static void writeInt(uint8_t *buf, uint64_t data, uint64_t size) {
 }
 
 template <class ELFT>
-void OutputSection::writeTo(uint8_t *buf, parallel::TaskGroup &tg) {
+void OutputSection::writeTo(Ctx &ctx, uint8_t *buf, parallel::TaskGroup &tg) {
   llvm::TimeTraceScope timeScope("Write sections", name);
   if (type == SHT_NOBITS)
     return;
@@ -507,7 +508,7 @@ void OutputSection::writeTo(uint8_t *buf, parallel::TaskGroup &tg) {
 
   // Write leading padding.
   ArrayRef<InputSection *> sections = getInputSections(*this, storage);
-  std::array<uint8_t, 4> filler = getFiller();
+  std::array<uint8_t, 4> filler = getFiller(ctx);
   bool nonZeroFiller = read32(filler.data()) != 0;
   if (nonZeroFiller)
     fill(buf, sections.empty() ? size : sections[0]->outSecOff, filler);
@@ -518,12 +519,12 @@ void OutputSection::writeTo(uint8_t *buf, parallel::TaskGroup &tg) {
     return;
   }
 
-  auto fn = [=](size_t begin, size_t end) {
+  auto fn = [=, &ctx](size_t begin, size_t end) {
     size_t numSections = sections.size();
     for (size_t i = begin; i != end; ++i) {
       InputSection *isec = sections[i];
       if (auto *s = dyn_cast<SyntheticSection>(isec))
-        s->writeTo(buf + isec->outSecOff);
+        s->writeTo(ctx, buf + isec->outSecOff);
       else
         isec->writeTo<ELFT>(buf + isec->outSecOff);
 
@@ -543,7 +544,7 @@ void OutputSection::writeTo(uint8_t *buf, parallel::TaskGroup &tg) {
           end = buf + sections[i + 1]->outSecOff;
         if (isec->nopFiller) {
           assert(ctx.target->nopInstrs);
-          nopInstrFill(start, end - start);
+          nopInstrFill(ctx, start, end - start);
         } else
           fill(start, end - start, filler);
       }
@@ -582,7 +583,8 @@ void OutputSection::writeTo(uint8_t *buf, parallel::TaskGroup &tg) {
   }
 }
 
-static void finalizeShtGroup(OutputSection *os, InputSection *section) {
+static void finalizeShtGroup(Ctx &ctx, OutputSection *os,
+                             InputSection *section) {
   // sh_link field for SHT_GROUP sections should contain the section index of
   // the symbol table.
   os->link = ctx.in.symTab->getParent()->sectionIndex;
@@ -668,7 +670,7 @@ static size_t relToCrel(raw_svector_ostream &os, Elf_Crel<ELFT::Is64Bits> &out,
 
 // Compute the content of a non-alloc CREL section due to -r or --emit-relocs.
 // Input CREL sections are decoded while REL[A] need to be converted.
-template <bool is64> void OutputSection::finalizeNonAllocCrel() {
+template <bool is64> void OutputSection::finalizeNonAllocCrel(Ctx &ctx) {
   using uint = typename Elf_Crel_Impl<is64>::uint;
   raw_svector_ostream os(crelBody);
   uint64_t totalCount = 0;
@@ -702,7 +704,7 @@ template <bool is64> void OutputSection::finalizeNonAllocCrel() {
   size = getULEB128Size(crelHeader) + crelBody.size();
 }
 
-void OutputSection::finalize() {
+void OutputSection::finalize(Ctx &ctx) {
   InputSection *first = getFirstInputSection(this);
 
   if (flags & SHF_LINK_ORDER) {
@@ -718,7 +720,7 @@ void OutputSection::finalize() {
   }
 
   if (type == SHT_GROUP) {
-    finalizeShtGroup(this, first);
+    finalizeShtGroup(ctx, this, first);
     return;
   }
 
@@ -741,9 +743,9 @@ void OutputSection::finalize() {
   // Finalize the content of non-alloc CREL.
   if (type == SHT_CREL) {
     if (ctx.arg.is64)
-      finalizeNonAllocCrel<true>();
+      finalizeNonAllocCrel<true>(ctx);
     else
-      finalizeNonAllocCrel<false>();
+      finalizeNonAllocCrel<false>(ctx);
   }
 }
 
@@ -854,7 +856,7 @@ void OutputSection::sortInitFini() {
   sort([](InputSectionBase *s) { return getPriority(s->name); });
 }
 
-std::array<uint8_t, 4> OutputSection::getFiller() {
+std::array<uint8_t, 4> OutputSection::getFiller(Ctx &ctx) {
   if (filler)
     return *filler;
   if (flags & SHF_EXECINSTR)
@@ -862,7 +864,7 @@ std::array<uint8_t, 4> OutputSection::getFiller() {
   return {0, 0, 0, 0};
 }
 
-void OutputSection::checkDynRelAddends(const uint8_t *bufStart) {
+void OutputSection::checkDynRelAddends(Ctx &ctx) {
   assert(ctx.arg.writeAddends && ctx.arg.checkDynamicRelocs);
   assert(isStaticRelSecType(type));
   SmallVector<InputSection *, 0> storage;
@@ -885,8 +887,8 @@ void OutputSection::checkDynRelAddends(const uint8_t *bufStart) {
           (rel.inputSec == ctx.in.ppc64LongBranchTarget.get() ||
            rel.inputSec == ctx.in.igotPlt.get()))
         continue;
-      const uint8_t *relocTarget =
-          bufStart + relOsec->offset + rel.inputSec->getOffset(rel.offsetInSec);
+      const uint8_t *relocTarget = ctx.bufferStart + relOsec->offset +
+                                   rel.inputSec->getOffset(rel.offsetInSec);
       // For SHT_NOBITS the written addend is always zero.
       int64_t writtenAddend =
           relOsec->type == SHT_NOBITS
@@ -909,16 +911,16 @@ template void OutputSection::writeHeaderTo<ELF32BE>(ELF32BE::Shdr *Shdr);
 template void OutputSection::writeHeaderTo<ELF64LE>(ELF64LE::Shdr *Shdr);
 template void OutputSection::writeHeaderTo<ELF64BE>(ELF64BE::Shdr *Shdr);
 
-template void OutputSection::writeTo<ELF32LE>(uint8_t *,
+template void OutputSection::writeTo<ELF32LE>(Ctx &, uint8_t *,
                                               llvm::parallel::TaskGroup &);
-template void OutputSection::writeTo<ELF32BE>(uint8_t *,
+template void OutputSection::writeTo<ELF32BE>(Ctx &, uint8_t *,
                                               llvm::parallel::TaskGroup &);
-template void OutputSection::writeTo<ELF64LE>(uint8_t *,
+template void OutputSection::writeTo<ELF64LE>(Ctx &, uint8_t *,
                                               llvm::parallel::TaskGroup &);
-template void OutputSection::writeTo<ELF64BE>(uint8_t *,
+template void OutputSection::writeTo<ELF64BE>(Ctx &, uint8_t *,
                                               llvm::parallel::TaskGroup &);
 
-template void OutputSection::maybeCompress<ELF32LE>();
-template void OutputSection::maybeCompress<ELF32BE>();
-template void OutputSection::maybeCompress<ELF64LE>();
-template void OutputSection::maybeCompress<ELF64BE>();
+template void OutputSection::maybeCompress<ELF32LE>(Ctx &);
+template void OutputSection::maybeCompress<ELF32BE>(Ctx &);
+template void OutputSection::maybeCompress<ELF64LE>(Ctx &);
+template void OutputSection::maybeCompress<ELF64BE>(Ctx &);
diff --git a/lld/ELF/OutputSections.h b/lld/ELF/OutputSections.h
index 909b284a75a2e..904206b20bc1c 100644
--- a/lld/ELF/OutputSections.h
+++ b/lld/ELF/OutputSections.h
@@ -74,8 +74,8 @@ class OutputSection final : public SectionBase {
   uint32_t shName = 0;
 
   void recordSection(InputSectionBase *isec);
-  void commitSection(InputSection *isec);
-  void finalizeInputSections(LinkerScript *script = nullptr);
+  void commitSection(Ctx &ctx, InputSection *isec);
+  void finalizeInputSections(Ctx &ctx);
 
   // The following members are normally only used in linker scripts.
   MemoryRegion *memRegion = nullptr;
@@ -111,13 +111,13 @@ class OutputSection final : public SectionBase {
   // DATA_RELRO_END.
   bool relro = false;
 
-  template <bool is64> void finalizeNonAllocCrel();
-  void finalize();
+  template <bool is64> void finalizeNonAllocCrel(Ctx &);
+  void finalize(Ctx &);
   template <class ELFT>
-  void writeTo(uint8_t *buf, llvm::parallel::TaskGroup &tg);
+  void writeTo(Ctx &, uint8_t *buf, llvm::parallel::TaskGroup &tg);
   // Check that the addends for dynamic relocations were written correctly.
-  void checkDynRelAddends(const uint8_t *bufStart);
-  template <class ELFT> void maybeCompress();
+  void checkDynRelAddends(Ctx &);
+  template <class ELFT> void maybeCompress(Ctx &);
 
   void sort(llvm::function_ref<int(InputSectionBase *s)> order);
   void sortInitFini();
@@ -130,7 +130,7 @@ class OutputSection final : public SectionBase {
 private:
   SmallVector<InputSection *, 0> storage;
 
-  std::array<uint8_t, 4> getFiller();
+  std::array<uint8_t, 4> getFiller(Ctx &);
 };
 
 struct OutputDesc final : SectionCommand {
@@ -169,7 +169,7 @@ llvm::ArrayRef<InputSection *>
 getInputSections(const OutputSection &os,
                  SmallVector<InputSection *, 0> &storage);
 
-uint64_t getHeaderSize();
+uint64_t getHeaderSize(Ctx &);
 } // namespace lld::elf
 
 #endif
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index 078166e0d3f03..7bb657822ee19 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -392,7 +392,7 @@ template <class ELFT> static void addCopyRelSymbol(Ctx &ctx, SharedSymbol &ss) {
     osec->commands.push_back(make<InputSectionDescription>(""));
   auto *isd = cast<InputSectionDescription>(osec->commands.back());
   isd->sections.push_back(sec);
-  osec->commitSection(sec);
+  osec->commitSection(ctx, sec);
 
   // Look through the DSO's dynamic symbol table for aliases and create a
   // dynamic symbol for each one. This causes the copy relocation to correctly
@@ -2019,14 +2019,15 @@ static void forEachInputSectionDescription(
 // This may invalidate any output section offsets stored outside of InputSection
 void ThunkCreator::mergeThunks(ArrayRef<OutputSection *> outputSections) {
   forEachInputSectionDescription(
-      outputSections, [&](OutputSection *os, InputSectionDescription *isd) {
+      outputSections,
+      [&, &ctx = ctx](OutputSection *os, InputSectionDescription *isd) {
         if (isd->thunkSections.empty())
           return;
 
         // Remove any zero sized precreated Thunks.
         llvm::erase_if(isd->thunkSections,
-                       [](const std::pair<ThunkSection *, uint32_t> &ts) {
-                         return ts.first->getSize() == 0;
+                       [&ctx](const std::pair<ThunkSection *, uint32_t> &ts) {
+                         return ts.first->getSize(ctx) == 0;
                        });
 
         // ISD->ThunkSections contains all created ThunkSections, including
@@ -2079,7 +2080,7 @@ ThunkSection *ThunkCreator::getISDThunkSec(OutputSection *os,
   for (std::pair<ThunkSection *, uint32_t> tp : isd->thunkSections) {
     ThunkSection *ts = tp.first;
     uint64_t tsBase = os->addr + ts->outSecOff - pcBias;
-    uint64_t tsLimit = tsBase + ts->getSize();
+    uint64_t tsLimit = tsBase + ts->getSize(ctx);
     if (ctx.target->inBranchRange(rel.type, src,
                                   (src > tsLimit) ? tsBase : tsLimit))
       return ts;
diff --git a/lld/ELF/ScriptParser.cpp b/lld/ELF/ScriptParser.cpp
index b16b2e56473ad..3282551c441d1 100644
--- a/lld/ELF/ScriptParser.cpp
+++ b/lld/ELF/ScriptParser.cpp
@@ -1647,7 +1647,7 @@ Expr ScriptParser::readPrimary() {
     return [=] { return cmd->size; };
   }
   if (tok == "SIZEOF_HEADERS")
-    return [=] { return elf::getHeaderSize(); };
+    return [=, &ctx = ctx] { return elf::getHeaderSize(ctx); };
 
   // Tok is the dot.
   if (tok == ".")
diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index 8298420b8b01d..521873f4cb141 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -102,7 +102,8 @@ MipsAbiFlagsSection<ELFT>::MipsAbiFlagsSection(Elf_Mips_ABIFlags flags)
   this->entsize = sizeof(Elf_Mips_ABIFlags);
 }
 
-template <class ELFT> void MipsAbiFlagsSection<ELFT>::writeTo(uint8_t *buf) {
+template <class ELFT>
+void MipsAbiFlagsSection<ELFT>::writeTo(Ctx &ctx, uint8_t *buf) {
   memcpy(buf, &flags, sizeof(flags));
 }
 
@@ -162,10 +163,11 @@ MipsOptionsSection<ELFT>::MipsOptionsSection(Elf_Mips_RegInfo reginfo)
   this->entsize = sizeof(Elf_Mips_Options) + sizeof(Elf_Mips_RegInfo);
 }
 
-template <class ELFT> void MipsOptionsSection<ELFT>::writeTo(uint8_t *buf) {
+template <class ELFT>
+void MipsOptionsSection<ELFT>::writeTo(Ctx &ctx, uint8_t *buf) {
   auto *options = reinterpret_cast<Elf_Mips_Options *>(buf);
   options->kind = ODK_REGINFO;
-  options->size = getSize();
+  options->size = getSize(ctx);
 
   if (!ctx.arg.relocatable)
     reginfo.ri_gp_value = ctx.in.mipsGot->getGp();
@@ -223,7 +225,8 @@ MipsReginfoSection<ELFT>::MipsReginfoSection(Elf_Mips_RegInfo reginfo)
   this->entsize = sizeof(Elf_Mips_RegInfo);
 }
 
-template <class ELFT> void MipsReginfoSection<ELFT>::writeTo(uint8_t *buf) {
+template <class ELFT>
+void MipsReginfoSection<ELFT>::writeTo(Ctx &ctx, uint8_t *buf) {
   if (!ctx.arg.relocatable)
     reginfo.ri_gp_value = ctx.in.mipsGot->getGp();
   memcpy(buf, &reginfo, sizeof(reginfo));
@@ -317,9 +320,9 @@ GnuPropertySection::GnuPropertySection()
     : SyntheticSection(llvm::ELF::SHF_ALLOC, llvm::ELF::SHT_NOTE,
                        ctx.arg.wordsize, ".note.gnu.property") {}
 
-void GnuPropertySection::writeTo(uint8_t *buf) {
+void GnuPropertySection::writeTo(Ctx &ctx, uint8_t *buf) {
   write32(buf, 4);                          // Name size
-  write32(buf + 4, getSize() - 16);         // Content size
+  write32(buf + 4, getSize(ctx) - 16);      // Content size
   write32(buf + 8, NT_GNU_PROPERTY_TYPE_0); // Type
   memcpy(buf + 12, "GNU", 4);               // Name string
 
@@ -345,7 +348,7 @@ void GnuPropertySection::writeTo(uint8_t *buf) {
   }
 }
 
-size_t GnuPropertySection::getSize() const {
+size_t GnuPropertySection::getSize(Ctx &ctx) const {
   uint32_t contentSize = 0;
   if (ctx.arg.andFeatures != 0)
     contentSize += ctx.arg.is64 ? 16 : 12;
@@ -359,7 +362,7 @@ BuildIdSection::BuildIdSection()
     : SyntheticSection(SHF_ALLOC, SHT_NOTE, 4, ".note.gnu.build-id"),
       hashSize(getHashSize()) {}
 
-void BuildIdSection::writeTo(uint8_t *buf) {
+void BuildIdSection::writeTo(Ctx &ctx, uint8_t *buf) {
   write32(buf, 4);                      // Name size
   write32(buf + 4, hashSize);           // Content size
   write32(buf + 8, NT_GNU_BUILD_ID);    // Type
@@ -505,7 +508,7 @@ static void writeCieFde(uint8_t *buf, ArrayRef<uint8_t> d) {
   write32(buf, d.size() - 4);
 }
 
-void EhFrameSection::finalizeContents() {
+void EhFrameSection::finalizeContents(Ctx &) {
   assert(!this->size); // Not finalized.
 
   switch (ctx.arg.ekind) {
@@ -621,7 +624,7 @@ uint64_t EhFrameSection::getFdePc(uint8_t *buf, size_t fdeOff,
   fatal("unknown FDE size relative encoding");
 }
 
-void EhFrameSection::writeTo(uint8_t *buf) {
+void EhFrameSection::writeTo(Ctx &ctx, uint8_t *buf) {
   // Write CIE and FDE records.
   for (CieRecord *rec : cieRecords) {
     size_t cieOffset = rec->cie->outputOff;
@@ -700,7 +703,7 @@ uint64_t GotSection::getGlobalDynOffset(const Symbol &b) const {
   return b.getTlsGdIdx() * ctx.arg.wordsize;
 }
 
-void GotSection::finalizeContents() {
+void GotSection::finalizeContents(Ctx &) {
   if (ctx.arg.emachine == EM_PPC64 &&
       numEntries <= ctx.target->gotHeaderEntriesNum &&
       !ctx.sym.globalOffsetTable)
@@ -715,7 +718,7 @@ bool GotSection::isNeeded() const {
   return hasGotOffRel || numEntries > ctx.target->gotHeaderEntriesNum;
 }
 
-void GotSection::writeTo(uint8_t *buf) {
+void GotSection::writeTo(Ctx &ctx, uint8_t *buf) {
   // On PPC64 .got may be needed but empty. Skip the write.
   if (size == 0)
     return;
@@ -869,7 +872,7 @@ bool MipsGotSection::tryMergeGots(FileGot &dst, FileGot &src, bool isPrimary) {
   return true;
 }
 
-void MipsGotSection::finalizeContents() { updateAllocSize(); }
+void MipsGotSection::finalizeContents(Ctx &) { updateAllocSize(); }
 
 bool MipsGotSection::updateAllocSize() {
   size = headerEntriesNum * ctx.arg.wordsize;
@@ -1096,7 +1099,7 @@ uint64_t MipsGotSection::getGp(const InputFile *f) const {
   return getVA() + gots[f->mipsGotIndex].startIndex * ctx.arg.wordsize + 0x7ff0;
 }
 
-void MipsGotSection::writeTo(uint8_t *buf) {
+void MipsGotSection::writeTo(Ctx &ctx, uint8_t *buf) {
   // Set the MSB of the second GOT slot. This is not required by any
   // MIPS ABI documentation, though.
   //
@@ -1180,12 +1183,12 @@ void GotPltSection::addEntry(Symbol &sym) {
   entries.push_back(&sym);
 }
 
-size_t GotPltSection::getSize() const {
+size_t GotPltSection::getSize(Ctx &ctx) const {
   return (ctx.target->gotPltHeaderEntriesNum + entries.size()) *
          ctx.target->gotEntrySize;
 }
 
-void GotPltSection::writeTo(uint8_t *buf) {
+void GotPltSection::writeTo(Ctx &ctx, uint8_t *buf) {
   ctx.target->writeGotPltHeader(buf);
   buf += ctx.target->gotPltHeaderEntriesNum * ctx.target->gotEntrySize;
   for (const Symbol *b : entries) {
@@ -1225,11 +1228,11 @@ void IgotPltSection::addEntry(Symbol &sym) {
   entries.push_back(&sym);
 }
 
-size_t IgotPltSection::getSize() const {
+size_t IgotPltSection::getSize(Ctx &ctx) const {
   return entries.size() * ctx.target->gotEntrySize;
 }
 
-void IgotPltSection::writeTo(uint8_t *buf) {
+void IgotPltSection::writeTo(Ctx &ctx, uint8_t *buf) {
   for (const Symbol *b : entries) {
     ctx.target->writeIgotPlt(buf, *b);
     buf += ctx.target->gotEntrySize;
@@ -1263,7 +1266,7 @@ unsigned StringTableSection::addString(StringRef s, bool hashIt) {
   return ret;
 }
 
-void StringTableSection::writeTo(uint8_t *buf) {
+void StringTableSection::writeTo(Ctx &ctx, uint8_t *buf) {
   for (StringRef s : strings) {
     memcpy(buf, s.data(), s.size());
     buf[s.size()] = '\0';
@@ -1300,9 +1303,9 @@ DynamicSection<ELFT>::DynamicSection()
 //
 // DT_RELASZ is the total size of the included sections.
 static uint64_t addRelaSz(const RelocationBaseSection &relaDyn) {
-  size_t size = relaDyn.getSize();
+  size_t size = relaDyn.getSize(ctx);
   if (ctx.in.relaPlt->getParent() == relaDyn.getParent())
-    size += ctx.in.relaPlt->getSize();
+    size += ctx.in.relaPlt->getSize(ctx);
   return size;
 }
 
@@ -1310,7 +1313,7 @@ static uint64_t addRelaSz(const RelocationBaseSection &relaDyn) {
 // output section. When this occurs we cannot just use the OutputSection
 // Size. Moreover the [DT_JMPREL, DT_JMPREL + DT_PLTRELSZ) is permitted to
 // overlap with the [DT_RELA, DT_RELA + DT_RELASZ).
-static uint64_t addPltRelSz() { return ctx.in.relaPlt->getSize(); }
+static uint64_t addPltRelSz() { return ctx.in.relaPlt->getSize(ctx); }
 
 // Add remaining entries to complete .dynamic contents.
 template <class ELFT>
@@ -1479,7 +1482,7 @@ DynamicSection<ELFT>::computeContents() {
         addInSec(DT_AARCH64_MEMTAG_GLOBALS,
                  *ctx.mainPart->memtagGlobalDescriptors);
         addInt(DT_AARCH64_MEMTAG_GLOBALSSZ,
-               ctx.mainPart->memtagGlobalDescriptors->getSize());
+               ctx.mainPart->memtagGlobalDescriptors->getSize(ctx));
       }
     }
   }
@@ -1487,7 +1490,7 @@ DynamicSection<ELFT>::computeContents() {
   addInSec(DT_SYMTAB, *part.dynSymTab);
   addInt(DT_SYMENT, sizeof(Elf_Sym));
   addInSec(DT_STRTAB, *part.dynStrTab);
-  addInt(DT_STRSZ, part.dynStrTab->getSize());
+  addInt(DT_STRSZ, part.dynStrTab->getSize(ctx));
   if (!ctx.arg.zText)
     addInt(DT_TEXTREL, 0);
   if (part.gnuHashTab && part.gnuHashTab->getParent())
@@ -1574,13 +1577,14 @@ DynamicSection<ELFT>::computeContents() {
   return entries;
 }
 
-template <class ELFT> void DynamicSection<ELFT>::finalizeContents() {
+template <class ELFT> void DynamicSection<ELFT>::finalizeContents(Ctx &) {
   if (OutputSection *sec = getPartition().dynStrTab->getParent())
     getParent()->link = sec->sectionIndex;
   this->size = computeContents().size() * this->entsize;
 }
 
-template <class ELFT> void DynamicSection<ELFT>::writeTo(uint8_t *buf) {
+template <class ELFT>
+void DynamicSection<ELFT>::writeTo(Ctx &ctx, uint8_t *buf) {
   auto *p = reinterpret_cast<Elf_Dyn *>(buf);
 
   for (std::pair<int32_t, uint64_t> kv : computeContents()) {
@@ -1675,7 +1679,7 @@ void RelocationBaseSection::partitionRels() {
       relocs.begin();
 }
 
-void RelocationBaseSection::finalizeContents() {
+void RelocationBaseSection::finalizeContents(Ctx &) {
   SymbolTableBaseSection *symTab = getPartition().dynSymTab.get();
 
   // When linking glibc statically, .rel{,a}.plt contains R_*_IRELATIVE
@@ -1732,7 +1736,8 @@ RelocationSection<ELFT>::RelocationSection(StringRef name, bool combreloc,
   this->entsize = ctx.arg.isRela ? sizeof(Elf_Rela) : sizeof(Elf_Rel);
 }
 
-template <class ELFT> void RelocationSection<ELFT>::writeTo(uint8_t *buf) {
+template <class ELFT>
+void RelocationSection<ELFT>::writeTo(Ctx &ctx, uint8_t *buf) {
   computeRels();
   for (const DynamicReloc &rel : relocs) {
     auto *p = reinterpret_cast<Elf_Rela *>(buf);
@@ -2127,7 +2132,7 @@ static bool sortMipsSymbols(const SymbolTableEntry &l,
   return !l.sym->isInGot();
 }
 
-void SymbolTableBaseSection::finalizeContents() {
+void SymbolTableBaseSection::finalizeContents(Ctx &) {
   if (OutputSection *sec = strTabSec.getParent())
     getParent()->link = sec->sectionIndex;
 
@@ -2243,7 +2248,8 @@ static uint32_t getSymSectionIndex(Symbol *sym) {
 }
 
 // Write the internal symbol table contents to the output symbol table.
-template <class ELFT> void SymbolTableSection<ELFT>::writeTo(uint8_t *buf) {
+template <class ELFT>
+void SymbolTableSection<ELFT>::writeTo(Ctx &ctx, uint8_t *buf) {
   // The first entry is a null entry as per the ELF spec.
   buf += sizeof(Elf_Sym);
 
@@ -2325,7 +2331,7 @@ SymtabShndxSection::SymtabShndxSection()
   this->entsize = 4;
 }
 
-void SymtabShndxSection::writeTo(uint8_t *buf) {
+void SymtabShndxSection::writeTo(Ctx &ctx, uint8_t *buf) {
   // We write an array of 32 bit values, where each value has 1:1 association
   // with an entry in ctx.in.symTab if the corresponding entry contains
   // SHN_XINDEX, we need to write actual index, otherwise, we must write
@@ -2351,11 +2357,11 @@ bool SymtabShndxSection::isNeeded() const {
   return size >= SHN_LORESERVE;
 }
 
-void SymtabShndxSection::finalizeContents() {
+void SymtabShndxSection::finalizeContents(Ctx &) {
   getParent()->link = ctx.in.symTab->getParent()->sectionIndex;
 }
 
-size_t SymtabShndxSection::getSize() const {
+size_t SymtabShndxSection::getSize(Ctx &ctx) const {
   return ctx.in.symTab->getNumSymbols() * 4;
 }
 
@@ -2393,7 +2399,7 @@ GnuHashTableSection::GnuHashTableSection()
     : SyntheticSection(SHF_ALLOC, SHT_GNU_HASH, ctx.arg.wordsize, ".gnu.hash") {
 }
 
-void GnuHashTableSection::finalizeContents() {
+void GnuHashTableSection::finalizeContents(Ctx &) {
   if (OutputSection *sec = getPartition().dynSymTab->getParent())
     getParent()->link = sec->sectionIndex;
 
@@ -2412,7 +2418,7 @@ void GnuHashTableSection::finalizeContents() {
   size += symbols.size() * 4;           // Hash values
 }
 
-void GnuHashTableSection::writeTo(uint8_t *buf) {
+void GnuHashTableSection::writeTo(Ctx &ctx, uint8_t *buf) {
   // Write a header.
   write32(buf, nBuckets);
   write32(buf + 4, getPartition().dynSymTab->getNumSymbols() - symbols.size());
@@ -2503,7 +2509,7 @@ HashTableSection::HashTableSection()
   this->entsize = 4;
 }
 
-void HashTableSection::finalizeContents() {
+void HashTableSection::finalizeContents(Ctx &) {
   SymbolTableBaseSection *symTab = getPartition().dynSymTab.get();
 
   if (OutputSection *sec = symTab->getParent())
@@ -2517,7 +2523,7 @@ void HashTableSection::finalizeContents() {
   this->size = numEntries * 4;
 }
 
-void HashTableSection::writeTo(uint8_t *buf) {
+void HashTableSection::writeTo(Ctx &ctx, uint8_t *buf) {
   SymbolTableBaseSection *symTab = getPartition().dynSymTab.get();
   unsigned numSymbols = symTab->getNumSymbols();
 
@@ -2559,7 +2565,7 @@ PltSection::PltSection()
     this->flags |= SHF_WRITE;
 }
 
-void PltSection::writeTo(uint8_t *buf) {
+void PltSection::writeTo(Ctx &ctx, uint8_t *buf) {
   // At beginning of PLT, we have code to call the dynamic
   // linker to resolve dynsyms at runtime. Write such code.
   ctx.target->writePltHeader(buf);
@@ -2577,7 +2583,7 @@ void PltSection::addEntry(Symbol &sym) {
   entries.push_back(&sym);
 }
 
-size_t PltSection::getSize() const {
+size_t PltSection::getSize(Ctx &ctx) const {
   return headerSize + entries.size() * ctx.target->pltEntrySize;
 }
 
@@ -2606,7 +2612,7 @@ IpltSection::IpltSection()
   }
 }
 
-void IpltSection::writeTo(uint8_t *buf) {
+void IpltSection::writeTo(Ctx &ctx, uint8_t *buf) {
   uint32_t off = 0;
   for (const Symbol *sym : entries) {
     ctx.target->writeIplt(buf + off, *sym, getVA() + off);
@@ -2614,7 +2620,7 @@ void IpltSection::writeTo(uint8_t *buf) {
   }
 }
 
-size_t IpltSection::getSize() const {
+size_t IpltSection::getSize(Ctx &ctx) const {
   return entries.size() * ctx.target->ipltEntrySize;
 }
 
@@ -2638,11 +2644,11 @@ PPC32GlinkSection::PPC32GlinkSection() {
   addralign = 4;
 }
 
-void PPC32GlinkSection::writeTo(uint8_t *buf) {
+void PPC32GlinkSection::writeTo(Ctx &ctx, uint8_t *buf) {
   writePPC32GlinkSection(buf, entries.size());
 }
 
-size_t PPC32GlinkSection::getSize() const {
+size_t PPC32GlinkSection::getSize(Ctx &ctx) const {
   return headerSize + entries.size() * ctx.target->pltEntrySize + footerSize;
 }
 
@@ -2707,11 +2713,11 @@ size_t PPC32GlinkSection::getSize() const {
 IBTPltSection::IBTPltSection()
     : SyntheticSection(SHF_ALLOC | SHF_EXECINSTR, SHT_PROGBITS, 16, ".plt") {}
 
-void IBTPltSection::writeTo(uint8_t *buf) {
+void IBTPltSection::writeTo(Ctx &ctx, uint8_t *buf) {
   ctx.target->writeIBTPlt(buf, ctx.in.plt->getNumEntries());
 }
 
-size_t IBTPltSection::getSize() const {
+size_t IBTPltSection::getSize(Ctx &ctx) const {
   // 16 is the header size of .plt.
   return 16 + ctx.in.plt->getNumEntries() * ctx.target->pltEntrySize;
 }
@@ -3217,7 +3223,7 @@ void DebugNamesSection<ELFT>::getNameRelocs(
   }
 }
 
-template <class ELFT> void DebugNamesSection<ELFT>::finalizeContents() {
+template <class ELFT> void DebugNamesSection<ELFT>::finalizeContents(Ctx &) {
   // Get relocations of .debug_names sections.
   auto relocs = std::make_unique<DenseMap<uint32_t, uint32_t>[]>(numChunks);
   parallelFor(0, numChunks, [&](size_t i) {
@@ -3237,7 +3243,8 @@ template <class ELFT> void DebugNamesSection<ELFT>::finalizeContents() {
   });
 }
 
-template <class ELFT> void DebugNamesSection<ELFT>::writeTo(uint8_t *buf) {
+template <class ELFT>
+void DebugNamesSection<ELFT>::writeTo(Ctx &ctx, uint8_t *buf) {
   [[maybe_unused]] const uint8_t *const beginBuf = buf;
   // Write the header.
   endian::writeNext<uint32_t, ELFT::Endianness>(buf, hdr.UnitLength);
@@ -3548,7 +3555,7 @@ std::unique_ptr<GdbIndexSection> GdbIndexSection::create() {
   return ret;
 }
 
-void GdbIndexSection::writeTo(uint8_t *buf) {
+void GdbIndexSection::writeTo(Ctx &ctx, uint8_t *buf) {
   // Write the header.
   auto *hdr = reinterpret_cast<GdbIndexHeader *>(buf);
   uint8_t *start = buf;
@@ -3622,7 +3629,7 @@ bool GdbIndexSection::isNeeded() const { return !chunks.empty(); }
 EhFrameHeader::EhFrameHeader()
     : SyntheticSection(SHF_ALLOC, SHT_PROGBITS, 4, ".eh_frame_hdr") {}
 
-void EhFrameHeader::writeTo(uint8_t *buf) {
+void EhFrameHeader::writeTo(Ctx &ctx, uint8_t *buf) {
   // Unlike most sections, the EhFrameHeader section is written while writing
   // another section, namely EhFrameSection, which calls the write() function
   // below from its writeTo() function. This is necessary because the contents
@@ -3655,7 +3662,7 @@ void EhFrameHeader::write() {
   }
 }
 
-size_t EhFrameHeader::getSize() const {
+size_t EhFrameHeader::getSize(Ctx &ctx) const {
   // .eh_frame_hdr has a 12 bytes header followed by an array of FDEs.
   return 12 + getPartition().ehFrame->numFdes * 8;
 }
@@ -3676,7 +3683,7 @@ StringRef VersionDefinitionSection::getFileDefName() {
   return ctx.arg.outputFile;
 }
 
-void VersionDefinitionSection::finalizeContents() {
+void VersionDefinitionSection::finalizeContents(Ctx &) {
   fileDefNameOff = getPartition().dynStrTab->addString(getFileDefName());
   for (const VersionDefinition &v : namedVersionDefs())
     verDefNameOffs.push_back(getPartition().dynStrTab->addString(v.name));
@@ -3708,7 +3715,7 @@ void VersionDefinitionSection::writeOne(uint8_t *buf, uint32_t index,
   write32(buf + 24, 0);       // vda_next
 }
 
-void VersionDefinitionSection::writeTo(uint8_t *buf) {
+void VersionDefinitionSection::writeTo(Ctx &ctx, uint8_t *buf) {
   writeOne(buf, 1, getFileDefName(), fileDefNameOff);
 
   auto nameOffIt = verDefNameOffs.begin();
@@ -3721,7 +3728,7 @@ void VersionDefinitionSection::writeTo(uint8_t *buf) {
   write32(buf + 16, 0); // vd_next
 }
 
-size_t VersionDefinitionSection::getSize() const {
+size_t VersionDefinitionSection::getSize(Ctx &ctx) const {
   return EntrySize * getVerDefNum();
 }
 
@@ -3732,17 +3739,17 @@ VersionTableSection::VersionTableSection()
   this->entsize = 2;
 }
 
-void VersionTableSection::finalizeContents() {
+void VersionTableSection::finalizeContents(Ctx &) {
   // At the moment of june 2016 GNU docs does not mention that sh_link field
   // should be set, but Sun docs do. Also readelf relies on this field.
   getParent()->link = getPartition().dynSymTab->getParent()->sectionIndex;
 }
 
-size_t VersionTableSection::getSize() const {
+size_t VersionTableSection::getSize(Ctx &ctx) const {
   return (getPartition().dynSymTab->getSymbols().size() + 1) * 2;
 }
 
-void VersionTableSection::writeTo(uint8_t *buf) {
+void VersionTableSection::writeTo(Ctx &ctx, uint8_t *buf) {
   buf += 2;
   for (const SymbolTableEntry &s : getPartition().dynSymTab->getSymbols()) {
     // For an unextracted lazy symbol (undefined weak), it must have been
@@ -3781,7 +3788,7 @@ VersionNeedSection<ELFT>::VersionNeedSection()
     : SyntheticSection(SHF_ALLOC, SHT_GNU_verneed, sizeof(uint32_t),
                        ".gnu.version_r") {}
 
-template <class ELFT> void VersionNeedSection<ELFT>::finalizeContents() {
+template <class ELFT> void VersionNeedSection<ELFT>::finalizeContents(Ctx &) {
   for (SharedFile *f : ctx.sharedFiles) {
     if (f->vernauxs.empty())
       continue;
@@ -3814,7 +3821,8 @@ template <class ELFT> void VersionNeedSection<ELFT>::finalizeContents() {
   getParent()->info = verneeds.size();
 }
 
-template <class ELFT> void VersionNeedSection<ELFT>::writeTo(uint8_t *buf) {
+template <class ELFT>
+void VersionNeedSection<ELFT>::writeTo(Ctx &ctx, uint8_t *buf) {
   // The Elf_Verneeds need to appear first, followed by the Elf_Vernauxs.
   auto *verneed = reinterpret_cast<Elf_Verneed *>(buf);
   auto *vernaux = reinterpret_cast<Elf_Vernaux *>(verneed + verneeds.size());
@@ -3844,7 +3852,7 @@ template <class ELFT> void VersionNeedSection<ELFT>::writeTo(uint8_t *buf) {
   verneed[-1].vn_next = 0;
 }
 
-template <class ELFT> size_t VersionNeedSection<ELFT>::getSize() const {
+template <class ELFT> size_t VersionNeedSection<ELFT>::getSize(Ctx &ctx) const {
   return verneeds.size() * sizeof(Elf_Verneed) +
          SharedFile::vernauxNum * sizeof(Elf_Vernaux);
 }
@@ -3865,11 +3873,11 @@ MergeTailSection::MergeTailSection(StringRef name, uint32_t type,
     : MergeSyntheticSection(name, type, flags, alignment),
       builder(StringTableBuilder::RAW, llvm::Align(alignment)) {}
 
-size_t MergeTailSection::getSize() const { return builder.getSize(); }
+size_t MergeTailSection::getSize(Ctx &) const { return builder.getSize(); }
 
-void MergeTailSection::writeTo(uint8_t *buf) { builder.write(buf); }
+void MergeTailSection::writeTo(Ctx &, uint8_t *buf) { builder.write(buf); }
 
-void MergeTailSection::finalizeContents() {
+void MergeTailSection::finalizeContents(Ctx &) {
   // Add all string pieces to the string table builder to create section
   // contents.
   for (MergeInputSection *sec : sections)
@@ -3889,7 +3897,7 @@ void MergeTailSection::finalizeContents() {
         sec->pieces[i].outputOff = builder.getOffset(sec->getData(i));
 }
 
-void MergeNoTailSection::writeTo(uint8_t *buf) {
+void MergeNoTailSection::writeTo(Ctx &ctx, uint8_t *buf) {
   parallelFor(0, numShards,
               [&](size_t i) { shards[i].write(buf + shardOffsets[i]); });
 }
@@ -3902,7 +3910,7 @@ void MergeNoTailSection::writeTo(uint8_t *buf) {
 // value is different from T's. If that's the case, we can safely put S and
 // T into different string builders without worrying about merge misses.
 // We do it in parallel.
-void MergeNoTailSection::finalizeContents() {
+void MergeNoTailSection::finalizeContents(Ctx &) {
   // Initializes string table builders.
   for (size_t i = 0; i < numShards; ++i)
     shards.emplace_back(StringTableBuilder::RAW, llvm::Align(addralign));
@@ -4084,7 +4092,7 @@ static bool isDuplicateArmExidxSec(InputSection *prev, InputSection *cur) {
 // must be sorted in ascending order of address, Sentinel is set to the
 // InputSection with the highest address and any InputSections that have
 // mergeable .ARM.exidx table entries are removed from it.
-void ARMExidxSyntheticSection::finalizeContents() {
+void ARMExidxSyntheticSection::finalizeContents(Ctx &) {
   // Ensure that any fixed-point iterations after the first see the original set
   // of sections.
   if (!originalExecutableSections.empty())
@@ -4171,7 +4179,7 @@ InputSection *ARMExidxSyntheticSection::getLinkOrderDep() const {
 //     section is to terminate the address range of the previous entry.
 // 3.) A trailing EXIDX_CANTUNWIND sentinel section is required at the end of
 //     the table to terminate the address range of the final entry.
-void ARMExidxSyntheticSection::writeTo(uint8_t *buf) {
+void ARMExidxSyntheticSection::writeTo(Ctx &ctx, uint8_t *buf) {
 
   // A linker generated CANTUNWIND entry is made up of two words:
   // 0x0 with R_ARM_PREL31 relocation to target.
@@ -4220,7 +4228,7 @@ ThunkSection::ThunkSection(OutputSection *os, uint64_t off)
   this->outSecOff = off;
 }
 
-size_t ThunkSection::getSize() const {
+size_t ThunkSection::getSize(Ctx &) const {
   if (roundUpSizeForErrata)
     return alignTo(size, 4096);
   return size;
@@ -4231,7 +4239,7 @@ void ThunkSection::addThunk(Thunk *t) {
   t->addSymbols(*this);
 }
 
-void ThunkSection::writeTo(uint8_t *buf) {
+void ThunkSection::writeTo(Ctx &ctx, uint8_t *buf) {
   for (Thunk *t : thunks)
     t->writeTo(buf + t->offset);
 }
@@ -4271,7 +4279,7 @@ bool PPC32Got2Section::isNeeded() const {
   return false;
 }
 
-void PPC32Got2Section::finalizeContents() {
+void PPC32Got2Section::finalizeContents(Ctx &) {
   // PPC32 may create multiple GOT sections for -fPIC/-fPIE, one per file in
   // .got2 . This function computes outSecOff of each .got2 to be used in
   // PPC32PltCallStub::writeTo(). The purpose of this empty synthetic section is
@@ -4310,11 +4318,11 @@ PPC64LongBranchTargetSection::addEntry(const Symbol *sym, int64_t addend) {
   return res.first->second;
 }
 
-size_t PPC64LongBranchTargetSection::getSize() const {
+size_t PPC64LongBranchTargetSection::getSize(Ctx &ctx) const {
   return entries.size() * 8;
 }
 
-void PPC64LongBranchTargetSection::writeTo(uint8_t *buf) {
+void PPC64LongBranchTargetSection::writeTo(Ctx &ctx, uint8_t *buf) {
   // If linking non-pic we have the final addresses of the targets and they get
   // written to the table directly. For pic the dynamic linker will allocate
   // the section and fill it.
@@ -4407,12 +4415,12 @@ PartitionElfHeaderSection<ELFT>::PartitionElfHeaderSection()
     : SyntheticSection(SHF_ALLOC, SHT_LLVM_PART_EHDR, 1, "") {}
 
 template <typename ELFT>
-size_t PartitionElfHeaderSection<ELFT>::getSize() const {
+size_t PartitionElfHeaderSection<ELFT>::getSize(Ctx &ctx) const {
   return sizeof(typename ELFT::Ehdr);
 }
 
 template <typename ELFT>
-void PartitionElfHeaderSection<ELFT>::writeTo(uint8_t *buf) {
+void PartitionElfHeaderSection<ELFT>::writeTo(Ctx &ctx, uint8_t *buf) {
   writeEhdr<ELFT>(buf, getPartition());
 
   // Loadable partitions are always ET_DYN.
@@ -4425,29 +4433,29 @@ PartitionProgramHeadersSection<ELFT>::PartitionProgramHeadersSection()
     : SyntheticSection(SHF_ALLOC, SHT_LLVM_PART_PHDR, 1, ".phdrs") {}
 
 template <typename ELFT>
-size_t PartitionProgramHeadersSection<ELFT>::getSize() const {
+size_t PartitionProgramHeadersSection<ELFT>::getSize(Ctx &ctx) const {
   return sizeof(typename ELFT::Phdr) * getPartition().phdrs.size();
 }
 
 template <typename ELFT>
-void PartitionProgramHeadersSection<ELFT>::writeTo(uint8_t *buf) {
+void PartitionProgramHeadersSection<ELFT>::writeTo(Ctx &ctx, uint8_t *buf) {
   writePhdrs<ELFT>(buf, getPartition());
 }
 
 PartitionIndexSection::PartitionIndexSection()
     : SyntheticSection(SHF_ALLOC, SHT_PROGBITS, 4, ".rodata") {}
 
-size_t PartitionIndexSection::getSize() const {
+size_t PartitionIndexSection::getSize(Ctx &ctx) const {
   return 12 * (ctx.partitions.size() - 1);
 }
 
-void PartitionIndexSection::finalizeContents() {
+void PartitionIndexSection::finalizeContents(Ctx &) {
   for (size_t i = 1; i != ctx.partitions.size(); ++i)
     ctx.partitions[i].nameStrTab =
         ctx.mainPart->dynStrTab->addString(ctx.partitions[i].name);
 }
 
-void PartitionIndexSection::writeTo(uint8_t *buf) {
+void PartitionIndexSection::writeTo(Ctx &ctx, uint8_t *buf) {
   uint64_t va = getVA();
   for (size_t i = 1; i != ctx.partitions.size(); ++i) {
     write32(buf, ctx.mainPart->dynStrTab->getVA() +
@@ -4517,7 +4525,7 @@ bool elf::canHaveMemtagGlobals() {
 }
 
 constexpr char kMemtagAndroidNoteName[] = "Android";
-void MemtagAndroidNote::writeTo(uint8_t *buf) {
+void MemtagAndroidNote::writeTo(Ctx &ctx, uint8_t *buf) {
   static_assert(
       sizeof(kMemtagAndroidNoteName) == 8,
       "Android 11 & 12 have an ABI that the note name is 8 bytes long. Keep it "
@@ -4540,13 +4548,13 @@ void MemtagAndroidNote::writeTo(uint8_t *buf) {
   write32(buf, value); // note value
 }
 
-size_t MemtagAndroidNote::getSize() const {
+size_t MemtagAndroidNote::getSize(Ctx &ctx) const {
   return sizeof(llvm::ELF::Elf64_Nhdr) +
          /*namesz=*/alignTo(sizeof(kMemtagAndroidNoteName), 4) +
          /*descsz=*/sizeof(uint32_t);
 }
 
-void PackageMetadataNote::writeTo(uint8_t *buf) {
+void PackageMetadataNote::writeTo(Ctx &ctx, uint8_t *buf) {
   write32(buf, 4);
   write32(buf + 4, ctx.arg.packageMetadata.size() + 1);
   write32(buf + 8, FDO_PACKAGING_METADATA);
@@ -4555,7 +4563,7 @@ void PackageMetadataNote::writeTo(uint8_t *buf) {
          ctx.arg.packageMetadata.size());
 }
 
-size_t PackageMetadataNote::getSize() const {
+size_t PackageMetadataNote::getSize(Ctx &ctx) const {
   return sizeof(llvm::ELF::Elf64_Nhdr) + 4 +
          alignTo(ctx.arg.packageMetadata.size() + 1, 4);
 }
@@ -4615,19 +4623,19 @@ createMemtagGlobalDescriptors(const SmallVector<const Symbol *, 0> &symbols,
 }
 
 bool MemtagGlobalDescriptors::updateAllocSize() {
-  size_t oldSize = getSize();
+  size_t oldSize = getSize(ctx);
   std::stable_sort(symbols.begin(), symbols.end(),
                    [](const Symbol *s1, const Symbol *s2) {
                      return s1->getVA() < s2->getVA();
                    });
-  return oldSize != getSize();
+  return oldSize != getSize(ctx);
 }
 
-void MemtagGlobalDescriptors::writeTo(uint8_t *buf) {
+void MemtagGlobalDescriptors::writeTo(Ctx &ctx, uint8_t *buf) {
   createMemtagGlobalDescriptors(symbols, buf);
 }
 
-size_t MemtagGlobalDescriptors::getSize() const {
+size_t MemtagGlobalDescriptors::getSize(Ctx &ctx) const {
   return createMemtagGlobalDescriptors(symbols);
 }
 
@@ -4823,7 +4831,7 @@ template <class ELFT> void elf::createSyntheticSections(Ctx &ctx) {
     ctx.in.partIndex = std::make_unique<PartitionIndexSection>();
     addOptionalRegular("__part_index_begin", ctx.in.partIndex.get(), 0);
     addOptionalRegular("__part_index_end", ctx.in.partIndex.get(),
-                       ctx.in.partIndex->getSize());
+                       ctx.in.partIndex->getSize(ctx));
     add(*ctx.in.partIndex);
   }
 
diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h
index 759a908b202a9..37724d783162c 100644
--- a/lld/ELF/SyntheticSections.h
+++ b/lld/ELF/SyntheticSections.h
@@ -51,10 +51,10 @@ struct CieRecord {
 class EhFrameSection final : public SyntheticSection {
 public:
   EhFrameSection();
-  void writeTo(uint8_t *buf) override;
-  void finalizeContents() override;
+  void writeTo(Ctx &, uint8_t *buf) override;
+  void finalizeContents(Ctx &) override;
   bool isNeeded() const override { return !sections.empty(); }
-  size_t getSize() const override { return size; }
+  size_t getSize(Ctx &ctx) const override { return size; }
 
   static bool classof(const SectionBase *d) {
     return SyntheticSection::classof(d) && d->name == ".eh_frame";
@@ -105,10 +105,10 @@ class EhFrameSection final : public SyntheticSection {
 class GotSection final : public SyntheticSection {
 public:
   GotSection();
-  size_t getSize() const override { return size; }
-  void finalizeContents() override;
+  size_t getSize(Ctx &ctx) const override { return size; }
+  void finalizeContents(Ctx &) override;
   bool isNeeded() const override;
-  void writeTo(uint8_t *buf) override;
+  void writeTo(Ctx &, uint8_t *buf) override;
 
   void addConstant(const Relocation &r);
   void addEntry(const Symbol &sym);
@@ -138,15 +138,15 @@ class GnuStackSection : public SyntheticSection {
 public:
   GnuStackSection()
       : SyntheticSection(0, llvm::ELF::SHT_PROGBITS, 1, ".note.GNU-stack") {}
-  void writeTo(uint8_t *buf) override {}
-  size_t getSize() const override { return 0; }
+  void writeTo(Ctx &, uint8_t *buf) override {}
+  size_t getSize(Ctx &ctx) const override { return 0; }
 };
 
 class GnuPropertySection final : public SyntheticSection {
 public:
   GnuPropertySection();
-  void writeTo(uint8_t *buf) override;
-  size_t getSize() const override;
+  void writeTo(Ctx &, uint8_t *buf) override;
+  size_t getSize(Ctx &) const override;
 };
 
 // .note.gnu.build-id section.
@@ -157,8 +157,8 @@ class BuildIdSection : public SyntheticSection {
 public:
   const size_t hashSize;
   BuildIdSection();
-  void writeTo(uint8_t *buf) override;
-  size_t getSize() const override { return headerSize + hashSize; }
+  void writeTo(Ctx &, uint8_t *buf) override;
+  size_t getSize(Ctx &ctx) const override { return headerSize + hashSize; }
   void writeBuildId(llvm::ArrayRef<uint8_t> buf);
 
 private:
@@ -172,9 +172,9 @@ class BuildIdSection : public SyntheticSection {
 class BssSection final : public SyntheticSection {
 public:
   BssSection(StringRef name, uint64_t size, uint32_t addralign);
-  void writeTo(uint8_t *) override {}
+  void writeTo(Ctx &, uint8_t *) override {}
   bool isNeeded() const override { return size != 0; }
-  size_t getSize() const override { return size; }
+  size_t getSize(Ctx &ctx) const override { return size; }
 
   static bool classof(const SectionBase *s) { return s->bss; }
   uint64_t size;
@@ -183,10 +183,10 @@ class BssSection final : public SyntheticSection {
 class MipsGotSection final : public SyntheticSection {
 public:
   MipsGotSection();
-  void writeTo(uint8_t *buf) override;
-  size_t getSize() const override { return size; }
+  void writeTo(Ctx &, uint8_t *buf) override;
+  size_t getSize(Ctx &ctx) const override { return size; }
   bool updateAllocSize() override;
-  void finalizeContents() override;
+  void finalizeContents(Ctx &) override;
   bool isNeeded() const override;
 
   // Join separate GOTs built for each input file to generate
@@ -361,8 +361,8 @@ class GotPltSection final : public SyntheticSection {
 public:
   GotPltSection();
   void addEntry(Symbol &sym);
-  size_t getSize() const override;
-  void writeTo(uint8_t *buf) override;
+  size_t getSize(Ctx &) const override;
+  void writeTo(Ctx &, uint8_t *buf) override;
   bool isNeeded() const override;
 
   // Flag to force GotPlt to be in output if we have relocations
@@ -381,8 +381,8 @@ class IgotPltSection final : public SyntheticSection {
 public:
   IgotPltSection();
   void addEntry(Symbol &sym);
-  size_t getSize() const override;
-  void writeTo(uint8_t *buf) override;
+  size_t getSize(Ctx &) const override;
+  void writeTo(Ctx &, uint8_t *buf) override;
   bool isNeeded() const override { return !entries.empty(); }
 
 private:
@@ -393,8 +393,8 @@ class StringTableSection final : public SyntheticSection {
 public:
   StringTableSection(StringRef name, bool dynamic);
   unsigned addString(StringRef s, bool hashIt = true);
-  void writeTo(uint8_t *buf) override;
-  size_t getSize() const override { return size; }
+  void writeTo(Ctx &, uint8_t *buf) override;
+  size_t getSize(Ctx &ctx) const override { return size; }
   bool isDynamic() const { return dynamic; }
 
 private:
@@ -484,9 +484,9 @@ template <class ELFT> class DynamicSection final : public SyntheticSection {
 
 public:
   DynamicSection();
-  void finalizeContents() override;
-  void writeTo(uint8_t *buf) override;
-  size_t getSize() const override { return size; }
+  void finalizeContents(Ctx &) override;
+  void writeTo(Ctx &, uint8_t *buf) override;
+  size_t getSize(Ctx &ctx) const override { return size; }
 
 private:
   std::vector<std::pair<int32_t, uint64_t>> computeContents();
@@ -541,11 +541,13 @@ class RelocationBaseSection : public SyntheticSection {
     return !relocs.empty() ||
            llvm::any_of(relocsVec, [](auto &v) { return !v.empty(); });
   }
-  size_t getSize() const override { return relocs.size() * this->entsize; }
+  size_t getSize(Ctx &ctx) const override {
+    return relocs.size() * this->entsize;
+  }
   size_t getRelativeRelocCount() const { return numRelativeRelocs; }
   void mergeRels();
   void partitionRels();
-  void finalizeContents() override;
+  void finalizeContents(Ctx &) override;
   static bool classof(const SectionBase *d) {
     return SyntheticSection::classof(d) &&
            (d->type == llvm::ELF::SHT_RELA || d->type == llvm::ELF::SHT_REL ||
@@ -577,7 +579,7 @@ class RelocationSection final : public RelocationBaseSection {
 
 public:
   RelocationSection(StringRef name, bool combreloc, unsigned concurrency);
-  void writeTo(uint8_t *buf) override;
+  void writeTo(Ctx &, uint8_t *buf) override;
 };
 
 template <class ELFT>
@@ -589,8 +591,8 @@ class AndroidPackedRelocationSection final : public RelocationBaseSection {
   AndroidPackedRelocationSection(StringRef name, unsigned concurrency);
 
   bool updateAllocSize() override;
-  size_t getSize() const override { return relocData.size(); }
-  void writeTo(uint8_t *buf) override {
+  size_t getSize(Ctx &ctx) const override { return relocData.size(); }
+  void writeTo(Ctx &, uint8_t *buf) override {
     memcpy(buf, relocData.data(), relocData.size());
   }
 
@@ -630,9 +632,11 @@ template <class ELFT> class RelrSection final : public RelrBaseSection {
   RelrSection(unsigned concurrency, bool isAArch64Auth = false);
 
   bool updateAllocSize() override;
-  size_t getSize() const override { return relrRelocs.size() * this->entsize; }
-  void writeTo(uint8_t *buf) override {
-    memcpy(buf, relrRelocs.data(), getSize());
+  size_t getSize(Ctx &ctx) const override {
+    return relrRelocs.size() * this->entsize;
+  }
+  void writeTo(Ctx &ctx, uint8_t *buf) override {
+    memcpy(buf, relrRelocs.data(), getSize(ctx));
   }
 
 private:
@@ -647,8 +651,8 @@ struct SymbolTableEntry {
 class SymbolTableBaseSection : public SyntheticSection {
 public:
   SymbolTableBaseSection(StringTableSection &strTabSec);
-  void finalizeContents() override;
-  size_t getSize() const override { return getNumSymbols() * entsize; }
+  void finalizeContents(Ctx &) override;
+  size_t getSize(Ctx &ctx) const override { return getNumSymbols() * entsize; }
   void addSymbol(Symbol *sym);
   unsigned getNumSymbols() const { return symbols.size() + 1; }
   size_t getSymbolIndex(const Symbol &sym);
@@ -673,17 +677,17 @@ class SymbolTableSection final : public SymbolTableBaseSection {
 
 public:
   SymbolTableSection(StringTableSection &strTabSec);
-  void writeTo(uint8_t *buf) override;
+  void writeTo(Ctx &, uint8_t *buf) override;
 };
 
 class SymtabShndxSection final : public SyntheticSection {
 public:
   SymtabShndxSection();
 
-  void writeTo(uint8_t *buf) override;
-  size_t getSize() const override;
+  void writeTo(Ctx &, uint8_t *buf) override;
+  size_t getSize(Ctx &) const override;
   bool isNeeded() const override;
-  void finalizeContents() override;
+  void finalizeContents(Ctx &) override;
 };
 
 // Outputs GNU Hash section. For detailed explanation see:
@@ -691,9 +695,9 @@ class SymtabShndxSection final : public SyntheticSection {
 class GnuHashTableSection final : public SyntheticSection {
 public:
   GnuHashTableSection();
-  void finalizeContents() override;
-  void writeTo(uint8_t *buf) override;
-  size_t getSize() const override { return size; }
+  void finalizeContents(Ctx &) override;
+  void writeTo(Ctx &, uint8_t *buf) override;
+  size_t getSize(Ctx &ctx) const override { return size; }
 
   // Adds symbols to the hash table.
   // Sorts the input to satisfy GNU hash section requirements.
@@ -719,9 +723,9 @@ class GnuHashTableSection final : public SyntheticSection {
 class HashTableSection final : public SyntheticSection {
 public:
   HashTableSection();
-  void finalizeContents() override;
-  void writeTo(uint8_t *buf) override;
-  size_t getSize() const override { return size; }
+  void finalizeContents(Ctx &) override;
+  void writeTo(Ctx &, uint8_t *buf) override;
+  size_t getSize(Ctx &ctx) const override { return size; }
 
 private:
   size_t size = 0;
@@ -741,8 +745,8 @@ class HashTableSection final : public SyntheticSection {
 class PltSection : public SyntheticSection {
 public:
   PltSection();
-  void writeTo(uint8_t *buf) override;
-  size_t getSize() const override;
+  void writeTo(Ctx &, uint8_t *buf) override;
+  size_t getSize(Ctx &) const override;
   bool isNeeded() const override;
   void addSymbols();
   void addEntry(Symbol &sym);
@@ -762,8 +766,8 @@ class IpltSection final : public SyntheticSection {
 
 public:
   IpltSection();
-  void writeTo(uint8_t *buf) override;
-  size_t getSize() const override;
+  void writeTo(Ctx &, uint8_t *buf) override;
+  size_t getSize(Ctx &) const override;
   bool isNeeded() const override { return !entries.empty(); }
   void addSymbols();
   void addEntry(Symbol &sym);
@@ -772,8 +776,8 @@ class IpltSection final : public SyntheticSection {
 class PPC32GlinkSection : public PltSection {
 public:
   PPC32GlinkSection();
-  void writeTo(uint8_t *buf) override;
-  size_t getSize() const override;
+  void writeTo(Ctx &, uint8_t *buf) override;
+  size_t getSize(Ctx &) const override;
 
   SmallVector<const Symbol *, 0> canonical_plts;
   static constexpr size_t footerSize = 64;
@@ -783,9 +787,9 @@ class PPC32GlinkSection : public PltSection {
 class IBTPltSection : public SyntheticSection {
 public:
   IBTPltSection();
-  void writeTo(uint8_t *Buf) override;
+  void writeTo(Ctx &, uint8_t *Buf) override;
   bool isNeeded() const override;
-  size_t getSize() const override;
+  size_t getSize(Ctx &) const override;
 };
 
 // Used to align the end of the PT_GNU_RELRO segment and the associated PT_LOAD
@@ -794,8 +798,8 @@ class IBTPltSection : public SyntheticSection {
 class RelroPaddingSection final : public SyntheticSection {
 public:
   RelroPaddingSection();
-  size_t getSize() const override { return 0; }
-  void writeTo(uint8_t *buf) override {}
+  size_t getSize(Ctx &ctx) const override { return 0; }
+  void writeTo(Ctx &, uint8_t *buf) override {}
 };
 
 // Used by the merged DWARF32 .debug_names (a per-module index). If we
@@ -869,7 +873,7 @@ class DebugNamesBaseSection : public SyntheticSection {
   };
 
   DebugNamesBaseSection();
-  size_t getSize() const override { return size; }
+  size_t getSize(Ctx &ctx) const override { return size; }
   bool isNeeded() const override { return numChunks > 0; }
 
 protected:
@@ -913,8 +917,8 @@ template <class ELFT>
 class DebugNamesSection final : public DebugNamesBaseSection {
 public:
   DebugNamesSection();
-  void finalizeContents() override;
-  void writeTo(uint8_t *buf) override;
+  void finalizeContents(Ctx &) override;
+  void writeTo(Ctx &, uint8_t *buf) override;
 
   template <class RelTy>
   void getNameRelocs(const InputFile &file,
@@ -961,8 +965,8 @@ class GdbIndexSection final : public SyntheticSection {
 
   GdbIndexSection();
   template <typename ELFT> static std::unique_ptr<GdbIndexSection> create();
-  void writeTo(uint8_t *buf) override;
-  size_t getSize() const override { return size; }
+  void writeTo(Ctx &, uint8_t *buf) override;
+  size_t getSize(Ctx &ctx) const override { return size; }
   bool isNeeded() const override;
 
 private:
@@ -1000,8 +1004,8 @@ class EhFrameHeader final : public SyntheticSection {
 public:
   EhFrameHeader();
   void write();
-  void writeTo(uint8_t *buf) override;
-  size_t getSize() const override;
+  void writeTo(Ctx &, uint8_t *buf) override;
+  size_t getSize(Ctx &) const override;
   bool isNeeded() const override;
 };
 
@@ -1016,9 +1020,9 @@ class EhFrameHeader final : public SyntheticSection {
 class VersionDefinitionSection final : public SyntheticSection {
 public:
   VersionDefinitionSection();
-  void finalizeContents() override;
-  size_t getSize() const override;
-  void writeTo(uint8_t *buf) override;
+  void finalizeContents(Ctx &) override;
+  size_t getSize(Ctx &) const override;
+  void writeTo(Ctx &, uint8_t *buf) override;
 
 private:
   enum { EntrySize = 28 };
@@ -1038,9 +1042,9 @@ class VersionDefinitionSection final : public SyntheticSection {
 class VersionTableSection final : public SyntheticSection {
 public:
   VersionTableSection();
-  void finalizeContents() override;
-  size_t getSize() const override;
-  void writeTo(uint8_t *buf) override;
+  void finalizeContents(Ctx &) override;
+  size_t getSize(Ctx &) const override;
+  void writeTo(Ctx &, uint8_t *buf) override;
   bool isNeeded() const override;
 };
 
@@ -1069,9 +1073,9 @@ class VersionNeedSection final : public SyntheticSection {
 
 public:
   VersionNeedSection();
-  void finalizeContents() override;
-  void writeTo(uint8_t *buf) override;
-  size_t getSize() const override;
+  void finalizeContents(Ctx &) override;
+  void writeTo(Ctx &, uint8_t *buf) override;
+  size_t getSize(Ctx &) const override;
   bool isNeeded() const override;
 };
 
@@ -1095,9 +1099,9 @@ class MergeTailSection final : public MergeSyntheticSection {
   MergeTailSection(StringRef name, uint32_t type, uint64_t flags,
                    uint32_t addralign);
 
-  size_t getSize() const override;
-  void writeTo(uint8_t *buf) override;
-  void finalizeContents() override;
+  size_t getSize(Ctx &) const override;
+  void writeTo(Ctx &, uint8_t *buf) override;
+  void finalizeContents(Ctx &) override;
 
 private:
   llvm::StringTableBuilder builder;
@@ -1109,9 +1113,9 @@ class MergeNoTailSection final : public MergeSyntheticSection {
                      uint32_t addralign)
       : MergeSyntheticSection(name, type, flags, addralign) {}
 
-  size_t getSize() const override { return size; }
-  void writeTo(uint8_t *buf) override;
-  void finalizeContents() override;
+  size_t getSize(Ctx &ctx) const override { return size; }
+  void writeTo(Ctx &, uint8_t *buf) override;
+  void finalizeContents(Ctx &) override;
 
 private:
   // We use the most significant bits of a hash as a shard ID.
@@ -1142,8 +1146,8 @@ class MipsAbiFlagsSection final : public SyntheticSection {
   static std::unique_ptr<MipsAbiFlagsSection> create();
 
   MipsAbiFlagsSection(Elf_Mips_ABIFlags flags);
-  size_t getSize() const override { return sizeof(Elf_Mips_ABIFlags); }
-  void writeTo(uint8_t *buf) override;
+  size_t getSize(Ctx &ctx) const override { return sizeof(Elf_Mips_ABIFlags); }
+  void writeTo(Ctx &, uint8_t *buf) override;
 
 private:
   Elf_Mips_ABIFlags flags;
@@ -1158,9 +1162,9 @@ template <class ELFT> class MipsOptionsSection final : public SyntheticSection {
   static std::unique_ptr<MipsOptionsSection<ELFT>> create();
 
   MipsOptionsSection(Elf_Mips_RegInfo reginfo);
-  void writeTo(uint8_t *buf) override;
+  void writeTo(Ctx &, uint8_t *buf) override;
 
-  size_t getSize() const override {
+  size_t getSize(Ctx &ctx) const override {
     return sizeof(Elf_Mips_Options) + sizeof(Elf_Mips_RegInfo);
   }
 
@@ -1176,8 +1180,8 @@ template <class ELFT> class MipsReginfoSection final : public SyntheticSection {
   static std::unique_ptr<MipsReginfoSection> create();
 
   MipsReginfoSection(Elf_Mips_RegInfo reginfo);
-  size_t getSize() const override { return sizeof(Elf_Mips_RegInfo); }
-  void writeTo(uint8_t *buf) override;
+  size_t getSize(Ctx &ctx) const override { return sizeof(Elf_Mips_RegInfo); }
+  void writeTo(Ctx &, uint8_t *buf) override;
 
 private:
   Elf_Mips_RegInfo reginfo;
@@ -1190,8 +1194,8 @@ template <class ELFT> class MipsReginfoSection final : public SyntheticSection {
 class MipsRldMapSection final : public SyntheticSection {
 public:
   MipsRldMapSection();
-  size_t getSize() const override { return ctx.arg.wordsize; }
-  void writeTo(uint8_t *buf) override {}
+  size_t getSize(Ctx &ctx) const override { return ctx.arg.wordsize; }
+  void writeTo(Ctx &, uint8_t *buf) override {}
 };
 
 // Representation of the combined .ARM.Exidx input sections. We process these
@@ -1236,11 +1240,11 @@ class ARMExidxSyntheticSection : public SyntheticSection {
   // section needs to be removed from the main input section list.
   bool addSection(InputSection *isec);
 
-  size_t getSize() const override { return size; }
-  void writeTo(uint8_t *buf) override;
+  size_t getSize(Ctx &ctx) const override { return size; }
+  void writeTo(Ctx &, uint8_t *buf) override;
   bool isNeeded() const override;
   // Sort and remove duplicate entries.
-  void finalizeContents() override;
+  void finalizeContents(Ctx &) override;
   InputSection *getLinkOrderDep() const;
 
   static bool classof(const SectionBase *sec) {
@@ -1284,8 +1288,8 @@ class ThunkSection final : public SyntheticSection {
   // Thunk defines a symbol in this InputSection that can be used as target
   // of a relocation
   void addThunk(Thunk *t);
-  size_t getSize() const override;
-  void writeTo(uint8_t *buf) override;
+  size_t getSize(Ctx &) const override;
+  void writeTo(Ctx &, uint8_t *buf) override;
   InputSection *getTargetInputSection() const;
   bool assignOffsets();
 
@@ -1309,11 +1313,11 @@ class ArmCmseSGSection final : public SyntheticSection {
 public:
   ArmCmseSGSection();
   bool isNeeded() const override { return !entries.empty(); }
-  size_t getSize() const override;
-  void writeTo(uint8_t *buf) override;
+  size_t getSize(Ctx &) const override;
+  void writeTo(Ctx &, uint8_t *buf) override;
   void addSGVeneer(Symbol *sym, Symbol *ext_sym);
   void addMappingSymbol();
-  void finalizeContents() override;
+  void finalizeContents(Ctx &) override;
   void exportEntries(SymbolTableBaseSection *symTab);
   uint64_t impLibMaxAddr = 0;
 
@@ -1328,10 +1332,10 @@ class ArmCmseSGSection final : public SyntheticSection {
 class PPC32Got2Section final : public SyntheticSection {
 public:
   PPC32Got2Section();
-  size_t getSize() const override { return 0; }
+  size_t getSize(Ctx &ctx) const override { return 0; }
   bool isNeeded() const override;
-  void finalizeContents() override;
-  void writeTo(uint8_t *buf) override {}
+  void finalizeContents(Ctx &) override;
+  void writeTo(Ctx &, uint8_t *buf) override {}
 };
 
 // This section is used to store the addresses of functions that are called
@@ -1344,10 +1348,10 @@ class PPC64LongBranchTargetSection final : public SyntheticSection {
   PPC64LongBranchTargetSection();
   uint64_t getEntryVA(const Symbol *sym, int64_t addend);
   std::optional<uint32_t> addEntry(const Symbol *sym, int64_t addend);
-  size_t getSize() const override;
-  void writeTo(uint8_t *buf) override;
+  size_t getSize(Ctx &) const override;
+  void writeTo(Ctx &, uint8_t *buf) override;
   bool isNeeded() const override;
-  void finalizeContents() override { finalized = true; }
+  void finalizeContents(Ctx &) override { finalized = true; }
 
 private:
   SmallVector<std::pair<const Symbol *, int64_t>, 0> entries;
@@ -1359,24 +1363,24 @@ template <typename ELFT>
 class PartitionElfHeaderSection final : public SyntheticSection {
 public:
   PartitionElfHeaderSection();
-  size_t getSize() const override;
-  void writeTo(uint8_t *buf) override;
+  size_t getSize(Ctx &) const override;
+  void writeTo(Ctx &, uint8_t *buf) override;
 };
 
 template <typename ELFT>
 class PartitionProgramHeadersSection final : public SyntheticSection {
 public:
   PartitionProgramHeadersSection();
-  size_t getSize() const override;
-  void writeTo(uint8_t *buf) override;
+  size_t getSize(Ctx &) const override;
+  void writeTo(Ctx &, uint8_t *buf) override;
 };
 
 class PartitionIndexSection final : public SyntheticSection {
 public:
   PartitionIndexSection();
-  size_t getSize() const override;
-  void finalizeContents() override;
-  void writeTo(uint8_t *buf) override;
+  size_t getSize(Ctx &) const override;
+  void finalizeContents(Ctx &) override;
+  void writeTo(Ctx &, uint8_t *buf) override;
 };
 
 // See the following link for the Android-specific loader code that operates on
@@ -1387,8 +1391,8 @@ class MemtagAndroidNote final : public SyntheticSection {
   MemtagAndroidNote()
       : SyntheticSection(llvm::ELF::SHF_ALLOC, llvm::ELF::SHT_NOTE,
                          /*alignment=*/4, ".note.android.memtag") {}
-  void writeTo(uint8_t *buf) override;
-  size_t getSize() const override;
+  void writeTo(Ctx &, uint8_t *buf) override;
+  size_t getSize(Ctx &) const override;
 };
 
 class PackageMetadataNote final : public SyntheticSection {
@@ -1396,8 +1400,8 @@ class PackageMetadataNote final : public SyntheticSection {
   PackageMetadataNote()
       : SyntheticSection(llvm::ELF::SHF_ALLOC, llvm::ELF::SHT_NOTE,
                          /*alignment=*/4, ".note.package") {}
-  void writeTo(uint8_t *buf) override;
-  size_t getSize() const override;
+  void writeTo(Ctx &, uint8_t *buf) override;
+  size_t getSize(Ctx &) const override;
 };
 
 class MemtagGlobalDescriptors final : public SyntheticSection {
@@ -1406,12 +1410,12 @@ class MemtagGlobalDescriptors final : public SyntheticSection {
       : SyntheticSection(llvm::ELF::SHF_ALLOC,
                          llvm::ELF::SHT_AARCH64_MEMTAG_GLOBALS_DYNAMIC,
                          /*alignment=*/4, ".memtag.globals.dynamic") {}
-  void writeTo(uint8_t *buf) override;
+  void writeTo(Ctx &, uint8_t *buf) override;
   // The size of the section is non-computable until all addresses are
   // synthetized, because the section's contents contain a sorted
   // varint-compressed list of pointers to global variables. We only know the
   // final size after `finalizeAddressDependentContent()`.
-  size_t getSize() const override;
+  size_t getSize(Ctx &) const override;
   bool updateAllocSize() override;
 
   void addSymbol(const Symbol &sym) {
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index 2ad7de7324a84..c413ebd349e6d 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -315,7 +315,7 @@ template <class ELFT> void Writer<ELFT>::run() {
   // If --compressed-debug-sections is specified, compress .debug_* sections.
   // Do it right now because it changes the size of output sections.
   for (OutputSection *sec : ctx.outputSections)
-    sec->maybeCompress<ELFT>();
+    sec->maybeCompress<ELFT>(ctx);
 
   if (ctx.script->hasSectionsCommand)
     ctx.script->allocateHeaders(ctx.mainPart->phdrs);
@@ -840,7 +840,7 @@ template <class ELFT> void Writer<ELFT>::setReservedSymbolSections() {
   if (ctx.sym.relaIpltStart && ctx.mainPart->relaDyn->isNeeded()) {
     ctx.sym.relaIpltStart->section = ctx.mainPart->relaDyn.get();
     ctx.sym.relaIpltEnd->section = ctx.mainPart->relaDyn.get();
-    ctx.sym.relaIpltEnd->value = ctx.mainPart->relaDyn->getSize();
+    ctx.sym.relaIpltEnd->value = ctx.mainPart->relaDyn->getSize(ctx);
   }
 
   PhdrEntry *last = nullptr;
@@ -1427,7 +1427,7 @@ template <class ELFT> void Writer<ELFT>::resolveShfLinkOrder() {
 static void finalizeSynthetic(SyntheticSection *sec) {
   if (sec && sec->isNeeded() && sec->getParent()) {
     llvm::TimeTraceScope timeScope("Finalize synthetic sections", sec->name);
-    sec->finalizeContents();
+    sec->finalizeContents(ctx);
   }
 }
 
@@ -2077,7 +2077,7 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() {
   // at the end because some tags like RELSZ depend on result
   // of finalizing other sections.
   for (OutputSection *sec : ctx.outputSections)
-    sec->finalize();
+    sec->finalize(ctx);
 
   ctx.script->checkFinalScriptConditions();
 
@@ -2574,8 +2574,8 @@ template <class ELFT> void Writer<ELFT>::setPhdrs(Partition &part) {
     // output section. We always want to describe just the
     // SyntheticSection.
     if (part.armExidx && p->p_type == PT_ARM_EXIDX) {
-      p->p_filesz = part.armExidx->getSize();
-      p->p_memsz = part.armExidx->getSize();
+      p->p_filesz = part.armExidx->getSize(ctx);
+      p->p_memsz = p->p_filesz;
       p->p_offset = first->offset + part.armExidx->outSecOff;
       p->p_vaddr = first->addr + part.armExidx->outSecOff;
       p->p_align = part.armExidx->addralign;
@@ -2806,7 +2806,7 @@ template <class ELFT> void Writer<ELFT>::writeSectionsBinary() {
   parallel::TaskGroup tg;
   for (OutputSection *sec : ctx.outputSections)
     if (sec->flags & SHF_ALLOC)
-      sec->writeTo<ELFT>(ctx.bufferStart + sec->offset, tg);
+      sec->writeTo<ELFT>(ctx, ctx.bufferStart + sec->offset, tg);
 }
 
 static void fillTrap(std::array<uint8_t, 4> trapInstr, uint8_t *i,
@@ -2857,20 +2857,20 @@ template <class ELFT> void Writer<ELFT>::writeSections() {
     parallel::TaskGroup tg;
     for (OutputSection *sec : ctx.outputSections)
       if (isStaticRelSecType(sec->type))
-        sec->writeTo<ELFT>(ctx.bufferStart + sec->offset, tg);
+        sec->writeTo<ELFT>(ctx, ctx.bufferStart + sec->offset, tg);
   }
   {
     parallel::TaskGroup tg;
     for (OutputSection *sec : ctx.outputSections)
       if (!isStaticRelSecType(sec->type))
-        sec->writeTo<ELFT>(ctx.bufferStart + sec->offset, tg);
+        sec->writeTo<ELFT>(ctx, ctx.bufferStart + sec->offset, tg);
   }
 
   // Finally, check that all dynamic relocation addends were written correctly.
   if (ctx.arg.checkDynamicRelocs && ctx.arg.writeAddends) {
     for (OutputSection *sec : ctx.outputSections)
       if (isStaticRelSecType(sec->type))
-        sec->checkDynRelAddends(ctx.bufferStart);
+        sec->checkDynRelAddends(ctx);
   }
 }
 
diff --git a/lld/test/COFF/undefined_lazy.test b/lld/test/COFF/undefined_lazy.test
new file mode 100644
index 0000000000000..ed5cd358b5cd9
--- /dev/null
+++ b/lld/test/COFF/undefined_lazy.test
@@ -0,0 +1,26 @@
+# REQUIRES: x86
+
+# RUN: split-file %s %t.dir
+# RUN: llvm-mc --filetype=obj -triple=x86_64-windows-msvc %t.dir/foo.s -o %t.foo.obj
+# RUN: llvm-mc --filetype=obj -triple=x86_64-windows-msvc %t.dir/bar.s -o %t.bar.obj
+# RUN: llvm-mc --filetype=obj -triple=x86_64-windows-msvc %t.dir/qux.s -o %t.qux.obj
+# RUN: llvm-lib %t.foo.obj -out:%t.foo.lib
+# RUN: llvm-lib %t.bar.obj -out:%t.bar.lib
+# RUN: lld-link %t.foo.lib %t.bar.lib %t.qux.obj -out:%t.dll -dll
+#
+#--- foo.s
+.text
+.globl foo
+foo:
+  call bar
+#--- bar.s
+.text
+.globl bar
+bar:
+  ret
+#--- qux.s
+.text
+.global _DllMainCRTStartup
+_DllMainCRTStartup:
+  call *__imp_foo(%rip)
+  ret
diff --git a/lldb/docs/man/lldb-server.rst b/lldb/docs/man/lldb-server.rst
index a67c00b305f6d..da6b2c7181842 100644
--- a/lldb/docs/man/lldb-server.rst
+++ b/lldb/docs/man/lldb-server.rst
@@ -147,21 +147,8 @@ GDB-SERVER CONNECTIONS
 
 .. option:: --gdbserver-port <port>
 
- Define a port to be used for gdb-server connections. Can be specified multiple
- times to allow multiple ports. Has no effect if --min-gdbserver-port
- and --max-gdbserver-port are specified.
-
-.. option:: --min-gdbserver-port <port>
-.. option:: --max-gdbserver-port <port>
-
- Specify the range of ports that can be used for gdb-server connections. Both
- options need to be specified simultaneously. Overrides --gdbserver-port.
-
-.. option:: --port-offset <offset>
-
- Add the specified offset to port numbers returned by server. This is useful
- if the server is running behind a firewall, and a range of ports is redirected
- to it with an offset.
+ Define a port to be used for gdb-server connections. This port is used for
+ multiple connections.
 
 EXAMPLES
 --------
diff --git a/lldb/docs/resources/lldbdap.md b/lldb/docs/resources/lldbdap.md
index 2d345e26b7e57..955713dd1b594 100644
--- a/lldb/docs/resources/lldbdap.md
+++ b/lldb/docs/resources/lldbdap.md
@@ -1,16 +1,95 @@
-# LLDB-DAP
-
-The `lldb-dap` tool (formerly `lldb-vscode`) is a command line tool that
-implements the [Debug Adapter
-Protocol](https://microsoft.github.io/debug-adapter-protocol/). It can be
-installed as an extension for Visual Studio Code and other IDEs supporting DAP.
-The protocol is easy to run remotely and also can allow other tools and IDEs to
-get a full featured debugger with a well defined protocol.
-
-## Local Installation for Visual Studio Code
+# Contributing to LLDB-DAP
+
+This guide describes how to extend and contribute to lldb-dap.
+For documentation on how to use lldb-dap, see [lldb-dap's README](https://github.com/llvm/llvm-project/blob/main/lldb/tools/lldb-dap/README.md).
+
+lldb-dap and LLDB are developed under the umbrella of the
+[LLVM project](https://llvm.org/). As such, the
+"[Getting Started with the LLVM System](https://llvm.org/docs/GettingStarted.html)",
+"[Contributing to LLVM](https://llvm.org/docs/Contributing.html)" and
+"[LLVM coding standard](https://llvm.org/docs/CodingStandards.html)"
+guides might also be relevant, if you are a first-time contributor to the LLVM
+project.
+
+lldb-dap's source code is [part of the LLVM
+repository](https://github.com/llvm/llvm-project/tree/main/lldb/tools/lldb-dap)
+on Github. We use Github's [issue
+tracker](https://github.com/llvm/llvm-project/tree/main/lldb/tools/lldb-dap)
+and patches can be submitted via [pull
+requests](https://github.com/llvm/llvm-project/pulls).
+
+## Building `lldb-dap` from soruce
+
+To build lldb-dap from source, first need to [setup a LLDB build](https://lldb.llvm.org/resources/build.html).
+After doing so, run `ninja lldb-dap`. To use your freshly built `lldb-dap`
+binary, install the VS Code extension and point it to lldb-dap by setting the
+`lldb-dap.executable-path` setting.
+
+## Responsibilities of LLDB, lldb-dap and the Visual Studio Code Extension
+
+Under the hood, the UI-based debugging experience is fueled by three separate
+components:
+
+* LLDB provides general, IDE-indepedent debugging features, such as:
+  loading binaries / core dumps, interpreting debug info, setting breakpoints,
+  pretty-printing variables, etc. The `lldb` binary exposes this functionality
+  via a command line interface.
+* `lldb-dap` exposes LLDB's functionality via the
+  "[Debug Adapter Protocol](https://microsoft.github.io/debug-adapter-protocol/)",
+  i.e. a protocol through which various IDEs (VS Code, Emacs, vim, neovim, ...)
+  can interact with a wide range of debuggers (`lldb-dap` and many others).
+* The VS Code extension exposes the lldb-dap binary within VS Code. It acts
+  as a thin wrapper around the lldb-dap binary, and adds VS-Code-specific UI
+  integration on top of lldb-dap, such as autocompletion for `launch.json`
+  configuration files.
+
+Since lldb-dap builds on top of LLDB, all of LLDB's extensibility mechanisms
+such as [Variable Pretty-Printing](https://lldb.llvm.org/use/variable.html),
+[Frame recognizers](https://lldb.llvm.org/use/python-reference.html#writing-lldb-frame-recognizers-in-python)
+and [Python Scripting](https://lldb.llvm.org/use/python.html) are available
+also in lldb-dap.
+
+When adding new functionality, you generally want to add it on the lowest
+applicable level. I.e., quite frequently you actually want to add functionality
+to LLDB's core in order to improve your debugging experience in VS Code.
+
+### The Debug Adapter Protocol
+
+The most relevant resources for the Debug Adapter Protocol are:
+* [The overview](https://microsoft.github.io/debug-adapter-protocol/overview)
+  which provides a high-level introduction,
+* the [human-readable specification](https://microsoft.github.io/debug-adapter-protocol/specification), and
+* the [JSON-schema specification](https://github.com/microsoft/debug-adapter-protocol/blob/main/debugAdapterProtocol.json).
+
+lldb-dap adds some additional non-standard extensions to the protocol. To take
+advantage of those extensions, IDE-specific support code is needed, usually
+inside the VS Code extension. When adding a new extension, please first look
+through the [issue tracker of the Debug Adapter
+Protocol](https://github.com/microsoft/debug-adapter-protocol/issues) to check
+if there already is a proposal serving your use case. If so, try to take
+inspiration from it. If not, consider opening an upstream issue.
+
+To avoid naming collisions with potential future extensions of the Debug
+Adapter protocol, all non-standard extensions should use the prefix
+`$__lldb_extension` in their JSON property names.
+
+### Debugging the Debug Adapter Protocol
+
+To debug the Debug Adapter Protocol, point the `LLDBDAP_LOG` environment
+variable to a file on your disk. lldb-dap will log all communication received
+from / sent to the IDE to the provided path. In the VS Code extension, you
+can also set the log path through the `lldb-dap.log-path` VS Code setting.
+
+## Building the VS Code extension from source
 
 Installing the plug-in is very straightforward and involves just a few steps.
 
+In most cases, installing the VS Code extension from the [VS Code
+Marketplace](https://marketplace.visualstudio.com/items?itemName=llvm-vs-code-extensions.lldb-dap)
+and pointing it to a locally built lldb-dap binary is sufficient. Building
+the VS Code extension from source is only necessary if the TypeScript code is
+changed.
+
 ### Pre-requisites
 
 - Install a modern version of node (e.g. `v20.0.0`).
@@ -33,10 +112,6 @@ And then you are ready!
 
 ### Updating the extension
 
-*Note: It's not necessary to update the extension if there has been changes
-to  `lldb-dap`. The extension needs to be updated only if the TypesScript code
-has changed.*
-
 Updating the extension is pretty much the same process as installing it from
 scratch. However, VS Code expects the version number of the upgraded extension
 to be greater than the previous one, otherwise the installation step might have
diff --git a/lldb/docs/resources/qemu-testing.rst b/lldb/docs/resources/qemu-testing.rst
index 51a30b11717a8..e102f84a1d31f 100644
--- a/lldb/docs/resources/qemu-testing.rst
+++ b/lldb/docs/resources/qemu-testing.rst
@@ -149,7 +149,6 @@ to the host (refer to QEMU's manuals for the specific options).
 * At least one to connect to the intial ``lldb-server``.
 * One more if you want to use ``lldb-server`` in ``platform mode``, and have it
   start a ``gdbserver`` instance for you.
-* A bunch more if you want to run tests against the ``lldb-server`` platform.
 
 If you are doing either of the latter 2 you should also restrict what ports
 ``lldb-server tries`` to use, otherwise it will randomly pick one that is almost
@@ -157,22 +156,14 @@ certainly not forwarded. An example of this is shown below.
 
 ::
 
-  $ lldb-server plaform --server --listen 0.0.0.0:54321 \
-    --min-gdbserver-port 49140 --max-gdbserver-port 49150
+  $ lldb-server plaform --server --listen 0.0.0.0:54321 --gdbserver-port 49140
 
 The result of this is that:
 
 * ``lldb-server`` platform mode listens externally on port ``54321``.
 
-* When it is asked to start a new gdbserver mode instance, it will use a port
-  in the range ``49140`` to ``49150``.
+* When it is asked to start a new gdbserver mode instance, it will use the port
+  ``49140``.
 
-Your VM configuration should have ports ``54321``, and ``49140`` to ``49150``
-forwarded for this to work.
-
-.. note::
-  These options are used to create a "port map" within ``lldb-server``.
-  Unfortunately this map is not cleaned up on Windows on connection close,
-  and across a few uses you may run out of valid ports. To work around this,
-  restart the platform every so often, especially after running a set of tests.
-  This is tracked here: https://github.com/llvm/llvm-project/issues/90923
+Your VM configuration should have ports ``54321`` and ``49140`` forwarded for
+this to work.
diff --git a/lldb/include/lldb/Interpreter/CommandReturnObject.h b/lldb/include/lldb/Interpreter/CommandReturnObject.h
index 8c4dcb54d708f..8f6c9f123b769 100644
--- a/lldb/include/lldb/Interpreter/CommandReturnObject.h
+++ b/lldb/include/lldb/Interpreter/CommandReturnObject.h
@@ -131,7 +131,7 @@ class CommandReturnObject {
     AppendError(llvm::formatv(format, std::forward<Args>(args)...).str());
   }
 
-  void SetError(const Status &error, const char *fallback_error_cstr = nullptr);
+  void SetError(Status error);
 
   void SetError(llvm::Error error);
 
diff --git a/lldb/include/lldb/Symbol/UnwindPlan.h b/lldb/include/lldb/Symbol/UnwindPlan.h
index e1567c7357d0b..48c9bef76857c 100644
--- a/lldb/include/lldb/Symbol/UnwindPlan.h
+++ b/lldb/include/lldb/Symbol/UnwindPlan.h
@@ -215,6 +215,7 @@ class UnwindPlan {
         isRegisterDereferenced, // FA = [reg]
         isDWARFExpression,      // FA = eval(dwarf_expr)
         isRaSearch,             // FA = SP + offset + ???
+        isConstant,             // FA = constant
       };
 
       FAValue() : m_value() {}
@@ -259,6 +260,15 @@ class UnwindPlan {
         m_value.expr.length = len;
       }
 
+      bool IsConstant() const { return m_type == isConstant; }
+
+      void SetIsConstant(uint64_t constant) {
+        m_type = isConstant;
+        m_value.constant = constant;
+      }
+
+      uint64_t GetConstant() const { return m_value.constant; }
+
       uint32_t GetRegisterNumber() const {
         if (m_type == isRegisterDereferenced || m_type == isRegisterPlusOffset)
           return m_value.reg.reg_num;
@@ -329,6 +339,8 @@ class UnwindPlan {
         } expr;
         // For m_type == isRaSearch
         int32_t ra_search_offset;
+        // For m_type = isConstant
+        uint64_t constant;
       } m_value;
     }; // class FAValue
 
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
index 449af1b67d802..1d5e6e0d75c7c 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
@@ -1006,7 +1006,7 @@ def request_compileUnits(self, moduleId):
         return response
 
     def request_completions(self, text, frameId=None):
-        args_dict = {"text": text, "column": len(text)}
+        args_dict = {"text": text, "column": len(text) + 1}
         if frameId:
             args_dict["frameId"] = frameId
         command_dict = {
diff --git a/lldb/source/API/SBCommandReturnObject.cpp b/lldb/source/API/SBCommandReturnObject.cpp
index 7d2c102b3d8c1..d0cdebe8c6491 100644
--- a/lldb/source/API/SBCommandReturnObject.cpp
+++ b/lldb/source/API/SBCommandReturnObject.cpp
@@ -326,10 +326,10 @@ void SBCommandReturnObject::SetError(lldb::SBError &error,
                                      const char *fallback_error_cstr) {
   LLDB_INSTRUMENT_VA(this, error, fallback_error_cstr);
 
-  if (error.IsValid())
-    ref().SetError(error.ref(), fallback_error_cstr);
+  if (error.IsValid() && !error.Fail())
+    ref().SetError(error.ref().Clone());
   else if (fallback_error_cstr)
-    ref().SetError(Status(), fallback_error_cstr);
+    ref().SetError(Status::FromErrorString(fallback_error_cstr));
 }
 
 void SBCommandReturnObject::SetError(const char *error_cstr) {
diff --git a/lldb/source/Commands/CommandObjectBreakpointCommand.cpp b/lldb/source/Commands/CommandObjectBreakpointCommand.cpp
index b668cd0f7c22f..ac2db5973effa 100644
--- a/lldb/source/Commands/CommandObjectBreakpointCommand.cpp
+++ b/lldb/source/Commands/CommandObjectBreakpointCommand.cpp
@@ -389,7 +389,7 @@ are no syntax errors may indicate that a function was declared but never called.
               m_bp_options_vec, result);
         }
         if (!error.Success())
-          result.SetError(error);
+          result.SetError(std::move(error));
       } else {
         // Special handling for one-liner specified inline.
         if (m_options.m_use_one_liner)
diff --git a/lldb/source/Commands/CommandObjectDWIMPrint.cpp b/lldb/source/Commands/CommandObjectDWIMPrint.cpp
index b7cd955e00203..2e1d6e6e5af99 100644
--- a/lldb/source/Commands/CommandObjectDWIMPrint.cpp
+++ b/lldb/source/Commands/CommandObjectDWIMPrint.cpp
@@ -202,7 +202,7 @@ void CommandObjectDWIMPrint::DoExecute(StringRef command,
     // If the expression failed, return an error.
     if (expr_result != eExpressionCompleted) {
       if (valobj_sp)
-        result.SetError(valobj_sp->GetError());
+        result.SetError(valobj_sp->GetError().Clone());
       else
         result.AppendErrorWithFormatv(
             "unknown error evaluating expression `{0}`", expr);
diff --git a/lldb/source/Commands/CommandObjectExpression.cpp b/lldb/source/Commands/CommandObjectExpression.cpp
index 9722c85a79b78..a72b409d21ed8 100644
--- a/lldb/source/Commands/CommandObjectExpression.cpp
+++ b/lldb/source/Commands/CommandObjectExpression.cpp
@@ -647,8 +647,8 @@ void CommandObjectExpression::DoExecute(llvm::StringRef command,
           initialize = true;
           repl_sp = target.GetREPL(repl_error, m_command_options.language,
                                     nullptr, true);
-          if (!repl_error.Success()) {
-            result.SetError(repl_error);
+          if (repl_error.Fail()) {
+            result.SetError(std::move(repl_error));
             return;
           }
         }
@@ -668,7 +668,7 @@ void CommandObjectExpression::DoExecute(llvm::StringRef command,
           repl_error = Status::FromErrorStringWithFormat(
               "Couldn't create a REPL for %s",
               Language::GetNameForLanguageType(m_command_options.language));
-          result.SetError(repl_error);
+          result.SetError(std::move(repl_error));
           return;
         }
       }
diff --git a/lldb/source/Commands/CommandObjectMemoryTag.cpp b/lldb/source/Commands/CommandObjectMemoryTag.cpp
index bc76319018da9..23fad06518366 100644
--- a/lldb/source/Commands/CommandObjectMemoryTag.cpp
+++ b/lldb/source/Commands/CommandObjectMemoryTag.cpp
@@ -290,7 +290,7 @@ class CommandObjectMemoryTagWrite : public CommandObjectParsed {
                                              tagged_range->GetByteSize(), tags);
 
     if (status.Fail()) {
-      result.SetError(status);
+      result.SetError(std::move(status));
       return;
     }
 
diff --git a/lldb/source/Commands/CommandObjectTarget.cpp b/lldb/source/Commands/CommandObjectTarget.cpp
index 10e761fa8de1a..e950fb346c253 100644
--- a/lldb/source/Commands/CommandObjectTarget.cpp
+++ b/lldb/source/Commands/CommandObjectTarget.cpp
@@ -2775,7 +2775,7 @@ class CommandObjectTargetModulesAdd : public CommandObjectParsed {
           result.AppendErrorWithFormat(
               "Unable to locate the executable or symbol file with UUID %s",
               strm.GetData());
-          result.SetError(error);
+          result.SetError(std::move(error));
           return;
         }
       } else {
@@ -4409,7 +4409,7 @@ class CommandObjectTargetSymbolsAdd : public CommandObjectParsed {
         return AddModuleSymbols(m_exe_ctx.GetTargetPtr(), module_spec, flush,
                                 result);
     } else {
-      result.SetError(error);
+      result.SetError(std::move(error));
     }
     return false;
   }
diff --git a/lldb/source/Commands/CommandObjectThread.cpp b/lldb/source/Commands/CommandObjectThread.cpp
index edbec0e305db7..bc1f5c39e702c 100644
--- a/lldb/source/Commands/CommandObjectThread.cpp
+++ b/lldb/source/Commands/CommandObjectThread.cpp
@@ -622,7 +622,7 @@ class CommandObjectThreadStepWithTypeAndScope : public CommandObjectParsed {
         result.SetStatus(eReturnStatusSuccessContinuingNoResult);
       }
     } else {
-      result.SetError(new_plan_status);
+      result.SetError(std::move(new_plan_status));
     }
   }
 
@@ -1046,7 +1046,7 @@ class CommandObjectThreadUntil : public CommandObjectParsed {
           new_plan_sp->SetIsControllingPlan(true);
           new_plan_sp->SetOkayToDiscard(false);
         } else {
-          result.SetError(new_plan_status);
+          result.SetError(std::move(new_plan_status));
           return;
         }
       } else {
@@ -1734,7 +1734,7 @@ class CommandObjectThreadJump : public CommandObjectParsed {
       Status err = thread->JumpToLine(file, line, m_options.m_force, &warnings);
 
       if (err.Fail()) {
-        result.SetError(err);
+        result.SetError(std::move(err));
         return;
       }
 
diff --git a/lldb/source/Expression/IRExecutionUnit.cpp b/lldb/source/Expression/IRExecutionUnit.cpp
index 15ca2ddbbae04..baf70cbccc8c5 100644
--- a/lldb/source/Expression/IRExecutionUnit.cpp
+++ b/lldb/source/Expression/IRExecutionUnit.cpp
@@ -277,6 +277,12 @@ void IRExecutionUnit::GetRunnableInfo(Status &error, lldb::addr_t &func_addr,
       .setMCJITMemoryManager(std::make_unique<MemoryManager>(*this))
       .setOptLevel(llvm::CodeGenOptLevel::Less);
 
+  // Resulted jitted code can be placed too far from the code in the binary
+  // and thus can contain more than +-2GB jumps, that are not available
+  // in RISC-V without large code model.
+  if (triple.isRISCV64())
+    builder.setCodeModel(llvm::CodeModel::Large);
+
   llvm::StringRef mArch;
   llvm::StringRef mCPU;
   llvm::SmallVector<std::string, 0> mAttrs;
diff --git a/lldb/source/Interpreter/CommandReturnObject.cpp b/lldb/source/Interpreter/CommandReturnObject.cpp
index 0bc58124f3941..d5da73c00a520 100644
--- a/lldb/source/Interpreter/CommandReturnObject.cpp
+++ b/lldb/source/Interpreter/CommandReturnObject.cpp
@@ -107,10 +107,8 @@ void CommandReturnObject::AppendError(llvm::StringRef in_string) {
   error(GetErrorStream()) << msg << '\n';
 }
 
-void CommandReturnObject::SetError(const Status &error,
-                                   const char *fallback_error_cstr) {
-  if (error.Fail())
-    AppendError(error.AsCString(fallback_error_cstr));
+void CommandReturnObject::SetError(Status error) {
+  SetError(error.takeError());
 }
 
 void CommandReturnObject::SetError(llvm::Error error) {
diff --git a/lldb/source/Plugins/ABI/RISCV/ABISysV_riscv.cpp b/lldb/source/Plugins/ABI/RISCV/ABISysV_riscv.cpp
index de304183f6717..38d452b6ab560 100644
--- a/lldb/source/Plugins/ABI/RISCV/ABISysV_riscv.cpp
+++ b/lldb/source/Plugins/ABI/RISCV/ABISysV_riscv.cpp
@@ -10,7 +10,9 @@
 
 #include <array>
 #include <limits>
+#include <sstream>
 
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/IR/DerivedTypes.h"
 
 #include "Utility/RISCV_DWARF_Registers.h"
@@ -20,6 +22,7 @@
 #include "lldb/Target/RegisterContext.h"
 #include "lldb/Target/StackFrame.h"
 #include "lldb/Target/Thread.h"
+#include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/RegisterValue.h"
 
 #define DEFINE_REG_NAME(reg_num) ConstString(#reg_num).GetCString()
@@ -120,6 +123,10 @@ static const std::array<RegisterInfo, 33> g_register_infos = {
 } // namespace dwarf
 } // namespace
 
+// Number of argument registers (the base integer calling convention
+// provides 8 argument registers, a0-a7)
+static constexpr size_t g_regs_for_args_count = 8U;
+
 const RegisterInfo *ABISysV_riscv::GetRegisterInfoArray(uint32_t &count) {
   count = dwarf::g_register_infos.size();
   return dwarf::g_register_infos.data();
@@ -164,11 +171,81 @@ TotalArgsSizeInWords(bool is_rv64,
   return total_size;
 }
 
+static bool UpdateRegister(RegisterContext *reg_ctx,
+                           const lldb::RegisterKind reg_kind,
+                           const uint32_t reg_num, const addr_t value) {
+  Log *log = GetLog(LLDBLog::Expressions);
+
+  const RegisterInfo *reg_info = reg_ctx->GetRegisterInfo(reg_kind, reg_num);
+
+  LLDB_LOG(log, "Writing {0}: 0x{1:x}", reg_info->name,
+           static_cast<uint64_t>(value));
+  if (!reg_ctx->WriteRegisterFromUnsigned(reg_info, value)) {
+    LLDB_LOG(log, "Writing {0}: failed", reg_info->name);
+    return false;
+  }
+  return true;
+}
+
+static void LogInitInfo(Log &log, const Thread &thread, addr_t sp,
+                        addr_t func_addr, addr_t return_addr,
+                        const llvm::ArrayRef<addr_t> args) {
+  std::stringstream ss;
+  ss << "ABISysV_riscv::PrepareTrivialCall"
+     << " (tid = 0x" << std::hex << thread.GetID() << ", sp = 0x" << sp
+     << ", func_addr = 0x" << func_addr << ", return_addr = 0x" << return_addr;
+
+  for (auto [idx, arg] : enumerate(args))
+    ss << ", arg" << std::dec << idx << " = 0x" << std::hex << arg;
+  ss << ")";
+  log.PutString(ss.str());
+}
+
 bool ABISysV_riscv::PrepareTrivialCall(Thread &thread, addr_t sp,
                                        addr_t func_addr, addr_t return_addr,
                                        llvm::ArrayRef<addr_t> args) const {
-  // TODO: Implement
-  return false;
+  Log *log = GetLog(LLDBLog::Expressions);
+  if (log)
+    LogInitInfo(*log, thread, sp, func_addr, return_addr, args);
+
+  const auto reg_ctx_sp = thread.GetRegisterContext();
+  if (!reg_ctx_sp) {
+    LLDB_LOG(log, "Failed to get RegisterContext");
+    return false;
+  }
+
+  if (args.size() > g_regs_for_args_count) {
+    LLDB_LOG(log, "Function has {0} arguments, but only {1} are allowed!",
+             args.size(), g_regs_for_args_count);
+    return false;
+  }
+
+  // Write arguments to registers
+  for (auto [idx, arg] : enumerate(args)) {
+    const RegisterInfo *reg_info = reg_ctx_sp->GetRegisterInfo(
+        eRegisterKindGeneric, LLDB_REGNUM_GENERIC_ARG1 + idx);
+    LLDB_LOG(log, "About to write arg{0} (0x{1:x}) into {2}", idx, arg,
+             reg_info->name);
+
+    if (!reg_ctx_sp->WriteRegisterFromUnsigned(reg_info, arg)) {
+      LLDB_LOG(log, "Failed to write arg{0} (0x{1:x}) into {2}", idx, arg,
+               reg_info->name);
+      return false;
+    }
+  }
+
+  if (!UpdateRegister(reg_ctx_sp.get(), eRegisterKindGeneric,
+                      LLDB_REGNUM_GENERIC_PC, func_addr))
+    return false;
+  if (!UpdateRegister(reg_ctx_sp.get(), eRegisterKindGeneric,
+                      LLDB_REGNUM_GENERIC_SP, sp))
+    return false;
+  if (!UpdateRegister(reg_ctx_sp.get(), eRegisterKindGeneric,
+                      LLDB_REGNUM_GENERIC_RA, return_addr))
+    return false;
+
+  LLDB_LOG(log, "ABISysV_riscv::{0}() success", __FUNCTION__);
+  return true;
 }
 
 bool ABISysV_riscv::PrepareTrivialCall(
@@ -222,14 +299,14 @@ bool ABISysV_riscv::PrepareTrivialCall(
   assert(prototype.getFunctionNumParams() == args.size());
 
   const size_t num_args = args.size();
-  const size_t regs_for_args_count = 8U;
   const size_t num_args_in_regs =
-      num_args > regs_for_args_count ?  regs_for_args_count : num_args;
+      num_args > g_regs_for_args_count ? g_regs_for_args_count : num_args;
 
   // Number of arguments passed on stack.
   size_t args_size = TotalArgsSizeInWords(m_is_rv64, args);
-  auto on_stack =
-      args_size <= regs_for_args_count ? 0 : args_size - regs_for_args_count;
+  auto on_stack = args_size <= g_regs_for_args_count
+                      ? 0
+                      : args_size - g_regs_for_args_count;
   auto offset = on_stack * word_size;
 
   uint8_t reg_value[8];
@@ -260,7 +337,7 @@ bool ABISysV_riscv::PrepareTrivialCall(
       ++reg_index;
     }
 
-    if (reg_index < regs_for_args_count || size == 0)
+    if (reg_index < g_regs_for_args_count || size == 0)
       continue;
 
     // Remaining arguments are passed on the stack.
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp
index 9b056ea73a77f..39f0e6269cd0f 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp
@@ -44,6 +44,7 @@
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/TargetSelect.h"
+#include "llvm/TargetParser/Triple.h"
 
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
@@ -442,24 +443,50 @@ static void SetupDefaultClangDiagnostics(CompilerInstance &compiler) {
 /// \return
 ///     A string representing target ABI for the current architecture.
 static std::string GetClangTargetABI(const ArchSpec &target_arch) {
-  std::string abi;
-
   if (target_arch.IsMIPS()) {
     switch (target_arch.GetFlags() & ArchSpec::eMIPSABI_mask) {
     case ArchSpec::eMIPSABI_N64:
-      abi = "n64";
-      break;
+      return "n64";
     case ArchSpec::eMIPSABI_N32:
-      abi = "n32";
-      break;
+      return "n32";
     case ArchSpec::eMIPSABI_O32:
-      abi = "o32";
-      break;
+      return "o32";
     default:
-      break;
+      return {};
     }
   }
-  return abi;
+
+  if (target_arch.GetTriple().isRISCV64()) {
+    switch (target_arch.GetFlags() & ArchSpec::eRISCV_float_abi_mask) {
+    case ArchSpec::eRISCV_float_abi_soft:
+      return "lp64";
+    case ArchSpec::eRISCV_float_abi_single:
+      return "lp64f";
+    case ArchSpec::eRISCV_float_abi_double:
+      return "lp64d";
+    case ArchSpec::eRISCV_float_abi_quad:
+      return "lp64q";
+    default:
+      return {};
+    }
+  }
+
+  if (target_arch.GetTriple().isRISCV32()) {
+    switch (target_arch.GetFlags() & ArchSpec::eRISCV_float_abi_mask) {
+    case ArchSpec::eRISCV_float_abi_soft:
+      return "ilp32";
+    case ArchSpec::eRISCV_float_abi_single:
+      return "ilp32f";
+    case ArchSpec::eRISCV_float_abi_double:
+      return "ilp32d";
+    case ArchSpec::eRISCV_float_abi_soft | ArchSpec::eRISCV_rve:
+      return "ilp32e";
+    default:
+      return {};
+    }
+  }
+
+  return {};
 }
 
 static void SetupTargetOpts(CompilerInstance &compiler,
@@ -506,6 +533,18 @@ static void SetupTargetOpts(CompilerInstance &compiler,
   // Set the target ABI
   if (std::string abi = GetClangTargetABI(target_arch); !abi.empty())
     compiler.getTargetOpts().ABI = std::move(abi);
+
+  if ((target_machine == llvm::Triple::riscv64 &&
+       compiler.getTargetOpts().ABI == "lp64f") ||
+      (target_machine == llvm::Triple::riscv32 &&
+       compiler.getTargetOpts().ABI == "ilp32f"))
+    compiler.getTargetOpts().FeaturesAsWritten.emplace_back("+f");
+
+  if ((target_machine == llvm::Triple::riscv64 &&
+       compiler.getTargetOpts().ABI == "lp64d") ||
+      (target_machine == llvm::Triple::riscv32 &&
+       compiler.getTargetOpts().ABI == "ilp32d"))
+    compiler.getTargetOpts().FeaturesAsWritten.emplace_back("+d");
 }
 
 static void SetupLangOpts(CompilerInstance &compiler,
@@ -757,7 +796,7 @@ ClangExpressionParser::ClangExpressionParser(
   m_compiler->getCodeGenOpts().EmitDeclMetadata = true;
   m_compiler->getCodeGenOpts().InstrumentFunctions = false;
   m_compiler->getCodeGenOpts().setFramePointer(
-                                    CodeGenOptions::FramePointerKind::All);
+      CodeGenOptions::FramePointerKind::All);
   if (generate_debug_info)
     m_compiler->getCodeGenOpts().setDebugInfo(codegenoptions::FullDebugInfo);
   else
@@ -771,7 +810,7 @@ ClangExpressionParser::ClangExpressionParser(
   // FIXME: We shouldn't need to do this, the target should be immutable once
   // created. This complexity should be lifted elsewhere.
   m_compiler->getTarget().adjust(m_compiler->getDiagnostics(),
-		                 m_compiler->getLangOpts());
+                                 m_compiler->getLangOpts());
 
   // 5. Set up the diagnostic buffer for reporting errors
   auto diag_mgr = new ClangDiagnosticManagerAdapter(
@@ -1191,8 +1230,7 @@ ClangExpressionParser::ParseInternal(DiagnosticManager &diagnostic_manager,
           if (auto fileEntry = m_compiler->getFileManager().getOptionalFileRef(
                   result_path)) {
             source_mgr.setMainFileID(source_mgr.createFileID(
-                *fileEntry,
-                SourceLocation(), SrcMgr::C_User));
+                *fileEntry, SourceLocation(), SrcMgr::C_User));
             created_main_file = true;
           }
         }
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp
index c60a83d351ba0..89fdfa74bc025 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp
@@ -44,79 +44,11 @@ using namespace lldb;
 using namespace lldb_private::process_gdb_remote;
 using namespace lldb_private;
 
-GDBRemoteCommunicationServerPlatform::PortMap::PortMap(uint16_t min_port,
-                                                       uint16_t max_port) {
-  assert(min_port);
-  for (; min_port < max_port; ++min_port)
-    m_port_map[min_port] = LLDB_INVALID_PROCESS_ID;
-}
-
-void GDBRemoteCommunicationServerPlatform::PortMap::AllowPort(uint16_t port) {
-  assert(port);
-  // Do not modify existing mappings
-  m_port_map.insert({port, LLDB_INVALID_PROCESS_ID});
-}
-
-llvm::Expected<uint16_t>
-GDBRemoteCommunicationServerPlatform::PortMap::GetNextAvailablePort() {
-  if (m_port_map.empty())
-    return 0; // Bind to port zero and get a port, we didn't have any
-              // limitations
-
-  for (auto &pair : m_port_map) {
-    if (pair.second == LLDB_INVALID_PROCESS_ID) {
-      pair.second = ~(lldb::pid_t)LLDB_INVALID_PROCESS_ID;
-      return pair.first;
-    }
-  }
-  return llvm::createStringError(llvm::inconvertibleErrorCode(),
-                                 "No free port found in port map");
-}
-
-bool GDBRemoteCommunicationServerPlatform::PortMap::AssociatePortWithProcess(
-    uint16_t port, lldb::pid_t pid) {
-  auto pos = m_port_map.find(port);
-  if (pos != m_port_map.end()) {
-    pos->second = pid;
-    return true;
-  }
-  return false;
-}
-
-bool GDBRemoteCommunicationServerPlatform::PortMap::FreePort(uint16_t port) {
-  std::map<uint16_t, lldb::pid_t>::iterator pos = m_port_map.find(port);
-  if (pos != m_port_map.end()) {
-    pos->second = LLDB_INVALID_PROCESS_ID;
-    return true;
-  }
-  return false;
-}
-
-bool GDBRemoteCommunicationServerPlatform::PortMap::FreePortForProcess(
-    lldb::pid_t pid) {
-  if (!m_port_map.empty()) {
-    for (auto &pair : m_port_map) {
-      if (pair.second == pid) {
-        pair.second = LLDB_INVALID_PROCESS_ID;
-        return true;
-      }
-    }
-  }
-  return false;
-}
-
-bool GDBRemoteCommunicationServerPlatform::PortMap::empty() const {
-  return m_port_map.empty();
-}
-
 // GDBRemoteCommunicationServerPlatform constructor
 GDBRemoteCommunicationServerPlatform::GDBRemoteCommunicationServerPlatform(
-    const Socket::SocketProtocol socket_protocol, const char *socket_scheme)
-    : GDBRemoteCommunicationServerCommon(),
-      m_socket_protocol(socket_protocol), m_socket_scheme(socket_scheme),
-      m_spawned_pids_mutex(), m_port_map(), m_port_offset(0) {
-  m_pending_gdb_server.pid = LLDB_INVALID_PROCESS_ID;
-  m_pending_gdb_server.port = 0;
+    const Socket::SocketProtocol socket_protocol, uint16_t gdbserver_port)
+    : GDBRemoteCommunicationServerCommon(), m_socket_protocol(socket_protocol),
+      m_gdbserver_port(gdbserver_port) {
 
   RegisterMemberFunctionHandler(
       StringExtractorGDBRemote::eServerPacketType_qC,
@@ -160,67 +92,55 @@ GDBRemoteCommunicationServerPlatform::~GDBRemoteCommunicationServerPlatform() =
     default;
 
 Status GDBRemoteCommunicationServerPlatform::LaunchGDBServer(
-    const lldb_private::Args &args, std::string hostname, lldb::pid_t &pid,
-    std::optional<uint16_t> &port, std::string &socket_name) {
-  if (!port) {
-    llvm::Expected<uint16_t> available_port = m_port_map.GetNextAvailablePort();
-    if (available_port)
-      port = *available_port;
-    else
-      return Status::FromError(available_port.takeError());
-  }
-
-  // Spawn a new thread to accept the port that gets bound after binding to
-  // port 0 (zero).
+    const lldb_private::Args &args, lldb::pid_t &pid, std::string &socket_name,
+    shared_fd_t fd) {
+  std::ostringstream url;
+  if (fd == SharedSocket::kInvalidFD) {
+    if (m_socket_protocol == Socket::ProtocolTcp) {
+      // Just check that GDBServer exists. GDBServer must be launched after
+      // accepting the connection.
+      if (!GetDebugserverPath(nullptr))
+        return Status::FromErrorString("unable to locate debugserver");
+      return Status();
+    }
 
-  // ignore the hostname send from the remote end, just use the ip address that
-  // we're currently communicating with as the hostname
+    // debugserver does not accept the URL scheme prefix.
+#if !defined(__APPLE__)
+    url << Socket::FindSchemeByProtocol(m_socket_protocol) << "://";
+#endif
+    socket_name = GetDomainSocketPath("gdbserver").GetPath();
+    url << socket_name;
+  } else {
+    if (m_socket_protocol != Socket::ProtocolTcp)
+      return Status::FromErrorString("protocol must be tcp");
+  }
 
   // Spawn a debugserver and try to get the port it listens to.
   ProcessLaunchInfo debugserver_launch_info;
-  if (hostname.empty())
-    hostname = "127.0.0.1";
-
   Log *log = GetLog(LLDBLog::Platform);
-  LLDB_LOGF(log, "Launching debugserver with: %s:%u...", hostname.c_str(),
-            *port);
+  LLDB_LOG(log, "Launching debugserver url='{0}', fd={1}...", url.str(), fd);
 
   // Do not run in a new session so that it can not linger after the platform
   // closes.
   debugserver_launch_info.SetLaunchInSeparateProcessGroup(false);
   debugserver_launch_info.SetMonitorProcessCallback(
-      std::bind(&GDBRemoteCommunicationServerPlatform::DebugserverProcessReaped,
-                this, std::placeholders::_1));
-
-  std::ostringstream url;
-// debugserver does not accept the URL scheme prefix.
-#if !defined(__APPLE__)
-  url << m_socket_scheme << "://";
-#endif
-  uint16_t *port_ptr = &*port;
-  if (m_socket_protocol == Socket::ProtocolTcp) {
-    std::string platform_uri = GetConnection()->GetURI();
-    std::optional<URI> parsed_uri = URI::Parse(platform_uri);
-    url << '[' << parsed_uri->hostname.str() << "]:" << *port;
-  } else {
-    socket_name = GetDomainSocketPath("gdbserver").GetPath();
-    url << socket_name;
-    port_ptr = nullptr;
-  }
+      [](lldb::pid_t, int, int) {});
 
-  Status error = StartDebugserverProcess(url.str().c_str(), nullptr,
-                                         debugserver_launch_info, port_ptr,
-                                         &args, SharedSocket::kInvalidFD);
+  Status error = StartDebugserverProcess(
+      url.str().c_str(), nullptr, debugserver_launch_info, nullptr, &args, fd);
 
-  pid = debugserver_launch_info.GetProcessID();
-  if (pid != LLDB_INVALID_PROCESS_ID) {
-    std::lock_guard<std::recursive_mutex> guard(m_spawned_pids_mutex);
-    m_spawned_pids.insert(pid);
-    if (*port > 0)
-      m_port_map.AssociatePortWithProcess(*port, pid);
+  if (error.Success()) {
+    pid = debugserver_launch_info.GetProcessID();
+    AddSpawnedProcess(pid);
+    LLDB_LOGF(log,
+              "GDBRemoteCommunicationServerPlatform::%s() "
+              "debugserver launched successfully as pid %" PRIu64,
+              __FUNCTION__, pid);
   } else {
-    if (*port > 0)
-      m_port_map.FreePort(*port);
+    LLDB_LOGF(log,
+              "GDBRemoteCommunicationServerPlatform::%s() "
+              "debugserver launch failed: %s",
+              __FUNCTION__, error.AsCString());
   }
   return error;
 }
@@ -251,27 +171,18 @@ GDBRemoteCommunicationServerPlatform::Handle_qLaunchGDBServer(
     }
   }
 
+  // Ignore client's hostname and the port.
+
   lldb::pid_t debugserver_pid = LLDB_INVALID_PROCESS_ID;
   std::string socket_name;
-  Status error =
-      LaunchGDBServer(Args(), hostname, debugserver_pid, port, socket_name);
-  if (error.Fail()) {
-    LLDB_LOGF(log,
-              "GDBRemoteCommunicationServerPlatform::%s() debugserver "
-              "launch failed: %s",
-              __FUNCTION__, error.AsCString());
-    return SendErrorResponse(9);
-  }
-
-  LLDB_LOGF(log,
-            "GDBRemoteCommunicationServerPlatform::%s() debugserver "
-            "launched successfully as pid %" PRIu64,
-            __FUNCTION__, debugserver_pid);
+  Status error = LaunchGDBServer(Args(), debugserver_pid, socket_name,
+                                 SharedSocket::kInvalidFD);
+  if (error.Fail())
+    return SendErrorResponse(9); // EBADF
 
   StreamGDBRemote response;
-  assert(port);
-  response.Printf("pid:%" PRIu64 ";port:%u;", debugserver_pid,
-                  *port + m_port_offset);
+  uint16_t gdbserver_port = socket_name.empty() ? m_gdbserver_port : 0;
+  response.Printf("pid:%" PRIu64 ";port:%u;", debugserver_pid, gdbserver_port);
   if (!socket_name.empty()) {
     response.PutCString("socket_name:");
     response.PutStringAsRawHex8(socket_name);
@@ -291,13 +202,15 @@ GDBRemoteCommunicationServerPlatform::Handle_qQueryGDBServer(
     StringExtractorGDBRemote &packet) {
   namespace json = llvm::json;
 
-  if (m_pending_gdb_server.pid == LLDB_INVALID_PROCESS_ID)
+  if (!m_pending_gdb_server_socket_name)
     return SendErrorResponse(4);
 
-  json::Object server{{"port", m_pending_gdb_server.port}};
+  json::Object server{{"port", m_pending_gdb_server_socket_name->empty()
+                                   ? m_gdbserver_port
+                                   : 0}};
 
-  if (!m_pending_gdb_server.socket_name.empty())
-    server.try_emplace("socket_name", m_pending_gdb_server.socket_name);
+  if (!m_pending_gdb_server_socket_name->empty())
+    server.try_emplace("socket_name", *m_pending_gdb_server_socket_name);
 
   json::Array server_list;
   server_list.push_back(std::move(server));
@@ -318,13 +231,10 @@ GDBRemoteCommunicationServerPlatform::Handle_qKillSpawnedProcess(
 
   lldb::pid_t pid = packet.GetU64(LLDB_INVALID_PROCESS_ID);
 
-  // verify that we know anything about this pid. Scope for locker
-  {
-    std::lock_guard<std::recursive_mutex> guard(m_spawned_pids_mutex);
-    if (m_spawned_pids.find(pid) == m_spawned_pids.end()) {
-      // not a pid we know about
-      return SendErrorResponse(10);
-    }
+  // verify that we know anything about this pid.
+  if (!SpawnedProcessIsRunning(pid)) {
+    // not a pid we know about
+    return SendErrorResponse(10);
   }
 
   // go ahead and attempt to kill the spawned process
@@ -334,12 +244,23 @@ GDBRemoteCommunicationServerPlatform::Handle_qKillSpawnedProcess(
     return SendErrorResponse(11);
 }
 
+void GDBRemoteCommunicationServerPlatform::AddSpawnedProcess(lldb::pid_t pid) {
+  assert(pid != LLDB_INVALID_PROCESS_ID);
+  std::lock_guard<std::recursive_mutex> guard(m_spawned_pids_mutex);
+  m_spawned_pids.insert(pid);
+}
+
+bool GDBRemoteCommunicationServerPlatform::SpawnedProcessIsRunning(
+    lldb::pid_t pid) {
+  std::lock_guard<std::recursive_mutex> guard(m_spawned_pids_mutex);
+  return (m_spawned_pids.find(pid) != m_spawned_pids.end());
+}
+
 bool GDBRemoteCommunicationServerPlatform::KillSpawnedProcess(lldb::pid_t pid) {
   // make sure we know about this process
-  {
-    std::lock_guard<std::recursive_mutex> guard(m_spawned_pids_mutex);
-    if (m_spawned_pids.find(pid) == m_spawned_pids.end())
-      return false;
+  if (!SpawnedProcessIsRunning(pid)) {
+    // it seems the process has been finished recently
+    return true;
   }
 
   // first try a SIGTERM (standard kill)
@@ -347,46 +268,30 @@ bool GDBRemoteCommunicationServerPlatform::KillSpawnedProcess(lldb::pid_t pid) {
 
   // check if that worked
   for (size_t i = 0; i < 10; ++i) {
-    {
-      std::lock_guard<std::recursive_mutex> guard(m_spawned_pids_mutex);
-      if (m_spawned_pids.find(pid) == m_spawned_pids.end()) {
-        // it is now killed
-        return true;
-      }
+    if (!SpawnedProcessIsRunning(pid)) {
+      // it is now killed
+      return true;
     }
     std::this_thread::sleep_for(std::chrono::milliseconds(10));
   }
 
-  {
-    std::lock_guard<std::recursive_mutex> guard(m_spawned_pids_mutex);
-    if (m_spawned_pids.find(pid) == m_spawned_pids.end())
-      return true;
-  }
+  if (!SpawnedProcessIsRunning(pid))
+    return true;
 
   // the launched process still lives.  Now try killing it again, this time
   // with an unblockable signal.
   Host::Kill(pid, SIGKILL);
 
   for (size_t i = 0; i < 10; ++i) {
-    {
-      std::lock_guard<std::recursive_mutex> guard(m_spawned_pids_mutex);
-      if (m_spawned_pids.find(pid) == m_spawned_pids.end()) {
-        // it is now killed
-        return true;
-      }
+    if (!SpawnedProcessIsRunning(pid)) {
+      // it is now killed
+      return true;
     }
     std::this_thread::sleep_for(std::chrono::milliseconds(10));
   }
 
   // check one more time after the final sleep
-  {
-    std::lock_guard<std::recursive_mutex> guard(m_spawned_pids_mutex);
-    if (m_spawned_pids.find(pid) == m_spawned_pids.end())
-      return true;
-  }
-
-  // no luck - the process still lives
-  return false;
+  return !SpawnedProcessIsRunning(pid);
 }
 
 GDBRemoteCommunication::PacketResult
@@ -522,7 +427,6 @@ GDBRemoteCommunicationServerPlatform::Handle_jSignalsInfo(
 void GDBRemoteCommunicationServerPlatform::DebugserverProcessReaped(
     lldb::pid_t pid) {
   std::lock_guard<std::recursive_mutex> guard(m_spawned_pids_mutex);
-  m_port_map.FreePortForProcess(pid);
   m_spawned_pids.erase(pid);
 }
 
@@ -552,19 +456,11 @@ Status GDBRemoteCommunicationServerPlatform::LaunchProcess() {
   // add to list of spawned processes.  On an lldb-gdbserver, we would expect
   // there to be only one.
   const auto pid = m_process_launch_info.GetProcessID();
-  if (pid != LLDB_INVALID_PROCESS_ID) {
-    // add to spawned pids
-    std::lock_guard<std::recursive_mutex> guard(m_spawned_pids_mutex);
-    m_spawned_pids.insert(pid);
-  }
+  AddSpawnedProcess(pid);
 
   return error;
 }
 
-void GDBRemoteCommunicationServerPlatform::SetPortMap(PortMap &&port_map) {
-  m_port_map = std::move(port_map);
-}
-
 const FileSpec &GDBRemoteCommunicationServerPlatform::GetDomainSocketDir() {
   static FileSpec g_domainsocket_dir;
   static llvm::once_flag g_once_flag;
@@ -595,13 +491,7 @@ GDBRemoteCommunicationServerPlatform::GetDomainSocketPath(const char *prefix) {
   return FileSpec(socket_path.c_str());
 }
 
-void GDBRemoteCommunicationServerPlatform::SetPortOffset(uint16_t port_offset) {
-  m_port_offset = port_offset;
-}
-
 void GDBRemoteCommunicationServerPlatform::SetPendingGdbServer(
-    lldb::pid_t pid, uint16_t port, const std::string &socket_name) {
-  m_pending_gdb_server.pid = pid;
-  m_pending_gdb_server.port = port;
-  m_pending_gdb_server.socket_name = socket_name;
+    const std::string &socket_name) {
+  m_pending_gdb_server_socket_name = socket_name;
 }
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.h b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.h
index 1853025466cf0..beee61ea8048e 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.h
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.h
@@ -25,90 +25,27 @@ namespace process_gdb_remote {
 class GDBRemoteCommunicationServerPlatform
     : public GDBRemoteCommunicationServerCommon {
 public:
-  class PortMap {
-  public:
-    // This class is used to restrict the range of ports that
-    // platform created debugserver/gdbserver processes will
-    // communicate on.
-
-    // Construct an empty map, where empty means any port is allowed.
-    PortMap() = default;
-
-    // Make a port map with a range of free ports
-    // from min_port to max_port-1.
-    PortMap(uint16_t min_port, uint16_t max_port);
-
-    // Add a port to the map. If it is already in the map do not modify
-    // its mapping. (used ports remain used, new ports start as free)
-    void AllowPort(uint16_t port);
-
-    // If we are using a port map where we can only use certain ports,
-    // get the next available port.
-    //
-    // If we are using a port map and we are out of ports, return an error.
-    //
-    // If we aren't using a port map, return 0 to indicate we should bind to
-    // port 0 and then figure out which port we used.
-    llvm::Expected<uint16_t> GetNextAvailablePort();
-
-    // Tie a port to a process ID. Returns false if the port is not in the port
-    // map. If the port is already in use it will be moved to the given pid.
-    // FIXME: This is and GetNextAvailablePort make create a race condition if
-    // the portmap is shared between processes.
-    bool AssociatePortWithProcess(uint16_t port, lldb::pid_t pid);
-
-    // Free the given port. Returns false if the port is not in the map.
-    bool FreePort(uint16_t port);
-
-    // Free the port associated with the given pid. Returns false if there is
-    // no port associated with the pid.
-    bool FreePortForProcess(lldb::pid_t pid);
-
-    // Returns true if there are no ports in the map, regardless of the state
-    // of those ports. Meaning a map with 1 used port is not empty.
-    bool empty() const;
-
-  private:
-    std::map<uint16_t, lldb::pid_t> m_port_map;
-  };
-
   GDBRemoteCommunicationServerPlatform(
-      const Socket::SocketProtocol socket_protocol, const char *socket_scheme);
+      const Socket::SocketProtocol socket_protocol, uint16_t gdbserver_port);
 
   ~GDBRemoteCommunicationServerPlatform() override;
 
   Status LaunchProcess() override;
 
-  // Set both ports to zero to let the platform automatically bind to
-  // a port chosen by the OS.
-  void SetPortMap(PortMap &&port_map);
-
-  void SetPortOffset(uint16_t port_offset);
-
   void SetInferiorArguments(const lldb_private::Args &args);
 
-  // Set port if you want to use a specific port number.
-  // Otherwise port will be set to the port that was chosen for you.
-  Status LaunchGDBServer(const lldb_private::Args &args, std::string hostname,
-                         lldb::pid_t &pid, std::optional<uint16_t> &port,
-                         std::string &socket_name);
+  Status LaunchGDBServer(const lldb_private::Args &args, lldb::pid_t &pid,
+                         std::string &socket_name, shared_fd_t fd);
 
-  void SetPendingGdbServer(lldb::pid_t pid, uint16_t port,
-                           const std::string &socket_name);
+  void SetPendingGdbServer(const std::string &socket_name);
 
 protected:
   const Socket::SocketProtocol m_socket_protocol;
-  const std::string m_socket_scheme;
   std::recursive_mutex m_spawned_pids_mutex;
   std::set<lldb::pid_t> m_spawned_pids;
 
-  PortMap m_port_map;
-  uint16_t m_port_offset;
-  struct {
-    lldb::pid_t pid;
-    uint16_t port;
-    std::string socket_name;
-  } m_pending_gdb_server;
+  uint16_t m_gdbserver_port;
+  std::optional<std::string> m_pending_gdb_server_socket_name;
 
   PacketResult Handle_qLaunchGDBServer(StringExtractorGDBRemote &packet);
 
@@ -130,6 +67,8 @@ class GDBRemoteCommunicationServerPlatform
 
 private:
   bool KillSpawnedProcess(lldb::pid_t pid);
+  bool SpawnedProcessIsRunning(lldb::pid_t pid);
+  void AddSpawnedProcess(lldb::pid_t pid);
 
   void DebugserverProcessReaped(lldb::pid_t pid);
 
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/CMakeLists.txt b/lldb/source/Plugins/SymbolFile/DWARF/CMakeLists.txt
index 0e4fd5b995d1b..83c4f1a4cf892 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/CMakeLists.txt
+++ b/lldb/source/Plugins/SymbolFile/DWARF/CMakeLists.txt
@@ -18,7 +18,6 @@ add_lldb_library(lldbPluginSymbolFileDWARF PLUGIN
   DWARFContext.cpp
   DWARFDataExtractor.cpp
   DWARFDebugAranges.cpp
-  DWARFDebugArangeSet.cpp
   DWARFDebugInfo.cpp
   DWARFDebugInfoEntry.cpp
   DWARFDebugMacro.cpp
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index 70540fe7fada6..a30d898a93cc4 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -2128,14 +2128,10 @@ bool DWARFASTParserClang::CompleteRecordType(const DWARFDIE &die,
   TypeSystemClang::BuildIndirectFields(clang_type);
   TypeSystemClang::CompleteTagDeclarationDefinition(clang_type);
 
-  if (type)
-    layout_info.bit_size = type->GetByteSize(nullptr).value_or(0) * 8;
-  if (layout_info.bit_size == 0)
-    layout_info.bit_size =
-        die.GetAttributeValueAsUnsigned(DW_AT_byte_size, 0) * 8;
-  if (layout_info.alignment == 0)
-    layout_info.alignment =
-        die.GetAttributeValueAsUnsigned(llvm::dwarf::DW_AT_alignment, 0) * 8;
+  layout_info.bit_size =
+      die.GetAttributeValueAsUnsigned(DW_AT_byte_size, 0) * 8;
+  layout_info.alignment =
+      die.GetAttributeValueAsUnsigned(llvm::dwarf::DW_AT_alignment, 0) * 8;
 
   clang::CXXRecordDecl *record_decl =
       m_ast.GetAsCXXRecordDecl(clang_type.GetOpaqueQualType());
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugArangeSet.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugArangeSet.cpp
deleted file mode 100644
index 8461b94abca63..0000000000000
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugArangeSet.cpp
+++ /dev/null
@@ -1,176 +0,0 @@
-//===-- DWARFDebugArangeSet.cpp -------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "DWARFDebugArangeSet.h"
-#include "DWARFDataExtractor.h"
-#include "LogChannelDWARF.h"
-#include "llvm/Object/Error.h"
-#include <cassert>
-
-using namespace lldb_private;
-using namespace lldb_private::plugin::dwarf;
-
-DWARFDebugArangeSet::DWARFDebugArangeSet()
-    : m_offset(DW_INVALID_OFFSET), m_next_offset(DW_INVALID_OFFSET) {}
-
-void DWARFDebugArangeSet::Clear() {
-  m_offset = DW_INVALID_OFFSET;
-  m_next_offset = DW_INVALID_OFFSET;
-  m_header.length = 0;
-  m_header.version = 0;
-  m_header.cu_offset = 0;
-  m_header.addr_size = 0;
-  m_header.seg_size = 0;
-  m_arange_descriptors.clear();
-}
-
-llvm::Error DWARFDebugArangeSet::extract(const DWARFDataExtractor &data,
-                                         lldb::offset_t *offset_ptr) {
-  assert(data.ValidOffset(*offset_ptr));
-
-  m_arange_descriptors.clear();
-  m_offset = *offset_ptr;
-
-  // 7.20 Address Range Table
-  //
-  // Each set of entries in the table of address ranges contained in the
-  // .debug_aranges section begins with a header consisting of: a 4-byte
-  // length containing the length of the set of entries for this compilation
-  // unit, not including the length field itself; a 2-byte version identifier
-  // containing the value 2 for DWARF Version 2; a 4-byte offset into
-  // the.debug_infosection; a 1-byte unsigned integer containing the size in
-  // bytes of an address (or the offset portion of an address for segmented
-  // addressing) on the target system; and a 1-byte unsigned integer
-  // containing the size in bytes of a segment descriptor on the target
-  // system. This header is followed by a series of tuples. Each tuple
-  // consists of an address and a length, each in the size appropriate for an
-  // address on the target architecture.
-  m_header.length = data.GetDWARFInitialLength(offset_ptr);
-  // The length could be 4 bytes or 12 bytes, so use the current offset to
-  // determine the next offset correctly.
-  if (m_header.length > 0)
-    m_next_offset = *offset_ptr + m_header.length;
-  else
-    m_next_offset = DW_INVALID_OFFSET;
-  m_header.version = data.GetU16(offset_ptr);
-  m_header.cu_offset = data.GetDWARFOffset(offset_ptr);
-  m_header.addr_size = data.GetU8(offset_ptr);
-  m_header.seg_size = data.GetU8(offset_ptr);
-
-  // Try to avoid reading invalid arange sets by making sure:
-  // 1 - the version looks good
-  // 2 - the address byte size looks plausible
-  // 3 - the length seems to make sense
-  // 4 - size looks plausible
-  // 5 - the arange tuples do not contain a segment field
-  if (m_header.version < 2 || m_header.version > 5)
-    return llvm::make_error<llvm::object::GenericBinaryError>(
-        "Invalid arange header version");
-
-  if (m_header.addr_size != 4 && m_header.addr_size != 8)
-    return llvm::make_error<llvm::object::GenericBinaryError>(
-        "Invalid arange header address size");
-
-  if (m_header.length == 0)
-    return llvm::make_error<llvm::object::GenericBinaryError>(
-        "Invalid arange header length");
-
-  if (!data.ValidOffset(m_offset + sizeof(m_header.length) + m_header.length -
-                        1))
-    return llvm::make_error<llvm::object::GenericBinaryError>(
-        "Invalid arange header length");
-
-  if (m_header.seg_size)
-    return llvm::make_error<llvm::object::GenericBinaryError>(
-        "segmented arange entries are not supported");
-
-  // The first tuple following the header in each set begins at an offset
-  // that is a multiple of the size of a single tuple (that is, twice the
-  // size of an address). The header is padded, if necessary, to the
-  // appropriate boundary.
-  const uint32_t header_size = *offset_ptr - m_offset;
-  const uint32_t tuple_size = m_header.addr_size << 1;
-  uint32_t first_tuple_offset = 0;
-  while (first_tuple_offset < header_size)
-    first_tuple_offset += tuple_size;
-
-  *offset_ptr = m_offset + first_tuple_offset;
-
-  Descriptor arangeDescriptor;
-
-  static_assert(sizeof(arangeDescriptor.address) ==
-                    sizeof(arangeDescriptor.length),
-                "DWARFDebugArangeSet::Descriptor.address and "
-                "DWARFDebugArangeSet::Descriptor.length must have same size");
-
-  const lldb::offset_t next_offset = GetNextOffset();
-  assert(next_offset != DW_INVALID_OFFSET);
-  uint32_t num_terminators = 0;
-  bool last_was_terminator = false;
-  while (*offset_ptr < next_offset) {
-    arangeDescriptor.address = data.GetMaxU64(offset_ptr, m_header.addr_size);
-    arangeDescriptor.length = data.GetMaxU64(offset_ptr, m_header.addr_size);
-
-    // Each set of tuples is terminated by a 0 for the address and 0 for
-    // the length. Some linkers can emit .debug_aranges with multiple
-    // terminator pair entries that are still withing the length of the
-    // DWARFDebugArangeSet. We want to be sure to parse all entries for
-    // this DWARFDebugArangeSet so that we don't stop parsing early and end up
-    // treating addresses as a header of the next DWARFDebugArangeSet. We also
-    // need to make sure we parse all valid address pairs so we don't omit them
-    // from the aranges result, so we can't stop at the first terminator entry
-    // we find.
-    if (arangeDescriptor.address == 0 && arangeDescriptor.length == 0) {
-      ++num_terminators;
-      last_was_terminator = true;
-    } else {
-      last_was_terminator = false;
-      // Only add .debug_aranges address entries that have a non zero size.
-      // Some linkers will zero out the length field for some .debug_aranges
-      // entries if they were stripped. We also could watch out for multiple
-      // entries at address zero and remove those as well.
-      if (arangeDescriptor.length > 0)
-        m_arange_descriptors.push_back(arangeDescriptor);
-    }
-  }
-  if (num_terminators > 1) {
-    Log *log = GetLog(DWARFLog::DebugInfo);
-    LLDB_LOG(log,
-             "warning: DWARFDebugArangeSet at %#" PRIx64 " contains %u "
-             "terminator entries",
-             m_offset, num_terminators);
-  }
-  if (last_was_terminator)
-    return llvm::ErrorSuccess();
-
-  return llvm::make_error<llvm::object::GenericBinaryError>(
-      "arange descriptors not terminated by null entry");
-}
-
-class DescriptorContainsAddress {
-public:
-  DescriptorContainsAddress(dw_addr_t address) : m_address(address) {}
-  bool operator()(const DWARFDebugArangeSet::Descriptor &desc) const {
-    return (m_address >= desc.address) &&
-           (m_address < (desc.address + desc.length));
-  }
-
-private:
-  const dw_addr_t m_address;
-};
-
-dw_offset_t DWARFDebugArangeSet::FindAddress(dw_addr_t address) const {
-  DescriptorConstIter end = m_arange_descriptors.end();
-  DescriptorConstIter pos =
-      std::find_if(m_arange_descriptors.begin(), end,   // Range
-                   DescriptorContainsAddress(address)); // Predicate
-  if (pos != end)
-    return m_header.cu_offset;
-
-  return DW_INVALID_OFFSET;
-}
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugArangeSet.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugArangeSet.h
deleted file mode 100644
index ecdbe953f58b0..0000000000000
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugArangeSet.h
+++ /dev/null
@@ -1,70 +0,0 @@
-//===-- DWARFDebugArangeSet.h -----------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFDEBUGARANGESET_H
-#define LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFDEBUGARANGESET_H
-
-#include "lldb/Core/dwarf.h"
-#include <cstdint>
-#include <vector>
-
-namespace lldb_private::plugin {
-namespace dwarf {
-class DWARFDebugArangeSet {
-public:
-  struct Header {
-    /// The total length of the entries for that set, not including the length
-    /// field itself.
-    uint32_t length = 0;
-    /// The DWARF version number.
-    uint16_t version = 0;
-    /// The offset from the beginning of the .debug_info section of the
-    /// compilation unit entry referenced by the table.
-    uint32_t cu_offset = 0;
-    /// The size in bytes of an address on the target architecture. For
-    /// segmented addressing, this is the size of the offset portion of the
-    /// address.
-    uint8_t addr_size = 0;
-    /// The size in bytes of a segment descriptor on the target architecture.
-    /// If the target system uses a flat address space, this value is 0.
-    uint8_t seg_size = 0;
-  };
-
-  struct Descriptor {
-    dw_addr_t address;
-    dw_addr_t length;
-    dw_addr_t end_address() const { return address + length; }
-  };
-
-  DWARFDebugArangeSet();
-  void Clear();
-  void SetOffset(uint32_t offset) { m_offset = offset; }
-  llvm::Error extract(const DWARFDataExtractor &data,
-                      lldb::offset_t *offset_ptr);
-  dw_offset_t FindAddress(dw_addr_t address) const;
-  size_t NumDescriptors() const { return m_arange_descriptors.size(); }
-  const Header &GetHeader() const { return m_header; }
-  dw_offset_t GetNextOffset() const { return m_next_offset; }
-  const Descriptor &GetDescriptorRef(uint32_t i) const {
-    return m_arange_descriptors[i];
-  }
-
-protected:
-  typedef std::vector<Descriptor> DescriptorColl;
-  typedef DescriptorColl::iterator DescriptorIter;
-  typedef DescriptorColl::const_iterator DescriptorConstIter;
-
-  dw_offset_t m_offset;
-  dw_offset_t m_next_offset;
-  Header m_header;
-  DescriptorColl m_arange_descriptors;
-};
-} // namespace dwarf
-} // namespace lldb_private::plugin
-
-#endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFDEBUGARANGESET_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugAranges.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugAranges.cpp
index f383261e8a5fc..a1a70ac1c9e85 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugAranges.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugAranges.cpp
@@ -7,65 +7,44 @@
 //===----------------------------------------------------------------------===//
 
 #include "DWARFDebugAranges.h"
-#include "DWARFDebugArangeSet.h"
 #include "DWARFUnit.h"
 #include "LogChannelDWARF.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/Timer.h"
+#include "llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h"
 
 using namespace lldb;
 using namespace lldb_private;
 using namespace lldb_private::plugin::dwarf;
+using llvm::DWARFDebugArangeSet;
 
 // Constructor
 DWARFDebugAranges::DWARFDebugAranges() : m_aranges() {}
 
-// CountArangeDescriptors
-class CountArangeDescriptors {
-public:
-  CountArangeDescriptors(uint32_t &count_ref) : count(count_ref) {
-    //      printf("constructor CountArangeDescriptors()\n");
-  }
-  void operator()(const DWARFDebugArangeSet &set) {
-    count += set.NumDescriptors();
-  }
-  uint32_t &count;
-};
-
 // Extract
 void DWARFDebugAranges::extract(const DWARFDataExtractor &debug_aranges_data) {
+  llvm::DWARFDataExtractor dwarf_data = debug_aranges_data.GetAsLLVMDWARF();
   lldb::offset_t offset = 0;
 
   DWARFDebugArangeSet set;
   Range range;
-  while (debug_aranges_data.ValidOffset(offset)) {
+  while (dwarf_data.isValidOffset(offset)) {
     const lldb::offset_t set_offset = offset;
-    if (llvm::Error error = set.extract(debug_aranges_data, &offset)) {
+    if (llvm::Error error = set.extract(dwarf_data, &offset)) {
       Log *log = GetLog(DWARFLog::DebugInfo);
       LLDB_LOG_ERROR(log, std::move(error),
                      "DWARFDebugAranges::extract failed to extract "
                      ".debug_aranges set at offset {1:x}: {0}",
                      set_offset);
-    } else {
-      const uint32_t num_descriptors = set.NumDescriptors();
-      if (num_descriptors > 0) {
-        const dw_offset_t cu_offset = set.GetHeader().cu_offset;
-
-        for (uint32_t i = 0; i < num_descriptors; ++i) {
-          const DWARFDebugArangeSet::Descriptor &descriptor =
-              set.GetDescriptorRef(i);
-          m_aranges.Append(RangeToDIE::Entry(descriptor.address,
-                                             descriptor.length, cu_offset));
-        }
-      }
+      set.clear();
+      return;
+    }
+    const uint64_t cu_offset = set.getCompileUnitDIEOffset();
+    for (const auto &desc : set.descriptors()) {
+      if (desc.Length != 0)
+        m_aranges.Append(
+            RangeToDIE::Entry(desc.Address, desc.Length, cu_offset));
     }
-    // Always use the previous DWARFDebugArangeSet's information to calculate
-    // the offset of the next DWARFDebugArangeSet in case we entouncter an
-    // error in the current DWARFDebugArangeSet and our offset position is
-    // still in the middle of the data. If we do this, we can parse all valid
-    // DWARFDebugArangeSet objects without returning invalid errors.
-    offset = set.GetNextOffset();
-    set.Clear();
   }
 }
 
diff --git a/lldb/source/Plugins/UnwindAssembly/x86/x86AssemblyInspectionEngine.cpp b/lldb/source/Plugins/UnwindAssembly/x86/x86AssemblyInspectionEngine.cpp
index 81b7f138fe7ca..45d2f95433d5b 100644
--- a/lldb/source/Plugins/UnwindAssembly/x86/x86AssemblyInspectionEngine.cpp
+++ b/lldb/source/Plugins/UnwindAssembly/x86/x86AssemblyInspectionEngine.cpp
@@ -809,7 +809,7 @@ bool x86AssemblyInspectionEngine::local_branch_p (
       // Branch target is before the start of this function
       return false;
     }
-    if (offset + next_pc_value > func_range.GetByteSize()) {
+    if (offset + next_pc_value >= func_range.GetByteSize()) {
       // Branch targets outside this function's bounds
       return false;
     }
@@ -967,6 +967,8 @@ bool x86AssemblyInspectionEngine::GetNonCallSiteUnwindPlanFromAssembly(
 
     bool in_epilogue = false; // we're in the middle of an epilogue sequence
     bool row_updated = false; // The UnwindPlan::Row 'row' has been updated
+    bool current_sp_offset_updated =
+        false; // current_sp_bytes_offset_from_fa has been updated this insn
 
     m_cur_insn = data + current_func_text_offset;
     if (!instruction_length(m_cur_insn, insn_len, size - current_func_text_offset)
@@ -1013,8 +1015,10 @@ bool x86AssemblyInspectionEngine::GetNonCallSiteUnwindPlanFromAssembly(
         afa_value.SetUnspecified();
         row_updated = true;
       }
-      if (fa_value_ptr->GetRegisterNumber() == m_lldb_fp_regnum)
+      if (fa_value_ptr->GetRegisterNumber() == m_lldb_fp_regnum) {
         current_sp_bytes_offset_from_fa = fa_value_ptr->GetOffset();
+        current_sp_offset_updated = true;
+      }
     }
 
     else if (mov_rbx_rsp_pattern_p()) {
@@ -1025,8 +1029,10 @@ bool x86AssemblyInspectionEngine::GetNonCallSiteUnwindPlanFromAssembly(
         afa_value.SetUnspecified();
         row_updated = true;
       }
-      if (fa_value_ptr->GetRegisterNumber() == m_lldb_alt_fp_regnum)
+      if (fa_value_ptr->GetRegisterNumber() == m_lldb_alt_fp_regnum) {
         current_sp_bytes_offset_from_fa = fa_value_ptr->GetOffset();
+        current_sp_offset_updated = true;
+      }
     }
 
     // This is the start() function (or a pthread equivalent), it starts with a
@@ -1039,6 +1045,7 @@ bool x86AssemblyInspectionEngine::GetNonCallSiteUnwindPlanFromAssembly(
 
     else if (push_reg_p(machine_regno)) {
       current_sp_bytes_offset_from_fa += m_wordsize;
+      current_sp_offset_updated = true;
       // the PUSH instruction has moved the stack pointer - if the FA is set
       // in terms of the stack pointer, we need to add a new row of
       // instructions.
@@ -1064,6 +1071,7 @@ bool x86AssemblyInspectionEngine::GetNonCallSiteUnwindPlanFromAssembly(
 
     else if (pop_reg_p(machine_regno)) {
       current_sp_bytes_offset_from_fa -= m_wordsize;
+      current_sp_offset_updated = true;
 
       if (nonvolatile_reg_p(machine_regno) &&
           machine_regno_to_lldb_regno(machine_regno, lldb_regno) &&
@@ -1091,6 +1099,7 @@ bool x86AssemblyInspectionEngine::GetNonCallSiteUnwindPlanFromAssembly(
 
     else if (pop_misc_reg_p()) {
       current_sp_bytes_offset_from_fa -= m_wordsize;
+      current_sp_offset_updated = true;
       if (fa_value_ptr->GetRegisterNumber() == m_lldb_sp_regnum) {
         fa_value_ptr->SetIsRegisterPlusOffset(
             m_lldb_sp_regnum, current_sp_bytes_offset_from_fa);
@@ -1126,6 +1135,7 @@ bool x86AssemblyInspectionEngine::GetNonCallSiteUnwindPlanFromAssembly(
       }
 
       current_sp_bytes_offset_from_fa -= m_wordsize;
+      current_sp_offset_updated = true;
 
       if (fa_value_ptr->GetRegisterNumber() == m_lldb_sp_regnum) {
         fa_value_ptr->SetIsRegisterPlusOffset(
@@ -1161,6 +1171,7 @@ bool x86AssemblyInspectionEngine::GetNonCallSiteUnwindPlanFromAssembly(
 
     else if (sub_rsp_pattern_p(stack_offset)) {
       current_sp_bytes_offset_from_fa += stack_offset;
+      current_sp_offset_updated = true;
       if (fa_value_ptr->GetRegisterNumber() == m_lldb_sp_regnum) {
         fa_value_ptr->SetOffset(current_sp_bytes_offset_from_fa);
         row_updated = true;
@@ -1169,6 +1180,7 @@ bool x86AssemblyInspectionEngine::GetNonCallSiteUnwindPlanFromAssembly(
 
     else if (add_rsp_pattern_p(stack_offset)) {
       current_sp_bytes_offset_from_fa -= stack_offset;
+      current_sp_offset_updated = true;
       if (fa_value_ptr->GetRegisterNumber() == m_lldb_sp_regnum) {
         fa_value_ptr->SetOffset(current_sp_bytes_offset_from_fa);
         row_updated = true;
@@ -1179,6 +1191,7 @@ bool x86AssemblyInspectionEngine::GetNonCallSiteUnwindPlanFromAssembly(
     else if (push_extended_pattern_p() || push_imm_pattern_p() ||
              push_misc_reg_p()) {
       current_sp_bytes_offset_from_fa += m_wordsize;
+      current_sp_offset_updated = true;
       if (fa_value_ptr->GetRegisterNumber() == m_lldb_sp_regnum) {
         fa_value_ptr->SetOffset(current_sp_bytes_offset_from_fa);
         row_updated = true;
@@ -1187,6 +1200,7 @@ bool x86AssemblyInspectionEngine::GetNonCallSiteUnwindPlanFromAssembly(
 
     else if (lea_rsp_pattern_p(stack_offset)) {
       current_sp_bytes_offset_from_fa -= stack_offset;
+      current_sp_offset_updated = true;
       if (fa_value_ptr->GetRegisterNumber() == m_lldb_sp_regnum) {
         fa_value_ptr->SetOffset(current_sp_bytes_offset_from_fa);
         row_updated = true;
@@ -1206,6 +1220,7 @@ bool x86AssemblyInspectionEngine::GetNonCallSiteUnwindPlanFromAssembly(
       if (fa_value_ptr->GetRegisterNumber() == m_lldb_fp_regnum) {
         current_sp_bytes_offset_from_fa =
           fa_value_ptr->GetOffset() - stack_offset;
+        current_sp_offset_updated = true;
       }
     }
 
@@ -1219,6 +1234,7 @@ bool x86AssemblyInspectionEngine::GetNonCallSiteUnwindPlanFromAssembly(
       }
       if (fa_value_ptr->GetRegisterNumber() == m_lldb_alt_fp_regnum) {
         current_sp_bytes_offset_from_fa = fa_value_ptr->GetOffset() - stack_offset;
+        current_sp_offset_updated = true;
       }
     }
 
@@ -1251,6 +1267,7 @@ bool x86AssemblyInspectionEngine::GetNonCallSiteUnwindPlanFromAssembly(
           row.reset(newrow);
           current_sp_bytes_offset_from_fa =
               prologue_completed_sp_bytes_offset_from_cfa;
+          current_sp_offset_updated = true;
           is_aligned = prologue_completed_is_aligned;
 
           saved_registers.clear();
@@ -1272,6 +1289,7 @@ bool x86AssemblyInspectionEngine::GetNonCallSiteUnwindPlanFromAssembly(
     // global data
     else if (call_next_insn_pattern_p()) {
       current_sp_bytes_offset_from_fa += m_wordsize;
+      current_sp_offset_updated = true;
       if (fa_value_ptr->GetRegisterNumber() == m_lldb_sp_regnum) {
         fa_value_ptr->SetOffset(current_sp_bytes_offset_from_fa);
         row_updated = true;
@@ -1304,7 +1322,7 @@ bool x86AssemblyInspectionEngine::GetNonCallSiteUnwindPlanFromAssembly(
 
     // We may change the sp value without adding a new Row necessarily -- keep
     // track of it either way.
-    if (!in_epilogue) {
+    if (!in_epilogue && current_sp_offset_updated) {
       prologue_completed_sp_bytes_offset_from_cfa =
           current_sp_bytes_offset_from_fa;
       prologue_completed_is_aligned = is_aligned;
diff --git a/lldb/source/Symbol/UnwindPlan.cpp b/lldb/source/Symbol/UnwindPlan.cpp
index a06e7cfd7f544..25e3676761436 100644
--- a/lldb/source/Symbol/UnwindPlan.cpp
+++ b/lldb/source/Symbol/UnwindPlan.cpp
@@ -187,6 +187,8 @@ operator==(const UnwindPlan::Row::FAValue &rhs) const {
         return !memcmp(m_value.expr.opcodes, rhs.m_value.expr.opcodes,
                        m_value.expr.length);
       break;
+    case isConstant:
+      return m_value.constant == rhs.m_value.constant;
     }
   }
   return false;
@@ -214,6 +216,8 @@ void UnwindPlan::Row::FAValue::Dump(Stream &s, const UnwindPlan *unwind_plan,
   case isRaSearch:
     s.Printf("RaSearch@SP%+d", m_value.ra_search_offset);
     break;
+  case isConstant:
+    s.Printf("0x%" PRIx64, m_value.constant);
   }
 }
 
diff --git a/lldb/source/Target/RegisterContextUnwind.cpp b/lldb/source/Target/RegisterContextUnwind.cpp
index b683ea7237de0..9a4a8db84a9fa 100644
--- a/lldb/source/Target/RegisterContextUnwind.cpp
+++ b/lldb/source/Target/RegisterContextUnwind.cpp
@@ -2088,6 +2088,12 @@ bool RegisterContextUnwind::ReadFrameAddress(
     UnwindLogMsg("No suitable CFA found");
     break;
   }
+  case UnwindPlan::Row::FAValue::isConstant: {
+    address = fa.GetConstant();
+    address = m_thread.GetProcess()->FixDataAddress(address);
+    UnwindLogMsg("CFA value set by constant is 0x%" PRIx64, address);
+    return true;
+  }
   default:
     return false;
   }
diff --git a/lldb/test/API/commands/platform/launchgdbserver/TestPlatformLaunchGDBServer.py b/lldb/test/API/commands/platform/launchgdbserver/TestPlatformLaunchGDBServer.py
index ea849ab1fa236..c365bc993e338 100644
--- a/lldb/test/API/commands/platform/launchgdbserver/TestPlatformLaunchGDBServer.py
+++ b/lldb/test/API/commands/platform/launchgdbserver/TestPlatformLaunchGDBServer.py
@@ -16,7 +16,8 @@ class TestPlatformProcessLaunchGDBServer(TestBase):
     NO_DEBUG_INFO_TESTCASE = True
 
     @skipIfRemote
-    # Windows doesn't use lldb-server and on Darwin we may be using debugserver.
+    # Windows cannot delete the executable while it is running.
+    # On Darwin we may be using debugserver.
     @skipUnlessPlatform(["linux"])
     @add_test_categories(["lldb-server"])
     def test_platform_process_launch_gdb_server(self):
@@ -44,15 +45,16 @@ def test_platform_process_launch_gdb_server(self):
         self.spawnSubprocess(new_lldb_server, commandline_args)
         socket_id = lldbutil.wait_for_file_on_target(self, port_file)
 
-        # Remove our new lldb-server so that when it tries to invoke itself as a
-        # gdbserver, it fails.
-        os.remove(new_lldb_server)
-
         new_platform = lldb.SBPlatform("remote-" + self.getPlatform())
         self.dbg.SetSelectedPlatform(new_platform)
 
         connect_url = "connect://[%s]:%s" % (hostname, socket_id)
         self.runCmd("platform connect %s" % connect_url)
 
+        # First connect to lldb-server which spawn a process to handle the connection.
+        # Then remove our new lldb-server so that when it tries to invoke itself as a
+        # gdbserver, it fails.
+        os.remove(new_lldb_server)
+
         self.runCmd("target create {}".format(self.getBuildArtifact("a.out")))
         self.expect("run", substrs=["unable to launch a GDB server on"], error=True)
diff --git a/lldb/test/API/tools/lldb-dap/completions/TestDAP_completions.py b/lldb/test/API/tools/lldb-dap/completions/TestDAP_completions.py
index 2b3ec656c107a..8ad12b3fc2f6b 100644
--- a/lldb/test/API/tools/lldb-dap/completions/TestDAP_completions.py
+++ b/lldb/test/API/tools/lldb-dap/completions/TestDAP_completions.py
@@ -9,6 +9,28 @@
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 
+session_completion = {
+    "text": "session",
+    "label": "session -- Commands controlling LLDB session.",
+}
+settings_completion = {
+    "text": "settings",
+    "label": "settings -- Commands for managing LLDB settings.",
+}
+memory_completion = {
+    "text": "memory",
+    "label": "memory -- Commands for operating on memory in the current target process.",
+}
+command_var_completion = {
+    "text": "var",
+    "label": "var -- Show variables for the current stack frame. Defaults to all arguments and local variables in scope. Names of argument, local, file static and file global variables can be specified.",
+}
+variable_var_completion = {
+    "text": "var",
+    "label": "var -- vector<baz> &",
+}
+variable_var1_completion = {"text": "var1", "label": "var1 -- int &"}
+variable_var2_completion = {"text": "var2", "label": "var2 -- int &"}
 
 class TestDAP_completions(lldbdap_testcase.DAPTestCaseBase):
     def verify_completions(self, actual_list, expected_list, not_expected_list=[]):
@@ -18,12 +40,8 @@ def verify_completions(self, actual_list, expected_list, not_expected_list=[]):
         for not_expected_item in not_expected_list:
             self.assertNotIn(not_expected_item, actual_list)
 
-    @skipIfWindows
-    @skipIf(compiler="clang", compiler_version=["<", "17.0"])
-    def test_completions(self):
-        """
-        Tests the completion request at different breakpoints
-        """
+
+    def setup_debugee(self):
         program = self.getBuildArtifact("a.out")
         self.build_and_launch(program)
 
@@ -32,90 +50,146 @@ def test_completions(self):
         breakpoint2_line = line_number(source, "// breakpoint 2")
 
         self.set_source_breakpoints(source, [breakpoint1_line, breakpoint2_line])
+
+    def test_command_completions(self):
+        """
+        Tests completion requests for lldb commands, within "repl-mode=command"
+        """
+        self.setup_debugee()
         self.continue_to_next_stop()
 
-        # shouldn't see variables inside main
+        res = self.dap_server.request_evaluate(
+            "`lldb-dap repl-mode command", context="repl"
+        )
+        self.assertTrue(res["success"])
+
+        # Provides completion for top-level commands
         self.verify_completions(
-            self.dap_server.get_completions("var"),
+            self.dap_server.get_completions("se"),
+            [session_completion, settings_completion],
+        )
+
+        # Provides completions for sub-commands
+        self.verify_completions(
+            self.dap_server.get_completions("memory "),
             [
                 {
-                    "text": "var",
-                    "label": "var -- vector<baz> &",
+                    "text": "read",
+                    "label": "read -- Read from the memory of the current target process.",
                 },
                 {
-                    "text": "var",
-                    "label": "var -- Show variables for the current stack frame. Defaults to all arguments and local variables in scope. Names of argument, local, file static and file global variables can be specified.",
+                    "text": "region",
+                    "label": "region -- Get information on the memory region containing an address in the current target process.",
                 },
             ],
-            [
-                {"text": "var1", "label": "var1 -- int &"},
-            ],
         )
 
-        # should see global keywords but not variables inside main
+        # Provides completions for parameter values of commands
         self.verify_completions(
-            self.dap_server.get_completions("str"),
-            [{"text": "struct", "label": "struct"}],
-            [{"text": "str1", "label": "str1 -- std::string &"}],
+            self.dap_server.get_completions("`log enable  "),
+            [{"text": "gdb-remote", "label": "gdb-remote"}],
         )
 
-        self.continue_to_next_stop()
+        # Also works if the escape prefix is used
+        self.verify_completions(
+            self.dap_server.get_completions("`mem"), [memory_completion]
+        )
 
-        # should see variables from main but not from the other function
         self.verify_completions(
-            self.dap_server.get_completions("var"),
-            [
-                {"text": "var1", "label": "var1 -- int &"},
-                {"text": "var2", "label": "var2 -- int &"},
-            ],
+            self.dap_server.get_completions("`"),
+            [session_completion, settings_completion, memory_completion],
+        )
+
+        # Completes an incomplete quoted token
+        self.verify_completions(
+            self.dap_server.get_completions('setting "se'),
             [
                 {
-                    "text": "var",
-                    "label": "var -- vector<baz> &",
+                    "text": "set",
+                    "label": "set -- Set the value of the specified debugger setting.",
                 }
             ],
         )
 
+        # Completes an incomplete quoted token
         self.verify_completions(
-            self.dap_server.get_completions("str"),
-            [
-                {"text": "struct", "label": "struct"},
-                {"text": "str1", "label": "str1 -- string &"},
-            ],
+            self.dap_server.get_completions("'mem"),
+            [memory_completion],
         )
 
-        # should complete arbitrary commands including word starts
+        # Completes expressions with quotes inside
         self.verify_completions(
-            self.dap_server.get_completions("`log enable  "),
-            [{"text": "gdb-remote", "label": "gdb-remote"}],
+            self.dap_server.get_completions('expr " "; typed'),
+            [{"text": "typedef", "label": "typedef"}],
         )
 
-        # should complete expressions with quotes inside
+        # Provides completions for commands, but not variables
         self.verify_completions(
-            self.dap_server.get_completions('`expr " "; typed'),
-            [{"text": "typedef", "label": "typedef"}],
+            self.dap_server.get_completions("var"),
+            [command_var_completion],
+            [variable_var_completion],
+        )
+
+    def test_variable_completions(self):
+        """
+        Tests completion requests in "repl-mode=variable"
+        """
+        self.setup_debugee()
+        self.continue_to_next_stop()
+
+        res = self.dap_server.request_evaluate(
+            "`lldb-dap repl-mode variable", context="repl"
+        )
+        self.assertTrue(res["success"])
+
+        # Provides completions for varibles, but not command
+        self.verify_completions(
+            self.dap_server.get_completions("var"),
+            [variable_var_completion],
+            [command_var_completion],
         )
 
-        # should complete an incomplete quoted token
+        # We stopped inside `fun`, so we shouldn't see variables from main
         self.verify_completions(
-            self.dap_server.get_completions('`setting "se'),
+            self.dap_server.get_completions("var"),
+            [variable_var_completion],
             [
-                {
-                    "text": "set",
-                    "label": "set -- Set the value of the specified debugger setting.",
-                }
+                variable_var1_completion,
+                variable_var2_completion,
             ],
         )
+
+        # We should see global keywords but not variables inside main
         self.verify_completions(
-            self.dap_server.get_completions("`'comm"),
+            self.dap_server.get_completions("str"),
+            [{"text": "struct", "label": "struct"}],
+            [{"text": "str1", "label": "str1 -- std::string &"}],
+        )
+
+        self.continue_to_next_stop()
+
+        # We stopped in `main`, so we should see variables from main but
+        # not from the other function
+        self.verify_completions(
+            self.dap_server.get_completions("var"),
             [
-                {
-                    "text": "command",
-                    "label": "command -- Commands for managing custom LLDB commands.",
-                }
+                variable_var1_completion,
+                variable_var2_completion,
+            ],
+            [
+                variable_var_completion,
+            ],
+        )
+
+        self.verify_completions(
+            self.dap_server.get_completions("str"),
+            [
+                {"text": "struct", "label": "struct"},
+                {"text": "str1", "label": "str1 -- string &"},
             ],
         )
 
+        # Completion also works for more complex expressions
         self.verify_completions(
             self.dap_server.get_completions("foo1.v"),
             [{"text": "var1", "label": "foo1.var1 -- int"}],
@@ -148,3 +222,45 @@ def test_completions(self):
             [{"text": "var1", "label": "var1 -- int"}],
             [{"text": "var2", "label": "var2 -- int"}],
         )
+
+        # Even in variable mode, we can still use the escape prefix
+        self.verify_completions(
+            self.dap_server.get_completions("`mem"), [memory_completion]
+        )
+
+    def test_auto_completions(self):
+        """
+        Tests completion requests in "repl-mode=auto"
+        """
+        self.setup_debugee()
+
+        res = self.dap_server.request_evaluate(
+            "`lldb-dap repl-mode auto", context="repl"
+        )
+        self.assertTrue(res["success"])
+
+        self.continue_to_next_stop()
+        self.continue_to_next_stop()
+
+        # We are stopped inside `main`. Variables `var1` and `var2` are in scope.
+        # Make sure, we offer all completions
+        self.verify_completions(
+            self.dap_server.get_completions("va"),
+            [
+                command_var_completion,
+                variable_var1_completion,
+                variable_var2_completion,
+            ],
+        )
+
+        # If we are using the escape prefix, only commands are suggested, but no variables
+        self.verify_completions(
+            self.dap_server.get_completions("`va"),
+            [
+                command_var_completion,
+            ],
+            [
+                variable_var1_completion,
+                variable_var2_completion,
+            ],
+        )
diff --git a/lldb/test/API/tools/lldb-dap/optimized/TestDAP_optimized.py b/lldb/test/API/tools/lldb-dap/optimized/TestDAP_optimized.py
index 42a95de309a08..ae144ebdca46b 100644
--- a/lldb/test/API/tools/lldb-dap/optimized/TestDAP_optimized.py
+++ b/lldb/test/API/tools/lldb-dap/optimized/TestDAP_optimized.py
@@ -28,6 +28,7 @@ def test_stack_frame_name(self):
         parent_frame = self.dap_server.get_stackFrame(frameIndex=1)
         self.assertTrue(parent_frame["name"].endswith(" [opt]"))
 
+    @skipIfAsan # On ASAN builds this test intermittently fails https://github.com/llvm/llvm-project/issues/111061
     @skipIfWindows
     def test_optimized_variable(self):
         """Test optimized variable value contains error."""
diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/DW_AT_declaration-unusual-layout.s b/lldb/test/Shell/SymbolFile/DWARF/x86/DW_AT_declaration-unusual-layout.s
new file mode 100644
index 0000000000000..27715c5004720
--- /dev/null
+++ b/lldb/test/Shell/SymbolFile/DWARF/x86/DW_AT_declaration-unusual-layout.s
@@ -0,0 +1,132 @@
+## Test that lldb respects the layout defined in DWARF even when starting out
+## with a declaration of the class.
+
+# RUN: split-file %s %t
+# RUN: llvm-mc --triple x86_64-pc-linux %t/asm --filetype=obj -o %t.o
+# RUN: %lldb -s %t/commands -o exit %t.o 2>&1 | FileCheck %s
+
+#--- commands
+target var a -fx
+# CHECK-LABEL: target var a
+# CHECK: (A) a = (i = 0xbaadf00d)
+
+#--- asm
+        .data
+        .p2align 4
+        .long 0
+a:
+        .long  0xdeadbeef
+        .long  0xbaadf00d
+
+        .section        .debug_abbrev,"",@progbits
+        .byte   1                               # Abbreviation Code
+        .byte   17                              # DW_TAG_compile_unit
+        .byte   1                               # DW_CHILDREN_yes
+        .byte   37                              # DW_AT_producer
+        .byte   8                               # DW_FORM_string
+        .byte   0                               # EOM(1)
+        .byte   0                               # EOM(2)
+        .byte   2                               # Abbreviation Code
+        .byte   52                              # DW_TAG_variable
+        .byte   0                               # DW_CHILDREN_no
+        .byte   3                               # DW_AT_name
+        .byte   8                               # DW_FORM_string
+        .byte   73                              # DW_AT_type
+        .byte   19                              # DW_FORM_ref4
+        .byte   2                               # DW_AT_location
+        .byte   24                              # DW_FORM_exprloc
+        .byte   0                               # EOM(1)
+        .byte   0                               # EOM(2)
+        .byte   3                               # Abbreviation Code
+        .byte   19                              # DW_TAG_structure_type
+        .byte   0                               # DW_CHILDREN_no
+        .byte   3                               # DW_AT_name
+        .byte   8                               # DW_FORM_string
+        .byte   60                              # DW_AT_declaration
+        .byte   25                              # DW_FORM_flag_present
+        .byte   0                               # EOM(1)
+        .byte   0                               # EOM(2)
+        .byte   4                               # Abbreviation Code
+        .byte   19                              # DW_TAG_structure_type
+        .byte   1                               # DW_CHILDREN_yes
+        .byte   3                               # DW_AT_name
+        .byte   8                               # DW_FORM_string
+        .byte   11                              # DW_AT_byte_size
+        .byte   11                              # DW_FORM_data1
+        .byte   0                               # EOM(1)
+        .byte   0                               # EOM(2)
+        .byte   5                               # Abbreviation Code
+        .byte   13                              # DW_TAG_member
+        .byte   0                               # DW_CHILDREN_no
+        .byte   3                               # DW_AT_name
+        .byte   8                               # DW_FORM_string
+        .byte   73                              # DW_AT_type
+        .byte   19                              # DW_FORM_ref4
+        .byte   56                              # DW_AT_data_member_location
+        .byte   11                              # DW_FORM_data1
+        .byte   0                               # EOM(1)
+        .byte   0                               # EOM(2)
+        .byte   6                               # Abbreviation Code
+        .byte   36                              # DW_TAG_base_type
+        .byte   0                               # DW_CHILDREN_no
+        .byte   3                               # DW_AT_name
+        .byte   8                               # DW_FORM_string
+        .byte   62                              # DW_AT_encoding
+        .byte   11                              # DW_FORM_data1
+        .byte   11                              # DW_AT_byte_size
+        .byte   11                              # DW_FORM_data1
+        .byte   0                               # EOM(1)
+        .byte   0                               # EOM(2)
+        .byte   0                               # EOM(3)
+
+        .section        .debug_info,"",@progbits
+.Lcu_begin0:
+        .long   .Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit
+.Ldebug_info_start0:
+        .short  4                               # DWARF version number
+        .long   .debug_abbrev                   # Offset Into Abbrev. Section
+        .byte   8                               # Address Size (in bytes)
+        .byte   1                               # Abbrev [1] DW_TAG_compile_unit
+        .asciz  "Hand-written DWARF"            # DW_AT_producer
+
+        .byte   2                               # Abbrev [2] DW_TAG_variable
+        .asciz  "a"                             # DW_AT_name
+        .long   .LA_decl-.Lcu_begin0            # DW_AT_type
+        .byte   9                               # DW_AT_location
+        .byte   3
+        .quad   a
+.LA_decl:
+        .byte   3                               # Abbrev [3] DW_TAG_structure_type
+        .asciz  "A"                             # DW_AT_name
+                                                # DW_AT_declaration
+        .byte   0                               # End Of Children Mark
+.Ldebug_info_end0:
+
+.Lcu_begin1:
+        .long   .Ldebug_info_end1-.Ldebug_info_start1 # Length of Unit
+.Ldebug_info_start1:
+        .short  4                               # DWARF version number
+        .long   .debug_abbrev                   # Offset Into Abbrev. Section
+        .byte   8                               # Address Size (in bytes)
+        .byte   1                               # Abbrev [1] DW_TAG_compile_unit
+        .asciz  "Hand-written DWARF"            # DW_AT_producer
+
+        .byte   4                               # Abbrev [4] DW_TAG_structure_type
+        .asciz  "A"                             # DW_AT_name
+                                                # DW_AT_declaration
+        .byte   8                               # DW_AT_byte_size
+        .byte   5                               # Abbrev [5] DW_TAG_member
+        .asciz  "i"                             # DW_AT_name
+        .long   .Lint-.Lcu_begin1               # DW_AT_type
+## NB: empty padding before this member
+        .byte   4                               # DW_AT_data_member_location
+        .byte   0                               # End Of Children Mark
+
+.Lint:
+        .byte   6                               # Abbrev [6] DW_TAG_base_type
+        .asciz  "int"                           # DW_AT_name
+        .byte   5                               # DW_AT_encoding
+        .byte   4                               # DW_AT_byte_size
+
+        .byte   0                               # End Of Children Mark
+.Ldebug_info_end1:
diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/DW_AT_declaration-with-children.s b/lldb/test/Shell/SymbolFile/DWARF/x86/DW_AT_declaration-with-children.s
index 8633d02f492e6..16bdbd3cfaad6 100644
--- a/lldb/test/Shell/SymbolFile/DWARF/x86/DW_AT_declaration-with-children.s
+++ b/lldb/test/Shell/SymbolFile/DWARF/x86/DW_AT_declaration-with-children.s
@@ -105,6 +105,8 @@ c1:
         .byte   1                               # DW_CHILDREN_yes
         .byte   3                               # DW_AT_name
         .byte   8                               # DW_FORM_string
+        .byte   11                              # DW_AT_byte_size
+        .byte   11                              # DW_FORM_data1
         .byte   0                               # EOM(1)
         .byte   0                               # EOM(2)
         .byte   7                               # Abbreviation Code
@@ -251,6 +253,7 @@ c1:
 .LB1:
         .byte   6                               # Abbrev [6] DW_TAG_class_type
         .asciz  "B1"                            # DW_AT_name
+        .byte   8                               # DW_AT_byte_size
         .byte   7                               # Abbrev [5] 0x58:0xc DW_TAG_member
         .asciz  "ptr"                           # DW_AT_name
         .long   .LAptr                          # DW_AT_type
diff --git a/lldb/test/Shell/lldb-server/TestGdbserverPort.test b/lldb/test/Shell/lldb-server/TestGdbserverPort.test
deleted file mode 100644
index 04facfec831ca..0000000000000
--- a/lldb/test/Shell/lldb-server/TestGdbserverPort.test
+++ /dev/null
@@ -1,4 +0,0 @@
-# Windows does not build lldb-server.
-# UNSUPPORTED: system-windows
-# RUN: %platformserver --server --listen :1234 --min-gdbserver-port 1234 --max-gdbserver-port 1234 2>&1 | FileCheck %s
-# CHECK: error: --min-gdbserver-port (1234) is not lower than --max-gdbserver-port (1234)
diff --git a/lldb/tools/debugserver/source/DNB.cpp b/lldb/tools/debugserver/source/DNB.cpp
index 1ac9a8040c616..10c55dc3f769f 100644
--- a/lldb/tools/debugserver/source/DNB.cpp
+++ b/lldb/tools/debugserver/source/DNB.cpp
@@ -475,9 +475,17 @@ nub_process_t DNBProcessAttach(nub_process_t attach_pid,
         extern int communication_fd;
 
         if (communication_fd == -1) {
-          fprintf(stderr, "Trying to attach to a translated process with the "
-                          "native debugserver, exiting...\n");
-          exit(1);
+          DNBLogError("Trying to attach to a translated process with the "
+                      "native debugserver, exiting...\n");
+          return INVALID_NUB_PROCESS_ARCH;
+        }
+
+        struct stat st;
+        if (::stat(translated_debugserver, &st) != 0) {
+          DNBLogError("Translated inferior process but Rosetta debugserver not "
+                      "found at %s",
+                      translated_debugserver);
+          return INVALID_NUB_PROCESS_ARCH;
         }
 
         snprintf(fdstr, sizeof(fdstr), "--fd=%d", communication_fd);
diff --git a/lldb/tools/lldb-dap/DAP.cpp b/lldb/tools/lldb-dap/DAP.cpp
index 884a71ff6693f..5e75d84cf8243 100644
--- a/lldb/tools/lldb-dap/DAP.cpp
+++ b/lldb/tools/lldb-dap/DAP.cpp
@@ -483,20 +483,20 @@ llvm::json::Value DAP::CreateTopLevelScopes() {
   return llvm::json::Value(std::move(scopes));
 }
 
-ExpressionContext DAP::DetectExpressionContext(lldb::SBFrame frame,
-                                               std::string &expression) {
-  // Include the escape hatch prefix.
+ReplMode DAP::DetectReplMode(lldb::SBFrame frame, std::string &expression,
+                             bool partial_expression) {
+  // Check for the escape hatch prefix.
   if (!expression.empty() &&
       llvm::StringRef(expression).starts_with(g_dap.command_escape_prefix)) {
     expression = expression.substr(g_dap.command_escape_prefix.size());
-    return ExpressionContext::Command;
+    return ReplMode::Command;
   }
 
   switch (repl_mode) {
   case ReplMode::Variable:
-    return ExpressionContext::Variable;
+    return ReplMode::Variable;
   case ReplMode::Command:
-    return ExpressionContext::Command;
+    return ReplMode::Command;
   case ReplMode::Auto:
     // To determine if the expression is a command or not, check if the first
     // term is a variable or command. If it's a variable in scope we will prefer
@@ -509,6 +509,12 @@ ExpressionContext DAP::DetectExpressionContext(lldb::SBFrame frame,
     //   int var and expression "va" > command
     std::pair<llvm::StringRef, llvm::StringRef> token =
         llvm::getToken(expression);
+
+    // If the first token is not fully finished yet, we can't
+    // determine whether this will be a variable or a lldb command.
+    if (partial_expression && token.second.empty())
+      return ReplMode::Auto;
+
     std::string term = token.first.str();
     lldb::SBCommandInterpreter interpreter = debugger.GetCommandInterpreter();
     bool term_is_command = interpreter.CommandExists(term.c_str()) ||
@@ -527,9 +533,9 @@ ExpressionContext DAP::DetectExpressionContext(lldb::SBFrame frame,
 
     // Variables take preference to commands in auto, since commands can always
     // be called using the command_escape_prefix
-    return term_is_variable  ? ExpressionContext::Variable
-           : term_is_command ? ExpressionContext::Command
-                             : ExpressionContext::Variable;
+    return term_is_variable  ? ReplMode::Variable
+           : term_is_command ? ReplMode::Command
+                             : ReplMode::Variable;
   }
 
   llvm_unreachable("enum cases exhausted.");
diff --git a/lldb/tools/lldb-dap/DAP.h b/lldb/tools/lldb-dap/DAP.h
index 57719f98d2aa1..ba6d3d80410e3 100644
--- a/lldb/tools/lldb-dap/DAP.h
+++ b/lldb/tools/lldb-dap/DAP.h
@@ -94,12 +94,6 @@ enum class PacketStatus {
 
 enum class ReplMode { Variable = 0, Command, Auto };
 
-/// The detected context of an expression based off the current repl mode.
-enum class ExpressionContext {
-  Variable = 0,
-  Command,
-};
-
 struct Variables {
   /// Variable_reference start index of permanent expandable variable.
   static constexpr int64_t PermanentVariableStartIndex = (1ll << 32);
@@ -245,12 +239,24 @@ struct DAP {
 
   void PopulateExceptionBreakpoints();
 
-  /// \return
-  ///   Attempt to determine if an expression is a variable expression or
-  ///   lldb command using a hueristic based on the first term of the
-  ///   expression.
-  ExpressionContext DetectExpressionContext(lldb::SBFrame frame,
-                                            std::string &expression);
+  /// Attempt to determine if an expression is a variable expression or
+  /// lldb command using a heuristic based on the first term of the
+  /// expression.
+  ///
+  /// \param[in] frame
+  ///     The frame, used as context to detect local variable names
+  /// \param[inout] expression
+  ///     The expression string. Might be modified by this function to
+  ///     remove the leading escape character.
+  /// \param[in] partial_expression
+  ///     Whether the provided `expression` is only a prefix of the
+  ///     final expression. If `true`, this function might return
+  ///     `ReplMode::Auto` to indicate that the expression could be
+  ///     either an expression or a statement, depending on the rest of
+  ///     the expression.
+  /// \return the expression mode
+  ReplMode DetectReplMode(lldb::SBFrame frame, std::string &expression,
+                          bool partial_expression);
 
   /// \return
   ///   \b false if a fatal error was found while executing these commands,
diff --git a/lldb/tools/lldb-dap/README.md b/lldb/tools/lldb-dap/README.md
index c3bed593154e0..7248df310c14a 100644
--- a/lldb/tools/lldb-dap/README.md
+++ b/lldb/tools/lldb-dap/README.md
@@ -6,9 +6,9 @@ The extension requires the `lldb-dap` (formerly `lldb-vscode`) binary.
 This binary is not packaged with the VS Code extension.
 
 There are multiple ways to obtain this binary:
-* build it from source (see [LLDB's build instructions](https://lldb.llvm.org/resources/build.html))
-* download one of the relase packages from the [LLVM release page](https://github.com/llvm/llvm-project/releases/). The `LLVM-19.1.0-{operating_system}.tar.xz` packages contain a prebuilt `lldb-dap` binary.
 * use the binary provided by your toolchain (for example `xcrun -f lldb-dap` on macOS) or contact your toolchain vendor to include it.
+* download one of the relase packages from the [LLVM release page](https://github.com/llvm/llvm-project/releases/). The `LLVM-19.1.0-{operating_system}.tar.xz` packages contain a prebuilt `lldb-dap` binary.
+* build it from source (see [LLDB's build instructions](https://lldb.llvm.org/resources/build.html))
 
 By default, the VS Code extension will expect to find `lldb-dap` in your `PATH`.
 Alternatively, you can explictly specify the location of the `lldb-dap` binary using the `lldb-dap.executable-path` setting.
@@ -185,7 +185,7 @@ specific key/value pairs:
 |**sourcePath**     |string| | Specify a source path to remap \"./\" to allow full paths to be used when setting breakpoints in binaries that have relative source paths.
 |**sourceMap**      |[string[2]]| | Specify an array of path re-mappings. Each element in the array must be a two element array containing a source and destination pathname. Overrides sourcePath.
 |**debuggerRoot**   | string| |Specify a working directory to use when launching lldb-dap. If the debug information in your executable contains relative paths, this option can be used so that `lldb-dap` can find source files and object files that have relative paths.
-|**commandEscapePrefix** | string | | The escape prefix to use for executing regular LLDB commands in the Debug Console, instead of printing variables. Defaults to a back-tick (`\``). If it's an empty string, then all expression in the Debug Console are treated as regular LLDB commands.
+|**commandEscapePrefix** | string | | The escape prefix to use for executing regular LLDB commands in the Debug Console, instead of printing variables. Defaults to a backtick. If it's an empty string, then all expression in the Debug Console are treated as regular LLDB commands.
 |**customFrameFormat** | string | | If non-empty, stack frames will have descriptions generated based on the provided format. See https://lldb.llvm.org/use/formatting.html for an explanation on format strings for frames. If the format string contains errors, an error message will be displayed on the Debug Console and the default frame names will be used. This might come with a performance cost because debug information might need to be processed to generate the description.
 |**customThreadFormat** | string | | Same as `customFrameFormat`, but for threads instead of stack frames.
 |**displayExtendedBacktrace**|bool| | Enable language specific extended backtraces.
@@ -227,10 +227,14 @@ the following `lldb-dap` specific key/value pairs:
 
 ## Debug Console
 
-The debug console allows printing variables / expressions and executing lldb commands.
-By default, all provided commands are interpreteted as variable names / expressions whose values will be printed to the Debug Console.
-To execute regular LLDB commands, prefix them with the `\`` character.
-The escape character can be changed via the `commandEscapePrefix` configuration option.
+The Debug Console allows printing variables / expressions and executing lldb commands.
+By default, lldb-dap tries to auto-detect whether a provided command is a variable
+name / expression whose values will be printed to the Debug Console or a LLDB command.
+To side-step this auto-detection and execute a LLDB command, prefix it with the
+`commandEscapePrefix`.
+
+The auto-detection can be disabled using the `lldb-dap repl-mode` command.
+The escape character can be adjusted via the `commandEscapePrefix` configuration option.
 
 ### lldb-dap specific commands
 
@@ -268,15 +272,26 @@ Inspect or adjust the behavior of lldb-dap repl evaluation requests. The
 supported modes are `variable`, `command` and `auto`.
 
 - `variable` - Variable mode expressions are evaluated in the context of the
-   current frame. Use a `\`` prefix on the command to run an lldb command.
+   current frame.
 - `command` - Command mode expressions are evaluated as lldb commands, as a
    result, values printed by lldb are always stringified representations of the
    expression output.
 - `auto` - Auto mode will attempt to infer if the expression represents an lldb
    command or a variable expression. A heuristic is used to infer if the input
-   represents a variable or a command. Use a `\`` prefix to ensure an expression
-   is evaluated as a command.
+   represents a variable or a command.
+
+In all three modes, you can use the `commandEscapePrefix` to ensure an expression
+is evaluated as a command.
 
 The initial repl-mode can be configured with the cli flag `--repl-mode=<mode>`
 and may also be adjusted at runtime using the lldb command
 `lldb-dap repl-mode <mode>`.
+
+## Contributing
+
+`lldb-dap` and `lldb` are developed under the umbrella of the [LLVM project](https://llvm.org/).
+The source code is [part of the LLVM repository](https://github.com/llvm/llvm-project/tree/main/lldb/tools/lldb-dap) on Github.
+We use Github's [issue tracker](https://github.com/llvm/llvm-project/tree/main/lldb/tools/lldb-dap) and patches can be submitted via [pull requests](https://github.com/llvm/llvm-project/pulls).
+Furthermore, there is a [LLDB category](https://discourse.llvm.org/c/subprojects/lldb/8) on the LLVM discourse forum.
+
+For instructions on how to get started with development on lldb-dap, see the "[Contributing to lldb-dap](https://lldb.llvm.org/resources/lldbdap.html)"
diff --git a/lldb/tools/lldb-dap/lldb-dap.cpp b/lldb/tools/lldb-dap/lldb-dap.cpp
index db4dbbd6f6200..0b6d640241065 100644
--- a/lldb/tools/lldb-dap/lldb-dap.cpp
+++ b/lldb/tools/lldb-dap/lldb-dap.cpp
@@ -1395,9 +1395,20 @@ void request_completions(const llvm::json::Object &request) {
   }
   llvm::json::Array targets;
 
-  if (!text.empty() &&
-      llvm::StringRef(text).starts_with(g_dap.command_escape_prefix)) {
-    text = text.substr(g_dap.command_escape_prefix.size());
+  bool had_escape_prefix =
+      llvm::StringRef(text).starts_with(g_dap.command_escape_prefix);
+  ReplMode completion_mode = g_dap.DetectReplMode(frame, text, true);
+
+  // Handle the offset change introduced by stripping out the
+  // `command_escape_prefix`.
+  if (had_escape_prefix) {
+    if (offset < static_cast<int64_t>(g_dap.command_escape_prefix.size())) {
+      body.try_emplace("targets", std::move(targets));
+      response.try_emplace("body", std::move(body));
+      g_dap.SendJSON(llvm::json::Value(std::move(response)));
+      return;
+    }
+    offset -= g_dap.command_escape_prefix.size();
   }
 
   // While the user is typing then we likely have an incomplete input and cannot
@@ -1409,7 +1420,7 @@ void request_completions(const llvm::json::Object &request) {
        std::make_tuple(ReplMode::Variable, expr_prefix + text,
                        offset + expr_prefix.size())}};
   for (const auto &[mode, line, cursor] : exprs) {
-    if (g_dap.repl_mode != ReplMode::Auto && g_dap.repl_mode != mode)
+    if (completion_mode != ReplMode::Auto && completion_mode != mode)
       continue;
 
     lldb::SBStringList matches;
@@ -1571,10 +1582,10 @@ void request_evaluate(const llvm::json::Object &request) {
   bool repeat_last_command =
       expression.empty() && g_dap.last_nonempty_var_expression.empty();
 
-  if (context == "repl" && (repeat_last_command ||
-                            (!expression.empty() &&
-                             g_dap.DetectExpressionContext(frame, expression) ==
-                                 ExpressionContext::Command))) {
+  if (context == "repl" &&
+      (repeat_last_command ||
+       (!expression.empty() &&
+        g_dap.DetectReplMode(frame, expression, false) == ReplMode::Command))) {
     // Since the current expression is not for a variable, clear the
     // last_nonempty_var_expression field.
     g_dap.last_nonempty_var_expression.clear();
diff --git a/lldb/tools/lldb-dap/package.json b/lldb/tools/lldb-dap/package.json
index 7b6133d1c20a7..480929703e4b5 100644
--- a/lldb/tools/lldb-dap/package.json
+++ b/lldb/tools/lldb-dap/package.json
@@ -8,7 +8,8 @@
   "license": "Apache 2.0 License with LLVM exceptions",
   "repository": {
     "type": "git",
-    "url": "https://github.com/llvm/vscode-lldb.git"
+    "url": "https://github.com/llvm/llvm-project.git",
+    "directory": "lldb/tools/lldb-dap/"
   },
   "bugs": {
     "url": "https://github.com/llvm/llvm-project/issues"
diff --git a/lldb/tools/lldb-server/Acceptor.cpp b/lldb/tools/lldb-server/Acceptor.cpp
deleted file mode 100644
index fc692a847d9e0..0000000000000
--- a/lldb/tools/lldb-server/Acceptor.cpp
+++ /dev/null
@@ -1,98 +0,0 @@
-//===-- Acceptor.cpp --------------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "Acceptor.h"
-
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Support/ScopedPrinter.h"
-
-#include "lldb/Host/ConnectionFileDescriptor.h"
-#include "lldb/Host/common/TCPSocket.h"
-#include "lldb/Utility/StreamString.h"
-#include "lldb/Utility/UriParser.h"
-#include <optional>
-
-using namespace lldb;
-using namespace lldb_private;
-using namespace lldb_private::lldb_server;
-using namespace llvm;
-
-Status Acceptor::Listen(int backlog) {
-  return m_listener_socket_up->Listen(StringRef(m_name), backlog);
-}
-
-Status Acceptor::Accept(const bool child_processes_inherit, Connection *&conn) {
-  Socket *conn_socket = nullptr;
-  auto error = m_listener_socket_up->Accept(conn_socket);
-  if (error.Success())
-    conn = new ConnectionFileDescriptor(conn_socket);
-
-  return error;
-}
-
-Socket::SocketProtocol Acceptor::GetSocketProtocol() const {
-  return m_listener_socket_up->GetSocketProtocol();
-}
-
-const char *Acceptor::GetSocketScheme() const {
-  return Socket::FindSchemeByProtocol(GetSocketProtocol());
-}
-
-std::string Acceptor::GetLocalSocketId() const { return m_local_socket_id(); }
-
-std::unique_ptr<Acceptor> Acceptor::Create(StringRef name,
-                                           const bool child_processes_inherit,
-                                           Status &error) {
-  error.Clear();
-
-  Socket::SocketProtocol socket_protocol = Socket::ProtocolUnixDomain;
-  // Try to match socket name as URL - e.g., tcp://localhost:5555
-  if (std::optional<URI> res = URI::Parse(name)) {
-    if (!Socket::FindProtocolByScheme(res->scheme.str().c_str(),
-                                      socket_protocol))
-      error = Status::FromErrorStringWithFormat(
-          "Unknown protocol scheme \"%s\"", res->scheme.str().c_str());
-    else
-      name = name.drop_front(res->scheme.size() + strlen("://"));
-  } else {
-    // Try to match socket name as $host:port - e.g., localhost:5555
-    if (!llvm::errorToBool(Socket::DecodeHostAndPort(name).takeError()))
-      socket_protocol = Socket::ProtocolTcp;
-  }
-
-  if (error.Fail())
-    return std::unique_ptr<Acceptor>();
-
-  std::unique_ptr<Socket> listener_socket_up =
-      Socket::Create(socket_protocol, child_processes_inherit, error);
-
-  LocalSocketIdFunc local_socket_id;
-  if (error.Success()) {
-    if (listener_socket_up->GetSocketProtocol() == Socket::ProtocolTcp) {
-      TCPSocket *tcp_socket =
-          static_cast<TCPSocket *>(listener_socket_up.get());
-      local_socket_id = [tcp_socket]() {
-        auto local_port = tcp_socket->GetLocalPortNumber();
-        return (local_port != 0) ? llvm::to_string(local_port) : "";
-      };
-    } else {
-      const std::string socket_name = std::string(name);
-      local_socket_id = [socket_name]() { return socket_name; };
-    }
-
-    return std::unique_ptr<Acceptor>(
-        new Acceptor(std::move(listener_socket_up), name, local_socket_id));
-  }
-
-  return std::unique_ptr<Acceptor>();
-}
-
-Acceptor::Acceptor(std::unique_ptr<Socket> &&listener_socket, StringRef name,
-                   const LocalSocketIdFunc &local_socket_id)
-    : m_listener_socket_up(std::move(listener_socket)), m_name(name.str()),
-      m_local_socket_id(local_socket_id) {}
diff --git a/lldb/tools/lldb-server/Acceptor.h b/lldb/tools/lldb-server/Acceptor.h
deleted file mode 100644
index b441e92dcd22f..0000000000000
--- a/lldb/tools/lldb-server/Acceptor.h
+++ /dev/null
@@ -1,60 +0,0 @@
-//===-- Acceptor.h ----------------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-#ifndef LLDB_TOOLS_LLDB_SERVER_ACCEPTOR_H
-#define LLDB_TOOLS_LLDB_SERVER_ACCEPTOR_H
-
-#include "lldb/Host/Socket.h"
-#include "lldb/Utility/Connection.h"
-#include "lldb/Utility/Status.h"
-
-#include <functional>
-#include <memory>
-#include <string>
-
-namespace llvm {
-class StringRef;
-}
-
-namespace lldb_private {
-namespace lldb_server {
-
-class Acceptor {
-public:
-  virtual ~Acceptor() = default;
-
-  Status Listen(int backlog);
-
-  Status Accept(const bool child_processes_inherit, Connection *&conn);
-
-  static std::unique_ptr<Acceptor> Create(llvm::StringRef name,
-                                          const bool child_processes_inherit,
-                                          Status &error);
-
-  Socket::SocketProtocol GetSocketProtocol() const;
-
-  const char *GetSocketScheme() const;
-
-  // Returns either TCP port number as string or domain socket path.
-  // Empty string is returned in case of error.
-  std::string GetLocalSocketId() const;
-
-private:
-  typedef std::function<std::string()> LocalSocketIdFunc;
-
-  Acceptor(std::unique_ptr<Socket> &&listener_socket, llvm::StringRef name,
-           const LocalSocketIdFunc &local_socket_id);
-
-  const std::unique_ptr<Socket> m_listener_socket_up;
-  const std::string m_name;
-  const LocalSocketIdFunc m_local_socket_id;
-};
-
-} // namespace lldb_server
-} // namespace lldb_private
-
-#endif // LLDB_TOOLS_LLDB_SERVER_ACCEPTOR_H
diff --git a/lldb/tools/lldb-server/CMakeLists.txt b/lldb/tools/lldb-server/CMakeLists.txt
index 9030ed709a647..e6d88a8579885 100644
--- a/lldb/tools/lldb-server/CMakeLists.txt
+++ b/lldb/tools/lldb-server/CMakeLists.txt
@@ -37,7 +37,6 @@ if(APPLE_EMBEDDED)
 endif()
 
 add_lldb_tool(lldb-server
-    Acceptor.cpp
     lldb-gdbserver.cpp
     lldb-platform.cpp
     lldb-server.cpp
diff --git a/lldb/tools/lldb-server/lldb-gdbserver.cpp b/lldb/tools/lldb-server/lldb-gdbserver.cpp
index d7503112697ef..bc93c837a107b 100644
--- a/lldb/tools/lldb-server/lldb-gdbserver.cpp
+++ b/lldb/tools/lldb-server/lldb-gdbserver.cpp
@@ -24,8 +24,8 @@
 #include "lldb/Host/ConnectionFileDescriptor.h"
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Host/Pipe.h"
-#include "lldb/Host/Socket.h"
 #include "lldb/Host/common/NativeProcessProtocol.h"
+#include "lldb/Host/common/TCPSocket.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Status.h"
@@ -202,10 +202,21 @@ void ConnectToRemote(MainLoop &mainloop,
   std::string url;
 
   if (connection_fd != SharedSocket::kInvalidFD) {
+#ifdef _WIN32
+    NativeSocket sockfd;
+    error = SharedSocket::GetNativeSocket(connection_fd, sockfd);
+    if (error.Fail()) {
+      llvm::errs() << llvm::formatv("error: GetNativeSocket failed: {0}\n",
+                                    error.AsCString());
+      exit(-1);
+    }
+    connection_up =
+        std::unique_ptr<Connection>(new ConnectionFileDescriptor(new TCPSocket(
+            sockfd, /*should_close=*/true, /*child_processes_inherit=*/false)));
+#else
     url = llvm::formatv("fd://{0}", connection_fd).str();
 
     // Create the connection.
-#if LLDB_ENABLE_POSIX && !defined _WIN32
     ::fcntl(connection_fd, F_SETFD, FD_CLOEXEC);
 #endif
   } else if (!host_and_port.empty()) {
diff --git a/lldb/tools/lldb-server/lldb-platform.cpp b/lldb/tools/lldb-server/lldb-platform.cpp
index c021bc7e30194..2ef780578d0a2 100644
--- a/lldb/tools/lldb-server/lldb-platform.cpp
+++ b/lldb/tools/lldb-server/lldb-platform.cpp
@@ -26,18 +26,22 @@
 #include "llvm/Support/WithColor.h"
 #include "llvm/Support/raw_ostream.h"
 
-#include "Acceptor.h"
 #include "LLDBServerUtilities.h"
 #include "Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.h"
 #include "Plugins/Process/gdb-remote/ProcessGDBRemoteLog.h"
 #include "lldb/Host/ConnectionFileDescriptor.h"
 #include "lldb/Host/HostGetOpt.h"
+#include "lldb/Host/MainLoop.h"
 #include "lldb/Host/OptionParser.h"
 #include "lldb/Host/Socket.h"
 #include "lldb/Host/common/TCPSocket.h"
+#if LLDB_ENABLE_POSIX
+#include "lldb/Host/posix/DomainSocket.h"
+#endif
 #include "lldb/Utility/FileSpec.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/Utility/UriParser.h"
 
 using namespace lldb;
 using namespace lldb_private;
@@ -45,19 +49,22 @@ using namespace lldb_private::lldb_server;
 using namespace lldb_private::process_gdb_remote;
 using namespace llvm;
 
-// option descriptors for getopt_long_only()
-
+// The test suite makes many connections in parallel, let's not miss any.
+// The highest this should get reasonably is a function of the number
+// of target CPUs. For now, let's just use 100.
+static const int backlog = 100;
+static const int socket_error = -1;
 static int g_debug = 0;
 static int g_verbose = 0;
 static int g_server = 0;
 
+// option descriptors for getopt_long_only()
 static struct option g_long_options[] = {
     {"debug", no_argument, &g_debug, 1},
     {"verbose", no_argument, &g_verbose, 1},
     {"log-file", required_argument, nullptr, 'l'},
     {"log-channels", required_argument, nullptr, 'c'},
     {"listen", required_argument, nullptr, 'L'},
-    {"port-offset", required_argument, nullptr, 'p'},
     {"gdbserver-port", required_argument, nullptr, 'P'},
     {"min-gdbserver-port", required_argument, nullptr, 'm'},
     {"max-gdbserver-port", required_argument, nullptr, 'M'},
@@ -98,6 +105,55 @@ static void display_usage(const char *progname, const char *subcommand) {
   exit(0);
 }
 
+static Status parse_listen_host_port(Socket::SocketProtocol &protocol,
+                                     const std::string &listen_host_port,
+                                     std::string &address,
+                                     uint16_t &platform_port,
+                                     std::string &gdb_address,
+                                     const uint16_t gdbserver_port) {
+  std::string hostname;
+  // Try to match socket name as URL - e.g., tcp://localhost:5555
+  if (std::optional<URI> uri = URI::Parse(listen_host_port)) {
+    if (!Socket::FindProtocolByScheme(uri->scheme.str().c_str(), protocol)) {
+      return Status::FromErrorStringWithFormat(
+          "Unknown protocol scheme \"%s\".", uri->scheme.str().c_str());
+    }
+    if (protocol == Socket::ProtocolTcp) {
+      hostname = uri->hostname;
+      if (uri->port) {
+        platform_port = *(uri->port);
+      }
+    } else
+      address = listen_host_port.substr(uri->scheme.size() + strlen("://"));
+  } else {
+    // Try to match socket name as $host:port - e.g., localhost:5555
+    llvm::Expected<Socket::HostAndPort> host_port =
+        Socket::DecodeHostAndPort(listen_host_port);
+    if (!llvm::errorToBool(host_port.takeError())) {
+      protocol = Socket::ProtocolTcp;
+      hostname = host_port->hostname;
+      platform_port = host_port->port;
+    } else
+      address = listen_host_port;
+  }
+
+  if (protocol == Socket::ProtocolTcp) {
+    if (platform_port != 0 && platform_port == gdbserver_port) {
+      return Status::FromErrorStringWithFormat(
+          "The same platform and gdb ports %u.", platform_port);
+    }
+    address = llvm::formatv("[{0}]:{1}", hostname, platform_port).str();
+    gdb_address = llvm::formatv("[{0}]:{1}", hostname, gdbserver_port).str();
+  } else {
+    if (gdbserver_port) {
+      return Status::FromErrorStringWithFormat(
+          "--gdbserver-port %u is redundant for non-tcp protocol %s.",
+          gdbserver_port, Socket::FindSchemeByProtocol(protocol));
+    }
+  }
+  return Status();
+}
+
 static Status save_socket_id_to_file(const std::string &socket_id,
                                      const FileSpec &file_spec) {
   FileSpec temp_file_spec(file_spec.GetDirectory().GetStringRef());
@@ -119,6 +175,59 @@ static Status save_socket_id_to_file(const std::string &socket_id,
   return status;
 }
 
+static Status ListenGdbConnectionsIfNeeded(
+    const Socket::SocketProtocol protocol, std::unique_ptr<TCPSocket> &gdb_sock,
+    const std::string &gdb_address, uint16_t &gdbserver_port) {
+  if (protocol != Socket::ProtocolTcp)
+    return Status();
+
+  gdb_sock = std::make_unique<TCPSocket>(
+      /*should_close=*/true, /*child_processes_inherit=*/false);
+  Status error = gdb_sock->Listen(gdb_address, backlog);
+  if (error.Fail())
+    return error;
+
+  if (gdbserver_port == 0)
+    gdbserver_port = gdb_sock->GetLocalPortNumber();
+
+  return Status();
+}
+
+static llvm::Expected<std::vector<MainLoopBase::ReadHandleUP>>
+AcceptGdbConnectionsIfNeeded(const Socket::SocketProtocol protocol,
+                             std::unique_ptr<TCPSocket> &gdb_sock,
+                             MainLoop &main_loop, const uint16_t gdbserver_port,
+                             const lldb_private::Args &args) {
+  if (protocol != Socket::ProtocolTcp)
+    return std::vector<MainLoopBase::ReadHandleUP>();
+
+  return gdb_sock->Accept(main_loop, [gdbserver_port,
+                                      &args](std::unique_ptr<Socket> sock_up) {
+    Log *log = GetLog(LLDBLog::Platform);
+    Status error;
+    SharedSocket shared_socket(sock_up.get(), error);
+    if (error.Fail()) {
+      LLDB_LOGF(log, "gdbserver SharedSocket failed: %s", error.AsCString());
+      return;
+    }
+    lldb::pid_t child_pid = LLDB_INVALID_PROCESS_ID;
+    std::string socket_name;
+    GDBRemoteCommunicationServerPlatform platform(Socket::ProtocolTcp,
+                                                  gdbserver_port);
+    error = platform.LaunchGDBServer(args, child_pid, socket_name,
+                                     shared_socket.GetSendableFD());
+    if (error.Success() && child_pid != LLDB_INVALID_PROCESS_ID) {
+      error = shared_socket.CompleteSending(child_pid);
+      if (error.Fail()) {
+        Host::Kill(child_pid, SIGTERM);
+        LLDB_LOGF(log, "gdbserver CompleteSending failed: %s",
+                  error.AsCString());
+        return;
+      }
+    }
+  });
+}
+
 static void client_handle(GDBRemoteCommunicationServerPlatform &platform,
                           const lldb_private::Args &args) {
   if (!platform.IsConnected())
@@ -126,13 +235,11 @@ static void client_handle(GDBRemoteCommunicationServerPlatform &platform,
 
   if (args.GetArgumentCount() > 0) {
     lldb::pid_t pid = LLDB_INVALID_PROCESS_ID;
-    std::optional<uint16_t> port;
     std::string socket_name;
-    Status error = platform.LaunchGDBServer(args,
-                                            "", // hostname
-                                            pid, port, socket_name);
+    Status error = platform.LaunchGDBServer(args, pid, socket_name,
+                                            SharedSocket::kInvalidFD);
     if (error.Success())
-      platform.SetPendingGdbServer(pid, *port, socket_name);
+      platform.SetPendingGdbServer(socket_name);
     else
       fprintf(stderr, "failed to start gdbserver: %s\n", error.AsCString());
   }
@@ -150,19 +257,11 @@ static void client_handle(GDBRemoteCommunicationServerPlatform &platform,
   printf("Disconnected.\n");
 }
 
-static GDBRemoteCommunicationServerPlatform::PortMap gdbserver_portmap;
-static std::mutex gdbserver_portmap_mutex;
-
-static void spawn_process_reaped(lldb::pid_t pid, int signal, int status) {
-  std::lock_guard<std::mutex> guard(gdbserver_portmap_mutex);
-  gdbserver_portmap.FreePortForProcess(pid);
-}
-
 static Status spawn_process(const char *progname, const Socket *conn_socket,
-                            uint16_t gdb_port, uint16_t port_offset,
-                            const lldb_private::Args &args,
+                            uint16_t gdb_port, const lldb_private::Args &args,
                             const std::string &log_file,
-                            const StringRef log_channels) {
+                            const StringRef log_channels, MainLoop &main_loop,
+                            std::promise<void> &child_exited) {
   Status error;
   SharedSocket shared_socket(conn_socket, error);
   if (error.Fail())
@@ -176,14 +275,14 @@ static Status spawn_process(const char *progname, const Socket *conn_socket,
   self_args.AppendArgument(llvm::StringRef("platform"));
   self_args.AppendArgument(llvm::StringRef("--child-platform-fd"));
   self_args.AppendArgument(llvm::to_string(shared_socket.GetSendableFD()));
+#ifndef _WIN32
+  launch_info.AppendDuplicateFileAction((int)shared_socket.GetSendableFD(),
+                                        (int)shared_socket.GetSendableFD());
+#endif
   if (gdb_port) {
     self_args.AppendArgument(llvm::StringRef("--gdbserver-port"));
     self_args.AppendArgument(llvm::to_string(gdb_port));
   }
-  if (port_offset > 0) {
-    self_args.AppendArgument(llvm::StringRef("--port-offset"));
-    self_args.AppendArgument(llvm::to_string(port_offset));
-  }
   if (!log_file.empty()) {
     self_args.AppendArgument(llvm::StringRef("--log-file"));
     self_args.AppendArgument(log_file);
@@ -198,7 +297,16 @@ static Status spawn_process(const char *progname, const Socket *conn_socket,
   }
 
   launch_info.SetLaunchInSeparateProcessGroup(false);
-  launch_info.SetMonitorProcessCallback(&spawn_process_reaped);
+
+  if (g_server)
+    launch_info.SetMonitorProcessCallback([](lldb::pid_t, int, int) {});
+  else
+    launch_info.SetMonitorProcessCallback(
+        [&child_exited, &main_loop](lldb::pid_t, int, int) {
+          main_loop.AddPendingCallback(
+              [](MainLoopBase &loop) { loop.RequestTermination(); });
+          child_exited.set_value();
+        });
 
   // Copy the current environment.
   launch_info.GetEnvironment() = Host::GetEnvironment();
@@ -229,11 +337,6 @@ static Status spawn_process(const char *progname, const Socket *conn_socket,
   LLDB_LOG(GetLog(LLDBLog::Platform), "lldb-platform launched '{0}', pid={1}",
            cmd, child_pid);
 
-  {
-    std::lock_guard<std::mutex> guard(gdbserver_portmap_mutex);
-    gdbserver_portmap.AssociatePortWithProcess(gdb_port, child_pid);
-  }
-
   error = shared_socket.CompleteSending(child_pid);
   if (error.Fail()) {
     Host::Kill(child_pid, SIGTERM);
@@ -264,14 +367,11 @@ int main_platform(int argc, char *argv[]) {
 
   shared_fd_t fd = SharedSocket::kInvalidFD;
 
-  int min_gdbserver_port = 0;
-  int max_gdbserver_port = 0;
-  uint16_t port_offset = 0;
+  uint16_t gdbserver_port = 0;
 
   FileSpec socket_file;
   bool show_usage = false;
   int option_error = 0;
-  int socket_error = -1;
 
   std::string short_options(OptionParser::GetShortOptionString(g_long_options));
 
@@ -307,21 +407,6 @@ int main_platform(int argc, char *argv[]) {
         socket_file.SetFile(optarg, FileSpec::Style::native);
       break;
 
-    case 'p': {
-      if (!llvm::to_integer(optarg, port_offset)) {
-        WithColor::error() << "invalid port offset string " << optarg << "\n";
-        option_error = 4;
-        break;
-      }
-      if (port_offset < LOW_PORT || port_offset > HIGH_PORT) {
-        WithColor::error() << llvm::formatv(
-            "port offset {0} is not in the "
-            "valid user port range of {1} - {2}\n",
-            port_offset, LOW_PORT, HIGH_PORT);
-        option_error = 5;
-      }
-    } break;
-
     case 'P':
     case 'm':
     case 'M': {
@@ -331,20 +416,12 @@ int main_platform(int argc, char *argv[]) {
         option_error = 2;
         break;
       }
-      if (portnum < LOW_PORT || portnum > HIGH_PORT) {
-        WithColor::error() << llvm::formatv(
-            "port number {0} is not in the "
-            "valid user port range of {1} - {2}\n",
-            portnum, LOW_PORT, HIGH_PORT);
-        option_error = 1;
-        break;
-      }
+      // Note the condition gdbserver_port > HIGH_PORT is valid in case of using
+      // --child-platform-fd. Check gdbserver_port later.
       if (ch == 'P')
-        gdbserver_portmap.AllowPort(portnum);
-      else if (ch == 'm')
-        min_gdbserver_port = portnum;
-      else
-        max_gdbserver_port = portnum;
+        gdbserver_port = portnum;
+      else if (gdbserver_port == 0)
+        gdbserver_port = portnum;
     } break;
 
     case 2: {
@@ -366,18 +443,6 @@ int main_platform(int argc, char *argv[]) {
   if (!LLDBServerUtilities::SetupLogging(log_file, log_channels, 0))
     return -1;
 
-  // Make a port map for a port range that was specified.
-  if (min_gdbserver_port && min_gdbserver_port < max_gdbserver_port) {
-    gdbserver_portmap = GDBRemoteCommunicationServerPlatform::PortMap(
-        min_gdbserver_port, max_gdbserver_port);
-  } else if (min_gdbserver_port || max_gdbserver_port) {
-    WithColor::error() << llvm::formatv(
-        "--min-gdbserver-port ({0}) is not lower than "
-        "--max-gdbserver-port ({1})\n",
-        min_gdbserver_port, max_gdbserver_port);
-    option_error = 3;
-  }
-
   // Print usage and exit if no listening port is specified.
   if (listen_host_port.empty() && fd == SharedSocket::kInvalidFD)
     show_usage = true;
@@ -393,55 +458,82 @@ int main_platform(int argc, char *argv[]) {
   lldb_private::Args inferior_arguments;
   inferior_arguments.SetArguments(argc, const_cast<const char **>(argv));
 
+  Socket::SocketProtocol protocol = Socket::ProtocolUnixDomain;
+
   if (fd != SharedSocket::kInvalidFD) {
     // Child process will handle the connection and exit.
+    if (gdbserver_port)
+      protocol = Socket::ProtocolTcp;
+
     Log *log = GetLog(LLDBLog::Platform);
-    if (!listen_host_port.empty()) {
-      LLDB_LOGF(log, "lldb-platform child: "
-                     "ambiguous parameters --listen and --child-platform-fd");
-      return socket_error;
-    }
 
-    NativeSocket socket;
-    error = SharedSocket::GetNativeSocket(fd, socket);
+    NativeSocket sockfd;
+    error = SharedSocket::GetNativeSocket(fd, sockfd);
     if (error.Fail()) {
       LLDB_LOGF(log, "lldb-platform child: %s", error.AsCString());
       return socket_error;
     }
 
-    GDBRemoteCommunicationServerPlatform platform(Socket::ProtocolTcp, "tcp");
-    if (port_offset > 0)
-      platform.SetPortOffset(port_offset);
-    platform.SetPortMap(std::move(gdbserver_portmap));
+    GDBRemoteCommunicationServerPlatform platform(protocol, gdbserver_port);
+    Socket *socket;
+    if (protocol == Socket::ProtocolTcp)
+      socket = new TCPSocket(sockfd, /*should_close=*/true,
+                             /*child_processes_inherit=*/false);
+    else {
+#if LLDB_ENABLE_POSIX
+      socket = new DomainSocket(sockfd, /*should_close=*/true,
+                                /*child_processes_inherit=*/false);
+#else
+      WithColor::error() << "lldb-platform child: Unix domain sockets are not "
+                            "supported on this platform.";
+      return socket_error;
+#endif
+    }
     platform.SetConnection(
-        std::unique_ptr<Connection>(new ConnectionFileDescriptor(
-            new TCPSocket(socket, /*should_close=*/true,
-                          /*child_processes_inherit=*/false))));
+        std::unique_ptr<Connection>(new ConnectionFileDescriptor(socket)));
     client_handle(platform, inferior_arguments);
     return 0;
   }
 
-  const bool children_inherit_listen_socket = false;
-  // the test suite makes many connections in parallel, let's not miss any.
-  // The highest this should get reasonably is a function of the number
-  // of target CPUs. For now, let's just use 100.
-  const int backlog = 100;
+  if (gdbserver_port != 0 &&
+      (gdbserver_port < LOW_PORT || gdbserver_port > HIGH_PORT)) {
+    WithColor::error() << llvm::formatv("Port number {0} is not in the "
+                                        "valid user port range of {1} - {2}\n",
+                                        gdbserver_port, LOW_PORT, HIGH_PORT);
+    return 1;
+  }
 
-  std::unique_ptr<Acceptor> acceptor_up(Acceptor::Create(
-      listen_host_port, children_inherit_listen_socket, error));
+  std::string address;
+  std::string gdb_address;
+  uint16_t platform_port = 0;
+  error = parse_listen_host_port(protocol, listen_host_port, address,
+                                 platform_port, gdb_address, gdbserver_port);
   if (error.Fail()) {
-    fprintf(stderr, "failed to create acceptor: %s", error.AsCString());
-    exit(socket_error);
+    printf("Failed to parse listen address: %s\n", error.AsCString());
+    return socket_error;
   }
 
-  error = acceptor_up->Listen(backlog);
+  std::unique_ptr<Socket> platform_sock =
+      Socket::Create(protocol, /*child_processes_inherit=*/false, error);
   if (error.Fail()) {
-    printf("failed to listen: %s\n", error.AsCString());
-    exit(socket_error);
+    printf("Failed to create platform socket: %s\n", error.AsCString());
+    return socket_error;
   }
+  error = platform_sock->Listen(address, backlog);
+  if (error.Fail()) {
+    printf("Failed to listen platform: %s\n", error.AsCString());
+    return socket_error;
+  }
+  if (protocol == Socket::ProtocolTcp && platform_port == 0)
+    platform_port =
+        static_cast<TCPSocket *>(platform_sock.get())->GetLocalPortNumber();
+
   if (socket_file) {
-    error =
-        save_socket_id_to_file(acceptor_up->GetLocalSocketId(), socket_file);
+    error = save_socket_id_to_file(
+        protocol == Socket::ProtocolTcp
+            ? (platform_port ? llvm::to_string(platform_port) : "")
+            : address,
+        socket_file);
     if (error.Fail()) {
       fprintf(stderr, "failed to write socket id to %s: %s\n",
               socket_file.GetPath().c_str(), error.AsCString());
@@ -449,69 +541,58 @@ int main_platform(int argc, char *argv[]) {
     }
   }
 
-  GDBRemoteCommunicationServerPlatform platform(
-      acceptor_up->GetSocketProtocol(), acceptor_up->GetSocketScheme());
-  if (port_offset > 0)
-    platform.SetPortOffset(port_offset);
+  std::unique_ptr<TCPSocket> gdb_sock;
+  // Update gdbserver_port if it is still 0 and protocol is tcp.
+  error = ListenGdbConnectionsIfNeeded(protocol, gdb_sock, gdb_address,
+                                       gdbserver_port);
+  if (error.Fail()) {
+    printf("Failed to listen gdb: %s\n", error.AsCString());
+    return socket_error;
+  }
 
-  do {
-    const bool children_inherit_accept_socket = true;
-    Connection *conn = nullptr;
-    error = acceptor_up->Accept(children_inherit_accept_socket, conn);
-    if (error.Fail()) {
-      WithColor::error() << error.AsCString() << '\n';
-      exit(socket_error);
+  std::promise<void> child_exited;
+  MainLoop main_loop;
+  {
+    llvm::Expected<std::vector<MainLoopBase::ReadHandleUP>> platform_handles =
+        platform_sock->Accept(
+            main_loop, [progname, gdbserver_port, &inferior_arguments, log_file,
+                        log_channels, &main_loop, &child_exited,
+                        &platform_handles](std::unique_ptr<Socket> sock_up) {
+              printf("Connection established.\n");
+              Status error = spawn_process(
+                  progname, sock_up.get(), gdbserver_port, inferior_arguments,
+                  log_file, log_channels, main_loop, child_exited);
+              if (error.Fail()) {
+                Log *log = GetLog(LLDBLog::Platform);
+                LLDB_LOGF(log, "spawn_process failed: %s", error.AsCString());
+                WithColor::error()
+                    << "spawn_process failed: " << error.AsCString() << "\n";
+                if (!g_server) {
+                  main_loop.RequestTermination();
+                  child_exited.set_value();
+                }
+              }
+              if (!g_server)
+                platform_handles->clear();
+            });
+    if (!platform_handles) {
+      printf("Failed to accept platform: %s\n",
+             llvm::toString(platform_handles.takeError()).c_str());
+      return socket_error;
     }
-    printf("Connection established.\n");
-
-    if (g_server) {
-      std::optional<uint16_t> available_port;
-      {
-        std::lock_guard<std::mutex> guard(gdbserver_portmap_mutex);
-        auto port = gdbserver_portmap.GetNextAvailablePort();
-        if (port)
-          available_port = *port;
-        else
-          llvm::consumeError(port.takeError());
-      }
-      if (!available_port) {
-        fprintf(stderr,
-                "no available gdbserver port for connection - dropping...\n");
-      } else {
-        const Socket *conn_socket =
-            static_cast<const Socket *>(conn->GetReadObject().get());
-        error =
-            spawn_process(progname, conn_socket, *available_port, port_offset,
-                          inferior_arguments, log_file, log_channels);
-        if (error.Fail()) {
-          {
-
-            std::lock_guard<std::mutex> guard(gdbserver_portmap_mutex);
-            gdbserver_portmap.FreePort(*available_port);
-          }
-          LLDB_LOGF(GetLog(LLDBLog::Platform), "spawn_process failed: %s",
-                    error.AsCString());
-          WithColor::error()
-              << "spawn_process failed: " << error.AsCString() << "\n";
-        }
-      }
-      // Parent doesn't need a connection to the lldb client
-      delete conn;
-
-      // Parent will continue to listen for new connections.
-      continue;
-    } else {
-      // If not running as a server, this process will not accept
-      // connections while a connection is active.
-      acceptor_up.reset();
-
-      // When not running in server mode, use all available ports
-      platform.SetPortMap(std::move(gdbserver_portmap));
+
+    llvm::Expected<std::vector<MainLoopBase::ReadHandleUP>> gdb_handles =
+        AcceptGdbConnectionsIfNeeded(protocol, gdb_sock, main_loop,
+                                     gdbserver_port, inferior_arguments);
+    if (!gdb_handles) {
+      printf("Failed to accept gdb: %s\n",
+             llvm::toString(gdb_handles.takeError()).c_str());
+      return socket_error;
     }
 
-    platform.SetConnection(std::unique_ptr<Connection>(conn));
-    client_handle(platform, inferior_arguments);
-  } while (g_server);
+    main_loop.Run();
+  }
+  child_exited.get_future().get();
 
   fprintf(stderr, "lldb-server exiting...\n");
 
diff --git a/lldb/unittests/Process/gdb-remote/CMakeLists.txt b/lldb/unittests/Process/gdb-remote/CMakeLists.txt
index de14dc0169c13..41bce6f31eafe 100644
--- a/lldb/unittests/Process/gdb-remote/CMakeLists.txt
+++ b/lldb/unittests/Process/gdb-remote/CMakeLists.txt
@@ -5,7 +5,6 @@ add_lldb_unittest(ProcessGdbRemoteTests
   GDBRemoteCommunicationServerTest.cpp
   GDBRemoteCommunicationTest.cpp
   GDBRemoteTestUtils.cpp
-  PortMapTest.cpp
 
   LINK_LIBS
     lldbCore
diff --git a/lldb/unittests/Process/gdb-remote/PortMapTest.cpp b/lldb/unittests/Process/gdb-remote/PortMapTest.cpp
deleted file mode 100644
index 8671ebe0fd089..0000000000000
--- a/lldb/unittests/Process/gdb-remote/PortMapTest.cpp
+++ /dev/null
@@ -1,115 +0,0 @@
-//===-- PortMapTest.cpp ---------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Testing/Support/Error.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-#include "Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.h"
-
-using namespace lldb_private::process_gdb_remote;
-
-TEST(PortMapTest, Constructors) {
-  // Default construct to empty map
-  GDBRemoteCommunicationServerPlatform::PortMap p1;
-  ASSERT_TRUE(p1.empty());
-
-  // Empty means no restrictions, return 0 and bind to get a port
-  llvm::Expected<uint16_t> available_port = p1.GetNextAvailablePort();
-  ASSERT_THAT_EXPECTED(available_port, llvm::HasValue(0));
-
-  // Adding any port makes it not empty
-  p1.AllowPort(1);
-  ASSERT_FALSE(p1.empty());
-
-  // So we will return the added port this time
-  available_port = p1.GetNextAvailablePort();
-  ASSERT_THAT_EXPECTED(available_port, llvm::HasValue(1));
-
-  // Construct from a range of ports
-  GDBRemoteCommunicationServerPlatform::PortMap p2(1, 4);
-  ASSERT_FALSE(p2.empty());
-
-  // Use up all the ports
-  for (uint16_t expected = 1; expected < 4; ++expected) {
-    available_port = p2.GetNextAvailablePort();
-    ASSERT_THAT_EXPECTED(available_port, llvm::HasValue(expected));
-    p2.AssociatePortWithProcess(*available_port, 1);
-  }
-
-  // Now we fail since we're not an empty port map but all ports are used
-  available_port = p2.GetNextAvailablePort();
-  ASSERT_THAT_EXPECTED(available_port, llvm::Failed());
-}
-
-TEST(PortMapTest, FreePort) {
-  GDBRemoteCommunicationServerPlatform::PortMap p(1, 4);
-  // Use up all the ports
-  for (uint16_t port = 1; port < 4; ++port) {
-    p.AssociatePortWithProcess(port, 1);
-  }
-
-  llvm::Expected<uint16_t> available_port = p.GetNextAvailablePort();
-  ASSERT_THAT_EXPECTED(available_port, llvm::Failed());
-
-  // Can't free a port that isn't in the map
-  ASSERT_FALSE(p.FreePort(0));
-  ASSERT_FALSE(p.FreePort(4));
-
-  // After freeing a port it becomes available
-  ASSERT_TRUE(p.FreePort(2));
-  available_port = p.GetNextAvailablePort();
-  ASSERT_THAT_EXPECTED(available_port, llvm::HasValue(2));
-}
-
-TEST(PortMapTest, FreePortForProcess) {
-  GDBRemoteCommunicationServerPlatform::PortMap p;
-  p.AllowPort(1);
-  p.AllowPort(2);
-  ASSERT_TRUE(p.AssociatePortWithProcess(1, 11));
-  ASSERT_TRUE(p.AssociatePortWithProcess(2, 22));
-
-  // All ports have been used
-  llvm::Expected<uint16_t> available_port = p.GetNextAvailablePort();
-  ASSERT_THAT_EXPECTED(available_port, llvm::Failed());
-
-  // Can't free a port for a process that doesn't have any
-  ASSERT_FALSE(p.FreePortForProcess(33));
-
-  // You can move a used port to a new pid
-  ASSERT_TRUE(p.AssociatePortWithProcess(1, 99));
-
-  ASSERT_TRUE(p.FreePortForProcess(22));
-  available_port = p.GetNextAvailablePort();
-  ASSERT_THAT_EXPECTED(available_port, llvm::Succeeded());
-  ASSERT_EQ(2, *available_port);
-
-  // proces 22 no longer has a port
-  ASSERT_FALSE(p.FreePortForProcess(22));
-}
-
-TEST(PortMapTest, AllowPort) {
-  GDBRemoteCommunicationServerPlatform::PortMap p;
-
-  // Allow port 1 and tie it to process 11
-  p.AllowPort(1);
-  ASSERT_TRUE(p.AssociatePortWithProcess(1, 11));
-
-  // Allowing it a second time shouldn't change existing mapping
-  p.AllowPort(1);
-  llvm::Expected<uint16_t> available_port = p.GetNextAvailablePort();
-  ASSERT_THAT_EXPECTED(available_port, llvm::Failed());
-
-  // A new port is marked as free when allowed
-  p.AllowPort(2);
-  available_port = p.GetNextAvailablePort();
-  ASSERT_THAT_EXPECTED(available_port, llvm::HasValue(2));
-
-  // 11 should still be tied to port 1
-  ASSERT_TRUE(p.FreePortForProcess(11));
-}
diff --git a/lldb/unittests/SymbolFile/DWARF/SymbolFileDWARFTests.cpp b/lldb/unittests/SymbolFile/DWARF/SymbolFileDWARFTests.cpp
index 3192efe8b339b..37ecfea26f36c 100644
--- a/lldb/unittests/SymbolFile/DWARF/SymbolFileDWARFTests.cpp
+++ b/lldb/unittests/SymbolFile/DWARF/SymbolFileDWARFTests.cpp
@@ -9,6 +9,7 @@
 #include "gtest/gtest.h"
 
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolData.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolExe.h"
 #include "llvm/Support/FileSystem.h"
@@ -16,7 +17,6 @@
 
 #include "Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.h"
 #include "Plugins/SymbolFile/DWARF/DWARFDataExtractor.h"
-#include "Plugins/SymbolFile/DWARF/DWARFDebugArangeSet.h"
 #include "Plugins/SymbolFile/DWARF/DWARFDebugAranges.h"
 #include "Plugins/SymbolFile/DWARF/SymbolFileDWARF.h"
 #include "Plugins/SymbolFile/PDB/SymbolFilePDB.h"
@@ -39,6 +39,7 @@ using namespace lldb;
 using namespace lldb_private;
 using namespace lldb_private::dwarf;
 using namespace lldb_private::plugin::dwarf;
+using llvm::DWARFDebugArangeSet;
 
 class SymbolFileDWARFTests : public testing::Test {
   SubsystemRAII<FileSystem, HostInfo, ObjectFilePECOFF, SymbolFileDWARF,
@@ -91,14 +92,15 @@ TEST_F(SymbolFileDWARFTests, ParseArangesNonzeroSegmentSize) {
       // END TUPLES
       0, 0, 0, 0, 0, 0, 0, 0, 0 // terminator
   };
-  DWARFDataExtractor data;
-  data.SetData(static_cast<const void *>(binary_data), sizeof binary_data,
-               lldb::ByteOrder::eByteOrderBig);
+  llvm::DWARFDataExtractor data(llvm::ArrayRef<unsigned char>(binary_data),
+                                /*isLittleEndian=*/false, /*AddrSize=*/4);
+
   DWARFDebugArangeSet debug_aranges;
   offset_t off = 0;
   llvm::Error error = debug_aranges.extract(data, &off);
   EXPECT_TRUE(bool(error));
-  EXPECT_EQ("segmented arange entries are not supported",
+  EXPECT_EQ("non-zero segment selector size in address range table at offset "
+            "0x0 is not supported",
             llvm::toString(std::move(error)));
   EXPECT_EQ(off, 12U); // Parser should read no further than the segment size
 }
@@ -115,12 +117,12 @@ TEST_F(SymbolFileDWARFTests, ParseArangesWithMultipleTerminators) {
   // DWARFDebugArangeSet header using the remaining segment + address pairs
   // from the remaining bytes.
   unsigned char binary_data[] = {
-      0, 0, 0, 0, // unit_length that will be set correctly after this
-      0, 2,       // DWARF version number (uint16_t)
-      0, 0, 0, 0, // CU offset (ignored for the purposes of this test)
-      4,          // address size
-      0,          // segment size
-      0, 0, 0, 0, // alignment for the first tuple
+      0, 0, 0, 0,   // unit_length that will be set correctly after this
+      0, 2,         // DWARF version number (uint16_t)
+      0, 0, 0, 255, // CU offset
+      4,            // address size
+      0,            // segment size
+      0, 0, 0, 0,   // alignment for the first tuple
       // BEGIN TUPLES
       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // premature terminator
       0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, // [0x1000-0x1100)
@@ -135,18 +137,18 @@ TEST_F(SymbolFileDWARFTests, ParseArangesWithMultipleTerminators) {
   DWARFDataExtractor data;
   data.SetData(static_cast<const void *>(binary_data), sizeof binary_data,
                lldb::ByteOrder::eByteOrderBig);
-  DWARFDebugArangeSet set;
-  offset_t off = 0;
-  llvm::Error error = set.extract(data, &off);
-  // Multiple terminators are not fatal as they do appear in binaries.
-  EXPECT_FALSE(bool(error));
+  DWARFDebugAranges debug_aranges;
+  debug_aranges.extract(data);
   // Parser should read all terminators to the end of the length specified.
-  EXPECT_EQ(off, binary_data_size);
-  ASSERT_EQ(set.NumDescriptors(), 2U);
-  ASSERT_EQ(set.GetDescriptorRef(0).address, (dw_addr_t)0x1000);
-  ASSERT_EQ(set.GetDescriptorRef(0).length, (dw_addr_t)0x100);
-  ASSERT_EQ(set.GetDescriptorRef(1).address, (dw_addr_t)0x2000);
-  ASSERT_EQ(set.GetDescriptorRef(1).length, (dw_addr_t)0x10);
+  ASSERT_EQ(debug_aranges.GetNumRanges(), 2U);
+  EXPECT_EQ(debug_aranges.FindAddress(0x0fff), DW_INVALID_OFFSET);
+  EXPECT_EQ(debug_aranges.FindAddress(0x1000), 255u);
+  EXPECT_EQ(debug_aranges.FindAddress(0x1100 - 1), 255u);
+  EXPECT_EQ(debug_aranges.FindAddress(0x1100), DW_INVALID_OFFSET);
+  EXPECT_EQ(debug_aranges.FindAddress(0x1fff), DW_INVALID_OFFSET);
+  EXPECT_EQ(debug_aranges.FindAddress(0x2000), 255u);
+  EXPECT_EQ(debug_aranges.FindAddress(0x2010 - 1), 255u);
+  EXPECT_EQ(debug_aranges.FindAddress(0x2010), DW_INVALID_OFFSET);
 }
 
 TEST_F(SymbolFileDWARFTests, ParseArangesIgnoreEmpty) {
@@ -154,12 +156,12 @@ TEST_F(SymbolFileDWARFTests, ParseArangesIgnoreEmpty) {
   // and we ensure that these are ignored by our DWARFDebugArangeSet parser
   // and not included in the descriptors that are returned.
   unsigned char binary_data[] = {
-      0, 0, 0, 0, // unit_length that will be set correctly after this
-      0, 2,       // DWARF version number (uint16_t)
-      0, 0, 0, 0, // CU offset (ignored for the purposes of this test)
-      4,          // address size
-      0,          // segment size
-      0, 0, 0, 0, // alignment for the first tuple
+      0, 0, 0, 0,   // unit_length that will be set correctly after this
+      0, 2,         // DWARF version number (uint16_t)
+      0, 0, 0, 255, // CU offset
+      4,            // address size
+      0,            // segment size
+      0, 0, 0, 0,   // alignment for the first tuple
       // BEGIN TUPLES
       0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, // [0x1000-0x1100)
       0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, // [0x1100-0x1100)
@@ -174,22 +176,22 @@ TEST_F(SymbolFileDWARFTests, ParseArangesIgnoreEmpty) {
   DWARFDataExtractor data;
   data.SetData(static_cast<const void *>(binary_data), sizeof binary_data,
                lldb::ByteOrder::eByteOrderBig);
-  DWARFDebugArangeSet set;
-  offset_t off = 0;
-  llvm::Error error = set.extract(data, &off);
-  // Multiple terminators are not fatal as they do appear in binaries.
-  EXPECT_FALSE(bool(error));
+  DWARFDebugAranges debug_aranges;
+  debug_aranges.extract(data);
   // Parser should read all terminators to the end of the length specified.
   // Previously the DWARFDebugArangeSet would stop at the first terminator
   // entry and leave the offset in the middle of the current
   // DWARFDebugArangeSet data, and that would cause the next extracted
   // DWARFDebugArangeSet to fail.
-  EXPECT_EQ(off, binary_data_size);
-  ASSERT_EQ(set.NumDescriptors(), 2U);
-  ASSERT_EQ(set.GetDescriptorRef(0).address, (dw_addr_t)0x1000);
-  ASSERT_EQ(set.GetDescriptorRef(0).length, (dw_addr_t)0x100);
-  ASSERT_EQ(set.GetDescriptorRef(1).address, (dw_addr_t)0x2000);
-  ASSERT_EQ(set.GetDescriptorRef(1).length, (dw_addr_t)0x10);
+  ASSERT_EQ(debug_aranges.GetNumRanges(), 2U);
+  EXPECT_EQ(debug_aranges.FindAddress(0x0fff), DW_INVALID_OFFSET);
+  EXPECT_EQ(debug_aranges.FindAddress(0x1000), 255u);
+  EXPECT_EQ(debug_aranges.FindAddress(0x1100 - 1), 255u);
+  EXPECT_EQ(debug_aranges.FindAddress(0x1100), DW_INVALID_OFFSET);
+  EXPECT_EQ(debug_aranges.FindAddress(0x1fff), DW_INVALID_OFFSET);
+  EXPECT_EQ(debug_aranges.FindAddress(0x2000), 255u);
+  EXPECT_EQ(debug_aranges.FindAddress(0x2010 - 1), 255u);
+  EXPECT_EQ(debug_aranges.FindAddress(0x2010), DW_INVALID_OFFSET);
 }
 
 TEST_F(SymbolFileDWARFTests, ParseAranges) {
@@ -212,6 +214,7 @@ TEST_F(SymbolFileDWARFTests, ParseAranges) {
       0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Size    0x0100
       // Terminating tuple
       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Terminator
+      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Terminator
   };
   // Set the little endian length correctly.
   binary_data[0] = sizeof(binary_data) - 4;
diff --git a/lldb/unittests/UnwindAssembly/x86/Testx86AssemblyInspectionEngine.cpp b/lldb/unittests/UnwindAssembly/x86/Testx86AssemblyInspectionEngine.cpp
index 3ff57b4f97f17..f5bb3b6d64236 100644
--- a/lldb/unittests/UnwindAssembly/x86/Testx86AssemblyInspectionEngine.cpp
+++ b/lldb/unittests/UnwindAssembly/x86/Testx86AssemblyInspectionEngine.cpp
@@ -2859,16 +2859,17 @@ TEST_F(Testx86AssemblyInspectionEngine, TestDisassemblyMidFunctionEpilogues) {
 
     0x90,                   // <+18>: nop             // prologue setup back
 
-    0x74, 0x7,              // <+19>: je 6 <+27>
+    0x74, 0x8,              // <+19>: je 7 <+28>
     0x48, 0x83, 0xc4, 0x70, // <+21>: addq $0x70, %rsp
     0x5d,                   // <+25>: popq %rbp
-    0xc3,                   // <+26>: retq            // epilogue completed
+    0x90,                   // <+26>: nop           // mid-epilogue non-epilogue
+    0xc3,                   // <+27>: retq            // epilogue completed
 
-    0x90,                   // <+27>: nop             // prologue setup back
+    0x90,                   // <+28>: nop             // prologue setup back
 
-    0x48, 0x83, 0xc4, 0x70, // <+28>: addq $0x70, %rsp
-    0x5d,                   // <+32>: popq %rbp
-    0xc3,                   // <+33>: retq            // epilogue completed
+    0x48, 0x83, 0xc4, 0x70, // <+29>: addq $0x70, %rsp
+    0x5d,                   // <+33>: popq %rbp
+    0xc3,                   // <+34>: retq            // epilogue completed
 
   };
 
@@ -2898,8 +2899,8 @@ TEST_F(Testx86AssemblyInspectionEngine, TestDisassemblyMidFunctionEpilogues) {
   // Check that we've reinstated the stack frame setup 
   // unwind instructions after a mid-function retq
   // row:   CFA=ebp +8 => esp=CFA+0 eip=[CFA-8]
-  row_sp = unwind_plan.GetRowForFunctionOffset(27);
-  EXPECT_EQ(27ull, row_sp->GetOffset());
+  row_sp = unwind_plan.GetRowForFunctionOffset(28);
+  EXPECT_EQ(28ull, row_sp->GetOffset());
   EXPECT_TRUE(row_sp->GetCFAValue().GetRegisterNumber() == k_ebp);
   EXPECT_TRUE(row_sp->GetCFAValue().IsRegisterPlusOffset() == true);
   EXPECT_EQ(wordsize * 2, row_sp->GetCFAValue().GetOffset());
@@ -2907,8 +2908,8 @@ TEST_F(Testx86AssemblyInspectionEngine, TestDisassemblyMidFunctionEpilogues) {
   // After last instruction in the function, verify that
   // the stack frame has been unwound
   // row:   CFA=esp +4 => esp=CFA+0 eip=[CFA-4]
-  row_sp = unwind_plan.GetRowForFunctionOffset(33);
-  EXPECT_EQ(33ull, row_sp->GetOffset());
+  row_sp = unwind_plan.GetRowForFunctionOffset(34);
+  EXPECT_EQ(34ull, row_sp->GetOffset());
   EXPECT_TRUE(row_sp->GetCFAValue().GetRegisterNumber() == k_esp);
   EXPECT_TRUE(row_sp->GetCFAValue().IsRegisterPlusOffset() == true);
   EXPECT_EQ(wordsize, row_sp->GetCFAValue().GetOffset());
@@ -2940,8 +2941,8 @@ TEST_F(Testx86AssemblyInspectionEngine, TestDisassemblyMidFunctionEpilogues) {
   // Check that we've reinstated the stack frame setup 
   // unwind instructions after a mid-function retq
   // row:   CFA=rbp+16 => rsp=CFA+0 rip=[CFA-16]
-  row_sp = unwind_plan.GetRowForFunctionOffset(27);
-  EXPECT_EQ(27ull, row_sp->GetOffset());
+  row_sp = unwind_plan.GetRowForFunctionOffset(28);
+  EXPECT_EQ(28ull, row_sp->GetOffset());
   EXPECT_TRUE(row_sp->GetCFAValue().GetRegisterNumber() == k_rbp);
   EXPECT_TRUE(row_sp->GetCFAValue().IsRegisterPlusOffset() == true);
   EXPECT_EQ(wordsize * 2, row_sp->GetCFAValue().GetOffset());
@@ -2949,8 +2950,8 @@ TEST_F(Testx86AssemblyInspectionEngine, TestDisassemblyMidFunctionEpilogues) {
   // After last instruction in the function, verify that
   // the stack frame has been unwound
   // row:   CFA=rsp +8 => esp=CFA+0 rip=[CFA-8]
-  row_sp = unwind_plan.GetRowForFunctionOffset(33);
-  EXPECT_EQ(33ull, row_sp->GetOffset());
+  row_sp = unwind_plan.GetRowForFunctionOffset(34);
+  EXPECT_EQ(34ull, row_sp->GetOffset());
   EXPECT_TRUE(row_sp->GetCFAValue().GetRegisterNumber() == k_rsp);
   EXPECT_TRUE(row_sp->GetCFAValue().IsRegisterPlusOffset() == true);
   EXPECT_EQ(wordsize, row_sp->GetCFAValue().GetOffset());
diff --git a/llvm/.gitignore b/llvm/.gitignore
index eb69323201a53..48dda423228c2 100644
--- a/llvm/.gitignore
+++ b/llvm/.gitignore
@@ -32,6 +32,7 @@ autoconf/autom4te.cache
 /CMakeSettings.json
 # CLion project configuration
 /.idea
+/cmake-build*
 # Qt Creator project configuration
 /CMakeLists.txt.user
 
diff --git a/llvm/CODE_OWNERS.TXT b/llvm/CODE_OWNERS.TXT
deleted file mode 100644
index 5b4df555fd666..0000000000000
--- a/llvm/CODE_OWNERS.TXT
+++ /dev/null
@@ -1,269 +0,0 @@
-This file is a list of the people responsible for ensuring that patches for a
-particular part of LLVM are reviewed, either by themself or by someone else.
-They are also the gatekeepers for their part of LLVM, with the final word on
-what goes in or not.
-
-The list is sorted by surname and formatted to allow easy grepping and
-beautification by scripts. The fields are: name (N), email (E), web-address
-(W), PGP key ID and fingerprint (P), description (D), snail-mail address
-(S) and (I) IRC handle. Each entry should contain at least the (N), (E) and
-(D) fields.
-
-N: Paul C. Anagnostopoulos
-E: paul@windfall.com
-D: TableGen
-
-N: Matt Arsenault
-E: Matthew.Arsenault@amd.com
-E: arsenm2@gmail.com
-I: arsenm
-D: AMDGPU, InferAddressSpaces
-
-N: Justin Bogner
-E: mail@justinbogner.com
-D: InstrProfiling and related parts of ProfileData
-D: SelectionDAG (lib/CodeGen/SelectionDAG/*)
-
-N: Alex Bradbury
-E: asb@igalia.com
-D: RISC-V backend (lib/Target/RISCV/*)
-
-N: Matthias Braun
-E: matze@braunis.de
-I: MatzeB
-D: Instruction Scheduling
-
-N: Chandler Carruth
-E: chandlerc@gmail.com
-E: chandlerc@google.com
-D: Config, ADT, Support, inlining & related passes, SROA/mem2reg & related passes, CMake, library layering
-
-N: Evan Cheng
-E: evan.cheng@apple.com
-D: parts of code generator not covered by someone else
-
-N: Eric Christopher
-E: echristo@gmail.com
-D: Debug Information, inline assembly
-
-N: Andrey Churbanov
-E: andrey.churbanov@intel.com
-D: OpenMP runtime library
-
-N: Johannes Doerfert
-E: jdoerfert@llnl.gov
-D: OpenMP target offloading, OpenMP-Opt, Attributor
-
-N: Greg Clayton
-E: clayborg@gmail.com
-D: LLDB
-
-N: Louis Dionne
-E: ldionne.2@gmail.com
-D: libc++
-
-N: Peter Collingbourne
-E: peter@pcc.me.uk
-D: libLTO (lib/LTO/* tools/lto/*), LLVM Bitcode (lib/Bitcode/* include/llvm/Bitcode/*)
-
-N: Quentin Colombet
-E: quentin.colombet@gmail.com
-D: Loop Strength Reduction, Register allocators
-
-N: Andrea Di Biagio
-E: andrea.dibiagio@sony.com
-E: andrea.dibiagio@gmail.com
-D: MCA, llvm-mca
-
-N: Hal Finkel
-E: hfinkel@anl.gov
-D: The loop reroller and alias analysis
-
-N: Zheng Chen
-E: czhengsz@cn.ibm.com
-D: PowerPC Backend
-
-N: Dan Gohman
-E: llvm@sunfishcode.online
-D: WebAssembly Backend (lib/Target/WebAssembly/*)
-
-N: Renato Golin
-E: rengolin@systemcall.eu
-D: ARM Linux support
-
-N: Venkatraman Govindaraju
-E: venkatra@cs.wisc.edu
-D: Sparc Backend (lib/Target/Sparc/*)
-
-N: Tobias Grosser
-E: tobias@grosser.es
-D: Polly
-
-N: James Grosbach
-E: grosbach@apple.com
-D: MC layer
-
-N: Justin Holewinski
-E: jholewinski@nvidia.com
-D: NVPTX Target (lib/Target/NVPTX/*)
-
-N: Lang Hames
-E: lhames@gmail.com
-D: MCJIT, RuntimeDyld and JIT event listeners, Orcish Warchief
-
-N: Min-Yih Hsu
-E: min@myhsu.dev
-D: M68k Target (lib/Target/M68k/*)
-
-N: Cyndy Ishida
-E: cyndyishida@gmail.com
-D: TextAPI (lib/TextAPI/*) and related TAPI tools
-
-N: Teresa Johnson
-E: tejohnson@google.com
-D: Gold plugin (tools/gold/*) and IR Linker
-
-N: Galina Kistanova
-E: gkistanova@gmail.com
-D: LLVM Buildbot
-
-N: Anton Korobeynikov
-E: anton@korobeynikov.info
-D: Exception handling, Windows codegen, ARM EABI, MSP430 Target (lib/Target/MSP430/*)
-
-N: Benjamin Kramer
-E: benny.kra@gmail.com
-D: DWARF Parser
-
-N: Sundeep Kushwaha
-E: sundeepk@quicinc.com
-D: Hexagon Backend
-
-N: Florian Hahn
-E: flo@fhahn.com
-D: Loop Vectorizer
-
-N: Alexey Bataev
-E: a.bataev@outlook.com
-D: SLP Vectorizer
-
-N: Sergei Larin
-E: slarin@codeaurora.org
-D: VLIW Instruction Scheduling, Packetization
-
-N: Chris Lattner
-E: sabre@nondot.org
-W: http://nondot.org/~sabre/
-D: Everything not covered by someone else
-
-N: Weining Lu
-E: luweining@loongson.cn
-D: LoongArch backend (lib/Target/LoongArch/*)
-
-N: David Majnemer
-E: david.majnemer@gmail.com
-D: IR Constant Folder, InstCombine
-
-N: Ben Shi
-E: 2283975856@qq.com, powerman1st@163.com
-D: AVR Backend
-
-N: Kazushi Marukawa
-E: marukawa@nec.com
-D: VE Backend
-
-N: Tim Northover
-E: t.p.northover@gmail.com
-D: AArch64 backend, misc ARM backend
-
-N: Diego Novillo
-E: dnovillo@google.com
-D: SampleProfile and related parts of ProfileData
-
-N: Nigel Perks
-E: nigelp@xmos.com
-D: XCore Backend
-
-N: Jacques Pienaar
-E: jpienaar@google.com
-D: Lanai Backend
-
-N: Simon Pilgrim
-E: llvm-dev@redking.me.uk
-D: X86 Backend
-
-N: Philip Reames
-E: listmail@philipreames.com
-D: IndVar Simplify, Scalar Evolution
-
-N: Paul Robinson
-E: paul.robinson@sony.com
-D: Sony PlayStation®4 support
-
-N: Chad Rosier
-E: mcrosier@codeaurora.org
-D: Fast-Isel
-
-N: Andrei Safronov
-E: andrei.safronov@espressif.com
-D: Xtensa backend (lib/Target/Xtensa/*)
-
-N: Duncan Sands
-E: baldrick@free.fr
-D: DragonEgg
-
-N: Mark Schimmel
-E: marksl@synopsys.com
-D: ARC backend (lib/Target/ARC/*)
-
-N: Kostya Serebryany
-E: kcc@google.com
-D: AddressSanitizer, ThreadSanitizer (LLVM parts)
-
-N: Michael Spencer
-E: bigcheesegs@gmail.com
-D: Windows parts of Support, Object, ar, nm, objdump, ranlib, size
-
-N: Alexei Starovoitov
-E: alexei.starovoitov@gmail.com
-D: BPF backend
-
-N: Tom Stellard
-E: tstellar@redhat.com
-D: Stable release management (x.y.[1-9] releases), libclc
-
-N: Evgeniy Stepanov
-E: eugenis@google.com
-D: MemorySanitizer (LLVM part)
-
-N: Phoebe Wang
-E: phoebe.wang@intel.com
-D: X86 Backend
-
-N: Ulrich Weigand
-E: uweigand@de.ibm.com
-D: SystemZ Backend
-
-N: Hans Wennborg
-E: hans@chromium.org
-D: Release management (x.y.0 releases)
-
-N: Jake Ehrlich
-E: jakehehrlich@google.com
-D: llvm-objcopy (tools/llvm-objcopy)
-
-N: Martin Storsjö
-E: martin@martin.st
-D: MinGW
-
-N: Zi Xuan Wu (Zeson)
-E: zixuan.wu@linux.alibaba.com
-D: C-SKY backend (lib/Target/CSKY/*)
-
-N: Ilia Diachkov
-E: ilia.diachkov@gmail.com
-D: SPIR-V backend (lib/Target/SPIRV/*)
-
-N: Christopher Apple, David Trevelyan
-E: cja-private@pm.me, david.trevelyan@gmail.com
-D: RealtimeSanitizer (LLVM part)
diff --git a/llvm/Maintainers.md b/llvm/Maintainers.md
new file mode 100644
index 0000000000000..b2599ab51fdd1
--- /dev/null
+++ b/llvm/Maintainers.md
@@ -0,0 +1,435 @@
+# LLVM Maintainers
+
+This file is a list of the
+[maintainers](https://llvm.org/docs/DeveloperPolicy.html#maintainers) for
+LLVM.
+
+## Current Maintainers
+
+The following people are the active maintainers for the project. Please reach
+out to them for code reviews, questions about their area of expertise, or other
+assistance.
+
+**Warning: The maintainer list for LLVM is currently not up to date.**
+
+### Lead maintainer
+
+The lead maintainer is responsible for all parts of LLVM not covered by somebody else.
+
+Chris Lattner \
+sabre@nondot.org (email), [lattner](https://github.com/lattner) (GitHub), clattner (Discourse)
+
+### Transforms and analyses
+
+#### AliasAnalysis
+
+Hal Finkel \
+hfinkel@anl.gov (email), [hfinkel](https://github.com/hfinkel) (GitHub)
+
+#### Attributor, OpenMPOpt
+
+Johannes Doerfert \
+jdoerfert@llnl.gov (email), [jdoerfert](https://github.com/jdoerfert) (GitHub)
+
+#### InferAddressSpaces
+
+Matt Arsenault \
+Matthew.Arsenault@amd.com, arsenm2@gmail.com (email), [arsenm](https://github.com/arsenm) (GitHub)
+
+#### Inlining
+
+Chandler Carruth \
+chandlerc@gmail.com, chandlerc@google.com (email), [chandlerc](https://github.com/chandlerc) (GitHub)
+
+#### InstCombine, ConstantFold
+
+David Majnemer \
+david.majnemer@gmail.com (email), [majnemer](https://github.com/majnemer) (GitHub)
+
+#### InstrProfiling and related parts of ProfileData
+
+Justin Bogner \
+mail@justinbogner.com (email), [bogner](https://github.com/bogner) (GitHub)
+
+#### SampleProfile and related parts of ProfileData
+
+Diego Novillo \
+dnovillo@google.com (email), [dnovillo](https://github.com/dnovillo) (GitHub)
+
+#### LoopStrengthReduce
+
+Quentin Colombet \
+quentin.colombet@gmail.com (email), [qcolombet](https://github.com/qcolombet) (GitHub)
+
+#### LoopVectorize
+
+Florian Hahn \
+flo@fhahn.com (email), [fhahn](https://github.com/fhahn) (GitHub)
+
+#### ScalarEvolution, IndVarSimplify
+
+Philip Reames \
+listmail@philipreames.com (email), [preames](https://github.com/preames) (GitHub)
+
+#### SLPVectorizer
+
+Alexey Bataev \
+a.bataev@outlook.com (email), [alexey-bataev](https://github.com/alexey-bataev) (GitHub)
+
+#### SROA, Mem2Reg
+
+Chandler Carruth \
+chandlerc@gmail.com, chandlerc@google.com (email), [chandlerc](https://github.com/chandlerc) (GitHub)
+
+### Instrumentation and sanitizers
+
+#### AddressSanitizer, ThreadSanitizer
+
+Kostya Serebryany \
+kcc@google.com (email), [kcc](https://github.com/kcc) (GitHub)
+
+#### MemorySanitizer
+
+Evgeniy Stepanov \
+eugenis@google.com (email), [eugenis](https://github.com/eugenis) (GitHub)
+
+#### RealtimeSanitizer
+
+Christopher Apple \
+cja-private@pm.me (email), [cjappl](https://github.com/cjappl) (GitHub) \
+David Trevelyan \
+david.trevelyan@gmail.com (email), [davidtrevelyan](https://github.com/davidtrevelyan) (GitHub)
+
+### Generic backend and code generation
+
+#### Parts of code generator not covered by someone else
+
+Evan Cheng \
+evan.cheng@apple.com (email)
+
+#### SelectionDAG
+
+Justin Bogner \
+mail@justinbogner.com (email), [bogner](https://github.com/bogner) (GitHub)
+
+#### FastISel
+
+Chad Rosier \
+mcrosier@codeaurora.org (email)
+
+#### Instruction scheduling
+
+Matthias Braun \
+matze@braunis.de (email), [MatzeB](https://github.com/MatzeB) (GitHub)
+
+#### VLIW Instruction Scheduling, Packetization
+
+Sergei Larin \
+slarin@codeaurora.org (email)
+
+#### Register allocation
+
+Quentin Colombet \
+quentin.colombet@gmail.com (email), [qcolombet](https://github.com/qcolombet) (GitHub)
+
+#### MC layer
+
+James Grosbach \
+grosbach@apple.com (email)
+
+#### Windows codegen
+
+Anton Korobeynikov \
+anton@korobeynikov.info (email), [asl](https://github.com/asl) (GitHub)
+
+### Backends / Targets
+
+#### AArch64 backend
+
+Tim Northover \
+t.p.northover@gmail.com (email), [TNorthover](https://github.com/TNorthover) (GitHub)
+
+#### AMDGPU backend
+
+Matt Arsenault \
+Matthew.Arsenault@amd.com, arsenm2@gmail.com (email), [arsenm](https://github.com/arsenm) (GitHub)
+
+#### ARC backend
+
+Mark Schimmel \
+marksl@synopsys.com (email), [markschimmel](https://github.com/markschimmel) (GitHub)
+
+#### ARM backend
+
+Renato Golin \
+rengolin@systemcall.eu (email), [rengolin](https://github.com/rengolin) (GitHub)
+
+#### AVR backend
+
+Ben Shi \
+2283975856@qq.com, powerman1st@163.com (email), [benshi001](https://github.com/benshi001) (GitHub)
+
+#### BPF backend
+
+Alexei Starovoitov \
+alexei.starovoitov@gmail.com (email), [4ast](https://github.com/4ast) (GitHub)
+
+#### CSKY backend
+
+Zi Xuan Wu (Zeson) \
+zixuan.wu@linux.alibaba.com (email), [zixuan-wu](https://github.com/zixuan-wu) (GitHub)
+
+#### Hexagon backend
+
+Sundeep Kushwaha \
+sundeepk@quicinc.com (email)
+
+#### Lanai backend
+
+Jacques Pienaar \
+jpienaar@google.com (email), [jpienaar](https://github.com/jpienaar) (GitHub)
+
+#### LoongArch backend
+
+Weining Lu \
+luweining@loongson.cn (email), [SixWeining](https://github.com/SixWeining) (GitHub)
+
+#### M68k backend
+
+Min-Yih Hsu \
+min@myhsu.dev (email), [mshockwave](https://github.com/mshockwave) (GitHub)
+
+#### MSP430 backend
+
+Anton Korobeynikov \
+anton@korobeynikov.info (email), [asl](https://github.com/asl) (GitHub)
+
+#### NVPTX backend
+
+Justin Holewinski \
+jholewinski@nvidia.com (email), [jholewinski](https://github.com/jholewinski) (GitHub)
+
+#### PowerPC backend
+
+Zheng Chen \
+czhengsz@cn.ibm.com (email), [chenzheng1030](https://github.com/chenzheng1030) (GitHub)
+
+#### RISCV backend
+
+Alex Bradbury \
+asb@igalia.com (email), [asb](https://github.com/asb) (GitHub)
+
+#### Sparc backend
+
+Venkatraman Govindaraju \
+venkatra@cs.wisc.edu (email), [vegovin](https://github.com/vegovin) (GitHub)
+
+#### SPIRV backend
+
+Ilia Diachkov \
+ilia.diachkov@gmail.com (email), [iliya-diyachkov](https://github.com/iliya-diyachkov) (GitHub)
+
+#### SystemZ backend
+
+Ulrich Weigand \
+uweigand@de.ibm.com (email), [uweigand](https://github.com/uweigand) (GitHub)
+
+#### VE backend
+
+Kazushi Marukawa \
+marukawa@nec.com (email), [kaz7](https://github.com/kaz7) (GitHub)
+
+#### WebAssembly backend
+
+Dan Gohman \
+llvm@sunfishcode.online (email), [sunfishcode](https://github.com/sunfishcode) (GitHub)
+
+#### X86 backend
+
+Simon Pilgrim \
+llvm-dev@redking.me.uk (email), [RKSimon](https://github.com/RKSimon) (GitHub) \
+Phoebe Wang \
+phoebe.wang@intel.com (email), [phoebewang](https://github.com/phoebewang) (GitHub)
+
+#### XCore backend
+
+Nigel Perks \
+nigelp@xmos.com (email), [nigelp-xmos](https://github.com/nigelp-xmos) (GitHub)
+
+#### Xtensa backend
+
+Andrei Safronov \
+andrei.safronov@espressif.com (email), [andreisfr](https://github.com/andreisfr) (GitHub)
+
+### Libraries and shared infrastructure
+
+#### ADT, Support
+
+Chandler Carruth \
+chandlerc@gmail.com, chandlerc@google.com (email), [chandlerc](https://github.com/chandlerc) (GitHub)
+
+#### Bitcode
+
+Peter Collingbourne \
+peter@pcc.me.uk (email), [pcc](https://github.com/pcc) (GitHub)
+
+#### CMake and library layering
+
+Chandler Carruth \
+chandlerc@gmail.com, chandlerc@google.com (email), [chandlerc](https://github.com/chandlerc) (GitHub)
+
+#### Debug info
+
+Eric Christopher \
+echristo@gmail.com (email), [echristo](https://github.com/echristo) (GitHub)
+
+#### DWARF Parser
+
+Benjamin Kramer \
+benny.kra@gmail.com (email), [d0k](https://github.com/d0k) (GitHub)
+
+#### IR Linker
+
+Teresa Johnson \
+tejohnson@google.com (email), [teresajohnson](https://github.com/teresajohnson) (GitHub)
+
+#### LTO
+
+Peter Collingbourne \
+peter@pcc.me.uk (email), [pcc](https://github.com/pcc) (GitHub)
+
+#### MCJIT, Orc, RuntimeDyld, PerfJITEvents
+
+Lang Hames \
+lhames@gmail.com (email), [lhames](https://github.com/lhames) (GitHub)
+
+#### TableGen
+
+Paul C. Anagnostopoulos \
+paul@windfall.com (email)
+
+#### TextAPI
+
+Cyndy Ishida \
+cyndyishida@gmail.com (email), [cyndyishida](https://github.com/cyndyishida) (GitHub)
+
+### Tools
+
+#### llvm-mca and MCA library
+
+Andrea Di Biagio \
+andrea.dibiagio@sony.com, andrea.dibiagio@gmail.com (email), [adibiagio](https://github.com/adibiagio) (GitHub)
+
+#### llvm-objcopy and ObjCopy library
+
+Jake Ehrlich \
+jakehehrlich@google.com (email), [jakehehrlich](https://github.com/jakehehrlich) (GitHub)
+
+#### Gold plugin
+
+Teresa Johnson \
+tejohnson@google.com (email), [teresajohnson](https://github.com/teresajohnson) (GitHub)
+
+### Other
+
+#### Release management
+
+For x.y.0 releases:
+
+Hans Wennborg \
+hans@chromium.org (email), [zmodem](https://github.com/zmodem) (GitHub)
+
+For x.y.\[1-9\] releases:
+
+Tom Stellard \
+tstellar@redhat.com (email), [tstellar](https://github.com/tstellar) (GitHub)
+
+#### MinGW support
+
+Martin Storsjö \
+martin@martin.st (email), [mstrorsjo](https://github.com/mstrorsjo) (GitHub)
+
+#### Windows support in object tools
+
+Michael Spencer \
+bigcheesegs@gmail.com (email), [Bigcheese](https://github.com/Bigcheese) (GitHub)
+
+#### Sony PlayStation®4 support
+
+Paul Robinson \
+paul.robinson@sony.com (email), [pogo59](https://github.com/pogo59) (GitHub)
+
+#### Inline assembly
+
+Eric Christopher \
+echristo@gmail.com (email), [echristo](https://github.com/echristo) (GitHub)
+
+#### Exception handling
+
+Anton Korobeynikov \
+anton@korobeynikov.info (email), [asl](https://github.com/asl) (GitHub)
+
+#### ARM EABI
+
+Anton Korobeynikov \
+anton@korobeynikov.info (email), [asl](https://github.com/asl) (GitHub)
+
+#### LLVM Buildbot
+
+Galina Kistanova \
+gkistanova@gmail.com (email), [gkistanova](https://github.com/gkistanova) (GitHub)
+
+### Other subprojects
+
+Some subprojects maintain their own list of per-component maintainers.
+Others only have a lead maintainer listed here.
+
+[Bolt maintainers](https://github.com/llvm/llvm-project/blob/main/bolt/CODE_OWNERS.TXT)
+
+[Clang maintainers](https://github.com/llvm/llvm-project/blob/main/clang/CodeOwners.rst)
+
+[Clang-tools-extra maintainers](https://github.com/llvm/llvm-project/blob/main/clang-tools-extra/CODE_OWNERS.TXT)
+
+[Compiler-rt maintainers](https://github.com/llvm/llvm-project/blob/main/compiler-rt/CODE_OWNERS.TXT)
+
+[Flang maintainers](https://github.com/llvm/llvm-project/blob/main/flang/CODE_OWNERS.TXT)
+
+[LLD maintainers](https://github.com/llvm/llvm-project/blob/main/lld/CODE_OWNERS.TXT)
+
+[LLDB maintainers](https://github.com/llvm/llvm-project/blob/main/lldb/CodeOwners.rst)
+
+#### libc++
+
+Louis Dionne \
+ldionne.2@gmail.com (email), [ldionne](https://github.com/ldionne) (GitHub)
+
+#### libclc
+
+Tom Stellard \
+tstellar@redhat.com (email), [tstellar](https://github.com/tstellar) (GitHub)
+
+#### OpenMP (runtime library)
+
+Andrey Churbanov \
+andrey.churbanov@intel.com (email), [AndreyChurbanov](https://github.com/AndreyChurbanov) (GitHub)
+
+#### Polly
+
+Tobias Grosser \
+tobias@grosser.es (email), [tobiasgrosser](https://github.com/tobiasgrosser) (GitHub)
+
+## Inactive Maintainers
+
+The following people have graciously spent time performing maintainer
+responsibilities but are no longer active in that role. Thank you for all your
+help with the success of the project!
+
+### Emeritus lead maintainers
+
+### Inactive component maintainers
+
+### Former maintainers of removed components
+
+Duncan Sands (baldrick@free.fr, [CunningBaldrick](https://github.com/CunningBaldrick)) -- DragonEgg \
+Hal Finkel (hfinkel@anl.gov, [hfinkel](https://github.com/hfinkel)) -- LoopReroll
+
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index c9d5baba3e2f4..bd1cfbbade511 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -535,7 +535,7 @@ TestRunner.py:
  %{fs-tmp-root}          root component of file system paths pointing to the test's temporary directory
  %{fs-sep}               file system path separator
  %t                      temporary file name unique to the test
- %basename_t             The last path component of %t but without the ``.tmp`` extension
+ %basename_t             The last path component of %t but without the ``.tmp`` extension (deprecated, use ``%{t:stem}`` instead)
  %T                      parent directory of %t (not unique, deprecated, do not use)
  %%                      %
  %/s                     %s but ``\`` is replaced by ``/``
@@ -543,6 +543,8 @@ TestRunner.py:
  %/p                     %p but ``\`` is replaced by ``/``
  %/t                     %t but ``\`` is replaced by ``/``
  %/T                     %T but ``\`` is replaced by ``/``
+ %{s:basename}           The last path component of %s
+ %{t:stem}               The last path component of %t but without the ``.tmp`` extension (alias for %basename_t)
  %{s:real}               %s after expanding all symbolic links and substitute drives
  %{S:real}               %S after expanding all symbolic links and substitute drives
  %{p:real}               %p after expanding all symbolic links and substitute drives
diff --git a/llvm/docs/CommandGuide/llvm-dwarfdump.rst b/llvm/docs/CommandGuide/llvm-dwarfdump.rst
index 8adccf1c0c0a3..426a6d596a1b8 100644
--- a/llvm/docs/CommandGuide/llvm-dwarfdump.rst
+++ b/llvm/docs/CommandGuide/llvm-dwarfdump.rst
@@ -45,6 +45,16 @@ OPTIONS
 
             Use colors in output.
 
+.. option:: --error-display=<value>       
+
+            Set the level of detail and summary to display when verifying.
+            Implies :option:`--verify`. The supported values are:
+
+            `quiet`   - Only display whether errors occurred.
+            `summary` - Display only a summary of the errors found.
+            `details` - Display each error in detail but no summary.
+            `full`    - Display each error as well as a summary. [default]
+
 .. option:: -f <name>, --find=<name>
 
             Search for the exact text <name> in the accelerator tables
@@ -150,6 +160,12 @@ OPTIONS
             compile unit chains, DIE relationships graph, address
             ranges, and more.
 
+.. option:: --verify-json=<path>
+
+            Output JSON-formatted error summary to the a file specfied by
+            <path>. Implies :option:`--verify`.  The output format is described
+            in the section below (:ref:`verify-json-format`).
+
 .. option:: --version
 
             Display the version of the tool.
@@ -196,6 +212,28 @@ For aggregated values, the following keys are used:
       - `#variables - entry values ...` ==> the number of variables excluding
         the entry values etc.
 
+.. _verify-json-format:
+
+FORMAT OF VERIFY JSON OUTPUT
+----------------------------
+
+The format of the JSON output created by the :option:`--verify-json` is::
+
+  { 
+    "error-categories": { 
+      "<first category description>": {"count": 1234},
+      "<next category description>": {"count": 4321}
+    },
+    "error-count": 5555
+  }
+
+The following is generated if there are no errors reported::
+
+  { 
+    "error-categories": {},
+    "error-count": 0
+  }
+
 EXIT STATUS
 -----------
 
diff --git a/llvm/docs/CommandGuide/llvm-objcopy.rst b/llvm/docs/CommandGuide/llvm-objcopy.rst
index 7f20a76550c83..e6af47ce9710a 100644
--- a/llvm/docs/CommandGuide/llvm-objcopy.rst
+++ b/llvm/docs/CommandGuide/llvm-objcopy.rst
@@ -119,8 +119,8 @@ multiple file formats.
 
 .. option:: --remove-symbol-prefix <prefix>
 
-Remove ``<prefix>`` from the start of every symbol name. No-op for symbols that do
-not start with ``<prefix>``.
+ Remove ``<prefix>`` from the start of every symbol name. No-op for symbols that do
+ not start with ``<prefix>``.
 
 .. option:: --remove-section <section>, -R
 
diff --git a/llvm/docs/GettingInvolved.rst b/llvm/docs/GettingInvolved.rst
index 32d3a83738a8e..d43390bdad455 100644
--- a/llvm/docs/GettingInvolved.rst
+++ b/llvm/docs/GettingInvolved.rst
@@ -351,8 +351,8 @@ The :doc:`CodeOfConduct` applies to all office hours.
     - English
   * - Renato Golin
     - General LLVM, MLIR & Linalg, distributed computing, research, socials.
-    - Every first Tuesday of the month, 11:00am UK time, for 60 minutes.
-    - `Google meet <https://meet.google.com/esg-fggc-hfe>`__
+    - Every first Friday of the month, 14:00 UK time, for 60 minutes.
+    - `Google meet <https://meet.google.com/jps-twgq-ivz>`__
     - English, Portuguese
   * - Rotating hosts
     - Getting Started, beginner questions, new contributors.
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 3f39d58b322a4..6fa35486669d6 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -1499,11 +1499,12 @@ Currently, only the following parameter attributes are defined:
 ``nofpclass(<test mask>)``
     This attribute applies to parameters and return values with
     floating-point and vector of floating-point types, as well as
-    arrays of such types. The test mask has the same format as the
-    second argument to the :ref:`llvm.is.fpclass <llvm.is.fpclass>`,
-    and indicates which classes of floating-point values are not
-    permitted for the value. For example a bitmask of 3 indicates
-    the parameter may not be a NaN.
+    :ref:`supported aggregates <fastmath_return_types>` of such types
+    (matching the supported types for :ref:`fast-math flags <fastmath>`).
+    The test mask has the same format as the second argument to the
+    :ref:`llvm.is.fpclass <llvm.is.fpclass>`, and indicates which classes
+    of floating-point values are not permitted for the value. For example
+    a bitmask of 3 indicates the parameter may not be a NaN.
 
     If the value is a floating-point class indicated by the
     ``nofpclass`` test mask, a :ref:`poison value <poisonvalues>` is
@@ -3685,9 +3686,9 @@ Fast-Math Flags
 
 LLVM IR floating-point operations (:ref:`fneg <i_fneg>`, :ref:`fadd <i_fadd>`,
 :ref:`fsub <i_fsub>`, :ref:`fmul <i_fmul>`, :ref:`fdiv <i_fdiv>`,
-:ref:`frem <i_frem>`, :ref:`fcmp <i_fcmp>`), :ref:`phi <i_phi>`,
-:ref:`select <i_select>` and :ref:`call <i_call>`
-may use the following flags to enable otherwise unsafe
+:ref:`frem <i_frem>`, :ref:`fcmp <i_fcmp>`), and :ref:`phi <i_phi>`,
+:ref:`select <i_select>`, or :ref:`call <i_call>` instructions that return
+floating-point types may use the following flags to enable otherwise unsafe
 floating-point transformations.
 
 ``fast``
@@ -3709,6 +3710,16 @@ floating-point transformations.
    argument or zero result as insignificant. This does not imply that -0.0
    is poison and/or guaranteed to not exist in the operation.
 
+Note: For :ref:`phi <i_phi>`, :ref:`select <i_select>`, and :ref:`call <i_call>`
+instructions, the following return types are considered to be floating-point
+types:
+
+.. _fastmath_return_types:
+
+- Floating-point scalar or vector types
+- Array types (nested to any depth) of floating-point scalar or vector types
+- Homogeneous literal struct types of floating-point scalar or vector types
+
 Rewrite-based flags
 ^^^^^^^^^^^^^^^^^^^
 
@@ -4343,7 +4354,7 @@ recursive, can be opaqued, and are never uniqued.
 :Examples:
 
 +------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| ``{ i32, i32, i32 }``        | A triple of three ``i32`` values                                                                                                                                                      |
+| ``{ i32, i32, i32 }``        | A triple of three ``i32`` values (this is a "homogeneous" struct as all element types are the same)                                                                                   |
 +------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 | ``{ float, ptr }``           | A pair, where the first element is a ``float`` and the second element is a :ref:`pointer <t_pointer>`.                                                                                |
 +------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
@@ -12472,9 +12483,8 @@ instruction's return value on the same edge).
 The optional ``fast-math-flags`` marker indicates that the phi has one
 or more :ref:`fast-math-flags <fastmath>`. These are optimization hints
 to enable otherwise unsafe floating-point optimizations. Fast-math-flags
-are only valid for phis that return a floating-point scalar or vector
-type, or an array (nested to any depth) of floating-point scalar or vector
-types.
+are only valid for phis that return :ref:`supported floating-point types
+<fastmath_return_types>`.
 
 Semantics:
 """"""""""
@@ -12523,8 +12533,8 @@ class <t_firstclass>` type.
 #. The optional ``fast-math flags`` marker indicates that the select has one or more
    :ref:`fast-math flags <fastmath>`. These are optimization hints to enable
    otherwise unsafe floating-point optimizations. Fast-math flags are only valid
-   for selects that return a floating-point scalar or vector type, or an array
-   (nested to any depth) of floating-point scalar or vector types.
+   for selects that return :ref:`supported floating-point types
+   <fastmath_return_types>`.
 
 Semantics:
 """"""""""
@@ -12762,8 +12772,7 @@ This instruction requires several arguments:
 #. The optional ``fast-math flags`` marker indicates that the call has one or more
    :ref:`fast-math flags <fastmath>`, which are optimization hints to enable
    otherwise unsafe floating-point optimizations. Fast-math flags are only valid
-   for calls that return a floating-point scalar or vector type, or an array
-   (nested to any depth) of floating-point scalar or vector types.
+   for calls that return :ref:`supported floating-point types <fastmath_return_types>`.
 
 #. The optional "cconv" marker indicates which :ref:`calling
    convention <callingconv>` the call should use. If none is
@@ -20527,8 +20536,8 @@ the explicit vector length.
 #. The optional ``fast-math flags`` marker indicates that the select has one or
    more :ref:`fast-math flags <fastmath>`. These are optimization hints to
    enable otherwise unsafe floating-point optimizations. Fast-math flags are
-   only valid for selects that return a floating-point scalar or vector type,
-   or an array (nested to any depth) of floating-point scalar or vector types.
+   only valid for selects that return :ref:`supported floating-point types
+   <fastmath_return_types>`.
 
 Semantics:
 """"""""""
@@ -20585,8 +20594,8 @@ is the pivot.
 #. The optional ``fast-math flags`` marker indicates that the merge has one or
    more :ref:`fast-math flags <fastmath>`. These are optimization hints to
    enable otherwise unsafe floating-point optimizations. Fast-math flags are
-   only valid for merges that return a floating-point scalar or vector type,
-   or an array (nested to any depth) of floating-point scalar or vector types.
+   only valid for merges that return :ref:`supported floating-point types
+   <fastmath_return_types>`.
 
 Semantics:
 """"""""""
diff --git a/llvm/docs/SPIRVUsage.rst b/llvm/docs/SPIRVUsage.rst
index 38c41b0fad12e..5485bb6195c3d 100644
--- a/llvm/docs/SPIRVUsage.rst
+++ b/llvm/docs/SPIRVUsage.rst
@@ -223,19 +223,19 @@ using target extension types and are represented as follows:
 
   .. table:: SPIR-V Opaque Types
 
-     ================== ====================== =========================================================================================
+     ================== ====================== ===========================================================================================
      SPIR-V Type        LLVM type name         LLVM type arguments
-     ================== ====================== =========================================================================================
-     OpTypeImage        ``spirv.Image``        sampled type, dimensionality, depth, arrayed, MS, sampled, image format, access qualifier
+     ================== ====================== ===========================================================================================
+     OpTypeImage        ``spirv.Image``        sampled type, dimensionality, depth, arrayed, MS, sampled, image format, [access qualifier]
      OpTypeSampler      ``spirv.Sampler``      (none)
-     OpTypeSampledImage ``spirv.SampledImage`` sampled type, dimensionality, depth, arrayed, MS, sampled, image format, access qualifier
+     OpTypeSampledImage ``spirv.SampledImage`` sampled type, dimensionality, depth, arrayed, MS, sampled, image format, [access qualifier]
      OpTypeEvent        ``spirv.Event``        (none)
      OpTypeDeviceEvent  ``spirv.DeviceEvent``  (none)
      OpTypeReserveId    ``spirv.ReserveId``    (none)
      OpTypeQueue        ``spirv.Queue``        (none)
      OpTypePipe         ``spirv.Pipe``         access qualifier
      OpTypePipeStorage  ``spirv.PipeStorage``  (none)
-     ================== ====================== =========================================================================================
+     ================== ====================== ===========================================================================================
 
 All integer arguments take the same value as they do in their `corresponding
 SPIR-V instruction <https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#_type_declaration_instructions>`_.
diff --git a/llvm/docs/TableGen/BackGuide.rst b/llvm/docs/TableGen/BackGuide.rst
index 60677a6dcd627..c26575272ad69 100644
--- a/llvm/docs/TableGen/BackGuide.rst
+++ b/llvm/docs/TableGen/BackGuide.rst
@@ -624,7 +624,7 @@ iterates over the pairs in the returned array.
 
 .. code-block:: text
 
-  ArrayRef<std::pair<Record *, SMRange>>
+  ArrayRef<std::pair<const Record *, SMRange>>
       Superclasses = Prototype->getSuperClasses();
   for (const auto &SuperPair : Superclasses) {
     ...
diff --git a/llvm/include/llvm/ADT/APFloat.h b/llvm/include/llvm/ADT/APFloat.h
index acb3b2e210300..40131c83a8ef4 100644
--- a/llvm/include/llvm/ADT/APFloat.h
+++ b/llvm/include/llvm/ADT/APFloat.h
@@ -195,6 +195,13 @@ struct APFloatBase {
     // improved range compared to half (16-bit) formats, at (potentially)
     // greater throughput than single precision (32-bit) formats.
     S_FloatTF32,
+    // 8-bit floating point number with (all the) 8 bits for the exponent
+    // like in FP32. There are no zeroes, no infinities, and no denormal values.
+    // This format has unsigned representation only. (U -> Unsigned only).
+    // NaN is represented with all bits set to 1. Bias is 127.
+    // This format represents the scale data type in the MX specification from:
+    // https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf
+    S_Float8E8M0FNU,
     // 6-bit floating point number with bit layout S1E3M2. Unlike IEEE-754
     // types, there are no infinity or NaN values. The format is detailed in
     // https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf
@@ -229,6 +236,7 @@ struct APFloatBase {
   static const fltSemantics &Float8E4M3B11FNUZ() LLVM_READNONE;
   static const fltSemantics &Float8E3M4() LLVM_READNONE;
   static const fltSemantics &FloatTF32() LLVM_READNONE;
+  static const fltSemantics &Float8E8M0FNU() LLVM_READNONE;
   static const fltSemantics &Float6E3M2FN() LLVM_READNONE;
   static const fltSemantics &Float6E2M3FN() LLVM_READNONE;
   static const fltSemantics &Float4E2M1FN() LLVM_READNONE;
@@ -581,7 +589,8 @@ class IEEEFloat final : public APFloatBase {
   integerPart addSignificand(const IEEEFloat &);
   integerPart subtractSignificand(const IEEEFloat &, integerPart);
   lostFraction addOrSubtractSignificand(const IEEEFloat &, bool subtract);
-  lostFraction multiplySignificand(const IEEEFloat &, IEEEFloat);
+  lostFraction multiplySignificand(const IEEEFloat &, IEEEFloat,
+                                   bool ignoreAddend = false);
   lostFraction multiplySignificand(const IEEEFloat&);
   lostFraction divideSignificand(const IEEEFloat &);
   void incrementSignificand();
@@ -591,6 +600,7 @@ class IEEEFloat final : public APFloatBase {
   unsigned int significandLSB() const;
   unsigned int significandMSB() const;
   void zeroSignificand();
+  unsigned int getNumHighBits() const;
   /// Return true if the significand excluding the integral bit is all ones.
   bool isSignificandAllOnes() const;
   bool isSignificandAllOnesExceptLSB() const;
@@ -652,6 +662,7 @@ class IEEEFloat final : public APFloatBase {
   APInt convertFloat8E4M3B11FNUZAPFloatToAPInt() const;
   APInt convertFloat8E3M4APFloatToAPInt() const;
   APInt convertFloatTF32APFloatToAPInt() const;
+  APInt convertFloat8E8M0FNUAPFloatToAPInt() const;
   APInt convertFloat6E3M2FNAPFloatToAPInt() const;
   APInt convertFloat6E2M3FNAPFloatToAPInt() const;
   APInt convertFloat4E2M1FNAPFloatToAPInt() const;
@@ -672,6 +683,7 @@ class IEEEFloat final : public APFloatBase {
   void initFromFloat8E4M3B11FNUZAPInt(const APInt &api);
   void initFromFloat8E3M4APInt(const APInt &api);
   void initFromFloatTF32APInt(const APInt &api);
+  void initFromFloat8E8M0FNUAPInt(const APInt &api);
   void initFromFloat6E3M2FNAPInt(const APInt &api);
   void initFromFloat6E2M3FNAPInt(const APInt &api);
   void initFromFloat4E2M1FNAPInt(const APInt &api);
@@ -1079,6 +1091,9 @@ class APFloat : public APFloatBase {
   /// \param Semantics - type float semantics
   static APFloat getAllOnesValue(const fltSemantics &Semantics);
 
+  /// Returns true if the given semantics supports either NaN or Infinity.
+  ///
+  /// \param Sem - type float semantics
   static bool hasNanOrInf(const fltSemantics &Sem) {
     switch (SemanticsToEnum(Sem)) {
     default:
@@ -1091,6 +1106,13 @@ class APFloat : public APFloatBase {
     }
   }
 
+  /// Returns true if the given semantics has actual significand.
+  ///
+  /// \param Sem - type float semantics
+  static bool hasSignificand(const fltSemantics &Sem) {
+    return &Sem != &Float8E8M0FNU();
+  }
+
   /// Used to insert APFloat objects, or objects that contain APFloat objects,
   /// into FoldingSets.
   void Profile(FoldingSetNodeID &NID) const;
diff --git a/llvm/include/llvm/ADT/Hashing.h b/llvm/include/llvm/ADT/Hashing.h
index 109966257b51c..17dcf31c616aa 100644
--- a/llvm/include/llvm/ADT/Hashing.h
+++ b/llvm/include/llvm/ADT/Hashing.h
@@ -311,10 +311,7 @@ struct hash_state {
 /// depend on the particular hash values. On platforms without ASLR, this is
 /// still likely non-deterministic per build.
 inline uint64_t get_execution_seed() {
-  // Work around x86-64 negative offset folding for old Clang -fno-pic
-  // https://reviews.llvm.org/D93931
-#if LLVM_ENABLE_ABI_BREAKING_CHECKS &&                                         \
-    (!defined(__clang__) || __clang_major__ > 11)
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
   return static_cast<uint64_t>(
       reinterpret_cast<uintptr_t>(&install_fatal_error_handler));
 #else
diff --git a/llvm/include/llvm/Analysis/CmpInstAnalysis.h b/llvm/include/llvm/Analysis/CmpInstAnalysis.h
index 406dacd930605..c7862a6d39d07 100644
--- a/llvm/include/llvm/Analysis/CmpInstAnalysis.h
+++ b/llvm/include/llvm/Analysis/CmpInstAnalysis.h
@@ -92,18 +92,21 @@ namespace llvm {
   Constant *getPredForFCmpCode(unsigned Code, Type *OpTy,
                                CmpInst::Predicate &Pred);
 
-  /// Represents the operation icmp (X & Mask) pred 0, where pred can only be
+  /// Represents the operation icmp (X & Mask) pred C, where pred can only be
   /// eq or ne.
   struct DecomposedBitTest {
     Value *X;
     CmpInst::Predicate Pred;
     APInt Mask;
+    APInt C;
   };
 
-  /// Decompose an icmp into the form ((X & Mask) pred 0) if possible.
+  /// Decompose an icmp into the form ((X & Mask) pred C) if possible.
+  /// Unless \p AllowNonZeroC is true, C will always be 0.
   std::optional<DecomposedBitTest>
   decomposeBitTestICmp(Value *LHS, Value *RHS, CmpInst::Predicate Pred,
-                       bool LookThroughTrunc = true);
+                       bool LookThroughTrunc = true,
+                       bool AllowNonZeroC = false);
 
 } // end namespace llvm
 
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index 132b7ec9aeef7..d9f3f4ab3935d 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -800,6 +800,17 @@ class GInsertVectorElement : public GenericMachineInstr {
   }
 };
 
+/// Represents an extract subvector.
+class GExtractSubvector : public GenericMachineInstr {
+public:
+  Register getSrcVec() const { return getOperand(1).getReg(); }
+  uint64_t getIndexImm() const { return getOperand(2).getImm(); }
+
+  static bool classof(const MachineInstr *MI) {
+    return MI->getOpcode() == TargetOpcode::G_EXTRACT_SUBVECTOR;
+  }
+};
+
 /// Represents a freeze.
 class GFreeze : public GenericMachineInstr {
 public:
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index ecade6b5caed6..ec8a299388376 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -378,6 +378,8 @@ class LegalizerHelper {
                                         LLT CastTy);
   LegalizeResult bitcastConcatVector(MachineInstr &MI, unsigned TypeIdx,
                                      LLT CastTy);
+  LegalizeResult bitcastExtractSubvector(MachineInstr &MI, unsigned TypeIdx,
+                                         LLT CastTy);
 
   LegalizeResult lowerConstant(MachineInstr &MI);
   LegalizeResult lowerFConstant(MachineInstr &MI);
diff --git a/llvm/include/llvm/CodeGen/MIRYamlMapping.h b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
index ab8dc442e04b7..c5bf6971df453 100644
--- a/llvm/include/llvm/CodeGen/MIRYamlMapping.h
+++ b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
@@ -734,6 +734,7 @@ struct MachineFunction {
   std::optional<bool> NoPHIs;
   std::optional<bool> IsSSA;
   std::optional<bool> NoVRegs;
+  std::optional<bool> HasFakeUses;
 
   bool CallsEHReturn = false;
   bool CallsUnwindInit = false;
@@ -780,6 +781,7 @@ template <> struct MappingTraits<MachineFunction> {
     YamlIO.mapOptional("noPhis", MF.NoPHIs, std::optional<bool>());
     YamlIO.mapOptional("isSSA", MF.IsSSA, std::optional<bool>());
     YamlIO.mapOptional("noVRegs", MF.NoVRegs, std::optional<bool>());
+    YamlIO.mapOptional("hasFakeUses", MF.HasFakeUses, std::optional<bool>());
 
     YamlIO.mapOptional("callsEHReturn", MF.CallsEHReturn, false);
     YamlIO.mapOptional("callsUnwindInit", MF.CallsUnwindInit, false);
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index 997960fcd5d09..053e7062fb499 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -376,6 +376,7 @@ class LLVM_ABI MachineFunction {
   bool HasEHCatchret = false;
   bool HasEHScopes = false;
   bool HasEHFunclets = false;
+  bool HasFakeUses = false;
   bool IsOutlined = false;
 
   /// BBID to assign to the next basic block of this function.
@@ -1195,6 +1196,9 @@ class LLVM_ABI MachineFunction {
   bool hasEHFunclets() const { return HasEHFunclets; }
   void setHasEHFunclets(bool V) { HasEHFunclets = V; }
 
+  bool hasFakeUses() const { return HasFakeUses; }
+  void setHasFakeUses(bool V) { HasFakeUses = V; }
+
   bool isOutlined() const { return IsOutlined; }
   void setIsOutlined(bool V) { IsOutlined = V; }
 
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index d6c2c36a0d482..b8f80738486a7 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -584,7 +584,7 @@ class SelectionDAG {
     return Root;
   }
 
-#ifndef NDEBUG
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
   void VerifyDAGDivergence();
 #endif
 
diff --git a/llvm/include/llvm/CodeGen/Spiller.h b/llvm/include/llvm/CodeGen/Spiller.h
index b2f5485eba02e..51ad36bc6b1f8 100644
--- a/llvm/include/llvm/CodeGen/Spiller.h
+++ b/llvm/include/llvm/CodeGen/Spiller.h
@@ -9,6 +9,9 @@
 #ifndef LLVM_CODEGEN_SPILLER_H
 #define LLVM_CODEGEN_SPILLER_H
 
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/CodeGen/Register.h"
+
 namespace llvm {
 
 class LiveRangeEdit;
@@ -30,6 +33,13 @@ class Spiller {
   /// spill - Spill the LRE.getParent() live interval.
   virtual void spill(LiveRangeEdit &LRE) = 0;
 
+  /// Return the registers that were spilled.
+  virtual ArrayRef<Register> getSpilledRegs() = 0;
+
+  /// Return registers that were not spilled, but otherwise replaced
+  /// (e.g. rematerialized).
+  virtual ArrayRef<Register> getReplacedRegs() = 0;
+
   virtual void postOptimization() {}
 };
 
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h
index 760d8826771c0..942de05f6bd84 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h
@@ -62,7 +62,7 @@ class DWARFDebugArangeSet {
 
   void clear();
   Error extract(DWARFDataExtractor data, uint64_t *offset_ptr,
-                function_ref<void(Error)> WarningHandler);
+                function_ref<void(Error)> WarningHandler = nullptr);
   void dump(raw_ostream &OS) const;
 
   uint64_t getCompileUnitDIEOffset() const { return HeaderData.CuOffset; }
diff --git a/llvm/include/llvm/Frontend/Atomic/Atomic.h b/llvm/include/llvm/Frontend/Atomic/Atomic.h
new file mode 100644
index 0000000000000..3942d06144ce1
--- /dev/null
+++ b/llvm/include/llvm/Frontend/Atomic/Atomic.h
@@ -0,0 +1,232 @@
+//===--- Atomic.h - Codegen of atomic operations
+//---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FRONTEND_ATOMIC_ATOMIC_H
+#define LLVM_FRONTEND_ATOMIC_ATOMIC_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Operator.h"
+#include "llvm/IR/RuntimeLibcalls.h"
+
+namespace llvm {
+
+template <typename IRBuilderTy> struct AtomicInfo {
+
+  IRBuilderTy *Builder;
+  Type *Ty;
+  uint64_t AtomicSizeInBits;
+  uint64_t ValueSizeInBits;
+  llvm::Align AtomicAlign;
+  llvm::Align ValueAlign;
+  bool UseLibcall;
+
+public:
+  AtomicInfo(IRBuilderTy *Builder, Type *Ty, uint64_t AtomicSizeInBits,
+             uint64_t ValueSizeInBits, llvm::Align AtomicAlign,
+             llvm::Align ValueAlign, bool UseLibcall)
+      : Builder(Builder), Ty(Ty), AtomicSizeInBits(AtomicSizeInBits),
+        ValueSizeInBits(ValueSizeInBits), AtomicAlign(AtomicAlign),
+        ValueAlign(ValueAlign), UseLibcall(UseLibcall) {}
+
+  virtual ~AtomicInfo() = default;
+
+  llvm::Align getAtomicAlignment() const { return AtomicAlign; }
+  uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
+  uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
+  bool shouldUseLibcall() const { return UseLibcall; }
+  llvm::Type *getAtomicTy() const { return Ty; }
+
+  virtual llvm::Value *getAtomicPointer() const = 0;
+  virtual void decorateWithTBAA(Instruction *I) = 0;
+  virtual llvm::AllocaInst *CreateAlloca(llvm::Type *Ty,
+                                         const llvm::Twine &Name) const = 0;
+
+  /*
+   * Is the atomic size larger than the underlying value type?
+   * Note that the absence of padding does not mean that atomic
+   * objects are completely interchangeable with non-atomic
+   * objects: we might have promoted the alignment of a type
+   * without making it bigger.
+   */
+  bool hasPadding() const { return (ValueSizeInBits != AtomicSizeInBits); }
+
+  LLVMContext &getLLVMContext() const { return Builder->getContext(); }
+
+  static bool shouldCastToInt(llvm::Type *ValTy, bool CmpXchg) {
+    if (ValTy->isFloatingPointTy())
+      return ValTy->isX86_FP80Ty() || CmpXchg;
+    return !ValTy->isIntegerTy() && !ValTy->isPointerTy();
+  }
+
+  llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile,
+                                bool CmpXchg = false) {
+    Value *Ptr = getAtomicPointer();
+    Type *AtomicTy = Ty;
+    if (shouldCastToInt(Ty, CmpXchg))
+      AtomicTy = llvm::IntegerType::get(getLLVMContext(), AtomicSizeInBits);
+    LoadInst *Load =
+        Builder->CreateAlignedLoad(AtomicTy, Ptr, AtomicAlign, "atomic-load");
+    Load->setAtomic(AO);
+    if (IsVolatile)
+      Load->setVolatile(true);
+    decorateWithTBAA(Load);
+    return Load;
+  }
+
+  static CallInst *EmitAtomicLibcall(IRBuilderTy *Builder, StringRef fnName,
+                                     Type *ResultType, ArrayRef<Value *> Args) {
+    LLVMContext &ctx = Builder->getContext();
+    SmallVector<Type *, 6> ArgTys;
+    for (Value *Arg : Args)
+      ArgTys.push_back(Arg->getType());
+    FunctionType *FnType = FunctionType::get(ResultType, ArgTys, false);
+    Module *M = Builder->GetInsertBlock()->getModule();
+
+    // TODO: Use llvm::TargetLowering for Libcall ABI
+    llvm::AttrBuilder fnAttrBuilder(ctx);
+    fnAttrBuilder.addAttribute(llvm::Attribute::NoUnwind);
+    fnAttrBuilder.addAttribute(llvm::Attribute::WillReturn);
+    llvm::AttributeList fnAttrs = llvm::AttributeList::get(
+        ctx, llvm::AttributeList::FunctionIndex, fnAttrBuilder);
+    FunctionCallee LibcallFn = M->getOrInsertFunction(fnName, FnType, fnAttrs);
+    CallInst *Call = Builder->CreateCall(LibcallFn, Args);
+    return Call;
+  }
+
+  llvm::Value *getAtomicSizeValue() const {
+    LLVMContext &ctx = getLLVMContext();
+
+    // TODO: Get from llvm::TargetMachine / clang::TargetInfo
+    // 	if clang shares this codegen in future
+    constexpr uint16_t SizeTBits = 64;
+    constexpr uint16_t BitsPerByte = 8;
+    return llvm::ConstantInt::get(llvm::IntegerType::get(ctx, SizeTBits),
+                                  AtomicSizeInBits / BitsPerByte);
+  }
+
+  std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeLibcall(
+      llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
+      llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure) {
+    LLVMContext &ctx = getLLVMContext();
+
+    // __atomic_compare_exchange's expected and desired are passed by pointers
+    // FIXME: types
+
+    // TODO: Get from llvm::TargetMachine / clang::TargetInfo
+    // 	if clang shares this codegen in future
+    constexpr uint64_t IntBits = 32;
+
+    // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
+    // 	void *desired, int success, int failure);
+    llvm::Value *Args[6] = {
+        getAtomicSizeValue(),
+        getAtomicPointer(),
+        ExpectedVal,
+        DesiredVal,
+        llvm::Constant::getIntegerValue(
+            llvm::IntegerType::get(ctx, IntBits),
+            llvm::APInt(IntBits, static_cast<uint64_t>(Success),
+                        /*signed=*/true)),
+        llvm::Constant::getIntegerValue(
+            llvm::IntegerType::get(ctx, IntBits),
+            llvm::APInt(IntBits, static_cast<uint64_t>(Failure),
+                        /*signed=*/true)),
+    };
+    auto Result = EmitAtomicLibcall(Builder, "__atomic_compare_exchange",
+                                    llvm::IntegerType::getInt1Ty(ctx), Args);
+    return std::make_pair(ExpectedVal, Result);
+  }
+
+  Value *castToAtomicIntPointer(Value *addr) const {
+    return addr; // opaque pointer
+  }
+
+  Value *getAtomicAddressAsAtomicIntPointer() const {
+    return castToAtomicIntPointer(getAtomicPointer());
+  }
+
+  std::pair<llvm::Value *, llvm::Value *>
+  EmitAtomicCompareExchangeOp(llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
+                              llvm::AtomicOrdering Success,
+                              llvm::AtomicOrdering Failure,
+                              bool IsVolatile = false, bool IsWeak = false) {
+    // Do the atomic store.
+    Value *Addr = getAtomicAddressAsAtomicIntPointer();
+    auto *Inst = Builder->CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal,
+                                              getAtomicAlignment(), Success,
+                                              Failure, llvm::SyncScope::System);
+    // Other decoration.
+    Inst->setVolatile(IsVolatile);
+    Inst->setWeak(IsWeak);
+
+    auto *PreviousVal = Builder->CreateExtractValue(Inst, /*Idxs=*/0);
+    auto *SuccessFailureVal = Builder->CreateExtractValue(Inst, /*Idxs=*/1);
+    return std::make_pair(PreviousVal, SuccessFailureVal);
+  }
+
+  std::pair<llvm::Value *, llvm::Value *>
+  EmitAtomicCompareExchange(llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
+                            llvm::AtomicOrdering Success,
+                            llvm::AtomicOrdering Failure, bool IsVolatile,
+                            bool IsWeak) {
+    if (shouldUseLibcall())
+      return EmitAtomicCompareExchangeLibcall(ExpectedVal, DesiredVal, Success,
+                                              Failure);
+
+    auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
+                                           Failure, IsVolatile, IsWeak);
+    return Res;
+  }
+
+  // void __atomic_load(size_t size, void *mem, void *return, int order);
+  std::pair<llvm::LoadInst *, llvm::AllocaInst *>
+  EmitAtomicLoadLibcall(llvm::AtomicOrdering AO) {
+    LLVMContext &Ctx = getLLVMContext();
+    Type *SizedIntTy = Type::getIntNTy(Ctx, getAtomicSizeInBits());
+    Type *ResultTy;
+    SmallVector<Value *, 6> Args;
+    AttributeList Attr;
+    Module *M = Builder->GetInsertBlock()->getModule();
+    const DataLayout &DL = M->getDataLayout();
+    Args.push_back(ConstantInt::get(DL.getIntPtrType(Ctx),
+                                    this->getAtomicSizeInBits() / 8));
+
+    Value *PtrVal = getAtomicPointer();
+    PtrVal = Builder->CreateAddrSpaceCast(PtrVal, PointerType::getUnqual(Ctx));
+    Args.push_back(PtrVal);
+    AllocaInst *AllocaResult =
+        CreateAlloca(Ty, getAtomicPointer()->getName() + "atomic.temp.load");
+    const Align AllocaAlignment = DL.getPrefTypeAlign(SizedIntTy);
+    AllocaResult->setAlignment(AllocaAlignment);
+    Args.push_back(AllocaResult);
+    Constant *OrderingVal =
+        ConstantInt::get(Type::getInt32Ty(Ctx), (int)toCABI(AO));
+    Args.push_back(OrderingVal);
+
+    ResultTy = Type::getVoidTy(Ctx);
+    SmallVector<Type *, 6> ArgTys;
+    for (Value *Arg : Args)
+      ArgTys.push_back(Arg->getType());
+    FunctionType *FnType = FunctionType::get(ResultTy, ArgTys, false);
+    FunctionCallee LibcallFn =
+        M->getOrInsertFunction("__atomic_load", FnType, Attr);
+    CallInst *Call = Builder->CreateCall(LibcallFn, Args);
+    Call->setAttributes(Attr);
+    return std::make_pair(
+        Builder->CreateAlignedLoad(Ty, AllocaResult, AllocaAlignment),
+        AllocaResult);
+  }
+};
+} // end namespace llvm
+
+#endif /* LLVM_FRONTEND_ATOMIC_ATOMIC_H */
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 01515108ca185..fcf087d1f9c6e 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -229,7 +229,7 @@ def OMPC_IsDevicePtr : Clause<"is_device_ptr"> {
 }
 def OMPC_LastPrivate : Clause<"lastprivate"> {
   let clangClass = "OMPLastprivateClause";
-  let flangClass = "OmpObjectList";
+  let flangClass = "OmpLastprivateClause";
 }
 def OMPC_Linear : Clause<"linear"> {
   let clangClass = "OMPLinearClause";
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 4be0159fb1dd9..1b8a6e47b3baf 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -15,6 +15,7 @@
 #define LLVM_FRONTEND_OPENMP_OMPIRBUILDER_H
 
 #include "llvm/Analysis/MemorySSAUpdater.h"
+#include "llvm/Frontend/Atomic/Atomic.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
 #include "llvm/Frontend/OpenMP/OMPGridValues.h"
 #include "llvm/IR/DebugLoc.h"
@@ -479,6 +480,27 @@ class OpenMPIRBuilder {
         T(Triple(M.getTargetTriple())) {}
   ~OpenMPIRBuilder();
 
+  class AtomicInfo : public llvm::AtomicInfo<IRBuilder<>> {
+    llvm::Value *AtomicVar;
+
+  public:
+    AtomicInfo(IRBuilder<> *Builder, llvm::Type *Ty, uint64_t AtomicSizeInBits,
+               uint64_t ValueSizeInBits, llvm::Align AtomicAlign,
+               llvm::Align ValueAlign, bool UseLibcall, llvm::Value *AtomicVar)
+        : llvm::AtomicInfo<IRBuilder<>>(Builder, Ty, AtomicSizeInBits,
+                                        ValueSizeInBits, AtomicAlign,
+                                        ValueAlign, UseLibcall),
+          AtomicVar(AtomicVar) {}
+
+    llvm::Value *getAtomicPointer() const override { return AtomicVar; }
+    void decorateWithTBAA(llvm::Instruction *I) override {}
+    llvm::AllocaInst *CreateAlloca(llvm::Type *Ty,
+                                   const llvm::Twine &Name) const override {
+      llvm::AllocaInst *allocaInst = Builder->CreateAlloca(Ty);
+      allocaInst->setName(Name);
+      return allocaInst;
+    }
+  };
   /// Initialize the internal state, this will put structures types and
   /// potentially other helpers into the underlying module. Must be called
   /// before any other method and only once! This internal state includes types
@@ -3039,6 +3061,15 @@ class OpenMPIRBuilder {
                    AtomicUpdateCallbackTy &UpdateOp, bool VolatileX,
                    bool IsXBinopExpr);
 
+  std::pair<llvm::LoadInst *, llvm::AllocaInst *>
+  EmitAtomicLoadLibcall(Value *X, Type *XElemTy, llvm::AtomicOrdering AO,
+                        uint64_t AtomicSizeInBits);
+
+  std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeLibcall(
+      Value *X, Type *XElemTy, uint64_t AtomicSizeInBits,
+      llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
+      llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure);
+
   /// Emit the binary op. described by \p RMWOp, using \p Src1 and \p Src2 .
   ///
   /// \Return The instruction
diff --git a/llvm/include/llvm/IR/ConstantFPRange.h b/llvm/include/llvm/IR/ConstantFPRange.h
index 67f9f945d748b..e240671296479 100644
--- a/llvm/include/llvm/IR/ConstantFPRange.h
+++ b/llvm/include/llvm/IR/ConstantFPRange.h
@@ -50,7 +50,6 @@ class [[nodiscard]] ConstantFPRange {
 
   void makeEmpty();
   void makeFull();
-  bool isNaNOnly() const;
 
   /// Initialize a full or empty set for the specified semantics.
   explicit ConstantFPRange(const fltSemantics &Sem, bool IsFullSet);
@@ -78,6 +77,9 @@ class [[nodiscard]] ConstantFPRange {
   /// Helper for (-inf, inf) to represent all finite values.
   static ConstantFPRange getFinite(const fltSemantics &Sem);
 
+  /// Helper for [-inf, inf] to represent all non-NaN values.
+  static ConstantFPRange getNonNaN(const fltSemantics &Sem);
+
   /// Create a range which doesn't contain NaNs.
   static ConstantFPRange getNonNaN(APFloat LowerVal, APFloat UpperVal) {
     return ConstantFPRange(std::move(LowerVal), std::move(UpperVal),
@@ -118,13 +120,13 @@ class [[nodiscard]] ConstantFPRange {
 
   /// Produce the exact range such that all values in the returned range satisfy
   /// the given predicate with any value contained within Other. Formally, this
-  /// returns the exact answer when the superset of 'union over all y in Other
-  /// is exactly same as the subset of intersection over all y in Other.
-  /// { x : fcmp op x y is true}'.
+  /// returns { x : fcmp op x Other is true }.
   ///
   /// Example: Pred = olt and Other = float 3 returns [-inf, 3)
-  static ConstantFPRange makeExactFCmpRegion(FCmpInst::Predicate Pred,
-                                             const APFloat &Other);
+  /// If the exact answer is not representable as a ConstantFPRange, returns
+  /// std::nullopt.
+  static std::optional<ConstantFPRange>
+  makeExactFCmpRegion(FCmpInst::Predicate Pred, const APFloat &Other);
 
   /// Does the predicate \p Pred hold between ranges this and \p Other?
   /// NOTE: false does not mean that inverse predicate holds!
@@ -139,6 +141,7 @@ class [[nodiscard]] ConstantFPRange {
   bool containsNaN() const { return MayBeQNaN || MayBeSNaN; }
   bool containsQNaN() const { return MayBeQNaN; }
   bool containsSNaN() const { return MayBeSNaN; }
+  bool isNaNOnly() const;
 
   /// Get the semantics of this ConstantFPRange.
   const fltSemantics &getSemantics() const { return Lower.getSemantics(); }
@@ -157,10 +160,15 @@ class [[nodiscard]] ConstantFPRange {
   bool contains(const ConstantFPRange &CR) const;
 
   /// If this set contains a single element, return it, otherwise return null.
-  const APFloat *getSingleElement() const;
+  /// If \p ExcludesNaN is true, return the non-NaN single element.
+  const APFloat *getSingleElement(bool ExcludesNaN = false) const;
 
   /// Return true if this set contains exactly one member.
-  bool isSingleElement() const { return getSingleElement() != nullptr; }
+  /// If \p ExcludesNaN is true, return true if this set contains exactly one
+  /// non-NaN member.
+  bool isSingleElement(bool ExcludesNaN = false) const {
+    return getSingleElement(ExcludesNaN) != nullptr;
+  }
 
   /// Return true if the sign bit of all values in this range is 1.
   /// Return false if the sign bit of all values in this range is 0.
@@ -185,7 +193,7 @@ class [[nodiscard]] ConstantFPRange {
   /// another range.
   ConstantFPRange intersectWith(const ConstantFPRange &CR) const;
 
-  /// Return the range that results from the union of this range
+  /// Return the smallest range that results from the union of this range
   /// with another range.  The resultant range is guaranteed to include the
   /// elements of both sets, but may contain more.
   ConstantFPRange unionWith(const ConstantFPRange &CR) const;
diff --git a/llvm/include/llvm/IR/DerivedTypes.h b/llvm/include/llvm/IR/DerivedTypes.h
index 975c142f1a457..a24801d8bdf83 100644
--- a/llvm/include/llvm/IR/DerivedTypes.h
+++ b/llvm/include/llvm/IR/DerivedTypes.h
@@ -301,6 +301,10 @@ class StructType : public Type {
   ///  {<vscale x 2 x i32>, <vscale x 4 x i64>}}
   bool containsHomogeneousScalableVectorTypes() const;
 
+  /// Return true if this struct is non-empty and all element types are the
+  /// same.
+  bool containsHomogeneousTypes() const;
+
   /// Return true if this is a named struct that has a non-empty name.
   bool hasName() const { return SymbolTableEntry != nullptr; }
 
diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h
index 4852f64d0977f..86d88da3d9460 100644
--- a/llvm/include/llvm/IR/InstrTypes.h
+++ b/llvm/include/llvm/IR/InstrTypes.h
@@ -1461,6 +1461,22 @@ class CallBase : public Instruction {
   ///
   void setAttributes(AttributeList A) { Attrs = A; }
 
+  /// Try to intersect the attributes from 'this' CallBase and the
+  /// 'Other' CallBase. Sets the intersected attributes to 'this' and
+  /// return true if successful. Doesn't modify 'this' and returns
+  /// false if unsuccessful.
+  bool tryIntersectAttributes(const CallBase *Other) {
+    if (this == Other)
+      return true;
+    AttributeList AL = getAttributes();
+    AttributeList ALOther = Other->getAttributes();
+    auto Intersected = AL.intersectWith(getContext(), ALOther);
+    if (!Intersected)
+      return false;
+    setAttributes(*Intersected);
+    return true;
+  }
+
   /// Determine whether this call has the given attribute. If it does not
   /// then determine if the called function has the attribute, but only if
   /// the attribute is allowed for the call.
diff --git a/llvm/include/llvm/IR/Instruction.h b/llvm/include/llvm/IR/Instruction.h
index 4a1e66a90831c..61dba265dc948 100644
--- a/llvm/include/llvm/IR/Instruction.h
+++ b/llvm/include/llvm/IR/Instruction.h
@@ -881,16 +881,20 @@ class Instruction : public User,
   /// This is like isIdenticalTo, except that it ignores the
   /// SubclassOptionalData flags, which may specify conditions under which the
   /// instruction's result is undefined.
-  bool isIdenticalToWhenDefined(const Instruction *I) const LLVM_READONLY;
+  bool
+  isIdenticalToWhenDefined(const Instruction *I,
+                           bool IntersectAttrs = false) const LLVM_READONLY;
 
   /// When checking for operation equivalence (using isSameOperationAs) it is
   /// sometimes useful to ignore certain attributes.
   enum OperationEquivalenceFlags {
     /// Check for equivalence ignoring load/store alignment.
-    CompareIgnoringAlignment = 1<<0,
+    CompareIgnoringAlignment = 1 << 0,
     /// Check for equivalence treating a type and a vector of that type
     /// as equivalent.
-    CompareUsingScalarTypes = 1<<1
+    CompareUsingScalarTypes = 1 << 1,
+    /// Check for equivalence with intersected callbase attrs.
+    CompareUsingIntersectedAttrs = 1 << 2,
   };
 
   /// This function determines if the specified instruction executes the same
@@ -911,8 +915,8 @@ class Instruction : public User,
   /// @returns true if the specific instruction has the same opcde specific
   /// characteristics as the current one. Determine if one instruction has the
   /// same state as another.
-  bool hasSameSpecialState(const Instruction *I2,
-                           bool IgnoreAlignment = false) const LLVM_READONLY;
+  bool hasSameSpecialState(const Instruction *I2, bool IgnoreAlignment = false,
+                           bool IntersectAttrs = false) const LLVM_READONLY;
 
   /// Return true if there are any uses of this instruction in blocks other than
   /// the specified block. Note that PHI nodes are considered to evaluate their
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 079ac61adef6e..20dd921ddbd23 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -64,36 +64,42 @@ def Throws : IntrinsicProperty;
 class AttrIndex<int idx> {
   int Value = idx;
 }
-def FuncIndex : AttrIndex<-1>;
 def RetIndex : AttrIndex<0>;
 class ArgIndex<int argNo> : AttrIndex<!add(argNo, 1)>;
 
+// Note: Properties that are applicable either to arguments or return values
+// use AttrIndex. Properties applicable only to arguments use ArgIndex. Please
+// refer to Attributes.td.
+
 // NoCapture - The specified argument pointer is not captured by the intrinsic.
-class NoCapture<AttrIndex idx> : IntrinsicProperty {
+class NoCapture<ArgIndex idx> : IntrinsicProperty {
   int ArgNo = idx.Value;
 }
 
-// NoAlias - The specified argument pointer is not aliasing other "noalias" pointer
-// arguments of the intrinsic wrt. the intrinsic scope.
+// NoAlias - The return value or the specified argument pointer is not aliasing
+// other "noalias" pointer arguments of the intrinsic wrt. the intrinsic scope.
 class NoAlias<AttrIndex idx> : IntrinsicProperty {
   int ArgNo = idx.Value;
 }
 
-// NoUndef - The specified argument is neither undef nor poison.
+// NoUndef - The return value or specified argument is neither undef nor poison.
 class NoUndef<AttrIndex idx> : IntrinsicProperty {
   int ArgNo = idx.Value;
 }
 
-// NonNull - The specified argument is not null.
+// NonNull - The return value or specified argument is not null.
 class NonNull<AttrIndex idx> : IntrinsicProperty {
   int ArgNo = idx.Value;
 }
 
+// Align - Alignment for return value or the specified argument.
 class Align<AttrIndex idx, int align> : IntrinsicProperty {
   int ArgNo = idx.Value;
   int Align = align;
 }
 
+// Dereferenceable -- Return value or the specified argument is dereferenceable
+// upto `bytes` bytes in size.
 class Dereferenceable<AttrIndex idx, int bytes> : IntrinsicProperty {
   int ArgNo = idx.Value;
   int Bytes = bytes;
@@ -101,30 +107,30 @@ class Dereferenceable<AttrIndex idx, int bytes> : IntrinsicProperty {
 
 // Returned - The specified argument is always the return value of the
 // intrinsic.
-class Returned<AttrIndex idx> : IntrinsicProperty {
+class Returned<ArgIndex idx> : IntrinsicProperty {
   int ArgNo = idx.Value;
 }
 
 // ImmArg - The specified argument must be an immediate.
-class ImmArg<AttrIndex idx> : IntrinsicProperty {
+class ImmArg<ArgIndex idx> : IntrinsicProperty {
   int ArgNo = idx.Value;
 }
 
 // ReadOnly - The specified argument pointer is not written to through the
 // pointer by the intrinsic.
-class ReadOnly<AttrIndex idx> : IntrinsicProperty {
+class ReadOnly<ArgIndex idx> : IntrinsicProperty {
   int ArgNo = idx.Value;
 }
 
 // WriteOnly - The intrinsic does not read memory through the specified
 // argument pointer.
-class WriteOnly<AttrIndex idx> : IntrinsicProperty {
+class WriteOnly<ArgIndex idx> : IntrinsicProperty {
   int ArgNo = idx.Value;
 }
 
 // ReadNone - The specified argument pointer is not dereferenced by the
 // intrinsic.
-class ReadNone<AttrIndex idx> : IntrinsicProperty {
+class ReadNone<ArgIndex idx> : IntrinsicProperty {
   int ArgNo = idx.Value;
 }
 
@@ -366,6 +372,10 @@ class LLVMType<ValueType vt> {
     []);
   assert !le(!size(IIT_Vecs), 1), "Duplicate type";
 
+  // For vector types, assert that the IIT_Vecs list is not empty.
+  assert !or(!not(vt.isVector), !not(!empty(IIT_Vecs))),
+         "Invalid IIT encoding for vector type v" # vt.nElem # vt.ElementType;
+
   list<int> Sig = !listconcat(
     !if(vt.isScalable, [IIT_SCALABLE_VEC.Number], []),
     !foreach(iit, IIT_Vecs, iit.Number),
@@ -512,7 +522,6 @@ def llvm_v128i1_ty     : LLVMType<v128i1>;   // 128 x i1
 def llvm_v256i1_ty     : LLVMType<v256i1>;   // 256 x i1
 def llvm_v512i1_ty     : LLVMType<v512i1>;   // 512 x i1
 def llvm_v1024i1_ty    : LLVMType<v1024i1>;  //1024 x i1
-def llvm_v2048i1_ty    : LLVMType<v2048i1>;  //2048 x i1
 
 def llvm_v1i8_ty       : LLVMType<v1i8>;     //  1 x i8
 def llvm_v2i8_ty       : LLVMType<v2i8>;     //  2 x i8
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index cd9e66b10d1de..594069c619ceb 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -2184,8 +2184,8 @@ def int_aarch64_sve_fcvtzs_i32f64   : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv2i1
 def int_aarch64_sve_fcvtzs_i64f16   : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
 def int_aarch64_sve_fcvtzs_i64f32   : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
 
-def int_aarch64_sve_fcvt_bf16f32    : Builtin_SVCVT<llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>;
-def int_aarch64_sve_fcvtnt_bf16f32  : Builtin_SVCVT<llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>;
+def int_aarch64_sve_fcvt_bf16f32_v2   : Builtin_SVCVT<llvm_nxv8bf16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>;
+def int_aarch64_sve_fcvtnt_bf16f32_v2 : Builtin_SVCVT<llvm_nxv8bf16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>;
 
 def int_aarch64_sve_fcvtzu_i32f16   : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
 def int_aarch64_sve_fcvtzu_i32f64   : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index 555877e7aaf0e..1adfcdc9a1ed2 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -42,6 +42,7 @@ def int_dx_all : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty], [IntrNoMem]>
 def int_dx_any : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty], [IntrNoMem]>;
 def int_dx_clamp : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem]>;
 def int_dx_uclamp : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem]>;
+def int_dx_cross : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem]>;
 def int_dx_saturate : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
 
 def int_dx_dot2 :
@@ -74,8 +75,8 @@ def int_dx_frac  : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>],
 def int_dx_isinf : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
     [llvm_anyfloat_ty], [IntrNoMem]>;
 
-def int_dx_lerp : Intrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty, LLVMMatchType<0>,LLVMMatchType<0>],
-    [IntrNoMem, IntrWillReturn] >;
+def int_dx_lerp : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty, LLVMMatchType<0>,LLVMMatchType<0>],
+    [IntrNoMem]>;
 
 def int_dx_length : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], [llvm_anyfloat_ty], [IntrNoMem]>;
 def int_dx_imad : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem]>;
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index 7ff3d58690ba7..88059aa837814 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -59,16 +59,17 @@ let TargetPrefix = "spv" in {
 
   // The following intrinsic(s) are mirrored from IntrinsicsDirectX.td for HLSL support.
   def int_spv_thread_id : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem, IntrWillReturn]>;
-  def int_spv_all : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty]>;
-  def int_spv_any : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty]>;
-  def int_spv_frac : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty]>;
-  def int_spv_lerp : Intrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty, LLVMMatchType<0>,LLVMMatchType<0>],
-    [IntrNoMem, IntrWillReturn] >;
-  def int_spv_length : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], [llvm_anyfloat_ty]>;
-  def int_spv_normalize : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty]>;
-  def int_spv_rsqrt : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty]>;
-  def int_spv_saturate : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>;
-  def int_spv_step : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [LLVMMatchType<0>, llvm_anyfloat_ty]>;
+  def int_spv_all : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty], [IntrNoMem]>;
+  def int_spv_any : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty], [IntrNoMem]>;
+  def int_spv_cross : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem]>;
+  def int_spv_frac : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty], [IntrNoMem]>;
+  def int_spv_lerp : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty, LLVMMatchType<0>,LLVMMatchType<0>],
+    [IntrNoMem] >;
+  def int_spv_length : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], [llvm_anyfloat_ty], [IntrNoMem]>;
+  def int_spv_normalize : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty], [IntrNoMem]>;
+  def int_spv_rsqrt : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty], [IntrNoMem]>;
+  def int_spv_saturate : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
+  def int_spv_step : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [LLVMMatchType<0>, llvm_anyfloat_ty], [IntrNoMem]>;
   def int_spv_fdot :
     DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
     [llvm_anyfloat_ty, LLVMScalarOrSameVectorWidth<0, LLVMVectorElementType<0>>],
@@ -82,5 +83,6 @@ let TargetPrefix = "spv" in {
     [llvm_anyint_ty, LLVMScalarOrSameVectorWidth<0, LLVMVectorElementType<0>>],
     [IntrNoMem, Commutative] >;
   def int_spv_wave_is_first_lane : DefaultAttrsIntrinsic<[llvm_i1_ty], [], [IntrConvergent]>;
-  def int_spv_sign : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>], [llvm_any_ty]>;
+  def int_spv_sign : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>], [llvm_any_ty], [IntrNoMem]>;
+  def int_spv_radians : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty], [IntrNoMem]>;
 }
diff --git a/llvm/include/llvm/IR/Operator.h b/llvm/include/llvm/IR/Operator.h
index 0e9f6ed35dcb4..5f3989362f9cd 100644
--- a/llvm/include/llvm/IR/Operator.h
+++ b/llvm/include/llvm/IR/Operator.h
@@ -268,6 +268,21 @@ class FPMathOperator : public Operator {
     SubclassOptionalData = FMF.Flags;
   }
 
+  /// Returns true if `Ty` is composed of a single kind of float-poing type
+  /// (possibly repeated within an aggregate).
+  static bool isComposedOfHomogeneousFloatingPointTypes(Type *Ty) {
+    if (auto *StructTy = dyn_cast<StructType>(Ty)) {
+      if (!StructTy->isLiteral() || !StructTy->containsHomogeneousTypes())
+        return false;
+      Ty = StructTy->elements().front();
+    } else if (auto *ArrayTy = dyn_cast<ArrayType>(Ty)) {
+      do {
+        Ty = ArrayTy->getElementType();
+      } while ((ArrayTy = dyn_cast<ArrayType>(Ty)));
+    }
+    return Ty->isFPOrFPVectorTy();
+  };
+
 public:
   /// Test if this operation allows all non-strict floating-point transforms.
   bool isFast() const {
@@ -326,6 +341,13 @@ class FPMathOperator : public Operator {
   /// precision.
   float getFPAccuracy() const;
 
+  /// Returns true if `Ty` is a supported floating-point type for phi, select,
+  /// or call FPMathOperators.
+  static bool isSupportedFloatingPointType(Type *Ty) {
+    return Ty->isFPOrFPVectorTy() ||
+           isComposedOfHomogeneousFloatingPointTypes(Ty);
+  }
+
   static bool classof(const Value *V) {
     unsigned Opcode;
     if (auto *I = dyn_cast<Instruction>(V))
@@ -350,10 +372,7 @@ class FPMathOperator : public Operator {
     case Instruction::PHI:
     case Instruction::Select:
     case Instruction::Call: {
-      Type *Ty = V->getType();
-      while (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty))
-        Ty = ArrTy->getElementType();
-      return Ty->isFPOrFPVectorTy();
+      return isSupportedFloatingPointType(V->getType());
     }
     default:
       return false;
diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h
index 214aa4e1c562d..a281c377f2601 100644
--- a/llvm/include/llvm/LTO/LTO.h
+++ b/llvm/include/llvm/LTO/LTO.h
@@ -298,7 +298,7 @@ class LTO {
   ///
   /// The client will receive at most one callback (via either AddStream or
   /// Cache) for each task identifier.
-  Error run(AddStreamFn AddStream, FileCache Cache = nullptr);
+  Error run(AddStreamFn AddStream, FileCache Cache = {});
 
   /// Static method that returns a list of libcall symbols that can be generated
   /// by LTO but might not be visible from bitcode symbol table.
diff --git a/llvm/include/llvm/LTO/LTOBackend.h b/llvm/include/llvm/LTO/LTOBackend.h
index de89f4bb10dff..098c0491dfe70 100644
--- a/llvm/include/llvm/LTO/LTOBackend.h
+++ b/llvm/include/llvm/LTO/LTOBackend.h
@@ -50,12 +50,14 @@ Error backend(const Config &C, AddStreamFn AddStream,
 /// already been mapped to memory and the corresponding BitcodeModule objects
 /// are saved in the ModuleMap. If \p ModuleMap is nullptr, module files will
 /// be mapped to memory on demand and at any given time during importing, only
-/// one source module will be kept open at the most.
+/// one source module will be kept open at the most. If \p CodeGenOnly is true,
+/// the backend will skip optimization and only perform code generation.
 Error thinBackend(const Config &C, unsigned Task, AddStreamFn AddStream,
                   Module &M, const ModuleSummaryIndex &CombinedIndex,
                   const FunctionImporter::ImportMapTy &ImportList,
                   const GVSummaryMapTy &DefinedGlobals,
                   MapVector<StringRef, BitcodeModule> *ModuleMap,
+                  bool CodeGenOnly,
                   const std::vector<uint8_t> &CmdArgs = std::vector<uint8_t>());
 
 Error finalizeOptimizationRemarks(
diff --git a/llvm/include/llvm/Object/Archive.h b/llvm/include/llvm/Object/Archive.h
index a3165c3235e0e..b2dd970b00c05 100644
--- a/llvm/include/llvm/Object/Archive.h
+++ b/llvm/include/llvm/Object/Archive.h
@@ -339,7 +339,7 @@ class Archive : public Binary {
   Kind kind() const { return (Kind)Format; }
   bool isThin() const { return IsThin; }
   static object::Archive::Kind getDefaultKind();
-  static object::Archive::Kind getDefaultKindForTriple(Triple &T);
+  static object::Archive::Kind getDefaultKindForTriple(const Triple &T);
 
   child_iterator child_begin(Error &Err, bool SkipInternal = true) const;
   child_iterator child_end() const;
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index fa9c744294a66..80eedc52bc324 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -171,7 +171,7 @@ class PreservedCFGCheckerInstrumentation {
                     FunctionAnalysisManager::Invalidator &);
   };
 
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
   SmallVector<StringRef, 8> PassStack;
 #endif
 
diff --git a/llvm/include/llvm/SandboxIR/Instruction.h b/llvm/include/llvm/SandboxIR/Instruction.h
index a34573a0bc1b0..a6dc0dbcdfc28 100644
--- a/llvm/include/llvm/SandboxIR/Instruction.h
+++ b/llvm/include/llvm/SandboxIR/Instruction.h
@@ -18,6 +18,24 @@
 
 namespace llvm::sandboxir {
 
+// Forward declaration for MSVC.
+class IntrinsicInst;
+
+class InsertPosition {
+  BBIterator InsertAt;
+
+public:
+  InsertPosition(BasicBlock *InsertAtEnd) {
+    assert(InsertAtEnd != nullptr && "Expected non-null!");
+    InsertAt = InsertAtEnd->end();
+  }
+  InsertPosition(BBIterator InsertAt) : InsertAt(InsertAt) {}
+  operator BBIterator() { return InsertAt; }
+  const BBIterator &getIterator() const { return InsertAt; }
+  Instruction &operator*() { return *InsertAt; }
+  BasicBlock *getBasicBlock() const { return InsertAt.getNodeParent(); }
+};
+
 /// A sandboxir::User with operands, opcode and linked with previous/next
 /// instructions in an instruction list.
 class Instruction : public User {
@@ -79,6 +97,20 @@ class Instruction : public User {
   virtual SmallVector<llvm::Instruction *, 1> getLLVMInstrs() const = 0;
   friend class EraseFromParent; // For getLLVMInstrs().
 
+  /// Helper function for create(). It sets the builder's insert position
+  /// according to \p Pos.
+  static IRBuilder<> &setInsertPos(InsertPosition Pos) {
+    auto *WhereBB = Pos.getBasicBlock();
+    auto WhereIt = Pos.getIterator();
+    auto &Ctx = WhereBB->getContext();
+    auto &Builder = Ctx.getLLVMIRBuilder();
+    if (WhereIt != WhereBB->end())
+      Builder.SetInsertPoint((*Pos).getTopmostLLVMInstruction());
+    else
+      Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+    return Builder;
+  }
+
 public:
   static const char *getOpcodeName(Opcode Opc);
   /// This is used by BasicBlock::iterator.
@@ -97,6 +129,9 @@ class Instruction : public User {
 
   const char *getOpcodeName() const { return getOpcodeName(Opc); }
 
+  const DataLayout &getDataLayout() const {
+    return cast<llvm::Instruction>(Val)->getModule()->getDataLayout();
+  }
   // Note that these functions below are calling into llvm::Instruction.
   // A sandbox IR instruction could introduce a new opcode that could change the
   // behavior of one of these functions. It is better that these functions are
@@ -332,26 +367,6 @@ class Instruction : public User {
 
   // TODO: Missing functions.
 
-  bool isStackSaveOrRestoreIntrinsic() const {
-    auto *I = cast<llvm::Instruction>(Val);
-    return match(I,
-                 PatternMatch::m_Intrinsic<llvm::Intrinsic::stackrestore>()) ||
-           match(I, PatternMatch::m_Intrinsic<llvm::Intrinsic::stacksave>());
-  }
-
-  /// We consider \p I as a Memory Dependency Candidate instruction if it
-  /// reads/write memory or if it has side-effects. This is used by the
-  /// dependency graph.
-  bool isMemDepCandidate() const {
-    auto *I = cast<llvm::Instruction>(Val);
-    return I->mayReadOrWriteMemory() &&
-           (!isa<llvm::IntrinsicInst>(I) ||
-            (cast<llvm::IntrinsicInst>(I)->getIntrinsicID() !=
-                 Intrinsic::sideeffect &&
-             cast<llvm::IntrinsicInst>(I)->getIntrinsicID() !=
-                 Intrinsic::pseudoprobe));
-  }
-
 #ifndef NDEBUG
   void dumpOS(raw_ostream &OS) const override;
 #endif
@@ -398,8 +413,8 @@ class FenceInst : public SingleLLVMInstructionImpl<llvm::FenceInst> {
   friend Context; // For constructor;
 
 public:
-  static FenceInst *create(AtomicOrdering Ordering, BBIterator WhereIt,
-                           BasicBlock *WhereBB, Context &Ctx,
+  static FenceInst *create(AtomicOrdering Ordering, InsertPosition Pos,
+                           Context &Ctx,
                            SyncScope::ID SSID = SyncScope::System);
   /// Returns the ordering constraint of this fence instruction.
   AtomicOrdering getOrdering() const {
@@ -425,16 +440,10 @@ class SelectInst : public SingleLLVMInstructionImpl<llvm::SelectInst> {
   SelectInst(llvm::SelectInst *CI, Context &Ctx)
       : SingleLLVMInstructionImpl(ClassID::Select, Opcode::Select, CI, Ctx) {}
   friend Context; // for SelectInst()
-  static Value *createCommon(Value *Cond, Value *True, Value *False,
-                             const Twine &Name, IRBuilder<> &Builder,
-                             Context &Ctx);
 
 public:
   static Value *create(Value *Cond, Value *True, Value *False,
-                       Instruction *InsertBefore, Context &Ctx,
-                       const Twine &Name = "");
-  static Value *create(Value *Cond, Value *True, Value *False,
-                       BasicBlock *InsertAtEnd, Context &Ctx,
+                       InsertPosition Pos, Context &Ctx,
                        const Twine &Name = "");
 
   const Value *getCondition() const { return getOperand(0); }
@@ -471,10 +480,7 @@ class InsertElementInst final
 
 public:
   static Value *create(Value *Vec, Value *NewElt, Value *Idx,
-                       Instruction *InsertBefore, Context &Ctx,
-                       const Twine &Name = "");
-  static Value *create(Value *Vec, Value *NewElt, Value *Idx,
-                       BasicBlock *InsertAtEnd, Context &Ctx,
+                       InsertPosition Pos, Context &Ctx,
                        const Twine &Name = "");
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::InsertElement;
@@ -496,10 +502,8 @@ class ExtractElementInst final
                         // create*()
 
 public:
-  static Value *create(Value *Vec, Value *Idx, Instruction *InsertBefore,
-                       Context &Ctx, const Twine &Name = "");
-  static Value *create(Value *Vec, Value *Idx, BasicBlock *InsertAtEnd,
-                       Context &Ctx, const Twine &Name = "");
+  static Value *create(Value *Vec, Value *Idx, InsertPosition Pos, Context &Ctx,
+                       const Twine &Name = "");
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::ExtractElement;
   }
@@ -523,17 +527,10 @@ class ShuffleVectorInst final
   friend class Context; // For accessing the constructor in create*()
 
 public:
-  static Value *create(Value *V1, Value *V2, Value *Mask,
-                       Instruction *InsertBefore, Context &Ctx,
-                       const Twine &Name = "");
-  static Value *create(Value *V1, Value *V2, Value *Mask,
-                       BasicBlock *InsertAtEnd, Context &Ctx,
-                       const Twine &Name = "");
-  static Value *create(Value *V1, Value *V2, ArrayRef<int> Mask,
-                       Instruction *InsertBefore, Context &Ctx,
-                       const Twine &Name = "");
+  static Value *create(Value *V1, Value *V2, Value *Mask, InsertPosition Pos,
+                       Context &Ctx, const Twine &Name = "");
   static Value *create(Value *V1, Value *V2, ArrayRef<int> Mask,
-                       BasicBlock *InsertAtEnd, Context &Ctx,
+                       InsertPosition Pos, Context &Ctx,
                        const Twine &Name = "");
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::ShuffleVector;
@@ -968,7 +965,7 @@ class InsertValueInst
 
 public:
   static Value *create(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs,
-                       BBIterator WhereIt, BasicBlock *WhereBB, Context &Ctx,
+                       InsertPosition Pos, Context &Ctx,
                        const Twine &Name = "");
 
   static bool classof(const Value *From) {
@@ -1026,15 +1023,10 @@ class BranchInst : public SingleLLVMInstructionImpl<llvm::BranchInst> {
   friend Context; // for BranchInst()
 
 public:
-  static BranchInst *create(BasicBlock *IfTrue, Instruction *InsertBefore,
-                            Context &Ctx);
-  static BranchInst *create(BasicBlock *IfTrue, BasicBlock *InsertAtEnd,
-                            Context &Ctx);
-  static BranchInst *create(BasicBlock *IfTrue, BasicBlock *IfFalse,
-                            Value *Cond, Instruction *InsertBefore,
+  static BranchInst *create(BasicBlock *IfTrue, InsertPosition Pos,
                             Context &Ctx);
   static BranchInst *create(BasicBlock *IfTrue, BasicBlock *IfFalse,
-                            Value *Cond, BasicBlock *InsertAtEnd, Context &Ctx);
+                            Value *Cond, InsertPosition Pos, Context &Ctx);
   /// For isa/dyn_cast.
   static bool classof(const Value *From);
   bool isUnconditional() const {
@@ -1116,9 +1108,8 @@ class ExtractValueInst : public UnaryInstruction {
   friend Context; // for ExtractValueInst()
 
 public:
-  static Value *create(Value *Agg, ArrayRef<unsigned> Idxs, BBIterator WhereIt,
-                       BasicBlock *WhereBB, Context &Ctx,
-                       const Twine &Name = "");
+  static Value *create(Value *Agg, ArrayRef<unsigned> Idxs, InsertPosition Pos,
+                       Context &Ctx, const Twine &Name = "");
 
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::ExtractValue;
@@ -1171,9 +1162,8 @@ class VAArgInst : public UnaryInstruction {
   friend Context; // For constructor;
 
 public:
-  static VAArgInst *create(Value *List, Type *Ty, BBIterator WhereIt,
-                           BasicBlock *WhereBB, Context &Ctx,
-                           const Twine &Name = "");
+  static VAArgInst *create(Value *List, Type *Ty, InsertPosition Pos,
+                           Context &Ctx, const Twine &Name = "");
   Value *getPointerOperand();
   const Value *getPointerOperand() const {
     return const_cast<VAArgInst *>(this)->getPointerOperand();
@@ -1192,8 +1182,8 @@ class FreezeInst : public UnaryInstruction {
   friend Context; // For constructor;
 
 public:
-  static FreezeInst *create(Value *V, BBIterator WhereIt, BasicBlock *WhereBB,
-                            Context &Ctx, const Twine &Name = "");
+  static FreezeInst *create(Value *V, InsertPosition Pos, Context &Ctx,
+                            const Twine &Name = "");
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::Freeze;
   }
@@ -1212,17 +1202,13 @@ class LoadInst final : public UnaryInstruction {
   void setVolatile(bool V);
 
   static LoadInst *create(Type *Ty, Value *Ptr, MaybeAlign Align,
-                          Instruction *InsertBefore, Context &Ctx,
-                          const Twine &Name = "");
-  static LoadInst *create(Type *Ty, Value *Ptr, MaybeAlign Align,
-                          Instruction *InsertBefore, bool IsVolatile,
-                          Context &Ctx, const Twine &Name = "");
-  static LoadInst *create(Type *Ty, Value *Ptr, MaybeAlign Align,
-                          BasicBlock *InsertAtEnd, Context &Ctx,
+                          InsertPosition Pos, bool IsVolatile, Context &Ctx,
                           const Twine &Name = "");
   static LoadInst *create(Type *Ty, Value *Ptr, MaybeAlign Align,
-                          BasicBlock *InsertAtEnd, bool IsVolatile,
-                          Context &Ctx, const Twine &Name = "");
+                          InsertPosition Pos, Context &Ctx,
+                          const Twine &Name = "") {
+    return create(Ty, Ptr, Align, Pos, /*IsVolatile=*/false, Ctx, Name);
+  }
 
   /// For isa/dyn_cast.
   static bool classof(const Value *From);
@@ -1245,15 +1231,12 @@ class StoreInst final : public SingleLLVMInstructionImpl<llvm::StoreInst> {
   void setVolatile(bool V);
 
   static StoreInst *create(Value *V, Value *Ptr, MaybeAlign Align,
-                           Instruction *InsertBefore, Context &Ctx);
-  static StoreInst *create(Value *V, Value *Ptr, MaybeAlign Align,
-                           Instruction *InsertBefore, bool IsVolatile,
-                           Context &Ctx);
+                           InsertPosition Pos, bool IsVolatile, Context &Ctx);
   static StoreInst *create(Value *V, Value *Ptr, MaybeAlign Align,
-                           BasicBlock *InsertAtEnd, Context &Ctx);
-  static StoreInst *create(Value *V, Value *Ptr, MaybeAlign Align,
-                           BasicBlock *InsertAtEnd, bool IsVolatile,
-                           Context &Ctx);
+                           InsertPosition Pos, Context &Ctx) {
+    return create(V, Ptr, Align, Pos, /*IsVolatile=*/false, Ctx);
+  }
+
   /// For isa/dyn_cast.
   static bool classof(const Value *From);
   Value *getValueOperand() const;
@@ -1276,8 +1259,7 @@ class UnreachableInst final : public Instruction {
   }
 
 public:
-  static UnreachableInst *create(Instruction *InsertBefore, Context &Ctx);
-  static UnreachableInst *create(BasicBlock *InsertAtEnd, Context &Ctx);
+  static UnreachableInst *create(InsertPosition Pos, Context &Ctx);
   static bool classof(const Value *From);
   unsigned getNumSuccessors() const { return 0; }
   unsigned getUseOperandNo(const Use &Use) const final {
@@ -1297,10 +1279,7 @@ class ReturnInst final : public SingleLLVMInstructionImpl<llvm::ReturnInst> {
                                   Context &Ctx);
 
 public:
-  static ReturnInst *create(Value *RetVal, Instruction *InsertBefore,
-                            Context &Ctx);
-  static ReturnInst *create(Value *RetVal, BasicBlock *InsertAtEnd,
-                            Context &Ctx);
+  static ReturnInst *create(Value *RetVal, InsertPosition Pos, Context &Ctx);
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::Ret;
   }
@@ -1439,24 +1418,17 @@ class CallBase : public SingleLLVMInstructionImpl<llvm::CallBase> {
   bool isInlineAsm() const { return cast<llvm::CallBase>(Val)->isInlineAsm(); }
 };
 
-class CallInst final : public CallBase {
+class CallInst : public CallBase {
   /// Use Context::createCallInst(). Don't call the
   /// constructor directly.
   CallInst(llvm::Instruction *I, Context &Ctx)
       : CallBase(ClassID::Call, Opcode::Call, I, Ctx) {}
-  friend class Context; // For accessing the constructor in
-                        // create*()
+  friend class Context;       // For accessing the constructor in create*()
+  friend class IntrinsicInst; // For constructor
 
 public:
   static CallInst *create(FunctionType *FTy, Value *Func,
-                          ArrayRef<Value *> Args, BBIterator WhereIt,
-                          BasicBlock *WhereBB, Context &Ctx,
-                          const Twine &NameStr = "");
-  static CallInst *create(FunctionType *FTy, Value *Func,
-                          ArrayRef<Value *> Args, Instruction *InsertBefore,
-                          Context &Ctx, const Twine &NameStr = "");
-  static CallInst *create(FunctionType *FTy, Value *Func,
-                          ArrayRef<Value *> Args, BasicBlock *InsertAtEnd,
+                          ArrayRef<Value *> Args, InsertPosition Pos,
                           Context &Ctx, const Twine &NameStr = "");
 
   static bool classof(const Value *From) {
@@ -1475,16 +1447,7 @@ class InvokeInst final : public CallBase {
 public:
   static InvokeInst *create(FunctionType *FTy, Value *Func,
                             BasicBlock *IfNormal, BasicBlock *IfException,
-                            ArrayRef<Value *> Args, BBIterator WhereIt,
-                            BasicBlock *WhereBB, Context &Ctx,
-                            const Twine &NameStr = "");
-  static InvokeInst *create(FunctionType *FTy, Value *Func,
-                            BasicBlock *IfNormal, BasicBlock *IfException,
-                            ArrayRef<Value *> Args, Instruction *InsertBefore,
-                            Context &Ctx, const Twine &NameStr = "");
-  static InvokeInst *create(FunctionType *FTy, Value *Func,
-                            BasicBlock *IfNormal, BasicBlock *IfException,
-                            ArrayRef<Value *> Args, BasicBlock *InsertAtEnd,
+                            ArrayRef<Value *> Args, InsertPosition Pos,
                             Context &Ctx, const Twine &NameStr = "");
 
   static bool classof(const Value *From) {
@@ -1520,18 +1483,7 @@ class CallBrInst final : public CallBase {
   static CallBrInst *create(FunctionType *FTy, Value *Func,
                             BasicBlock *DefaultDest,
                             ArrayRef<BasicBlock *> IndirectDests,
-                            ArrayRef<Value *> Args, BBIterator WhereIt,
-                            BasicBlock *WhereBB, Context &Ctx,
-                            const Twine &NameStr = "");
-  static CallBrInst *create(FunctionType *FTy, Value *Func,
-                            BasicBlock *DefaultDest,
-                            ArrayRef<BasicBlock *> IndirectDests,
-                            ArrayRef<Value *> Args, Instruction *InsertBefore,
-                            Context &Ctx, const Twine &NameStr = "");
-  static CallBrInst *create(FunctionType *FTy, Value *Func,
-                            BasicBlock *DefaultDest,
-                            ArrayRef<BasicBlock *> IndirectDests,
-                            ArrayRef<Value *> Args, BasicBlock *InsertAtEnd,
+                            ArrayRef<Value *> Args, InsertPosition Pos,
                             Context &Ctx, const Twine &NameStr = "");
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::CallBr;
@@ -1560,8 +1512,8 @@ class LandingPadInst : public SingleLLVMInstructionImpl<llvm::LandingPadInst> {
 
 public:
   static LandingPadInst *create(Type *RetTy, unsigned NumReservedClauses,
-                                BBIterator WhereIt, BasicBlock *WhereBB,
-                                Context &Ctx, const Twine &Name = "");
+                                InsertPosition Pos, Context &Ctx,
+                                const Twine &Name = "");
   /// Return 'true' if this landingpad instruction is a
   /// cleanup. I.e., it should be run when unwinding even if its landing pad
   /// doesn't catch the exception.
@@ -1637,8 +1589,8 @@ class CatchPadInst : public FuncletPadInst {
   // for now, as there is no CatchPadInst member function that can undo it.
 
   static CatchPadInst *create(Value *ParentPad, ArrayRef<Value *> Args,
-                              BBIterator WhereIt, BasicBlock *WhereBB,
-                              Context &Ctx, const Twine &Name = "");
+                              InsertPosition Pos, Context &Ctx,
+                              const Twine &Name = "");
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::CatchPad;
   }
@@ -1651,8 +1603,8 @@ class CleanupPadInst : public FuncletPadInst {
 
 public:
   static CleanupPadInst *create(Value *ParentPad, ArrayRef<Value *> Args,
-                                BBIterator WhereIt, BasicBlock *WhereBB,
-                                Context &Ctx, const Twine &Name = "");
+                                InsertPosition Pos, Context &Ctx,
+                                const Twine &Name = "");
   static bool classof(const Value *From) {
     return From->getSubclassID() == ClassID::CleanupPad;
   }
diff --git a/llvm/include/llvm/SandboxIR/IntrinsicInst.h b/llvm/include/llvm/SandboxIR/IntrinsicInst.h
new file mode 100644
index 0000000000000..33d3746f15119
--- /dev/null
+++ b/llvm/include/llvm/SandboxIR/IntrinsicInst.h
@@ -0,0 +1,44 @@
+//===- IntrinsicInst.h ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SANDBOXIR_INTRINSICINST_H
+#define LLVM_SANDBOXIR_INTRINSICINST_H
+
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/SandboxIR/Instruction.h"
+
+namespace llvm::sandboxir {
+
+class IntrinsicInst : public CallInst {
+  IntrinsicInst(llvm::IntrinsicInst *I, Context &Ctx) : CallInst(I, Ctx) {}
+
+public:
+  Intrinsic::ID getIntrinsicID() const {
+    return cast<llvm::IntrinsicInst>(Val)->getIntrinsicID();
+  }
+  bool isAssociative() const {
+    return cast<llvm::IntrinsicInst>(Val)->isAssociative();
+  }
+  bool isCommutative() const {
+    return cast<llvm::IntrinsicInst>(Val)->isCommutative();
+  }
+  bool isAssumeLikeIntrinsic() const {
+    return cast<llvm::IntrinsicInst>(Val)->isAssumeLikeIntrinsic();
+  }
+  static bool mayLowerToFunctionCall(Intrinsic::ID IID) {
+    return llvm::IntrinsicInst::mayLowerToFunctionCall(IID);
+  }
+  static bool classof(const Value *V) {
+    auto *LLVMV = V->Val;
+    return isa<llvm::IntrinsicInst>(LLVMV);
+  }
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_SANDBOXIR_INTRINSICINST_H
diff --git a/llvm/include/llvm/SandboxIR/Pass.h b/llvm/include/llvm/SandboxIR/Pass.h
index 34776e3529e5c..211f10f5d57c5 100644
--- a/llvm/include/llvm/SandboxIR/Pass.h
+++ b/llvm/include/llvm/SandboxIR/Pass.h
@@ -15,6 +15,7 @@
 namespace llvm::sandboxir {
 
 class Function;
+class Region;
 
 /// The base class of a Sandbox IR Pass.
 class Pass {
@@ -24,6 +25,7 @@ class Pass {
   const std::string Name;
 
 public:
+  /// \p Name can't contain any spaces or start with '-'.
   Pass(StringRef Name) : Name(Name) {
     assert(!Name.contains(' ') &&
            "A pass name should not contain whitespaces!");
@@ -47,11 +49,21 @@ class Pass {
 /// A pass that runs on a sandbox::Function.
 class FunctionPass : public Pass {
 public:
+  /// \p Name can't contain any spaces or start with '-'.
   FunctionPass(StringRef Name) : Pass(Name) {}
   /// \Returns true if it modifies \p F.
   virtual bool runOnFunction(Function &F) = 0;
 };
 
+/// A pass that runs on a sandbox::Region.
+class RegionPass : public Pass {
+public:
+  /// \p Name can't contain any spaces or start with '-'.
+  RegionPass(StringRef Name) : Pass(Name) {}
+  /// \Returns true if it modifies \p R.
+  virtual bool runOnRegion(Region &R) = 0;
+};
+
 } // namespace llvm::sandboxir
 
 #endif // LLVM_SANDBOXIR_PASS_H
diff --git a/llvm/include/llvm/SandboxIR/PassManager.h b/llvm/include/llvm/SandboxIR/PassManager.h
index 98b56ba08c4eb..54192c6bf1333 100644
--- a/llvm/include/llvm/SandboxIR/PassManager.h
+++ b/llvm/include/llvm/SandboxIR/PassManager.h
@@ -73,6 +73,12 @@ class FunctionPassManager final
   bool runOnFunction(Function &F) final;
 };
 
+class RegionPassManager final : public PassManager<RegionPass, RegionPass> {
+public:
+  RegionPassManager(StringRef Name) : PassManager(Name) {}
+  bool runOnRegion(Region &R) final;
+};
+
 /// Owns the passes and provides an API to get a pass by its name.
 class PassRegistry {
   SmallVector<std::unique_ptr<Pass>, 8> Passes;
diff --git a/llvm/include/llvm/SandboxIR/Utils.h b/llvm/include/llvm/SandboxIR/Utils.h
index 17fc837f555b8..e50621b4c1228 100644
--- a/llvm/include/llvm/SandboxIR/Utils.h
+++ b/llvm/include/llvm/SandboxIR/Utils.h
@@ -56,6 +56,12 @@ class Utils {
     return DL.getTypeSizeInBits(Ty->LLVMTy);
   }
 
+  /// \Returns the number of bits required to represent the operands or
+  /// return value of \p I.
+  static unsigned getNumBits(Instruction *I) {
+    return I->getDataLayout().getTypeSizeInBits(getExpectedType(I)->LLVMTy);
+  }
+
   /// Equivalent to MemoryLocation::getOrNone(I).
   static std::optional<llvm::MemoryLocation>
   memoryLocationGetOrNone(const Instruction *I) {
@@ -94,6 +100,7 @@ class Utils {
     return *Diff > 0;
   }
 };
+
 } // namespace llvm::sandboxir
 
 #endif // LLVM_SANDBOXIR_UTILS_H
diff --git a/llvm/include/llvm/SandboxIR/Value.h b/llvm/include/llvm/SandboxIR/Value.h
index b28f0e664f80b..3509f2a8d8365 100644
--- a/llvm/include/llvm/SandboxIR/Value.h
+++ b/llvm/include/llvm/SandboxIR/Value.h
@@ -27,6 +27,7 @@ class GlobalObject;
 class Module;
 class UnaryInstruction;
 class CmpInst;
+class IntrinsicInst;
 
 /// Iterator for the `Use` edges of a Value's users.
 /// \Returns a `Use` when dereferenced.
@@ -156,6 +157,7 @@ class Value {
   friend class ConstantExpr;          // For `Val`.
   friend class Utils;                 // For `Val`.
   friend class Module;                // For `Val`.
+  friend class IntrinsicInst;         // For `Val`.
   // Region needs to manipulate metadata in the underlying LLVM Value, we don't
   // expose metadata in sandboxir.
   friend class Region;
diff --git a/llvm/include/llvm/Support/Caching.h b/llvm/include/llvm/Support/Caching.h
index 4fa57cc92e51f..cf45145619d95 100644
--- a/llvm/include/llvm/Support/Caching.h
+++ b/llvm/include/llvm/Support/Caching.h
@@ -41,21 +41,48 @@ class CachedFileStream {
 using AddStreamFn = std::function<Expected<std::unique_ptr<CachedFileStream>>(
     unsigned Task, const Twine &ModuleName)>;
 
-/// This is the type of a file cache. To request an item from the cache, pass a
-/// unique string as the Key. For hits, the cached file will be added to the
-/// link and this function will return AddStreamFn(). For misses, the cache will
-/// return a stream callback which must be called at most once to produce
-/// content for the stream. The file stream produced by the stream callback will
-/// add the file to the link after the stream is written to. ModuleName is the
-/// unique module identifier for the bitcode module the cache is being checked
-/// for.
+/// This is a callable that manages file caching operations. It accepts a task
+/// ID \p Task, a unique key \p Key, and a module name \p ModuleName, and
+/// returns AddStreamFn(). This function determines whether a cache hit or miss
+/// occurs and handles the appropriate actions.
+using FileCacheFunction = std::function<Expected<AddStreamFn>(
+    unsigned Task, StringRef Key, const Twine &ModuleName)>;
+
+/// This type represents a file cache system that manages caching of files.
+/// It encapsulates a caching function and the directory path where the cache is
+/// stored. To request an item from the cache, pass a unique string as the Key.
+/// For hits, the cached file will be added to the link and this function will
+/// return AddStreamFn(). For misses, the cache will return a stream callback
+/// which must be called at most once to produce content for the stream. The
+/// file stream produced by the stream callback will add the file to the link
+/// after the stream is written to. ModuleName is the unique module identifier
+/// for the bitcode module the cache is being checked for.
 ///
 /// Clients generally look like this:
 ///
 /// if (AddStreamFn AddStream = Cache(Task, Key, ModuleName))
 ///   ProduceContent(AddStream);
-using FileCache = std::function<Expected<AddStreamFn>(
-    unsigned Task, StringRef Key, const Twine &ModuleName)>;
+///
+/// CacheDirectoryPath stores the directory path where cached files are kept.
+struct FileCache {
+  FileCache(FileCacheFunction CacheFn, const std::string &DirectoryPath)
+      : CacheFunction(std::move(CacheFn)), CacheDirectoryPath(DirectoryPath) {}
+  FileCache() = default;
+
+  Expected<AddStreamFn> operator()(unsigned Task, StringRef Key,
+                                   const Twine &ModuleName) {
+    assert(isValid() && "Invalid cache function");
+    return CacheFunction(Task, Key, ModuleName);
+  }
+  const std::string &getCacheDirectoryPath() const {
+    return CacheDirectoryPath;
+  }
+  bool isValid() const { return static_cast<bool>(CacheFunction); }
+
+private:
+  FileCacheFunction CacheFunction = nullptr;
+  std::string CacheDirectoryPath;
+};
 
 /// This type defines the callback to add a pre-existing file (e.g. in a cache).
 ///
diff --git a/llvm/include/llvm/Support/GenericDomTreeConstruction.h b/llvm/include/llvm/Support/GenericDomTreeConstruction.h
index 9aab5ec60f4a2..5c3c54c552398 100644
--- a/llvm/include/llvm/Support/GenericDomTreeConstruction.h
+++ b/llvm/include/llvm/Support/GenericDomTreeConstruction.h
@@ -640,7 +640,7 @@ struct SemiNCAInfo {
         Bucket;
     SmallDenseSet<TreeNodePtr, 8> Visited;
     SmallVector<TreeNodePtr, 8> Affected;
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
     SmallVector<TreeNodePtr, 8> VisitedUnaffected;
 #endif
   };
@@ -815,7 +815,7 @@ struct SemiNCAInfo {
             LLVM_DEBUG(dbgs() << "\t\tMarking visited not affected "
                               << BlockNamePrinter(Succ) << "\n");
             UnaffectedOnCurrentLevel.push_back(SuccTN);
-#ifndef NDEBUG
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
             II.VisitedUnaffected.push_back(SuccTN);
 #endif
           } else {
@@ -849,7 +849,7 @@ struct SemiNCAInfo {
       TN->setIDom(NCD);
     }
 
-#if defined(LLVM_ENABLE_ABI_BREAKING_CHECKS) && !defined(NDEBUG)
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS && !defined(NDEBUG)
     for (const TreeNodePtr TN : II.VisitedUnaffected)
       assert(TN->getLevel() == TN->getIDom()->getLevel() + 1 &&
              "TN should have been updated by an affected ancestor");
@@ -915,7 +915,7 @@ struct SemiNCAInfo {
     LLVM_DEBUG(dbgs() << "Deleting edge " << BlockNamePrinter(From) << " -> "
                       << BlockNamePrinter(To) << "\n");
 
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
     // Ensure that the edge was in fact deleted from the CFG before informing
     // the DomTree about it.
     // The check is O(N), so run it only in debug configuration.
diff --git a/llvm/include/llvm/Support/KnownBits.h b/llvm/include/llvm/Support/KnownBits.h
index e4ec202f36aae..a4b554fa2a0b7 100644
--- a/llvm/include/llvm/Support/KnownBits.h
+++ b/llvm/include/llvm/Support/KnownBits.h
@@ -29,6 +29,9 @@ struct KnownBits {
   KnownBits(APInt Zero, APInt One)
       : Zero(std::move(Zero)), One(std::move(One)) {}
 
+  // Flip the range of values: [-0x80000000, 0x7FFFFFFF] <-> [0, 0xFFFFFFFF]
+  static KnownBits flipSignBit(const KnownBits &Val);
+
 public:
   // Default construct Zero and One.
   KnownBits() = default;
diff --git a/llvm/include/llvm/TableGen/DirectiveEmitter.h b/llvm/include/llvm/TableGen/DirectiveEmitter.h
index 70d7d8fbe14f8..a2c9b2d427cce 100644
--- a/llvm/include/llvm/TableGen/DirectiveEmitter.h
+++ b/llvm/include/llvm/TableGen/DirectiveEmitter.h
@@ -136,28 +136,30 @@ class Directive : public BaseRecord {
   Directive(const Record *Def) : BaseRecord(Def) {}
 
   std::vector<const Record *> getAllowedClauses() const {
-    return Def->getValueAsListOfConstDefs("allowedClauses");
+    return Def->getValueAsListOfDefs("allowedClauses");
   }
 
   std::vector<const Record *> getAllowedOnceClauses() const {
-    return Def->getValueAsListOfConstDefs("allowedOnceClauses");
+    return Def->getValueAsListOfDefs("allowedOnceClauses");
   }
 
   std::vector<const Record *> getAllowedExclusiveClauses() const {
-    return Def->getValueAsListOfConstDefs("allowedExclusiveClauses");
+    return Def->getValueAsListOfDefs("allowedExclusiveClauses");
   }
 
   std::vector<const Record *> getRequiredClauses() const {
-    return Def->getValueAsListOfConstDefs("requiredClauses");
+    return Def->getValueAsListOfDefs("requiredClauses");
   }
 
   std::vector<const Record *> getLeafConstructs() const {
-    return Def->getValueAsListOfConstDefs("leafConstructs");
+    return Def->getValueAsListOfDefs("leafConstructs");
   }
 
-  Record *getAssociation() const { return Def->getValueAsDef("association"); }
+  const Record *getAssociation() const {
+    return Def->getValueAsDef("association");
+  }
 
-  Record *getCategory() const { return Def->getValueAsDef("category"); }
+  const Record *getCategory() const { return Def->getValueAsDef("category"); }
 };
 
 // Wrapper class that contains Clause's information defined in DirectiveBase.td
@@ -204,7 +206,7 @@ class Clause : public BaseRecord {
   }
 
   std::vector<const Record *> getClauseVals() const {
-    return Def->getValueAsListOfConstDefs("allowedClauseValues");
+    return Def->getValueAsListOfDefs("allowedClauseValues");
   }
 
   bool isValueOptional() const { return Def->getValueAsBit("isValueOptional"); }
diff --git a/llvm/include/llvm/TableGen/Error.h b/llvm/include/llvm/TableGen/Error.h
index a51ed48dba0e8..512249b0160c2 100644
--- a/llvm/include/llvm/TableGen/Error.h
+++ b/llvm/include/llvm/TableGen/Error.h
@@ -48,7 +48,8 @@ void PrintError(const RecordVal *RecVal, const Twine &Msg);
 [[noreturn]] void PrintFatalError(const RecordVal *RecVal, const Twine &Msg);
 [[noreturn]] void PrintFatalError(function_ref<void(raw_ostream &OS)> PrintMsg);
 
-void CheckAssert(SMLoc Loc, Init *Condition, Init *Message);
+// Returns true if the assert failed.
+bool CheckAssert(SMLoc Loc, Init *Condition, Init *Message);
 void dumpMessage(SMLoc Loc, Init *Message);
 
 extern SourceMgr SrcMgr;
diff --git a/llvm/include/llvm/TableGen/Record.h b/llvm/include/llvm/TableGen/Record.h
index f39a2ee9f1d9b..f856ff4cbd34b 100644
--- a/llvm/include/llvm/TableGen/Record.h
+++ b/llvm/include/llvm/TableGen/Record.h
@@ -51,6 +51,7 @@ class RecordVal;
 class Resolver;
 class StringInit;
 class TypedInit;
+class TGTimer;
 
 //===----------------------------------------------------------------------===//
 //  Type Classes
@@ -783,7 +784,7 @@ class ListInit final : public TypedInit, public FoldingSetNode,
     return cast<ListRecTy>(getType())->getElementType();
   }
 
-  Record *getElementAsRecord(unsigned i) const;
+  const Record *getElementAsRecord(unsigned i) const;
 
   Init *convertInitializerTo(const RecTy *Ty) const override;
 
@@ -1316,9 +1317,9 @@ class VarBitInit final : public TypedInit {
 class DefInit : public TypedInit {
   friend class Record;
 
-  Record *Def;
+  const Record *Def;
 
-  explicit DefInit(Record *D);
+  explicit DefInit(const Record *D);
 
 public:
   DefInit(const DefInit &) = delete;
@@ -1330,7 +1331,7 @@ class DefInit : public TypedInit {
 
   Init *convertInitializerTo(const RecTy *Ty) const override;
 
-  Record *getDef() const { return Def; }
+  const Record *getDef() const { return Def; }
 
   const RecTy *getFieldType(StringInit *FieldName) const override;
 
@@ -1347,11 +1348,12 @@ class DefInit : public TypedInit {
 class VarDefInit final : public TypedInit,
                          public FoldingSetNode,
                          public TrailingObjects<VarDefInit, ArgumentInit *> {
+  SMLoc Loc;
   Record *Class;
   DefInit *Def = nullptr; // after instantiation
   unsigned NumArgs;
 
-  explicit VarDefInit(Record *Class, unsigned N);
+  explicit VarDefInit(SMLoc Loc, Record *Class, unsigned N);
 
   DefInit *instantiate();
 
@@ -1365,7 +1367,8 @@ class VarDefInit final : public TypedInit,
   static bool classof(const Init *I) {
     return I->getKind() == IK_VarDefInit;
   }
-  static VarDefInit *get(Record *Class, ArrayRef<ArgumentInit *> Args);
+  static VarDefInit *get(SMLoc Loc, Record *Class,
+                         ArrayRef<ArgumentInit *> Args);
 
   void Profile(FoldingSetNodeID &ID) const;
 
@@ -1473,7 +1476,7 @@ class DagInit final : public TypedInit, public FoldingSetNode,
   void Profile(FoldingSetNodeID &ID) const;
 
   Init *getOperator() const { return Val; }
-  Record *getOperatorAsDef(ArrayRef<SMLoc> Loc) const;
+  const Record *getOperatorAsDef(ArrayRef<SMLoc> Loc) const;
 
   StringInit *getName() const { return ValName; }
 
@@ -1660,7 +1663,7 @@ class Record {
   // this record.
   SmallVector<SMLoc, 4> Locs;
   SmallVector<SMLoc, 0> ForwardDeclarationLocs;
-  SmallVector<SMRange, 0> ReferenceLocs;
+  mutable SmallVector<SMRange, 0> ReferenceLocs;
   SmallVector<Init *, 0> TemplateArgs;
   SmallVector<RecordVal, 0> Values;
   SmallVector<AssertionInfo, 0> Assertions;
@@ -1668,7 +1671,7 @@ class Record {
 
   // All superclasses in the inheritance forest in post-order (yes, it
   // must be a forest; diamond-shaped inheritance is not allowed).
-  SmallVector<std::pair<Record *, SMRange>, 0> SuperClasses;
+  SmallVector<std::pair<const Record *, SMRange>, 0> SuperClasses;
 
   // Tracks Record instances. Not owned by Record.
   RecordKeeper &TrackedRecords;
@@ -1729,7 +1732,7 @@ class Record {
   }
 
   /// Add a reference to this record value.
-  void appendReferenceLoc(SMRange Loc) { ReferenceLocs.push_back(Loc); }
+  void appendReferenceLoc(SMRange Loc) const { ReferenceLocs.push_back(Loc); }
 
   /// Return the references of this record value.
   ArrayRef<SMRange> getReferenceLocs() const { return ReferenceLocs; }
@@ -1758,7 +1761,7 @@ class Record {
   ArrayRef<AssertionInfo> getAssertions() const { return Assertions; }
   ArrayRef<DumpInfo> getDumps() const { return Dumps; }
 
-  ArrayRef<std::pair<Record *, SMRange>> getSuperClasses() const {
+  ArrayRef<std::pair<const Record *, SMRange>> getSuperClasses() const {
     return SuperClasses;
   }
 
@@ -1783,11 +1786,13 @@ class Record {
   }
 
   RecordVal *getValue(const Init *Name) {
-    return const_cast<RecordVal *>(static_cast<const Record *>(this)->getValue(Name));
+    return const_cast<RecordVal *>(
+        static_cast<const Record *>(this)->getValue(Name));
   }
 
   RecordVal *getValue(StringRef Name) {
-    return const_cast<RecordVal *>(static_cast<const Record *>(this)->getValue(Name));
+    return const_cast<RecordVal *>(
+        static_cast<const Record *>(this)->getValue(Name));
   }
 
   void addTemplateArg(Init *Name) {
@@ -1832,25 +1837,25 @@ class Record {
   void checkUnusedTemplateArgs();
 
   bool isSubClassOf(const Record *R) const {
-    for (const auto &SCPair : SuperClasses)
-      if (SCPair.first == R)
+    for (const auto &[SC, _] : SuperClasses)
+      if (SC == R)
         return true;
     return false;
   }
 
   bool isSubClassOf(StringRef Name) const {
-    for (const auto &SCPair : SuperClasses) {
-      if (const auto *SI = dyn_cast<StringInit>(SCPair.first->getNameInit())) {
+    for (const auto &[SC, _] : SuperClasses) {
+      if (const auto *SI = dyn_cast<StringInit>(SC->getNameInit())) {
         if (SI->getValue() == Name)
           return true;
-      } else if (SCPair.first->getNameInitAsString() == Name) {
+      } else if (SC->getNameInitAsString() == Name) {
         return true;
       }
     }
     return false;
   }
 
-  void addSuperClass(Record *R, SMRange Range) {
+  void addSuperClass(const Record *R, SMRange Range) {
     assert(!CorrespondingDefInit &&
            "changing type of record after it has been referenced");
     assert(!isSubClassOf(R) && "Already subclassing record!");
@@ -1916,10 +1921,7 @@ class Record {
   /// This method looks up the specified field and returns its value as a
   /// vector of records, throwing an exception if the field does not exist or
   /// if the value is not the right type.
-  std::vector<Record*> getValueAsListOfDefs(StringRef FieldName) const;
-  // Temporary function to help staged migration to const Record pointers.
-  std::vector<const Record *>
-  getValueAsListOfConstDefs(StringRef FieldName) const;
+  std::vector<const Record *> getValueAsListOfDefs(StringRef FieldName) const;
 
   /// This method looks up the specified field and returns its value as a
   /// vector of integers, throwing an exception if the field does not exist or
@@ -1934,13 +1936,13 @@ class Record {
   /// This method looks up the specified field and returns its value as a
   /// Record, throwing an exception if the field does not exist or if the value
   /// is not the right type.
-  Record *getValueAsDef(StringRef FieldName) const;
+  const Record *getValueAsDef(StringRef FieldName) const;
 
   /// This method looks up the specified field and returns its value as a
   /// Record, returning null if the field exists but is "uninitialized" (i.e.
   /// set to `?`), and throwing an exception if the field does not exist or if
   /// its value is not the right type.
-  Record *getValueAsOptionalDef(StringRef FieldName) const;
+  const Record *getValueAsOptionalDef(StringRef FieldName) const;
 
   /// This method looks up the specified field and returns its value as a bit,
   /// throwing an exception if the field does not exist or if the value is not
@@ -1994,14 +1996,14 @@ class RecordKeeper {
   }
 
   /// Get the concrete record with the specified name.
-  Record *getDef(StringRef Name) const {
+  const Record *getDef(StringRef Name) const {
     auto I = Defs.find(Name);
     return I == Defs.end() ? nullptr : I->second.get();
   }
 
   /// Get the \p Init value of the specified global variable.
   Init *getGlobal(StringRef Name) const {
-    if (Record *R = getDef(Name))
+    if (const Record *R = getDef(Name))
       return R->getDefInit();
     auto It = ExtraGlobals.find(Name);
     return It == ExtraGlobals.end() ? nullptr : It->second;
@@ -2034,55 +2036,24 @@ class RecordKeeper {
 
   Init *getNewAnonymousName();
 
-  /// Start phase timing; called if the --time-phases option is specified.
-  void startPhaseTiming() {
-    TimingGroup = new TimerGroup("TableGen", "TableGen Phase Timing");
-  }
-
-  /// Start timing a phase. Automatically stops any previous phase timer.
-  void startTimer(StringRef Name) const;
-
-  /// Stop timing a phase.
-  void stopTimer();
-
-  /// Start timing the overall backend. If the backend itself starts a timer,
-  /// then this timer is cleared.
-  void startBackendTimer(StringRef Name);
-
-  /// Stop timing the overall backend.
-  void stopBackendTimer();
-
-  /// Stop phase timing and print the report.
-  void stopPhaseTiming() {
-    if (TimingGroup)
-      delete TimingGroup;
-  }
+  TGTimer &getTimer() const { return *Timer; }
 
   //===--------------------------------------------------------------------===//
   // High-level helper methods, useful for tablegen backends.
 
-  // Non-const methods return std::vector<Record *> by value or reference.
-  // Const methods return std::vector<const Record *> by value or
-  // ArrayRef<const Record *>.
-
   /// Get all the concrete records that inherit from the one specified
   /// class. The class must be defined.
   ArrayRef<const Record *> getAllDerivedDefinitions(StringRef ClassName) const;
-  const std::vector<Record *> &getAllDerivedDefinitions(StringRef ClassName);
 
   /// Get all the concrete records that inherit from all the specified
   /// classes. The classes must be defined.
   std::vector<const Record *>
   getAllDerivedDefinitions(ArrayRef<StringRef> ClassNames) const;
-  std::vector<Record *>
-  getAllDerivedDefinitions(ArrayRef<StringRef> ClassNames);
 
   /// Get all the concrete records that inherit from specified class, if the
   /// class is defined. Returns an empty vector if the class is not defined.
   ArrayRef<const Record *>
   getAllDerivedDefinitionsIfDefined(StringRef ClassName) const;
-  const std::vector<Record *> &
-  getAllDerivedDefinitionsIfDefined(StringRef ClassName);
 
   void dump() const;
 
@@ -2094,36 +2065,14 @@ class RecordKeeper {
   RecordKeeper &operator=(RecordKeeper &&) = delete;
   RecordKeeper &operator=(const RecordKeeper &) = delete;
 
-  // Helper template functions for backend accessors.
-  template <typename VecTy>
-  const VecTy &
-  getAllDerivedDefinitionsImpl(StringRef ClassName,
-                               std::map<std::string, VecTy> &Cache) const;
-
-  template <typename VecTy>
-  VecTy getAllDerivedDefinitionsImpl(ArrayRef<StringRef> ClassNames) const;
-
-  template <typename VecTy>
-  const VecTy &getAllDerivedDefinitionsIfDefinedImpl(
-      StringRef ClassName, std::map<std::string, VecTy> &Cache) const;
-
   std::string InputFilename;
   RecordMap Classes, Defs;
-  mutable std::map<std::string, std::vector<const Record *>>
-      ClassRecordsMapConst;
-  mutable std::map<std::string, std::vector<Record *>> ClassRecordsMap;
+  mutable std::map<std::string, std::vector<const Record *>> Cache;
   GlobalMap ExtraGlobals;
 
-  // TODO: Move timing related code out of RecordKeeper.
-  // These members are for the phase timing feature. We need a timer group,
-  // the last timer started, and a flag to say whether the last timer
-  // is the special "backend overall timer."
-  mutable TimerGroup *TimingGroup = nullptr;
-  mutable Timer *LastTimer = nullptr;
-  mutable bool BackendTimer = false;
-
   /// The internal uniquer implementation of the RecordKeeper.
   std::unique_ptr<detail::RecordKeeperImpl> Impl;
+  std::unique_ptr<TGTimer> Timer;
 };
 
 /// Sorting predicate to sort record pointers by name.
diff --git a/llvm/include/llvm/TableGen/TGTimer.h b/llvm/include/llvm/TableGen/TGTimer.h
new file mode 100644
index 0000000000000..31753f54028cb
--- /dev/null
+++ b/llvm/include/llvm/TableGen/TGTimer.h
@@ -0,0 +1,59 @@
+//===- llvm/TableGen/TGTimer.h - Class for TableGen Timer -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the TableGen timer class. It's a thin wrapper around timer
+// support in llvm/Support/Timer.h.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TABLEGEN_TGTIMER_H
+#define LLVM_TABLEGEN_TGTIMER_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Timer.h"
+#include <memory>
+
+namespace llvm {
+
+// Timer related functionality f or TableGen backends.
+class TGTimer {
+private:
+  std::unique_ptr<TimerGroup> TimingGroup;
+  std::unique_ptr<Timer> LastTimer;
+  bool BackendTimer = false; // Is last timer special backend overall timer?
+
+public:
+  TGTimer() = default;
+  ~TGTimer() = default;
+
+  /// Start phase timing; called if the --time-phases option is specified.
+  void startPhaseTiming() {
+    TimingGroup =
+        std::make_unique<TimerGroup>("TableGen", "TableGen Phase Timing");
+  }
+
+  /// Start timing a phase. Automatically stops any previous phase timer.
+  void startTimer(StringRef Name);
+
+  /// Stop timing a phase.
+  void stopTimer();
+
+  /// Start timing the overall backend. If the backend itself starts a timer,
+  /// then this timer is cleared.
+  void startBackendTimer(StringRef Name);
+
+  /// Stop timing the overall backend.
+  void stopBackendTimer();
+
+  /// Stop phase timing and print the report.
+  void stopPhaseTiming() { TimingGroup.reset(); }
+};
+
+} // end namespace llvm
+
+#endif // LLVM_TABLEGEN_TGTIMER_H
diff --git a/llvm/include/llvm/TableGen/TableGenBackend.h b/llvm/include/llvm/TableGen/TableGenBackend.h
index 80134179850b0..7cb540f66ec70 100644
--- a/llvm/include/llvm/TableGen/TableGenBackend.h
+++ b/llvm/include/llvm/TableGen/TableGenBackend.h
@@ -23,8 +23,7 @@ class RecordKeeper;
 class raw_ostream;
 
 namespace TableGen::Emitter {
-// Supports const and non-const forms of callback functions.
-using FnT = function_ref<void(RecordKeeper &Records, raw_ostream &OS)>;
+using FnT = function_ref<void(const RecordKeeper &Records, raw_ostream &OS)>;
 
 /// Creating an `Opt` object registers the command line option \p Name with
 /// TableGen backend and associates the callback \p CB with that option. If
@@ -37,7 +36,9 @@ struct Opt {
 /// Convienence wrapper around `Opt` that registers `EmitterClass::run` as the
 /// callback.
 template <class EmitterC> class OptClass : Opt {
-  static void run(RecordKeeper &RK, raw_ostream &OS) { EmitterC(RK).run(OS); }
+  static void run(const RecordKeeper &RK, raw_ostream &OS) {
+    EmitterC(RK).run(OS);
+  }
 
 public:
   OptClass(StringRef Name, StringRef Desc) : Opt(Name, run, Desc) {}
@@ -45,7 +46,7 @@ template <class EmitterC> class OptClass : Opt {
 
 /// Apply callback for any command line option registered above. Returns false
 /// is no callback was applied.
-bool ApplyCallback(RecordKeeper &Records, raw_ostream &OS);
+bool ApplyCallback(const RecordKeeper &Records, raw_ostream &OS);
 
 } // namespace TableGen::Emitter
 
diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
index 2169e9c94b61f..12c3aa6024a39 100644
--- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h
+++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
@@ -118,9 +118,9 @@ struct ArchInfo {
   // Defines the following partial order, indicating when an architecture is
   // a superset of another:
   //
-  //   v9.5a > v9.4a > v9.3a > v9.2a > v9.1a > v9a;
-  //             v       v       v       v       v
-  //           v8.9a > v8.8a > v8.7a > v8.6a > v8.5a > v8.4a > ... > v8a;
+  // v9.6a > v9.5a > v9.4a > v9.3a > v9.2a > v9.1a > v9a;
+  //                   v       v       v       v       v
+  //                 v8.9a > v8.8a > v8.7a > v8.6a > v8.5a > v8.4a > ... > v8a;
   //
   // v8r has no relation to anything. This is used to determine which
   // features to enable for a given architecture. See
diff --git a/llvm/include/llvm/TargetParser/ARMTargetParser.def b/llvm/include/llvm/TargetParser/ARMTargetParser.def
index e5a1ce54fd46a..7480d45807f23 100644
--- a/llvm/include/llvm/TargetParser/ARMTargetParser.def
+++ b/llvm/include/llvm/TargetParser/ARMTargetParser.def
@@ -182,6 +182,11 @@ ARM_ARCH("armv9.5-a", ARMV9_5A, "9.5-A", "+v9.5a", ARMBuildAttrs::CPUArch::v9_A,
          (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM |
           ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS |
           ARM::AEK_DOTPROD | ARM::AEK_BF16 | ARM::AEK_I8MM))
+ARM_ARCH("armv9.6-a", ARMV9_6A, "9.6-A", "+v9.6a", ARMBuildAttrs::CPUArch::v9_A,
+         FK_NEON_FP_ARMV8,
+         (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM |
+          ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS |
+          ARM::AEK_DOTPROD | ARM::AEK_BF16 | ARM::AEK_I8MM))
 ARM_ARCH("armv8-r", ARMV8R, "8-R", "+v8r", ARMBuildAttrs::CPUArch::v8_R,
          FK_FPV5_SP_D16,
          (ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM |
diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index 7c7bf785cee94..280bd7a9febc1 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -110,6 +110,7 @@ class Triple {
   enum SubArchType {
     NoSubArch,
 
+    ARMSubArch_v9_6a,
     ARMSubArch_v9_5a,
     ARMSubArch_v9_4a,
     ARMSubArch_v9_3a,
diff --git a/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h b/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h
index a2be1099ff68f..52b6c4918ada9 100644
--- a/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h
+++ b/llvm/include/llvm/Transforms/Coroutines/CoroSplit.h
@@ -21,19 +21,26 @@
 
 namespace llvm {
 
+namespace coro {
+class BaseABI;
+struct Shape;
+} // namespace coro
+
 struct CoroSplitPass : PassInfoMixin<CoroSplitPass> {
-  const std::function<bool(Instruction &)> MaterializableCallback;
 
   CoroSplitPass(bool OptimizeFrame = false);
   CoroSplitPass(std::function<bool(Instruction &)> MaterializableCallback,
-                bool OptimizeFrame = false)
-      : MaterializableCallback(MaterializableCallback),
-        OptimizeFrame(OptimizeFrame) {}
+                bool OptimizeFrame = false);
 
   PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM,
                         LazyCallGraph &CG, CGSCCUpdateResult &UR);
   static bool isRequired() { return true; }
 
+  using BaseABITy =
+      std::function<std::unique_ptr<coro::BaseABI>(Function &, coro::Shape &)>;
+  // Generator for an ABI transformer
+  BaseABITy CreateAndInitABI;
+
   // Would be true if the Optimization level isn't O0.
   bool OptimizeFrame;
 };
diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index b32f07e6427e8..8915969f75466 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -5786,6 +5786,53 @@ struct AAPointerInfo : public AbstractAttribute {
     AK_MUST_READ_WRITE = AK_MUST | AK_R | AK_W,
   };
 
+  /// A helper containing a list of offsets computed for a Use. Ideally this
+  /// list should be strictly ascending, but we ensure that only when we
+  /// actually translate the list of offsets to a RangeList.
+  struct OffsetInfo {
+    using VecTy = SmallSet<int64_t, 4>;
+    using const_iterator = VecTy::const_iterator;
+    VecTy Offsets;
+
+    const_iterator begin() const { return Offsets.begin(); }
+    const_iterator end() const { return Offsets.end(); }
+
+    bool operator==(const OffsetInfo &RHS) const {
+      return Offsets == RHS.Offsets;
+    }
+
+    bool operator!=(const OffsetInfo &RHS) const { return !(*this == RHS); }
+
+    bool insert(int64_t Offset) { return Offsets.insert(Offset).second; }
+    bool isUnassigned() const { return Offsets.size() == 0; }
+
+    bool isUnknown() const {
+      if (isUnassigned())
+        return false;
+      if (Offsets.size() == 1)
+        return *Offsets.begin() == AA::RangeTy::Unknown;
+      return false;
+    }
+
+    void setUnknown() {
+      Offsets.clear();
+      Offsets.insert(AA::RangeTy::Unknown);
+    }
+
+    void addToAll(int64_t Inc) {
+      VecTy NewOffsets;
+      for (auto &Offset : Offsets)
+        NewOffsets.insert(Offset + Inc);
+      Offsets = std::move(NewOffsets);
+    }
+
+    /// Copy offsets from \p R into the current list.
+    ///
+    /// Ideally all lists should be strictly ascending, but we defer that to the
+    /// actual use of the list. So we just blindly append here.
+    bool merge(const OffsetInfo &R) { return set_union(Offsets, R.Offsets); }
+  };
+
   /// A container for a list of ranges.
   struct RangeList {
     // The set of ranges rarely contains more than one element, and is unlikely
@@ -6123,6 +6170,7 @@ struct AAPointerInfo : public AbstractAttribute {
   virtual const_bin_iterator end() const = 0;
   virtual int64_t numOffsetBins() const = 0;
   virtual bool reachesReturn() const = 0;
+  virtual void addReturnedOffsetsTo(OffsetInfo &) const = 0;
 
   /// Call \p CB on all accesses that might interfere with \p Range and return
   /// true if all such accesses were known and the callback returned true for
diff --git a/llvm/include/llvm/Transforms/Scalar/JumpThreading.h b/llvm/include/llvm/Transforms/Scalar/JumpThreading.h
index a3f2ce23f7d9a..7d11fc0ad6938 100644
--- a/llvm/include/llvm/Transforms/Scalar/JumpThreading.h
+++ b/llvm/include/llvm/Transforms/Scalar/JumpThreading.h
@@ -89,9 +89,9 @@ class JumpThreadingPass : public PassInfoMixin<JumpThreadingPass> {
   bool ChangedSinceLastAnalysisUpdate = false;
   bool HasGuards = false;
 #ifndef LLVM_ENABLE_ABI_BREAKING_CHECKS
-  SmallPtrSet<const BasicBlock *, 16> LoopHeaders;
-#else
   SmallSet<AssertingVH<const BasicBlock>, 16> LoopHeaders;
+#else
+  SmallPtrSet<const BasicBlock *, 16> LoopHeaders;
 #endif
 
   unsigned BBDupThreshold;
diff --git a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
index 3858be05c61fa..f55022fbff07c 100644
--- a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
+++ b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
@@ -256,7 +256,7 @@ class LPMUpdater {
   }
 
   void setParentLoop(Loop *L) {
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
     ParentL = L;
 #endif
   }
@@ -295,7 +295,7 @@ class LPMUpdater {
   /// loops within them will be visited in postorder as usual for the loop pass
   /// manager.
   void addSiblingLoops(ArrayRef<Loop *> NewSibLoops) {
-#if defined(LLVM_ENABLE_ABI_BREAKING_CHECKS) && !defined(NDEBUG)
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS && !defined(NDEBUG)
     for (Loop *NewL : NewSibLoops)
       assert(NewL->getParentLoop() == ParentL &&
              "All of the new loops must be siblings of the current loop!");
@@ -347,7 +347,7 @@ class LPMUpdater {
   const bool LoopNestMode;
   bool LoopNestChanged;
 
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
   // In debug builds we also track the parent loop to implement asserts even in
   // the face of loop deletion.
   Loop *ParentL;
diff --git a/llvm/include/llvm/Transforms/Scalar/Scalarizer.h b/llvm/include/llvm/Transforms/Scalar/Scalarizer.h
index 4d2a1a2f889a3..401ca4d995348 100644
--- a/llvm/include/llvm/Transforms/Scalar/Scalarizer.h
+++ b/llvm/include/llvm/Transforms/Scalar/Scalarizer.h
@@ -27,13 +27,25 @@ class Function;
 class FunctionPass;
 
 struct ScalarizerPassOptions {
-  // These options correspond 1:1 to cl::opt options defined in
-  // Scalarizer.cpp. When the cl::opt are specified, they take precedence.
-  // When the cl::opt are not specified, the present optional values allow to
-  // override the cl::opt's default values.
-  std::optional<bool> ScalarizeVariableInsertExtract;
-  std::optional<bool> ScalarizeLoadStore;
-  std::optional<unsigned> ScalarizeMinBits;
+  /// Instruct the scalarizer pass to attempt to keep values of a minimum number
+  /// of bits.
+
+  /// Split vectors larger than this size into fragments, where each fragment is
+  /// either a vector no larger than this size or a scalar.
+  ///
+  /// Instructions with operands or results of different sizes that would be
+  /// split into a different number of fragments are currently left as-is.
+  unsigned ScalarizeMinBits = 0;
+
+  /// Allow the scalarizer pass to scalarize insertelement/extractelement with
+  /// variable index.
+  bool ScalarizeVariableInsertExtract = true;
+
+  /// Allow the scalarizer pass to scalarize loads and store
+  ///
+  /// This is disabled by default because having separate loads and stores makes
+  /// it more likely that the -combiner-alias-analysis limits will be reached.
+  bool ScalarizeLoadStore = false;
 };
 
 class ScalarizerPass : public PassInfoMixin<ScalarizerPass> {
diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
index 0af3efeacd040..2806326593649 100644
--- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
+++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
@@ -168,7 +168,7 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
   /// consistent when instructions are moved.
   SmallVector<SCEVInsertPointGuard *, 8> InsertPointGuards;
 
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
   const char *DebugType;
 #endif
 
@@ -184,7 +184,7 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
         Builder(se.getContext(), InstSimplifyFolder(DL),
                 IRBuilderCallbackInserter(
                     [this](Instruction *I) { rememberInstruction(I); })) {
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
     DebugType = "";
 #endif
   }
@@ -194,7 +194,7 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
     assert(InsertPointGuards.empty());
   }
 
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
   void setDebugType(const char *s) { DebugType = s; }
 #endif
 
@@ -451,7 +451,7 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
 
   /// Expand a SCEVAddExpr with a pointer type into a GEP instead of using
   /// ptrtoint+arithmetic+inttoptr.
-  Value *expandAddToGEP(const SCEV *Op, Value *V);
+  Value *expandAddToGEP(const SCEV *Op, Value *V, SCEV::NoWrapFlags Flags);
 
   /// Find a previous Value in ExprValueMap for expand.
   /// DropPoisonGeneratingInsts is populated with instructions for which
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h
index 7f6e6d11e5f53..6333f0b81f9c3 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h
@@ -25,39 +25,91 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/SandboxIR/Instruction.h"
+#include "llvm/SandboxIR/IntrinsicInst.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h"
 
 namespace llvm::sandboxir {
 
+class DependencyGraph;
+class MemDGNode;
+
+/// SubclassIDs for isa/dyn_cast etc.
+enum class DGNodeID {
+  DGNode,
+  MemDGNode,
+};
+
 /// A DependencyGraph Node that points to an Instruction and contains memory
 /// dependency edges.
 class DGNode {
+protected:
   Instruction *I;
+  // TODO: Use a PointerIntPair for SubclassID and I.
+  /// For isa/dyn_cast etc.
+  DGNodeID SubclassID;
   /// Memory predecessors.
-  DenseSet<DGNode *> MemPreds;
-  /// This is true if this may read/write memory, or if it has some ordering
-  /// constraints, like with stacksave/stackrestore and alloca/inalloca.
-  bool IsMem;
+  DenseSet<MemDGNode *> MemPreds;
+
+  DGNode(Instruction *I, DGNodeID ID) : I(I), SubclassID(ID) {}
+  friend class MemDGNode; // For constructor.
 
 public:
-  DGNode(Instruction *I) : I(I) {
-    IsMem = I->isMemDepCandidate() ||
-            (isa<AllocaInst>(I) && cast<AllocaInst>(I)->isUsedWithInAlloca()) ||
-            I->isStackSaveOrRestoreIntrinsic();
+  DGNode(Instruction *I) : I(I), SubclassID(DGNodeID::DGNode) {
+    assert(!isMemDepNodeCandidate(I) && "Expected Non-Mem instruction, ");
   }
+  DGNode(const DGNode &Other) = delete;
+  virtual ~DGNode() = default;
+  /// \Returns true if this is before \p Other in program order.
+  bool comesBefore(const DGNode *Other) { return I->comesBefore(Other->I); }
+
+  static bool isStackSaveOrRestoreIntrinsic(Instruction *I) {
+    if (auto *II = dyn_cast<IntrinsicInst>(I)) {
+      auto IID = II->getIntrinsicID();
+      return IID == Intrinsic::stackrestore || IID == Intrinsic::stacksave;
+    }
+    return false;
+  }
+
+  /// \Returns true if intrinsic \p I touches memory. This is used by the
+  /// dependency graph.
+  static bool isMemIntrinsic(IntrinsicInst *I) {
+    auto IID = I->getIntrinsicID();
+    return IID != Intrinsic::sideeffect && IID != Intrinsic::pseudoprobe;
+  }
+
+  /// We consider \p I as a Memory Dependency Candidate instruction if it
+  /// reads/write memory or if it has side-effects. This is used by the
+  /// dependency graph.
+  static bool isMemDepCandidate(Instruction *I) {
+    IntrinsicInst *II;
+    return I->mayReadOrWriteMemory() &&
+           (!(II = dyn_cast<IntrinsicInst>(I)) || isMemIntrinsic(II));
+  }
+
+  /// \Returns true if \p I is a memory dependency candidate instruction.
+  static bool isMemDepNodeCandidate(Instruction *I) {
+    AllocaInst *Alloca;
+    return isMemDepCandidate(I) ||
+           ((Alloca = dyn_cast<AllocaInst>(I)) &&
+            Alloca->isUsedWithInAlloca()) ||
+           isStackSaveOrRestoreIntrinsic(I);
+  }
+
   Instruction *getInstruction() const { return I; }
-  void addMemPred(DGNode *PredN) { MemPreds.insert(PredN); }
+  void addMemPred(MemDGNode *PredN) { MemPreds.insert(PredN); }
   /// \Returns all memory dependency predecessors.
-  iterator_range<DenseSet<DGNode *>::const_iterator> memPreds() const {
+  iterator_range<DenseSet<MemDGNode *>::const_iterator> memPreds() const {
     return make_range(MemPreds.begin(), MemPreds.end());
   }
   /// \Returns true if there is a memory dependency N->this.
-  bool hasMemPred(DGNode *N) const { return MemPreds.count(N); }
-  /// \Returns true if this may read/write memory, or if it has some ordering
-  /// constraints, like with stacksave/stackrestore and alloca/inalloca.
-  bool isMem() const { return IsMem; }
+  bool hasMemPred(DGNode *N) const {
+    if (auto *MN = dyn_cast<MemDGNode>(N))
+      return MemPreds.count(MN);
+    return false;
+  }
+
 #ifndef NDEBUG
-  void print(raw_ostream &OS, bool PrintDeps = true) const;
+  virtual void print(raw_ostream &OS, bool PrintDeps = true) const;
   friend raw_ostream &operator<<(DGNode &N, raw_ostream &OS) {
     N.print(OS);
     return OS;
@@ -66,9 +118,46 @@ class DGNode {
 #endif // NDEBUG
 };
 
+/// A DependencyGraph Node for instructions that may read/write memory, or have
+/// some ordering constraints, like with stacksave/stackrestore and
+/// alloca/inalloca.
+class MemDGNode final : public DGNode {
+  MemDGNode *PrevMemN = nullptr;
+  MemDGNode *NextMemN = nullptr;
+
+  void setNextNode(MemDGNode *N) { NextMemN = N; }
+  void setPrevNode(MemDGNode *N) { PrevMemN = N; }
+  friend class DependencyGraph; // For setNextNode(), setPrevNode().
+
+public:
+  MemDGNode(Instruction *I) : DGNode(I, DGNodeID::MemDGNode) {
+    assert(isMemDepNodeCandidate(I) && "Expected Mem instruction!");
+  }
+  static bool classof(const DGNode *Other) {
+    return Other->SubclassID == DGNodeID::MemDGNode;
+  }
+  /// \Returns the previous Mem DGNode in instruction order.
+  MemDGNode *getPrevNode() const { return PrevMemN; }
+  /// \Returns the next Mem DGNode in instruction order.
+  MemDGNode *getNextNode() const { return NextMemN; }
+};
+
+/// Convenience builders for a MemDGNode interval.
+class MemDGNodeIntervalBuilder {
+public:
+  /// Given \p Instrs it finds their closest mem nodes in the interval and
+  /// returns the corresponding mem range. Note: BotN (or its neighboring mem
+  /// node) is included in the range.
+  static Interval<MemDGNode> make(const Interval<Instruction> &Instrs,
+                                  DependencyGraph &DAG);
+  static Interval<MemDGNode> makeEmpty() { return {}; }
+};
+
 class DependencyGraph {
 private:
   DenseMap<Instruction *, std::unique_ptr<DGNode>> InstrToNodeMap;
+  /// The DAG spans across all instructions in this interval.
+  Interval<Instruction> DAGInterval;
 
 public:
   DependencyGraph() {}
@@ -77,10 +166,20 @@ class DependencyGraph {
     auto It = InstrToNodeMap.find(I);
     return It != InstrToNodeMap.end() ? It->second.get() : nullptr;
   }
+  /// Like getNode() but returns nullptr if \p I is nullptr.
+  DGNode *getNodeOrNull(Instruction *I) const {
+    if (I == nullptr)
+      return nullptr;
+    return getNode(I);
+  }
   DGNode *getOrCreateNode(Instruction *I) {
     auto [It, NotInMap] = InstrToNodeMap.try_emplace(I);
-    if (NotInMap)
-      It->second = std::make_unique<DGNode>(I);
+    if (NotInMap) {
+      if (DGNode::isMemDepNodeCandidate(I))
+        It->second = std::make_unique<MemDGNode>(I);
+      else
+        It->second = std::make_unique<DGNode>(I);
+    }
     return It->second.get();
   }
   /// Build/extend the dependency graph such that it includes \p Instrs. Returns
diff --git a/llvm/lib/Analysis/CmpInstAnalysis.cpp b/llvm/lib/Analysis/CmpInstAnalysis.cpp
index 36d7aa510545a..2580ea7e97248 100644
--- a/llvm/lib/Analysis/CmpInstAnalysis.cpp
+++ b/llvm/lib/Analysis/CmpInstAnalysis.cpp
@@ -75,79 +75,90 @@ Constant *llvm::getPredForFCmpCode(unsigned Code, Type *OpTy,
 
 std::optional<DecomposedBitTest>
 llvm::decomposeBitTestICmp(Value *LHS, Value *RHS, CmpInst::Predicate Pred,
-                           bool LookThruTrunc) {
+                           bool LookThruTrunc, bool AllowNonZeroC) {
   using namespace PatternMatch;
 
-  const APInt *C;
-  if (!match(RHS, m_APIntAllowPoison(C)))
+  const APInt *OrigC;
+  if (!ICmpInst::isRelational(Pred) || !match(RHS, m_APIntAllowPoison(OrigC)))
     return std::nullopt;
 
+  bool Inverted = false;
+  if (ICmpInst::isGT(Pred) || ICmpInst::isGE(Pred)) {
+    Inverted = true;
+    Pred = ICmpInst::getInversePredicate(Pred);
+  }
+
+  APInt C = *OrigC;
+  if (ICmpInst::isLE(Pred)) {
+    if (ICmpInst::isSigned(Pred) ? C.isMaxSignedValue() : C.isMaxValue())
+      return std::nullopt;
+    ++C;
+    Pred = ICmpInst::getStrictPredicate(Pred);
+  }
+
   DecomposedBitTest Result;
   switch (Pred) {
   default:
-    return std::nullopt;
-  case ICmpInst::ICMP_SLT:
+    llvm_unreachable("Unexpected predicate");
+  case ICmpInst::ICMP_SLT: {
     // X < 0 is equivalent to (X & SignMask) != 0.
-    if (!C->isZero())
-      return std::nullopt;
-    Result.Mask = APInt::getSignMask(C->getBitWidth());
-    Result.Pred = ICmpInst::ICMP_NE;
-    break;
-  case ICmpInst::ICMP_SLE:
-    // X <= -1 is equivalent to (X & SignMask) != 0.
-    if (!C->isAllOnes())
-      return std::nullopt;
-    Result.Mask = APInt::getSignMask(C->getBitWidth());
-    Result.Pred = ICmpInst::ICMP_NE;
-    break;
-  case ICmpInst::ICMP_SGT:
-    // X > -1 is equivalent to (X & SignMask) == 0.
-    if (!C->isAllOnes())
-      return std::nullopt;
-    Result.Mask = APInt::getSignMask(C->getBitWidth());
-    Result.Pred = ICmpInst::ICMP_EQ;
-    break;
-  case ICmpInst::ICMP_SGE:
-    // X >= 0 is equivalent to (X & SignMask) == 0.
-    if (!C->isZero())
-      return std::nullopt;
-    Result.Mask = APInt::getSignMask(C->getBitWidth());
-    Result.Pred = ICmpInst::ICMP_EQ;
-    break;
+    if (C.isZero()) {
+      Result.Mask = APInt::getSignMask(C.getBitWidth());
+      Result.C = APInt::getZero(C.getBitWidth());
+      Result.Pred = ICmpInst::ICMP_NE;
+      break;
+    }
+
+    APInt FlippedSign = C ^ APInt::getSignMask(C.getBitWidth());
+    if (FlippedSign.isPowerOf2()) {
+      // X s< 10000100 is equivalent to (X & 11111100 == 10000000)
+      Result.Mask = -FlippedSign;
+      Result.C = APInt::getSignMask(C.getBitWidth());
+      Result.Pred = ICmpInst::ICMP_EQ;
+      break;
+    }
+
+    if (FlippedSign.isNegatedPowerOf2()) {
+      // X s< 01111100 is equivalent to (X & 11111100 != 01111100)
+      Result.Mask = FlippedSign;
+      Result.C = C;
+      Result.Pred = ICmpInst::ICMP_NE;
+      break;
+    }
+
+    return std::nullopt;
+  }
   case ICmpInst::ICMP_ULT:
     // X <u 2^n is equivalent to (X & ~(2^n-1)) == 0.
-    if (!C->isPowerOf2())
-      return std::nullopt;
-    Result.Mask = -*C;
-    Result.Pred = ICmpInst::ICMP_EQ;
-    break;
-  case ICmpInst::ICMP_ULE:
-    // X <=u 2^n-1 is equivalent to (X & ~(2^n-1)) == 0.
-    if (!(*C + 1).isPowerOf2())
-      return std::nullopt;
-    Result.Mask = ~*C;
-    Result.Pred = ICmpInst::ICMP_EQ;
-    break;
-  case ICmpInst::ICMP_UGT:
-    // X >u 2^n-1 is equivalent to (X & ~(2^n-1)) != 0.
-    if (!(*C + 1).isPowerOf2())
-      return std::nullopt;
-    Result.Mask = ~*C;
-    Result.Pred = ICmpInst::ICMP_NE;
-    break;
-  case ICmpInst::ICMP_UGE:
-    // X >=u 2^n is equivalent to (X & ~(2^n-1)) != 0.
-    if (!C->isPowerOf2())
-      return std::nullopt;
-    Result.Mask = -*C;
-    Result.Pred = ICmpInst::ICMP_NE;
-    break;
+    if (C.isPowerOf2()) {
+      Result.Mask = -C;
+      Result.C = APInt::getZero(C.getBitWidth());
+      Result.Pred = ICmpInst::ICMP_EQ;
+      break;
+    }
+
+    // X u< 11111100 is equivalent to (X & 11111100 != 11111100)
+    if (C.isNegatedPowerOf2()) {
+      Result.Mask = C;
+      Result.C = C;
+      Result.Pred = ICmpInst::ICMP_NE;
+      break;
+    }
+
+    return std::nullopt;
   }
 
+  if (!AllowNonZeroC && !Result.C.isZero())
+    return std::nullopt;
+
+  if (Inverted)
+    Result.Pred = ICmpInst::getInversePredicate(Result.Pred);
+
   Value *X;
   if (LookThruTrunc && match(LHS, m_Trunc(m_Value(X)))) {
     Result.X = X;
     Result.Mask = Result.Mask.zext(X->getType()->getScalarSizeInBits());
+    Result.C = Result.C.zext(X->getType()->getScalarSizeInBits());
   } else {
     Result.X = LHS;
   }
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 3f189724763d4..fe00ea0097a43 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1943,16 +1943,24 @@ MemoryDepChecker::getDependenceDistanceStrideAndSize(
   // required for correctness.
   if (SE.isLoopInvariant(Src, InnermostLoop) ||
       SE.isLoopInvariant(Sink, InnermostLoop)) {
-    const auto &[SrcStart, SrcEnd] =
+    const auto &[SrcStart_, SrcEnd_] =
         getStartAndEndForAccess(InnermostLoop, Src, ATy, PSE, PointerBounds);
-    const auto &[SinkStart, SinkEnd] =
+    const auto &[SinkStart_, SinkEnd_] =
         getStartAndEndForAccess(InnermostLoop, Sink, BTy, PSE, PointerBounds);
-    if (!isa<SCEVCouldNotCompute>(SrcStart) &&
-        !isa<SCEVCouldNotCompute>(SrcEnd) &&
-        !isa<SCEVCouldNotCompute>(SinkStart) &&
-        !isa<SCEVCouldNotCompute>(SinkEnd)) {
+    if (!isa<SCEVCouldNotCompute>(SrcStart_) &&
+        !isa<SCEVCouldNotCompute>(SrcEnd_) &&
+        !isa<SCEVCouldNotCompute>(SinkStart_) &&
+        !isa<SCEVCouldNotCompute>(SinkEnd_)) {
+      if (!LoopGuards)
+        LoopGuards.emplace(
+            ScalarEvolution::LoopGuards::collect(InnermostLoop, SE));
+      auto SrcEnd = SE.applyLoopGuards(SrcEnd_, *LoopGuards);
+      auto SinkStart = SE.applyLoopGuards(SinkStart_, *LoopGuards);
       if (SE.isKnownPredicate(CmpInst::ICMP_ULE, SrcEnd, SinkStart))
         return MemoryDepChecker::Dependence::NoDep;
+
+      auto SinkEnd = SE.applyLoopGuards(SinkEnd_, *LoopGuards);
+      auto SrcStart = SE.applyLoopGuards(SrcStart_, *LoopGuards);
       if (SE.isKnownPredicate(CmpInst::ICMP_ULE, SinkEnd, SrcStart))
         return MemoryDepChecker::Dependence::NoDep;
     }
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 56eb3f99b39d2..446ff42f3e243 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -369,9 +369,9 @@ static void computeKnownBitsAddSub(bool Add, const Value *Op0, const Value *Op1,
 }
 
 static void computeKnownBitsMul(const Value *Op0, const Value *Op1, bool NSW,
-                                const APInt &DemandedElts, KnownBits &Known,
-                                KnownBits &Known2, unsigned Depth,
-                                const SimplifyQuery &Q) {
+                                bool NUW, const APInt &DemandedElts,
+                                KnownBits &Known, KnownBits &Known2,
+                                unsigned Depth, const SimplifyQuery &Q) {
   computeKnownBits(Op1, DemandedElts, Known, Depth + 1, Q);
   computeKnownBits(Op0, DemandedElts, Known2, Depth + 1, Q);
 
@@ -390,6 +390,13 @@ static void computeKnownBitsMul(const Value *Op0, const Value *Op1, bool NSW,
       // The product of two numbers with the same sign is non-negative.
       isKnownNonNegative = (isKnownNegativeOp1 && isKnownNegativeOp0) ||
                            (isKnownNonNegativeOp1 && isKnownNonNegativeOp0);
+      if (!isKnownNonNegative && NUW) {
+        // mul nuw nsw with a factor > 1 is non-negative.
+        KnownBits One = KnownBits::makeConstant(APInt(Known.getBitWidth(), 1));
+        isKnownNonNegative = KnownBits::sgt(Known, One).value_or(false) ||
+                             KnownBits::sgt(Known2, One).value_or(false);
+      }
+
       // The product of a negative number and a non-negative number is either
       // negative or zero.
       if (!isKnownNonNegative)
@@ -821,9 +828,12 @@ void llvm::computeKnownBitsFromContext(const Value *V, KnownBits &Known,
         continue;
       if (RetainedKnowledge RK = getKnowledgeFromBundle(
               *I, I->bundle_op_info_begin()[Elem.Index])) {
+        // Allow AllowEphemerals in isValidAssumeForContext, as the CxtI might
+        // be the producer of the pointer in the bundle. At the moment, align
+        // assumptions aren't optimized away.
         if (RK.WasOn == V && RK.AttrKind == Attribute::Alignment &&
             isPowerOf2_64(RK.ArgValue) &&
-            isValidAssumeForContext(I, Q.CxtI, Q.DT))
+            isValidAssumeForContext(I, Q.CxtI, Q.DT, /*AllowEphemerals*/ true))
           Known.Zero.setLowBits(Log2_64(RK.ArgValue));
       }
       continue;
@@ -1090,8 +1100,9 @@ static void computeKnownBitsFromOperator(const Operator *I,
     break;
   case Instruction::Mul: {
     bool NSW = Q.IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(I));
-    computeKnownBitsMul(I->getOperand(0), I->getOperand(1), NSW, DemandedElts,
-                        Known, Known2, Depth, Q);
+    bool NUW = Q.IIQ.hasNoUnsignedWrap(cast<OverflowingBinaryOperator>(I));
+    computeKnownBitsMul(I->getOperand(0), I->getOperand(1), NSW, NUW,
+                        DemandedElts, Known, Known2, Depth, Q);
     break;
   }
   case Instruction::UDiv: {
@@ -1419,9 +1430,12 @@ static void computeKnownBitsFromOperator(const Operator *I,
       // If this is a shift recurrence, we know the bits being shifted in.
       // We can combine that with information about the start value of the
       // recurrence to conclude facts about the result.
-      if ((Opcode == Instruction::LShr || Opcode == Instruction::AShr ||
-           Opcode == Instruction::Shl) &&
-          BO->getOperand(0) == I) {
+      switch (Opcode) {
+      case Instruction::LShr:
+      case Instruction::AShr:
+      case Instruction::Shl: {
+        if (BO->getOperand(0) != I)
+          break;
 
         // We have matched a recurrence of the form:
         // %iv = [R, %entry], [%iv.next, %backedge]
@@ -1449,17 +1463,18 @@ static void computeKnownBitsFromOperator(const Operator *I,
           Known.Zero.setHighBits(Known2.countMinLeadingZeros());
           Known.One.setHighBits(Known2.countMinLeadingOnes());
           break;
-        };
+        }
+        break;
       }
 
       // Check for operations that have the property that if
       // both their operands have low zero bits, the result
       // will have low zero bits.
-      if (Opcode == Instruction::Add ||
-          Opcode == Instruction::Sub ||
-          Opcode == Instruction::And ||
-          Opcode == Instruction::Or ||
-          Opcode == Instruction::Mul) {
+      case Instruction::Add:
+      case Instruction::Sub:
+      case Instruction::And:
+      case Instruction::Or:
+      case Instruction::Mul: {
         // Change the context instruction to the "edge" that flows into the
         // phi. This is important because that is where the value is actually
         // "evaluated" even though it is used later somewhere else. (see also
@@ -1484,38 +1499,51 @@ static void computeKnownBitsFromOperator(const Operator *I,
                                        Known3.countMinTrailingZeros()));
 
         auto *OverflowOp = dyn_cast<OverflowingBinaryOperator>(BO);
-        if (OverflowOp && Q.IIQ.hasNoSignedWrap(OverflowOp)) {
-          // If initial value of recurrence is nonnegative, and we are adding
-          // a nonnegative number with nsw, the result can only be nonnegative
-          // or poison value regardless of the number of times we execute the
-          // add in phi recurrence. If initial value is negative and we are
-          // adding a negative number with nsw, the result can only be
-          // negative or poison value. Similar arguments apply to sub and mul.
-          //
-          // (add non-negative, non-negative) --> non-negative
-          // (add negative, negative) --> negative
-          if (Opcode == Instruction::Add) {
-            if (Known2.isNonNegative() && Known3.isNonNegative())
-              Known.makeNonNegative();
-            else if (Known2.isNegative() && Known3.isNegative())
-              Known.makeNegative();
-          }
+        if (!OverflowOp || !Q.IIQ.hasNoSignedWrap(OverflowOp))
+          break;
 
-          // (sub nsw non-negative, negative) --> non-negative
-          // (sub nsw negative, non-negative) --> negative
-          else if (Opcode == Instruction::Sub && BO->getOperand(0) == I) {
-            if (Known2.isNonNegative() && Known3.isNegative())
-              Known.makeNonNegative();
-            else if (Known2.isNegative() && Known3.isNonNegative())
-              Known.makeNegative();
-          }
+        switch (Opcode) {
+        // If initial value of recurrence is nonnegative, and we are adding
+        // a nonnegative number with nsw, the result can only be nonnegative
+        // or poison value regardless of the number of times we execute the
+        // add in phi recurrence. If initial value is negative and we are
+        // adding a negative number with nsw, the result can only be
+        // negative or poison value. Similar arguments apply to sub and mul.
+        //
+        // (add non-negative, non-negative) --> non-negative
+        // (add negative, negative) --> negative
+        case Instruction::Add: {
+          if (Known2.isNonNegative() && Known3.isNonNegative())
+            Known.makeNonNegative();
+          else if (Known2.isNegative() && Known3.isNegative())
+            Known.makeNegative();
+          break;
+        }
 
-          // (mul nsw non-negative, non-negative) --> non-negative
-          else if (Opcode == Instruction::Mul && Known2.isNonNegative() &&
-                   Known3.isNonNegative())
+        // (sub nsw non-negative, negative) --> non-negative
+        // (sub nsw negative, non-negative) --> negative
+        case Instruction::Sub: {
+          if (BO->getOperand(0) != I)
+            break;
+          if (Known2.isNonNegative() && Known3.isNegative())
             Known.makeNonNegative();
+          else if (Known2.isNegative() && Known3.isNonNegative())
+            Known.makeNegative();
+          break;
         }
 
+        // (mul nsw non-negative, non-negative) --> non-negative
+        case Instruction::Mul:
+          if (Known2.isNonNegative() && Known3.isNonNegative())
+            Known.makeNonNegative();
+          break;
+
+        default:
+          break;
+        }
+        break;
+      }
+      default:
         break;
       }
     }
@@ -1961,7 +1989,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
         case Intrinsic::umul_with_overflow:
         case Intrinsic::smul_with_overflow:
           computeKnownBitsMul(II->getArgOperand(0), II->getArgOperand(1), false,
-                              DemandedElts, Known, Known2, Depth, Q);
+                              false, DemandedElts, Known, Known2, Depth, Q);
           break;
         }
       }
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 317278911b28f..3a8cde7330efc 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -1703,6 +1703,20 @@ static bool needFuncLabels(const MachineFunction &MF, const AsmPrinter &Asm) {
       classifyEHPersonality(MF.getFunction().getPersonalityFn()));
 }
 
+// Return the mnemonic of a MachineInstr if available, or the MachineInstr
+// opcode name otherwise.
+static StringRef getMIMnemonic(const MachineInstr &MI, MCStreamer &Streamer) {
+  const TargetInstrInfo *TII =
+      MI.getParent()->getParent()->getSubtarget().getInstrInfo();
+  MCInst MCI;
+  MCI.setOpcode(MI.getOpcode());
+  if (StringRef Name = Streamer.getMnemonic(MCI); !Name.empty())
+    return Name;
+  StringRef Name = TII->getName(MI.getOpcode());
+  assert(!Name.empty() && "Missing mnemonic and name for opcode");
+  return Name;
+}
+
 /// EmitFunctionBody - This method emits the body and trailer for a
 /// function.
 void AsmPrinter::emitFunctionBody() {
@@ -1746,7 +1760,6 @@ void AsmPrinter::emitFunctionBody() {
       if (!MI.isPosition() && !MI.isImplicitDef() && !MI.isKill() &&
           !MI.isDebugInstr()) {
         HasAnyRealCode = true;
-        ++NumInstsInFunction;
       }
 
       // If there is a pre-instruction symbol, emit a label for it here.
@@ -1845,12 +1858,25 @@ void AsmPrinter::emitFunctionBody() {
         break;
       default:
         emitInstruction(&MI);
-        if (CanDoExtraAnalysis) {
-          MCInst MCI;
-          MCI.setOpcode(MI.getOpcode());
-          auto Name = OutStreamer->getMnemonic(MCI);
-          auto I = MnemonicCounts.insert({Name, 0u});
-          I.first->second++;
+
+        auto CountInstruction = [&](const MachineInstr &MI) {
+          // Skip Meta instructions inside bundles.
+          if (MI.isMetaInstruction())
+            return;
+          ++NumInstsInFunction;
+          if (CanDoExtraAnalysis) {
+            StringRef Name = getMIMnemonic(MI, *OutStreamer);
+            ++MnemonicCounts[Name];
+          }
+        };
+        if (!MI.isBundle()) {
+          CountInstruction(MI);
+          break;
+        }
+        // Separately count all the instructions in a bundle.
+        for (auto It = std::next(MI.getIterator());
+             It != MBB.end() && It->isInsideBundle(); ++It) {
+          CountInstruction(*It);
         }
         break;
       }
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index aa44d62da47be..27931e118504b 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -352,8 +352,6 @@ DwarfDebug::DwarfDebug(AsmPrinter *A)
   else
     UseInlineStrings = DwarfInlinedStrings == Enable;
 
-  UseLocSection = !TT.isNVPTX();
-
   // Always emit .debug_aranges for SCE tuning.
   UseARangesSection = GenerateARangeSection || tuneForSCE();
 
@@ -1325,15 +1323,22 @@ void DwarfDebug::finalizeModuleInfo() {
     DwarfCompileUnit &U = SkCU ? *SkCU : TheCU;
 
     if (unsigned NumRanges = TheCU.getRanges().size()) {
-      if (NumRanges > 1 && useRangesSection())
-        // A DW_AT_low_pc attribute may also be specified in combination with
-        // DW_AT_ranges to specify the default base address for use in
-        // location lists (see Section 2.6.2) and range lists (see Section
-        // 2.17.3).
-        U.addUInt(U.getUnitDie(), dwarf::DW_AT_low_pc, dwarf::DW_FORM_addr, 0);
-      else
-        U.setBaseAddress(TheCU.getRanges().front().Begin);
-      U.attachRangesOrLowHighPC(U.getUnitDie(), TheCU.takeRanges());
+      // PTX does not support subtracting labels from the code section in the
+      // debug_loc section.  To work around this, the NVPTX backend needs the
+      // compile unit to have no low_pc in order to have a zero base_address
+      // when handling debug_loc in cuda-gdb.
+      if (!(Asm->TM.getTargetTriple().isNVPTX() && tuneForGDB())) {
+        if (NumRanges > 1 && useRangesSection())
+          // A DW_AT_low_pc attribute may also be specified in combination with
+          // DW_AT_ranges to specify the default base address for use in
+          // location lists (see Section 2.6.2) and range lists (see Section
+          // 2.17.3).
+          U.addUInt(U.getUnitDie(), dwarf::DW_AT_low_pc, dwarf::DW_FORM_addr,
+                    0);
+        else
+          U.setBaseAddress(TheCU.getRanges().front().Begin);
+        U.attachRangesOrLowHighPC(U.getUnitDie(), TheCU.takeRanges());
+      }
     }
 
     // We don't keep track of which addresses are used in which CU so this
@@ -1920,10 +1925,6 @@ void DwarfDebug::collectEntityInfo(DwarfCompileUnit &TheCU,
       }
     }
 
-    // Do not emit location lists if .debug_loc secton is disabled.
-    if (!useLocSection())
-      continue;
-
     // Handle multiple DBG_VALUE instructions describing one variable.
     DebugLocStream::ListBuilder List(DebugLocs, TheCU, *Asm, *RegVar);
 
@@ -2841,7 +2842,17 @@ static void emitRangeList(
   bool BaseIsSet = false;
   for (const auto &P : SectionRanges) {
     auto *Base = CUBase;
-    if (!Base && ShouldUseBaseAddress) {
+    if ((Asm->TM.getTargetTriple().isNVPTX() && DD.tuneForGDB())) {
+      // PTX does not support subtracting labels from the code section in the
+      // debug_loc section.  To work around this, the NVPTX backend needs the
+      // compile unit to have no low_pc in order to have a zero base_address
+      // when handling debug_loc in cuda-gdb.  Additionally, cuda-gdb doesn't
+      // seem to handle setting a per-variable base to zero.  To make cuda-gdb
+      // happy, just emit labels with no base while having no compile unit
+      // low_pc.
+      BaseIsSet = false;
+      Base = nullptr;
+    } else if (!Base && ShouldUseBaseAddress) {
       const MCSymbol *Begin = P.second.front()->Begin;
       const MCSymbol *NewBase = DD.getSectionLabel(&Begin->getSection());
       if (!UseDwarf5) {
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
index 19f5b677bb8d0..6ed03124a2626 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
@@ -432,9 +432,6 @@ class DwarfDebug : public DebugHandlerBase {
   /// temp symbols inside DWARF sections.
   bool UseSectionsAsReferences = false;
 
-  ///Allow emission of the .debug_loc section.
-  bool UseLocSection = true;
-
   /// Allow emission of .debug_aranges section
   bool UseARangesSection = false;
 
@@ -791,9 +788,6 @@ class DwarfDebug : public DebugHandlerBase {
     return UseSectionsAsReferences;
   }
 
-  /// Returns whether .debug_loc section should be emitted.
-  bool useLocSection() const { return UseLocSection; }
-
   /// Returns whether to generate DWARF v4 type units.
   bool generateTypeUnits() const { return GenerateTypeUnits; }
 
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 40360b0b0f1d8..932287d140302 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -2201,6 +2201,7 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
       for (auto VReg : getOrCreateVRegs(*Arg))
         VRegs.push_back(VReg);
     MIRBuilder.buildInstr(TargetOpcode::FAKE_USE, {}, VRegs);
+    MF->setHasFakeUses(true);
     return true;
   }
   case Intrinsic::dbg_declare: {
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 2fb2d104f1ce3..3b2fd95076c46 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -3666,6 +3666,65 @@ LegalizerHelper::bitcastConcatVector(MachineInstr &MI, unsigned TypeIdx,
   return Legalized;
 }
 
+/// This attempts to bitcast G_EXTRACT_SUBVECTOR to CastTy.
+///
+///  <vscale x 8 x i1> = G_EXTRACT_SUBVECTOR <vscale x 16 x i1>, N
+///
+/// ===>
+///
+///  <vscale x 2 x i1> = G_BITCAST <vscale x 16 x i1>
+///  <vscale x 1 x i8> = G_EXTRACT_SUBVECTOR <vscale x 2 x i1>, N / 8
+///  <vscale x 8 x i1> = G_BITCAST <vscale x 1 x i8>
+LegalizerHelper::LegalizeResult
+LegalizerHelper::bitcastExtractSubvector(MachineInstr &MI, unsigned TypeIdx,
+                                         LLT CastTy) {
+  auto ES = cast<GExtractSubvector>(&MI);
+
+  if (!CastTy.isVector())
+    return UnableToLegalize;
+
+  if (TypeIdx != 0)
+    return UnableToLegalize;
+
+  Register Dst = ES->getReg(0);
+  Register Src = ES->getSrcVec();
+  uint64_t Idx = ES->getIndexImm();
+
+  MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
+
+  LLT DstTy = MRI.getType(Dst);
+  LLT SrcTy = MRI.getType(Src);
+  ElementCount DstTyEC = DstTy.getElementCount();
+  ElementCount SrcTyEC = SrcTy.getElementCount();
+  auto DstTyMinElts = DstTyEC.getKnownMinValue();
+  auto SrcTyMinElts = SrcTyEC.getKnownMinValue();
+
+  if (DstTy == CastTy)
+    return Legalized;
+
+  if (DstTy.getSizeInBits() != CastTy.getSizeInBits())
+    return UnableToLegalize;
+
+  unsigned CastEltSize = CastTy.getElementType().getSizeInBits();
+  unsigned DstEltSize = DstTy.getElementType().getSizeInBits();
+  if (CastEltSize < DstEltSize)
+    return UnableToLegalize;
+
+  auto AdjustAmt = CastEltSize / DstEltSize;
+  if (Idx % AdjustAmt != 0 || DstTyMinElts % AdjustAmt != 0 ||
+      SrcTyMinElts % AdjustAmt != 0)
+    return UnableToLegalize;
+
+  Idx /= AdjustAmt;
+  SrcTy = LLT::vector(SrcTyEC.divideCoefficientBy(AdjustAmt), AdjustAmt);
+  auto CastVec = MIRBuilder.buildBitcast(SrcTy, Src);
+  auto PromotedES = MIRBuilder.buildExtractSubvector(CastTy, CastVec, Idx);
+  MIRBuilder.buildBitcast(Dst, PromotedES);
+
+  ES->eraseFromParent();
+  return Legalized;
+}
+
 LegalizerHelper::LegalizeResult LegalizerHelper::lowerLoad(GAnyLoad &LoadMI) {
   // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT
   Register DstReg = LoadMI.getDstReg();
@@ -3972,6 +4031,8 @@ LegalizerHelper::bitcast(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) {
     return bitcastInsertVectorElt(MI, TypeIdx, CastTy);
   case TargetOpcode::G_CONCAT_VECTORS:
     return bitcastConcatVector(MI, TypeIdx, CastTy);
+  case TargetOpcode::G_EXTRACT_SUBVECTOR:
+    return bitcastExtractSubvector(MI, TypeIdx, CastTy);
   default:
     return UnableToLegalize;
   }
diff --git a/llvm/lib/CodeGen/InlineSpiller.cpp b/llvm/lib/CodeGen/InlineSpiller.cpp
index 409879a8b49bc..d1f67d1a3d4aa 100644
--- a/llvm/lib/CodeGen/InlineSpiller.cpp
+++ b/llvm/lib/CodeGen/InlineSpiller.cpp
@@ -167,6 +167,10 @@ class InlineSpiller : public Spiller {
   // All registers to spill to StackSlot, including the main register.
   SmallVector<Register, 8> RegsToSpill;
 
+  // All registers that were replaced by the spiller through some other method,
+  // e.g. rematerialization.
+  SmallVector<Register, 8> RegsReplaced;
+
   // All COPY instructions to/from snippets.
   // They are ignored since both operands refer to the same stack slot.
   // For bundled copies, this will only include the first header copy.
@@ -199,6 +203,8 @@ class InlineSpiller : public Spiller {
         HSpiller(Pass, MF, VRM), VRAI(VRAI) {}
 
   void spill(LiveRangeEdit &) override;
+  ArrayRef<Register> getSpilledRegs() override { return RegsToSpill; }
+  ArrayRef<Register> getReplacedRegs() override { return RegsReplaced; }
   void postOptimization() override;
 
 private:
@@ -385,6 +391,7 @@ void InlineSpiller::collectRegsToSpill() {
   // Main register always spills.
   RegsToSpill.assign(1, Reg);
   SnippetCopies.clear();
+  RegsReplaced.clear();
 
   // Snippets all have the same original, so there can't be any for an original
   // register.
@@ -796,6 +803,7 @@ void InlineSpiller::reMaterializeAll() {
   for (Register Reg : RegsToSpill) {
     if (MRI.reg_nodbg_empty(Reg)) {
       Edit->eraseVirtReg(Reg);
+      RegsReplaced.push_back(Reg);
       continue;
     }
 
@@ -1255,8 +1263,13 @@ void InlineSpiller::spillAll() {
   LLVM_DEBUG(dbgs() << "Merged spilled regs: " << *StackInt << '\n');
 
   // Spill around uses of all RegsToSpill.
-  for (Register Reg : RegsToSpill)
+  for (Register Reg : RegsToSpill) {
     spillAroundUses(Reg);
+    // Assign all of the spilled registers to the slot so that
+    // LiveDebugVariables knows about these locations later on.
+    if (VRM.getStackSlot(Reg) == VirtRegMap::NO_STACK_SLOT)
+      VRM.assignVirt2StackSlot(Reg, StackSlot);
+  }
 
   // Hoisted spills may cause dead code.
   if (!DeadDefs.empty()) {
diff --git a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
index 997c428ca77dc..30b1a717caaab 100644
--- a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
@@ -380,6 +380,7 @@ bool MIRParserImpl::computeFunctionProperties(
 
   bool HasPHI = false;
   bool HasInlineAsm = false;
+  bool HasFakeUses = false;
   bool AllTiedOpsRewritten = true, HasTiedOps = false;
   for (const MachineBasicBlock &MBB : MF) {
     for (const MachineInstr &MI : MBB) {
@@ -387,6 +388,8 @@ bool MIRParserImpl::computeFunctionProperties(
         HasPHI = true;
       if (MI.isInlineAsm())
         HasInlineAsm = true;
+      if (MI.isFakeUse())
+        HasFakeUses = true;
       for (unsigned I = 0; I < MI.getNumOperands(); ++I) {
         const MachineOperand &MO = MI.getOperand(I);
         if (!MO.isReg() || !MO.getReg())
@@ -441,6 +444,16 @@ bool MIRParserImpl::computeFunctionProperties(
         " has explicit property NoVRegs, but contains virtual registers");
   }
 
+  // For hasFakeUses we follow similar logic to the ComputedPropertyHelper,
+  // except for caring about the inverse case only, i.e. when the property is
+  // explicitly set to false and Fake Uses are present; having HasFakeUses=true
+  // on a function without fake uses is harmless.
+  if (YamlMF.HasFakeUses && !*YamlMF.HasFakeUses && HasFakeUses)
+    return error(
+        MF.getName() +
+        " has explicit property hasFakeUses=false, but contains fake uses");
+  MF.setHasFakeUses(YamlMF.HasFakeUses.value_or(HasFakeUses));
+
   return false;
 }
 
diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp
index cf6122bce2236..d52c1d831267f 100644
--- a/llvm/lib/CodeGen/MIRPrinter.cpp
+++ b/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -207,6 +207,7 @@ void MIRPrinter::print(const MachineFunction &MF) {
   YamlMF.HasEHCatchret = MF.hasEHCatchret();
   YamlMF.HasEHScopes = MF.hasEHScopes();
   YamlMF.HasEHFunclets = MF.hasEHFunclets();
+  YamlMF.HasFakeUses = MF.hasFakeUses();
   YamlMF.IsOutlined = MF.isOutlined();
   YamlMF.UseDebugInstrRef = MF.useDebugInstrRef();
 
diff --git a/llvm/lib/CodeGen/MachineBlockPlacement.cpp b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
index 7807875c06584..c42e63202c3b5 100644
--- a/llvm/lib/CodeGen/MachineBlockPlacement.cpp
+++ b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
@@ -218,6 +218,11 @@ static cl::opt<unsigned> ExtTspBlockPlacementMaxBlocks(
              "block placement."),
     cl::init(UINT_MAX), cl::Hidden);
 
+// Apply the ext-tsp algorithm minimizing the size of a binary.
+static cl::opt<bool>
+    ApplyExtTspForSize("apply-ext-tsp-for-size", cl::init(false), cl::Hidden,
+                       cl::desc("Use ext-tsp for size-aware block placement."));
+
 namespace llvm {
 extern cl::opt<bool> EnableExtTspBlockPlacement;
 extern cl::opt<bool> ApplyExtTspWithoutProfile;
@@ -595,7 +600,7 @@ class MachineBlockPlacement : public MachineFunctionPass {
   void precomputeTriangleChains();
 
   /// Apply a post-processing step optimizing block placement.
-  void applyExtTsp();
+  void applyExtTsp(bool OptForSize);
 
   /// Modify the existing block placement in the function and adjust all jumps.
   void assignBlockOrder(const std::vector<const MachineBasicBlock *> &NewOrder);
@@ -3505,20 +3510,36 @@ bool MachineBlockPlacement::runOnMachineFunction(MachineFunction &MF) {
   // Initialize tail duplication thresholds.
   initTailDupThreshold();
 
+  const bool OptForSize =
+      MF.getFunction().hasOptSize() ||
+      llvm::shouldOptimizeForSize(&MF, PSI, &MBFI->getMBFI());
+  // Determine whether to use ext-tsp for perf/size optimization. The method
+  // is beneficial only for instances with at least 3 basic blocks and it can be
+  // disabled for huge functions (exceeding a certain size).
+  bool UseExtTspForPerf = false;
+  bool UseExtTspForSize = false;
+  if (3 <= MF.size() && MF.size() <= ExtTspBlockPlacementMaxBlocks) {
+    UseExtTspForPerf =
+        EnableExtTspBlockPlacement &&
+        (ApplyExtTspWithoutProfile || MF.getFunction().hasProfileData());
+    UseExtTspForSize = OptForSize && ApplyExtTspForSize;
+  }
+
   // Apply tail duplication.
   if (allowTailDupPlacement()) {
     MPDT = &getAnalysis<MachinePostDominatorTreeWrapperPass>().getPostDomTree();
-    bool OptForSize = MF.getFunction().hasOptSize() ||
-                      llvm::shouldOptimizeForSize(&MF, PSI, &MBFI->getMBFI());
     if (OptForSize)
       TailDupSize = 1;
     const bool PreRegAlloc = false;
     TailDup.initMF(MF, PreRegAlloc, MBPI, MBFI.get(), PSI,
                    /* LayoutMode */ true, TailDupSize);
-    precomputeTriangleChains();
+    if (!UseExtTspForSize)
+      precomputeTriangleChains();
   }
 
-  buildCFGChains();
+  // Run the main block placement.
+  if (!UseExtTspForSize)
+    buildCFGChains();
 
   // Changing the layout can create new tail merging opportunities.
   // TailMerge can create jump into if branches that make CFG irreducible for
@@ -3545,14 +3566,14 @@ bool MachineBlockPlacement::runOnMachineFunction(MachineFunction &MF) {
     }
   }
 
-  // Apply a post-processing optimizing block placement.
-  if (MF.size() >= 3 && EnableExtTspBlockPlacement &&
-      (ApplyExtTspWithoutProfile || MF.getFunction().hasProfileData()) &&
-      MF.size() <= ExtTspBlockPlacementMaxBlocks) {
-    // Find a new placement and modify the layout of the blocks in the function.
-    applyExtTsp();
-
-    // Re-create CFG chain so that we can optimizeBranches and alignBlocks.
+  // Apply a post-processing optimizing block placement:
+  // - find a new placement and modify the layout of the blocks in the function;
+  // - re-create CFG chains so that we can optimizeBranches and alignBlocks.
+  if (UseExtTspForPerf || UseExtTspForSize) {
+    assert(
+        !(UseExtTspForPerf && UseExtTspForSize) &&
+        "UseExtTspForPerf and UseExtTspForSize can not be set simultaneosly");
+    applyExtTsp(/*OptForSize=*/UseExtTspForSize);
     createCFGChainExtTsp();
   }
 
@@ -3577,7 +3598,7 @@ bool MachineBlockPlacement::runOnMachineFunction(MachineFunction &MF) {
   return true;
 }
 
-void MachineBlockPlacement::applyExtTsp() {
+void MachineBlockPlacement::applyExtTsp(bool OptForSize) {
   // Prepare data; blocks are indexed by their index in the current ordering.
   DenseMap<const MachineBasicBlock *, uint64_t> BlockIndex;
   BlockIndex.reserve(F->size());
@@ -3589,13 +3610,15 @@ void MachineBlockPlacement::applyExtTsp() {
     CurrentBlockOrder.push_back(&MBB);
   }
 
-  auto BlockSizes = std::vector<uint64_t>(F->size());
-  auto BlockCounts = std::vector<uint64_t>(F->size());
-  std::vector<codelayout::EdgeCount> JumpCounts;
+  SmallVector<uint64_t, 0> BlockCounts(F->size());
+  SmallVector<uint64_t, 0> BlockSizes(F->size());
+  SmallVector<codelayout::EdgeCount, 0> JumpCounts;
+  SmallVector<MachineOperand, 4> Cond; // For analyzeBranch.
+  SmallVector<const MachineBasicBlock *, 4> Succs;
   for (MachineBasicBlock &MBB : *F) {
     // Getting the block frequency.
     BlockFrequency BlockFreq = MBFI->getBlockFreq(&MBB);
-    BlockCounts[BlockIndex[&MBB]] = BlockFreq.getFrequency();
+    BlockCounts[BlockIndex[&MBB]] = OptForSize ? 1 : BlockFreq.getFrequency();
     // Getting the block size:
     // - approximate the size of an instruction by 4 bytes, and
     // - ignore debug instructions.
@@ -3604,23 +3627,49 @@ void MachineBlockPlacement::applyExtTsp() {
     // not see a perf improvement with the exact block sizes.
     auto NonDbgInsts =
         instructionsWithoutDebug(MBB.instr_begin(), MBB.instr_end());
-    int NumInsts = std::distance(NonDbgInsts.begin(), NonDbgInsts.end());
+    size_t NumInsts = std::distance(NonDbgInsts.begin(), NonDbgInsts.end());
     BlockSizes[BlockIndex[&MBB]] = 4 * NumInsts;
+
     // Getting jump frequencies.
-    for (MachineBasicBlock *Succ : MBB.successors()) {
-      auto EP = MBPI->getEdgeProbability(&MBB, Succ);
-      BlockFrequency JumpFreq = BlockFreq * EP;
-      JumpCounts.push_back(
-          {BlockIndex[&MBB], BlockIndex[Succ], JumpFreq.getFrequency()});
+    if (OptForSize) {
+      Cond.clear();
+      MachineBasicBlock *TBB = nullptr, *FBB = nullptr; // For analyzeBranch.
+      if (TII->analyzeBranch(MBB, TBB, FBB, Cond))
+        continue;
+
+      const MachineBasicBlock *FTB = MBB.getFallThrough();
+      // Succs is a collection of distinct destinations of the block reachable
+      // from MBB via a jump instruction; initialize the list using the three
+      // (non-necessarily distinct) blocks, FTB, TBB, and FBB.
+      Succs.clear();
+      if (TBB && TBB != FTB)
+        Succs.push_back(TBB);
+      if (FBB && FBB != FTB)
+        Succs.push_back(FBB);
+      if (FTB)
+        Succs.push_back(FTB);
+      // Absolute magnitude of non-zero counts does not matter for the
+      // optimization; prioritize slightly jumps with a single successor, since
+      // the corresponding jump instruction will be removed from the binary.
+      const uint64_t Freq = Succs.size() == 1 ? 110 : 100;
+      for (const MachineBasicBlock *Succ : Succs)
+        JumpCounts.push_back({BlockIndex[&MBB], BlockIndex[Succ], Freq});
+    } else {
+      for (MachineBasicBlock *Succ : MBB.successors()) {
+        auto EP = MBPI->getEdgeProbability(&MBB, Succ);
+        BlockFrequency JumpFreq = BlockFreq * EP;
+        JumpCounts.push_back(
+            {BlockIndex[&MBB], BlockIndex[Succ], JumpFreq.getFrequency()});
+      }
     }
   }
 
   LLVM_DEBUG(dbgs() << "Applying ext-tsp layout for |V| = " << F->size()
                     << " with profile = " << F->getFunction().hasProfileData()
-                    << " (" << F->getName().str() << ")"
-                    << "\n");
-  LLVM_DEBUG(dbgs() << format("  original  layout score: %0.2f\n",
-                              calcExtTspScore(BlockSizes, JumpCounts)));
+                    << " (" << F->getName() << ")" << "\n");
+
+  const double OrgScore = calcExtTspScore(BlockSizes, JumpCounts);
+  LLVM_DEBUG(dbgs() << format("  original  layout score: %0.2f\n", OrgScore));
 
   // Run the layout algorithm.
   auto NewOrder = computeExtTspLayout(BlockSizes, BlockCounts, JumpCounts);
@@ -3629,12 +3678,14 @@ void MachineBlockPlacement::applyExtTsp() {
   for (uint64_t Node : NewOrder) {
     NewBlockOrder.push_back(CurrentBlockOrder[Node]);
   }
-  LLVM_DEBUG(
-      dbgs() << format("  optimized layout score: %0.2f\n",
-                       calcExtTspScore(NewOrder, BlockSizes, JumpCounts)));
+  const double OptScore = calcExtTspScore(NewOrder, BlockSizes, JumpCounts);
+  LLVM_DEBUG(dbgs() << format("  optimized layout score: %0.2f\n", OptScore));
 
-  // Assign new block order.
-  assignBlockOrder(NewBlockOrder);
+  // If the optimization is unsuccessful, fall back to the original block order.
+  if (OptForSize && OrgScore > OptScore)
+    assignBlockOrder(CurrentBlockOrder);
+  else
+    assignBlockOrder(NewBlockOrder);
 }
 
 void MachineBlockPlacement::assignBlockOrder(
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index 1ad70c86d68e3..8cfd2192de460 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -2504,7 +2504,10 @@ MCRegister RAGreedy::selectOrSplitImpl(const LiveInterval &VirtReg,
     // Tell LiveDebugVariables about the new ranges. Ranges not being covered by
     // the new regs are kept in LDV (still mapping to the old register), until
     // we rewrite spilled locations in LDV at a later stage.
-    DebugVars->splitRegister(VirtReg.reg(), LRE.regs(), *LIS);
+    for (Register r : spiller().getSpilledRegs())
+      DebugVars->splitRegister(r, LRE.regs(), *LIS);
+    for (Register r : spiller().getReplacedRegs())
+      DebugVars->splitRegister(r, LRE.regs(), *LIS);
 
     if (VerifyEnabled)
       MF->verify(this, "After spilling", &errs());
diff --git a/llvm/lib/CodeGen/RemoveLoadsIntoFakeUses.cpp b/llvm/lib/CodeGen/RemoveLoadsIntoFakeUses.cpp
index 232181a199b8c..ef7a58670c3ac 100644
--- a/llvm/lib/CodeGen/RemoveLoadsIntoFakeUses.cpp
+++ b/llvm/lib/CodeGen/RemoveLoadsIntoFakeUses.cpp
@@ -74,10 +74,8 @@ INITIALIZE_PASS_END(RemoveLoadsIntoFakeUses, DEBUG_TYPE,
                     "Remove Loads Into Fake Uses", false, false)
 
 bool RemoveLoadsIntoFakeUses::runOnMachineFunction(MachineFunction &MF) {
-  // Only `optdebug` functions should contain FAKE_USEs, so don't try to run
-  // this for other functions.
-  if (!MF.getFunction().hasFnAttribute(Attribute::OptimizeForDebugging) ||
-      skipFunction(MF.getFunction()))
+  // Only run this for functions that have fake uses.
+  if (!MF.hasFakeUses() || skipFunction(MF.getFunction()))
     return false;
 
   bool AnyChanges = false;
diff --git a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
index 8405ba9ac326c..6a6bc19314d5b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
@@ -31,6 +31,7 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -200,12 +201,22 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf,
             }
           }
         }
-        // Look for calls to the @llvm.va_start intrinsic. We can omit some
-        // prologue boilerplate for variadic functions that don't examine their
-        // arguments.
         if (const auto *II = dyn_cast<IntrinsicInst>(&I)) {
-          if (II->getIntrinsicID() == Intrinsic::vastart)
+          switch (II->getIntrinsicID()) {
+          case Intrinsic::vastart:
+            // Look for calls to the @llvm.va_start intrinsic. We can omit
+            // some prologue boilerplate for variadic functions that don't
+            // examine their arguments.
             MF->getFrameInfo().setHasVAStart(true);
+            break;
+          case Intrinsic::fake_use:
+            // Look for llvm.fake.uses, so that we can remove loads into fake
+            // uses later if necessary.
+            MF->setHasFakeUses(true);
+            break;
+          default:
+            break;
+          }
         }
 
         // If we have a musttail call in a variadic function, we need to ensure
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 12a48ab06f1c0..281d1578d0173 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -1102,6 +1102,9 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
 
     if (Flags.hasNoFPExcept())
       MI->setFlag(MachineInstr::MIFlag::NoFPExcept);
+
+    if (Flags.hasDisjoint())
+      MI->setFlag(MachineInstr::MIFlag::Disjoint);
   }
 
   // Emit all of the actual operands of this instruction, adding them to the
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 04eb891f719d2..42d031310d5e0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -5851,13 +5851,10 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
                     DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
     break;
   }
-  case ISD::VP_REDUCE_FADD:
-  case ISD::VP_REDUCE_FMUL:
   case ISD::VP_REDUCE_FMAX:
   case ISD::VP_REDUCE_FMIN:
   case ISD::VP_REDUCE_FMAXIMUM:
   case ISD::VP_REDUCE_FMINIMUM:
-  case ISD::VP_REDUCE_SEQ_FADD:
     Results.push_back(PromoteReduction(Node));
     break;
   }
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 64414e2d44e65..498debf095598 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -11725,7 +11725,7 @@ void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
   }
 }
 
-#ifndef NDEBUG
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
 void SelectionDAG::VerifyDAGDivergence() {
   std::vector<SDNode *> TopoOrder;
   CreateTopologicalOrder(TopoOrder);
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDebugArangeSet.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDebugArangeSet.cpp
index c60c9d9d72272..f64c71efd87e1 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFDebugArangeSet.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFDebugArangeSet.cpp
@@ -144,11 +144,13 @@ Error DWARFDebugArangeSet::extract(DWARFDataExtractor data,
     if (arangeDescriptor.Length == 0 && arangeDescriptor.Address == 0) {
       if (*offset_ptr == end_offset)
         return ErrorSuccess();
-      WarningHandler(createStringError(
-          errc::invalid_argument,
-          "address range table at offset 0x%" PRIx64
-          " has a premature terminator entry at offset 0x%" PRIx64,
-          Offset, EntryOffset));
+      if (WarningHandler) {
+        WarningHandler(createStringError(
+            errc::invalid_argument,
+            "address range table at offset 0x%" PRIx64
+            " has a premature terminator entry at offset 0x%" PRIx64,
+            Offset, EntryOffset));
+      }
     }
 
     ArangeDescriptors.push_back(arangeDescriptor);
diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
index b4e088d533946..92f37c22ae513 100644
--- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
+++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
@@ -1011,6 +1011,135 @@ void RuntimeDyldELF::resolveBPFRelocation(const SectionEntry &Section,
   }
 }
 
+static void applyUTypeImmRISCV(uint8_t *InstrAddr, uint32_t Imm) {
+  uint32_t UpperImm = (Imm + 0x800) & 0xfffff000;
+  auto Instr = support::ulittle32_t::ref(InstrAddr);
+  Instr = (Instr & 0xfff) | UpperImm;
+}
+
+static void applyITypeImmRISCV(uint8_t *InstrAddr, uint32_t Imm) {
+  uint32_t LowerImm = Imm & 0xfff;
+  auto Instr = support::ulittle32_t::ref(InstrAddr);
+  Instr = (Instr & 0xfffff) | (LowerImm << 20);
+}
+
+void RuntimeDyldELF::resolveRISCVRelocation(const SectionEntry &Section,
+                                            uint64_t Offset, uint64_t Value,
+                                            uint32_t Type, int64_t Addend,
+                                            SID SectionID) {
+  switch (Type) {
+  default: {
+    std::string Err = "Unimplemented reloc type: " + std::to_string(Type);
+    llvm::report_fatal_error(Err.c_str());
+  }
+    // 32-bit PC-relative function call, macros call, tail (PIC)
+    // Write first 20 bits of 32 bit value to the auipc instruction
+    // Last 12 bits to the jalr instruction
+  case ELF::R_RISCV_CALL:
+  case ELF::R_RISCV_CALL_PLT: {
+    uint64_t P = Section.getLoadAddressWithOffset(Offset);
+    uint64_t PCOffset = Value + Addend - P;
+    applyUTypeImmRISCV(Section.getAddressWithOffset(Offset), PCOffset);
+    applyITypeImmRISCV(Section.getAddressWithOffset(Offset + 4), PCOffset);
+    break;
+  }
+    // High 20 bits of 32-bit absolute address, %hi(symbol)
+  case ELF::R_RISCV_HI20: {
+    uint64_t PCOffset = Value + Addend;
+    applyUTypeImmRISCV(Section.getAddressWithOffset(Offset), PCOffset);
+    break;
+  }
+    // Low 12 bits of 32-bit absolute address, %lo(symbol)
+  case ELF::R_RISCV_LO12_I: {
+    uint64_t PCOffset = Value + Addend;
+    applyITypeImmRISCV(Section.getAddressWithOffset(Offset), PCOffset);
+    break;
+  }
+    // High 20 bits of 32-bit PC-relative reference, %pcrel_hi(symbol)
+  case ELF::R_RISCV_GOT_HI20:
+  case ELF::R_RISCV_PCREL_HI20: {
+    uint64_t P = Section.getLoadAddressWithOffset(Offset);
+    uint64_t PCOffset = Value + Addend - P;
+    applyUTypeImmRISCV(Section.getAddressWithOffset(Offset), PCOffset);
+    break;
+  }
+
+    // label:
+    //    auipc      a0, %pcrel_hi(symbol)    // R_RISCV_PCREL_HI20
+    //    addi       a0, a0, %pcrel_lo(label) // R_RISCV_PCREL_LO12_I
+    //
+    // The low 12 bits of relative address between pc and symbol.
+    // The symbol is related to the high part instruction which is marked by
+    // label.
+  case ELF::R_RISCV_PCREL_LO12_I: {
+    for (auto &&PendingReloc : PendingRelocs) {
+      const RelocationValueRef &MatchingValue = PendingReloc.first;
+      RelocationEntry &Reloc = PendingReloc.second;
+      uint64_t HIRelocPC =
+          getSectionLoadAddress(Reloc.SectionID) + Reloc.Offset;
+      if (Value + Addend == HIRelocPC) {
+        uint64_t Symbol = getSectionLoadAddress(MatchingValue.SectionID) +
+                          MatchingValue.Addend;
+        auto PCOffset = Symbol - HIRelocPC;
+        applyITypeImmRISCV(Section.getAddressWithOffset(Offset), PCOffset);
+        return;
+      }
+    }
+
+    llvm::report_fatal_error(
+        "R_RISCV_PCREL_LO12_I without matching R_RISCV_PCREL_HI20");
+  }
+  case ELF::R_RISCV_32_PCREL: {
+    uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset);
+    int64_t RealOffset = Value + Addend - FinalAddress;
+    int32_t TruncOffset = Lo_32(RealOffset);
+    support::ulittle32_t::ref(Section.getAddressWithOffset(Offset)) =
+        TruncOffset;
+    break;
+  }
+  case ELF::R_RISCV_32: {
+    auto Ref = support::ulittle32_t::ref(Section.getAddressWithOffset(Offset));
+    Ref = Value + Addend;
+    break;
+  }
+  case ELF::R_RISCV_64: {
+    auto Ref = support::ulittle64_t::ref(Section.getAddressWithOffset(Offset));
+    Ref = Value + Addend;
+    break;
+  }
+  case ELF::R_RISCV_ADD16: {
+    auto Ref = support::ulittle16_t::ref(Section.getAddressWithOffset(Offset));
+    Ref = Ref + Value + Addend;
+    break;
+  }
+  case ELF::R_RISCV_ADD32: {
+    auto Ref = support::ulittle32_t::ref(Section.getAddressWithOffset(Offset));
+    Ref = Ref + Value + Addend;
+    break;
+  }
+  case ELF::R_RISCV_ADD64: {
+    auto Ref = support::ulittle64_t::ref(Section.getAddressWithOffset(Offset));
+    Ref = Ref + Value + Addend;
+    break;
+  }
+  case ELF::R_RISCV_SUB16: {
+    auto Ref = support::ulittle16_t::ref(Section.getAddressWithOffset(Offset));
+    Ref = Ref - Value - Addend;
+    break;
+  }
+  case ELF::R_RISCV_SUB32: {
+    auto Ref = support::ulittle32_t::ref(Section.getAddressWithOffset(Offset));
+    Ref = Ref - Value - Addend;
+    break;
+  }
+  case ELF::R_RISCV_SUB64: {
+    auto Ref = support::ulittle64_t::ref(Section.getAddressWithOffset(Offset));
+    Ref = Ref - Value - Addend;
+    break;
+  }
+  }
+}
+
 // The target location for the relocation is described by RE.SectionID and
 // RE.Offset.  RE.SectionID can be used to find the SectionEntry.  Each
 // SectionEntry has three members describing its location.
@@ -1076,12 +1205,17 @@ void RuntimeDyldELF::resolveRelocation(const SectionEntry &Section,
   case Triple::bpfeb:
     resolveBPFRelocation(Section, Offset, Value, Type, Addend);
     break;
+  case Triple::riscv32: // Fall through.
+  case Triple::riscv64:
+    resolveRISCVRelocation(Section, Offset, Value, Type, Addend, SectionID);
+    break;
   default:
     llvm_unreachable("Unsupported CPU type!");
   }
 }
 
-void *RuntimeDyldELF::computePlaceholderAddress(unsigned SectionID, uint64_t Offset) const {
+void *RuntimeDyldELF::computePlaceholderAddress(unsigned SectionID,
+                                                uint64_t Offset) const {
   return (void *)(Sections[SectionID].getObjAddress() + Offset);
 }
 
@@ -1870,7 +2004,8 @@ RuntimeDyldELF::processRelocationRef(
       Value.Addend += support::ulittle32_t::ref(computePlaceholderAddress(SectionID, Offset));
       processSimpleRelocation(SectionID, Offset, RelType, Value);
     } else if (RelType == ELF::R_X86_64_PC64) {
-      Value.Addend += support::ulittle64_t::ref(computePlaceholderAddress(SectionID, Offset));
+      Value.Addend += support::ulittle64_t::ref(
+          computePlaceholderAddress(SectionID, Offset));
       processSimpleRelocation(SectionID, Offset, RelType, Value);
     } else if (RelType == ELF::R_X86_64_GOTTPOFF) {
       processX86_64GOTTPOFFRelocation(SectionID, Offset, Value, Addend);
@@ -1884,9 +2019,23 @@ RuntimeDyldELF::processRelocationRef(
     } else {
       processSimpleRelocation(SectionID, Offset, RelType, Value);
     }
+  } else if (Arch == Triple::riscv32 || Arch == Triple::riscv64) {
+    // *_LO12 relocation receive information about a symbol from the
+    // corresponding *_HI20 relocation, so we have to collect this information
+    // before resolving
+    if (RelType == ELF::R_RISCV_GOT_HI20 ||
+        RelType == ELF::R_RISCV_PCREL_HI20 ||
+        RelType == ELF::R_RISCV_TPREL_HI20 ||
+        RelType == ELF::R_RISCV_TLS_GD_HI20 ||
+        RelType == ELF::R_RISCV_TLS_GOT_HI20) {
+      RelocationEntry RE(SectionID, Offset, RelType, Addend);
+      PendingRelocs.push_back({Value, RE});
+    }
+    processSimpleRelocation(SectionID, Offset, RelType, Value);
   } else {
     if (Arch == Triple::x86) {
-      Value.Addend += support::ulittle32_t::ref(computePlaceholderAddress(SectionID, Offset));
+      Value.Addend += support::ulittle32_t::ref(
+          computePlaceholderAddress(SectionID, Offset));
     }
     processSimpleRelocation(SectionID, Offset, RelType, Value);
   }
diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h
index 1b90013dfe2df..97517884654bc 100644
--- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h
+++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h
@@ -58,6 +58,10 @@ class RuntimeDyldELF : public RuntimeDyldImpl {
   void resolveBPFRelocation(const SectionEntry &Section, uint64_t Offset,
                             uint64_t Value, uint32_t Type, int64_t Addend);
 
+  void resolveRISCVRelocation(const SectionEntry &Section, uint64_t Offset,
+                              uint64_t Value, uint32_t Type, int64_t Addend,
+                              SID SectionID);
+
   unsigned getMaxStubSize() const override {
     if (Arch == Triple::aarch64 || Arch == Triple::aarch64_be)
       return 20; // movz; movk; movk; movk; br
@@ -146,6 +150,9 @@ class RuntimeDyldELF : public RuntimeDyldImpl {
 
   // *HI16 relocations will be added for resolving when we find matching
   // *LO16 part. (Mips specific)
+  //
+  // *HI20 relocations will be added for resolving when we find matching
+  // *LO12 part. (RISC-V specific)
   SmallVector<std::pair<RelocationValueRef, RelocationEntry>, 8> PendingRelocs;
 
   // When a module is loaded we save the SectionID of the EH frame section
diff --git a/llvm/lib/Frontend/Atomic/Atomic.cpp b/llvm/lib/Frontend/Atomic/Atomic.cpp
new file mode 100644
index 0000000000000..03b476d113bd4
--- /dev/null
+++ b/llvm/lib/Frontend/Atomic/Atomic.cpp
@@ -0,0 +1,19 @@
+//===--- Atomic.h - Codegen of atomic operations
+//---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Frontend/Atomic/Atomic.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/Frontend/Atomic/Atomic.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/Operator.h"
+
+namespace {} // namespace
+
+namespace llvm {} // end namespace llvm
diff --git a/llvm/lib/Frontend/Atomic/CMakeLists.txt b/llvm/lib/Frontend/Atomic/CMakeLists.txt
new file mode 100644
index 0000000000000..0d0d3d445b726
--- /dev/null
+++ b/llvm/lib/Frontend/Atomic/CMakeLists.txt
@@ -0,0 +1,15 @@
+add_llvm_component_library(LLVMFrontendAtomic
+	Atomic.cpp
+
+	ADDITIONAL_HEADER_DIRS
+	${LLVM_MAIN_INCLUDE_DIR}/llvm/Frontend/Atomic
+
+	DEPENDS
+	LLVMAnalysis
+	LLVMTargetParser
+
+	LINK_COMPONENTS
+	Core
+	Support
+	Analysis
+)
diff --git a/llvm/lib/Frontend/CMakeLists.txt b/llvm/lib/Frontend/CMakeLists.txt
index 62dd0da1e6c2d..b305ce7d771ce 100644
--- a/llvm/lib/Frontend/CMakeLists.txt
+++ b/llvm/lib/Frontend/CMakeLists.txt
@@ -1,3 +1,4 @@
+add_subdirectory(Atomic)
 add_subdirectory(Driver)
 add_subdirectory(HLSL)
 add_subdirectory(OpenACC)
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 922c65d7fc3f5..47cc6ff7655ca 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -7977,6 +7977,54 @@ std::pair<Value *, Value *> OpenMPIRBuilder::emitAtomicUpdate(
       Res.second = Res.first;
     else
       Res.second = emitRMWOpAsInstruction(Res.first, Expr, RMWOp);
+  } else if (RMWOp == llvm::AtomicRMWInst::BinOp::BAD_BINOP &&
+             XElemTy->isStructTy()) {
+    LoadInst *OldVal =
+        Builder.CreateLoad(XElemTy, X, X->getName() + ".atomic.load");
+    OldVal->setAtomic(AO);
+    const DataLayout &LoadDL = OldVal->getModule()->getDataLayout();
+    unsigned LoadSize =
+        LoadDL.getTypeStoreSize(OldVal->getPointerOperand()->getType());
+
+    OpenMPIRBuilder::AtomicInfo atomicInfo(
+        &Builder, XElemTy, LoadSize * 8, LoadSize * 8, OldVal->getAlign(),
+        OldVal->getAlign(), true /* UseLibcall */, X);
+    auto AtomicLoadRes = atomicInfo.EmitAtomicLoadLibcall(AO);
+    BasicBlock *CurBB = Builder.GetInsertBlock();
+    Instruction *CurBBTI = CurBB->getTerminator();
+    CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
+    BasicBlock *ExitBB =
+        CurBB->splitBasicBlock(CurBBTI, X->getName() + ".atomic.exit");
+    BasicBlock *ContBB = CurBB->splitBasicBlock(CurBB->getTerminator(),
+                                                X->getName() + ".atomic.cont");
+    ContBB->getTerminator()->eraseFromParent();
+    Builder.restoreIP(AllocaIP);
+    AllocaInst *NewAtomicAddr = Builder.CreateAlloca(XElemTy);
+    NewAtomicAddr->setName(X->getName() + "x.new.val");
+    Builder.SetInsertPoint(ContBB);
+    llvm::PHINode *PHI = Builder.CreatePHI(OldVal->getType(), 2);
+    PHI->addIncoming(AtomicLoadRes.first, CurBB);
+    Value *OldExprVal = PHI;
+    Value *Upd = UpdateOp(OldExprVal, Builder);
+    Builder.CreateStore(Upd, NewAtomicAddr);
+    AtomicOrdering Failure =
+        llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
+    auto Result = atomicInfo.EmitAtomicCompareExchangeLibcall(
+        AtomicLoadRes.second, NewAtomicAddr, AO, Failure);
+    LoadInst *PHILoad = Builder.CreateLoad(XElemTy, Result.first);
+    PHI->addIncoming(PHILoad, Builder.GetInsertBlock());
+    Builder.CreateCondBr(Result.second, ExitBB, ContBB);
+    OldVal->eraseFromParent();
+    Res.first = OldExprVal;
+    Res.second = Upd;
+
+    if (UnreachableInst *ExitTI =
+            dyn_cast<UnreachableInst>(ExitBB->getTerminator())) {
+      CurBBTI->eraseFromParent();
+      Builder.SetInsertPoint(ExitBB);
+    } else {
+      Builder.SetInsertPoint(ExitTI);
+    }
   } else {
     IntegerType *IntCastTy =
         IntegerType::get(M.getContext(), XElemTy->getScalarSizeInBits());
diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp
index 692207e45be23..f2ba61ae51039 100644
--- a/llvm/lib/IR/Attributes.cpp
+++ b/llvm/lib/IR/Attributes.cpp
@@ -28,6 +28,7 @@
 #include "llvm/IR/ConstantRangeList.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Operator.h"
 #include "llvm/IR/Type.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -1799,12 +1800,10 @@ AttributeList::intersectWith(LLVMContext &C, AttributeList Other) const {
   if (*this == Other)
     return *this;
 
-  // At least for now, only intersect lists with same number of params.
-  if (getNumAttrSets() != Other.getNumAttrSets())
-    return std::nullopt;
-
   SmallVector<std::pair<unsigned, AttributeSet>> IntersectedAttrs;
-  for (unsigned Idx : indexes()) {
+  auto IndexIt =
+      index_iterator(std::max(getNumAttrSets(), Other.getNumAttrSets()));
+  for (unsigned Idx : IndexIt) {
     auto IntersectedAS =
         getAttributes(Idx).intersectWith(C, Other.getAttributes(Idx));
     // If any index fails to intersect, fail.
@@ -2296,12 +2295,8 @@ bool AttrBuilder::operator==(const AttrBuilder &B) const {
 
 /// Returns true if this is a type legal for the 'nofpclass' attribute. This
 /// follows the same type rules as FPMathOperator.
-///
-/// TODO: Consider relaxing to any FP type struct fields.
 bool AttributeFuncs::isNoFPClassCompatibleType(Type *Ty) {
-  while (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty))
-    Ty = ArrTy->getElementType();
-  return Ty->isFPOrFPVectorTy();
+  return FPMathOperator::isSupportedFloatingPointType(Ty);
 }
 
 /// Which attributes cannot be applied to a type.
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 6f833acd6dbc0..e469c2ae52eb7 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -846,6 +846,12 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
         return false; // No other 'aarch64.sve.bf*'.
       }
 
+      // 'aarch64.sve.fcvt.bf16f32' || 'aarch64.sve.fcvtnt.bf16f32'
+      if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
+        NewFn = nullptr;
+        return true;
+      }
+
       if (Name.consume_front("addqv")) {
         // 'aarch64.sve.addqv'.
         if (!F->getReturnType()->isFPOrFPVectorTy())
@@ -4072,6 +4078,35 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
   return Rep;
 }
 
+static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
+                                          Function *F, IRBuilder<> &Builder) {
+  Intrinsic::ID NewID =
+      StringSwitch<Intrinsic::ID>(Name)
+          .Case("sve.fcvt.bf16f32", Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
+          .Case("sve.fcvtnt.bf16f32", Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
+          .Default(Intrinsic::not_intrinsic);
+  if (NewID == Intrinsic::not_intrinsic)
+    llvm_unreachable("Unhandled Intrinsic!");
+
+  SmallVector<Value *, 3> Args(CI->args());
+
+  // The original intrinsics incorrectly used a predicate based on the smallest
+  // element type rather than the largest.
+  Type *BadPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 8);
+  Type *GoodPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 4);
+
+  if (Args[1]->getType() != BadPredTy)
+    llvm_unreachable("Unexpected predicate type!");
+
+  Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
+                                    BadPredTy, Args[1]);
+  Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool,
+                                    GoodPredTy, Args[1]);
+
+  Function *NewF = Intrinsic::getDeclaration(CI->getModule(), NewID);
+  return Builder.CreateCall(NewF, Args, CI->getName());
+}
+
 static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
                                       IRBuilder<> &Builder) {
   if (Name == "mve.vctp64.old") {
@@ -4325,6 +4360,7 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
 
     bool IsX86 = Name.consume_front("x86.");
     bool IsNVVM = Name.consume_front("nvvm.");
+    bool IsAArch64 = Name.consume_front("aarch64.");
     bool IsARM = Name.consume_front("arm.");
     bool IsAMDGCN = Name.consume_front("amdgcn.");
     bool IsDbg = Name.consume_front("dbg.");
@@ -4336,6 +4372,8 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
       Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
     } else if (IsX86) {
       Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
+    } else if (IsAArch64) {
+      Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
     } else if (IsARM) {
       Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
     } else if (IsAMDGCN) {
@@ -5523,8 +5561,8 @@ std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
     std::string I128 = "-i128:128";
     if (!StringRef(Res).contains(I128)) {
       size_t Pos = Res.find(I64);
-      assert(Pos != size_t(-1) && "no i64 data layout found!");
-      Res.insert(Pos + I64.size(), I128);
+      if (Pos != size_t(-1))
+        Res.insert(Pos + I64.size(), I128);
     }
     return Res;
   }
diff --git a/llvm/lib/IR/ConstantFPRange.cpp b/llvm/lib/IR/ConstantFPRange.cpp
index 957701891c8f3..74c9797d969f9 100644
--- a/llvm/lib/IR/ConstantFPRange.cpp
+++ b/llvm/lib/IR/ConstantFPRange.cpp
@@ -81,13 +81,12 @@ static void canonicalizeRange(APFloat &Lower, APFloat &Upper) {
 }
 
 ConstantFPRange::ConstantFPRange(APFloat LowerVal, APFloat UpperVal,
-                                 bool MayBeQNaN, bool MayBeSNaN)
-    : Lower(std::move(LowerVal)), Upper(std::move(UpperVal)) {
+                                 bool MayBeQNaNVal, bool MayBeSNaNVal)
+    : Lower(std::move(LowerVal)), Upper(std::move(UpperVal)),
+      MayBeQNaN(MayBeQNaNVal), MayBeSNaN(MayBeSNaNVal) {
   assert(&Lower.getSemantics() == &Upper.getSemantics() &&
          "Should only use the same semantics");
   assert(!isNonCanonicalEmptySet(Lower, Upper) && "Non-canonical form");
-  this->MayBeQNaN = MayBeQNaN;
-  this->MayBeSNaN = MayBeSNaN;
 }
 
 ConstantFPRange ConstantFPRange::getFinite(const fltSemantics &Sem) {
@@ -103,11 +102,120 @@ ConstantFPRange ConstantFPRange::getNaNOnly(const fltSemantics &Sem,
                          MayBeSNaN);
 }
 
+ConstantFPRange ConstantFPRange::getNonNaN(const fltSemantics &Sem) {
+  return ConstantFPRange(APFloat::getInf(Sem, /*Negative=*/true),
+                         APFloat::getInf(Sem, /*Negative=*/false),
+                         /*MayBeQNaN=*/false, /*MayBeSNaN=*/false);
+}
+
+/// Return true for ULT/UGT/OLT/OGT
+static bool fcmpPredExcludesEqual(FCmpInst::Predicate Pred) {
+  return !(Pred & FCmpInst::FCMP_OEQ);
+}
+
+/// Return [-inf, V) or [-inf, V]
+static ConstantFPRange makeLessThan(APFloat V, FCmpInst::Predicate Pred) {
+  const fltSemantics &Sem = V.getSemantics();
+  if (fcmpPredExcludesEqual(Pred)) {
+    if (V.isNegInfinity())
+      return ConstantFPRange::getEmpty(Sem);
+    V.next(/*nextDown=*/true);
+  }
+  return ConstantFPRange::getNonNaN(APFloat::getInf(Sem, /*Negative=*/true),
+                                    std::move(V));
+}
+
+/// Return (V, +inf] or [V, +inf]
+static ConstantFPRange makeGreaterThan(APFloat V, FCmpInst::Predicate Pred) {
+  const fltSemantics &Sem = V.getSemantics();
+  if (fcmpPredExcludesEqual(Pred)) {
+    if (V.isPosInfinity())
+      return ConstantFPRange::getEmpty(Sem);
+    V.next(/*nextDown=*/false);
+  }
+  return ConstantFPRange::getNonNaN(std::move(V),
+                                    APFloat::getInf(Sem, /*Negative=*/false));
+}
+
+/// Make sure that +0/-0 are both included in the range.
+static ConstantFPRange extendZeroIfEqual(const ConstantFPRange &CR,
+                                         FCmpInst::Predicate Pred) {
+  if (fcmpPredExcludesEqual(Pred))
+    return CR;
+
+  APFloat Lower = CR.getLower();
+  APFloat Upper = CR.getUpper();
+  if (Lower.isPosZero())
+    Lower = APFloat::getZero(Lower.getSemantics(), /*Negative=*/true);
+  if (Upper.isNegZero())
+    Upper = APFloat::getZero(Upper.getSemantics(), /*Negative=*/false);
+  return ConstantFPRange(std::move(Lower), std::move(Upper), CR.containsQNaN(),
+                         CR.containsSNaN());
+}
+
+static ConstantFPRange setNaNField(const ConstantFPRange &CR,
+                                   FCmpInst::Predicate Pred) {
+  bool ContainsNaN = FCmpInst::isUnordered(Pred);
+  return ConstantFPRange(CR.getLower(), CR.getUpper(),
+                         /*MayBeQNaN=*/ContainsNaN, /*MayBeSNaN=*/ContainsNaN);
+}
+
 ConstantFPRange
 ConstantFPRange::makeAllowedFCmpRegion(FCmpInst::Predicate Pred,
                                        const ConstantFPRange &Other) {
-  // TODO
-  return getFull(Other.getSemantics());
+  if (Other.isEmptySet())
+    return Other;
+  if (Other.containsNaN() && FCmpInst::isUnordered(Pred))
+    return getFull(Other.getSemantics());
+  if (Other.isNaNOnly() && FCmpInst::isOrdered(Pred))
+    return getEmpty(Other.getSemantics());
+
+  switch (Pred) {
+  case FCmpInst::FCMP_TRUE:
+    return getFull(Other.getSemantics());
+  case FCmpInst::FCMP_FALSE:
+    return getEmpty(Other.getSemantics());
+  case FCmpInst::FCMP_ORD:
+    return getNonNaN(Other.getSemantics());
+  case FCmpInst::FCMP_UNO:
+    return getNaNOnly(Other.getSemantics(), /*MayBeQNaN=*/true,
+                      /*MayBeSNaN=*/true);
+  case FCmpInst::FCMP_OEQ:
+  case FCmpInst::FCMP_UEQ:
+    return setNaNField(extendZeroIfEqual(Other, Pred), Pred);
+  case FCmpInst::FCMP_ONE:
+  case FCmpInst::FCMP_UNE:
+    if (const APFloat *SingleElement =
+            Other.getSingleElement(/*ExcludesNaN=*/true)) {
+      const fltSemantics &Sem = SingleElement->getSemantics();
+      if (SingleElement->isPosInfinity())
+        return setNaNField(
+            getNonNaN(APFloat::getInf(Sem, /*Negative=*/true),
+                      APFloat::getLargest(Sem, /*Negative=*/false)),
+            Pred);
+      if (SingleElement->isNegInfinity())
+        return setNaNField(
+            getNonNaN(APFloat::getLargest(Sem, /*Negative=*/true),
+                      APFloat::getInf(Sem, /*Negative=*/false)),
+            Pred);
+    }
+    return Pred == FCmpInst::FCMP_ONE ? getNonNaN(Other.getSemantics())
+                                      : getFull(Other.getSemantics());
+  case FCmpInst::FCMP_OLT:
+  case FCmpInst::FCMP_OLE:
+  case FCmpInst::FCMP_ULT:
+  case FCmpInst::FCMP_ULE:
+    return setNaNField(
+        extendZeroIfEqual(makeLessThan(Other.getUpper(), Pred), Pred), Pred);
+  case FCmpInst::FCMP_OGT:
+  case FCmpInst::FCMP_OGE:
+  case FCmpInst::FCMP_UGT:
+  case FCmpInst::FCMP_UGE:
+    return setNaNField(
+        extendZeroIfEqual(makeGreaterThan(Other.getLower(), Pred), Pred), Pred);
+  default:
+    llvm_unreachable("Unexpected predicate");
+  }
 }
 
 ConstantFPRange
@@ -117,9 +225,10 @@ ConstantFPRange::makeSatisfyingFCmpRegion(FCmpInst::Predicate Pred,
   return getEmpty(Other.getSemantics());
 }
 
-ConstantFPRange ConstantFPRange::makeExactFCmpRegion(FCmpInst::Predicate Pred,
-                                                     const APFloat &Other) {
-  return makeAllowedFCmpRegion(Pred, ConstantFPRange(Other));
+std::optional<ConstantFPRange>
+ConstantFPRange::makeExactFCmpRegion(FCmpInst::Predicate Pred,
+                                     const APFloat &Other) {
+  return std::nullopt;
 }
 
 bool ConstantFPRange::fcmp(FCmpInst::Predicate Pred,
@@ -161,8 +270,8 @@ bool ConstantFPRange::contains(const ConstantFPRange &CR) const {
          strictCompare(CR.Upper, Upper) != APFloat::cmpGreaterThan;
 }
 
-const APFloat *ConstantFPRange::getSingleElement() const {
-  if (MayBeSNaN || MayBeQNaN)
+const APFloat *ConstantFPRange::getSingleElement(bool ExcludesNaN) const {
+  if (!ExcludesNaN && (MayBeSNaN || MayBeQNaN))
     return nullptr;
   return Lower.bitwiseIsEqual(Upper) ? &Lower : nullptr;
 }
@@ -189,8 +298,8 @@ FPClassTest ConstantFPRange::classify() const {
     FPClassTest LowerMask = Lower.classify();
     FPClassTest UpperMask = Upper.classify();
     assert(LowerMask <= UpperMask && "Range is nan-only.");
-    for (uint32_t I = LowerMask; I <= UpperMask; I <<= 1)
-      Mask |= I;
+    // Set all bits from log2(LowerMask) to log2(UpperMask).
+    Mask |= (UpperMask << 1) - LowerMask;
   }
   return static_cast<FPClassTest>(Mask);
 }
diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp
index b1c2b0200c826..eec751078698b 100644
--- a/llvm/lib/IR/Instruction.cpp
+++ b/llvm/lib/IR/Instruction.cpp
@@ -785,11 +785,21 @@ const char *Instruction::getOpcodeName(unsigned OpCode) {
 /// This must be kept in sync with FunctionComparator::cmpOperations in
 /// lib/Transforms/IPO/MergeFunctions.cpp.
 bool Instruction::hasSameSpecialState(const Instruction *I2,
-                                      bool IgnoreAlignment) const {
+                                      bool IgnoreAlignment,
+                                      bool IntersectAttrs) const {
   auto I1 = this;
   assert(I1->getOpcode() == I2->getOpcode() &&
          "Can not compare special state of different instructions");
 
+  auto CheckAttrsSame = [IntersectAttrs](const CallBase *CB0,
+                                         const CallBase *CB1) {
+    return IntersectAttrs
+               ? CB0->getAttributes()
+                     .intersectWith(CB0->getContext(), CB1->getAttributes())
+                     .has_value()
+               : CB0->getAttributes() == CB1->getAttributes();
+  };
+
   if (const AllocaInst *AI = dyn_cast<AllocaInst>(I1))
     return AI->getAllocatedType() == cast<AllocaInst>(I2)->getAllocatedType() &&
            (AI->getAlign() == cast<AllocaInst>(I2)->getAlign() ||
@@ -811,15 +821,15 @@ bool Instruction::hasSameSpecialState(const Instruction *I2,
   if (const CallInst *CI = dyn_cast<CallInst>(I1))
     return CI->isTailCall() == cast<CallInst>(I2)->isTailCall() &&
            CI->getCallingConv() == cast<CallInst>(I2)->getCallingConv() &&
-           CI->getAttributes() == cast<CallInst>(I2)->getAttributes() &&
+           CheckAttrsSame(CI, cast<CallInst>(I2)) &&
            CI->hasIdenticalOperandBundleSchema(*cast<CallInst>(I2));
   if (const InvokeInst *CI = dyn_cast<InvokeInst>(I1))
     return CI->getCallingConv() == cast<InvokeInst>(I2)->getCallingConv() &&
-           CI->getAttributes() == cast<InvokeInst>(I2)->getAttributes() &&
+           CheckAttrsSame(CI, cast<InvokeInst>(I2)) &&
            CI->hasIdenticalOperandBundleSchema(*cast<InvokeInst>(I2));
   if (const CallBrInst *CI = dyn_cast<CallBrInst>(I1))
     return CI->getCallingConv() == cast<CallBrInst>(I2)->getCallingConv() &&
-           CI->getAttributes() == cast<CallBrInst>(I2)->getAttributes() &&
+           CheckAttrsSame(CI, cast<CallBrInst>(I2)) &&
            CI->hasIdenticalOperandBundleSchema(*cast<CallBrInst>(I2));
   if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(I1))
     return IVI->getIndices() == cast<InsertValueInst>(I2)->getIndices();
@@ -857,15 +867,16 @@ bool Instruction::isIdenticalTo(const Instruction *I) const {
          SubclassOptionalData == I->SubclassOptionalData;
 }
 
-bool Instruction::isIdenticalToWhenDefined(const Instruction *I) const {
+bool Instruction::isIdenticalToWhenDefined(const Instruction *I,
+                                           bool IntersectAttrs) const {
   if (getOpcode() != I->getOpcode() ||
-      getNumOperands() != I->getNumOperands() ||
-      getType() != I->getType())
+      getNumOperands() != I->getNumOperands() || getType() != I->getType())
     return false;
 
   // If both instructions have no operands, they are identical.
   if (getNumOperands() == 0 && I->getNumOperands() == 0)
-    return this->hasSameSpecialState(I);
+    return this->hasSameSpecialState(I, /*IgnoreAlignment=*/false,
+                                     IntersectAttrs);
 
   // We have two instructions of identical opcode and #operands.  Check to see
   // if all operands are the same.
@@ -879,7 +890,8 @@ bool Instruction::isIdenticalToWhenDefined(const Instruction *I) const {
                       otherPHI->block_begin());
   }
 
-  return this->hasSameSpecialState(I);
+  return this->hasSameSpecialState(I, /*IgnoreAlignment=*/false,
+                                   IntersectAttrs);
 }
 
 // Keep this in sync with FunctionComparator::cmpOperations in
@@ -887,7 +899,8 @@ bool Instruction::isIdenticalToWhenDefined(const Instruction *I) const {
 bool Instruction::isSameOperationAs(const Instruction *I,
                                     unsigned flags) const {
   bool IgnoreAlignment = flags & CompareIgnoringAlignment;
-  bool UseScalarTypes  = flags & CompareUsingScalarTypes;
+  bool UseScalarTypes = flags & CompareUsingScalarTypes;
+  bool IntersectAttrs = flags & CompareUsingIntersectedAttrs;
 
   if (getOpcode() != I->getOpcode() ||
       getNumOperands() != I->getNumOperands() ||
@@ -905,7 +918,7 @@ bool Instruction::isSameOperationAs(const Instruction *I,
         getOperand(i)->getType() != I->getOperand(i)->getType())
       return false;
 
-  return this->hasSameSpecialState(I, IgnoreAlignment);
+  return this->hasSameSpecialState(I, IgnoreAlignment, IntersectAttrs);
 }
 
 bool Instruction::isUsedOutsideOfBlock(const BasicBlock *BB) const {
diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp
index 3784ad28d7219..f618263f79c31 100644
--- a/llvm/lib/IR/Type.cpp
+++ b/llvm/lib/IR/Type.cpp
@@ -430,13 +430,14 @@ bool StructType::containsScalableVectorType(
 }
 
 bool StructType::containsHomogeneousScalableVectorTypes() const {
-  Type *FirstTy = getNumElements() > 0 ? elements()[0] : nullptr;
-  if (!FirstTy || !isa<ScalableVectorType>(FirstTy))
+  if (getNumElements() <= 0 || !isa<ScalableVectorType>(elements().front()))
     return false;
-  for (Type *Ty : elements())
-    if (Ty != FirstTy)
-      return false;
-  return true;
+  return containsHomogeneousTypes();
+}
+
+bool StructType::containsHomogeneousTypes() const {
+  ArrayRef<Type *> ElementTys = elements();
+  return !ElementTys.empty() && all_equal(ElementTys);
 }
 
 void StructType::setBody(ArrayRef<Type*> Elements, bool isPacked) {
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index a88124dacfaef..b5eb7953f23b0 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -1473,7 +1473,8 @@ class InProcessThinBackend : public ThinBackendProc {
         return MOrErr.takeError();
 
       return thinBackend(Conf, Task, AddStream, **MOrErr, CombinedIndex,
-                         ImportList, DefinedGlobals, &ModuleMap);
+                         ImportList, DefinedGlobals, &ModuleMap,
+                         Conf.CodeGenOnly);
     };
 
     auto ModuleID = BM.getModuleIdentifier();
@@ -1483,7 +1484,7 @@ class InProcessThinBackend : public ThinBackendProc {
         return E;
     }
 
-    if (!Cache || !CombinedIndex.modulePaths().count(ModuleID) ||
+    if (!Cache.isValid() || !CombinedIndex.modulePaths().count(ModuleID) ||
         all_of(CombinedIndex.getModuleHash(ModuleID),
                [](uint32_t V) { return V == 0; }))
       // Cache disabled or no entry for this module in the combined index or
@@ -1839,45 +1840,49 @@ Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache,
 
   TimeTraceScopeExit.release();
 
-  std::unique_ptr<ThinBackendProc> BackendProc =
-      ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
-                      AddStream, Cache);
-
   auto &ModuleMap =
       ThinLTO.ModulesToCompile ? *ThinLTO.ModulesToCompile : ThinLTO.ModuleMap;
 
-  auto ProcessOneModule = [&](int I) -> Error {
-    auto &Mod = *(ModuleMap.begin() + I);
-    // Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for
-    // combined module and parallel code generation partitions.
-    return BackendProc->start(RegularLTO.ParallelCodeGenParallelismLevel + I,
-                              Mod.second, ImportLists[Mod.first],
-                              ExportLists[Mod.first], ResolvedODR[Mod.first],
-                              ThinLTO.ModuleMap);
+  auto RunBackends = [&](ThinBackendProc *BackendProcess) -> Error {
+    auto ProcessOneModule = [&](int I) -> Error {
+      auto &Mod = *(ModuleMap.begin() + I);
+      // Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for
+      // combined module and parallel code generation partitions.
+      return BackendProcess->start(
+          RegularLTO.ParallelCodeGenParallelismLevel + I, Mod.second,
+          ImportLists[Mod.first], ExportLists[Mod.first],
+          ResolvedODR[Mod.first], ThinLTO.ModuleMap);
+    };
+
+    if (BackendProcess->getThreadCount() == 1) {
+      // Process the modules in the order they were provided on the
+      // command-line. It is important for this codepath to be used for
+      // WriteIndexesThinBackend, to ensure the emitted LinkedObjectsFile lists
+      // ThinLTO objects in the same order as the inputs, which otherwise would
+      // affect the final link order.
+      for (int I = 0, E = ModuleMap.size(); I != E; ++I)
+        if (Error E = ProcessOneModule(I))
+          return E;
+    } else {
+      // When executing in parallel, process largest bitsize modules first to
+      // improve parallelism, and avoid starving the thread pool near the end.
+      // This saves about 15 sec on a 36-core machine while link `clang.exe`
+      // (out of 100 sec).
+      std::vector<BitcodeModule *> ModulesVec;
+      ModulesVec.reserve(ModuleMap.size());
+      for (auto &Mod : ModuleMap)
+        ModulesVec.push_back(&Mod.second);
+      for (int I : generateModulesOrdering(ModulesVec))
+        if (Error E = ProcessOneModule(I))
+          return E;
+    }
+    return BackendProcess->wait();
   };
 
-  if (BackendProc->getThreadCount() == 1) {
-    // Process the modules in the order they were provided on the command-line.
-    // It is important for this codepath to be used for WriteIndexesThinBackend,
-    // to ensure the emitted LinkedObjectsFile lists ThinLTO objects in the same
-    // order as the inputs, which otherwise would affect the final link order.
-    for (int I = 0, E = ModuleMap.size(); I != E; ++I)
-      if (Error E = ProcessOneModule(I))
-        return E;
-  } else {
-    // When executing in parallel, process largest bitsize modules first to
-    // improve parallelism, and avoid starving the thread pool near the end.
-    // This saves about 15 sec on a 36-core machine while link `clang.exe` (out
-    // of 100 sec).
-    std::vector<BitcodeModule *> ModulesVec;
-    ModulesVec.reserve(ModuleMap.size());
-    for (auto &Mod : ModuleMap)
-      ModulesVec.push_back(&Mod.second);
-    for (int I : generateModulesOrdering(ModulesVec))
-      if (Error E = ProcessOneModule(I))
-        return E;
-  }
-  return BackendProc->wait();
+  std::unique_ptr<ThinBackendProc> BackendProc =
+      ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
+                      AddStream, Cache);
+  return RunBackends(BackendProc.get());
 }
 
 Expected<std::unique_ptr<ToolOutputFile>> lto::setupLLVMOptimizationRemarks(
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
index 4e58cd369c3ac..06eeed3e1bc41 100644
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -565,7 +565,7 @@ Error lto::thinBackend(const Config &Conf, unsigned Task, AddStreamFn AddStream,
                        const FunctionImporter::ImportMapTy &ImportList,
                        const GVSummaryMapTy &DefinedGlobals,
                        MapVector<StringRef, BitcodeModule> *ModuleMap,
-                       const std::vector<uint8_t> &CmdArgs) {
+                       bool CodeGenOnly, const std::vector<uint8_t> &CmdArgs) {
   Expected<const Target *> TOrErr = initAndLookupTarget(Conf, Mod);
   if (!TOrErr)
     return TOrErr.takeError();
@@ -586,7 +586,9 @@ Error lto::thinBackend(const Config &Conf, unsigned Task, AddStreamFn AddStream,
   Mod.setPartialSampleProfileRatio(CombinedIndex);
 
   LLVM_DEBUG(dbgs() << "Running ThinLTO\n");
-  if (Conf.CodeGenOnly) {
+  if (CodeGenOnly) {
+    // If CodeGenOnly is set, we only perform code generation and skip
+    // optimization. This value may differ from Conf.CodeGenOnly.
     codegen(Conf, TM.get(), AddStream, Task, Mod, CombinedIndex);
     return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
   }
diff --git a/llvm/lib/MC/XCOFFObjectWriter.cpp b/llvm/lib/MC/XCOFFObjectWriter.cpp
index 124b31e870884..c7f29c73eaac0 100644
--- a/llvm/lib/MC/XCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/XCOFFObjectWriter.cpp
@@ -666,8 +666,9 @@ void XCOFFObjectWriter::recordRelocation(MCAssembler &Asm,
     // If we could not find the symbol directly in SymbolIndexMap, this symbol
     // could either be a temporary symbol or an undefined symbol. In this case,
     // we would need to have the relocation reference its csect instead.
-    return SymbolIndexMap.contains(Sym)
-               ? SymbolIndexMap[Sym]
+    auto It = SymbolIndexMap.find(Sym);
+    return It != SymbolIndexMap.end()
+               ? It->second
                : SymbolIndexMap[ContainingCsect->getQualNameSymbol()];
   };
 
diff --git a/llvm/lib/Object/Archive.cpp b/llvm/lib/Object/Archive.cpp
index e798bbdd16f14..9857eb0de7a80 100644
--- a/llvm/lib/Object/Archive.cpp
+++ b/llvm/lib/Object/Archive.cpp
@@ -969,7 +969,7 @@ Archive::Archive(MemoryBufferRef Source, Error &Err)
   Err = Error::success();
 }
 
-object::Archive::Kind Archive::getDefaultKindForTriple(Triple &T) {
+object::Archive::Kind Archive::getDefaultKindForTriple(const Triple &T) {
   if (T.isOSDarwin())
     return object::Archive::K_DARWIN;
   if (T.isOSAIX())
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 402b2340e5c35..f47f8ca35d773 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -1047,6 +1047,40 @@ Expected<IPSCCPOptions> parseIPSCCPOptions(StringRef Params) {
   return Result;
 }
 
+Expected<ScalarizerPassOptions> parseScalarizerOptions(StringRef Params) {
+  ScalarizerPassOptions Result;
+  while (!Params.empty()) {
+    StringRef ParamName;
+    std::tie(ParamName, Params) = Params.split(';');
+
+    if (ParamName.consume_front("min-bits=")) {
+      if (ParamName.getAsInteger(0, Result.ScalarizeMinBits)) {
+        return make_error<StringError>(
+            formatv("invalid argument to Scalarizer pass min-bits "
+                    "parameter: '{0}' ",
+                    ParamName)
+                .str(),
+            inconvertibleErrorCode());
+      }
+
+      continue;
+    }
+
+    bool Enable = !ParamName.consume_front("no-");
+    if (ParamName == "load-store")
+      Result.ScalarizeLoadStore = Enable;
+    else if (ParamName == "variable-insert-extract")
+      Result.ScalarizeVariableInsertExtract = Enable;
+    else {
+      return make_error<StringError>(
+          formatv("invalid Scalarizer pass parameter '{0}' ", ParamName).str(),
+          inconvertibleErrorCode());
+    }
+  }
+
+  return Result;
+}
+
 Expected<SROAOptions> parseSROAOptions(StringRef Params) {
   if (Params.empty() || Params == "modify-cfg")
     return SROAOptions::ModifyCFG;
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 3dc7f185f330c..90859c18c4f49 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -456,7 +456,6 @@ FUNCTION_PASS("reg2mem", RegToMemPass())
 FUNCTION_PASS("safe-stack", SafeStackPass(TM))
 FUNCTION_PASS("sandbox-vectorizer", SandboxVectorizerPass())
 FUNCTION_PASS("scalarize-masked-mem-intrin", ScalarizeMaskedMemIntrinPass())
-FUNCTION_PASS("scalarizer", ScalarizerPass())
 FUNCTION_PASS("sccp", SCCPPass())
 FUNCTION_PASS("select-optimize", SelectOptimizePass(TM))
 FUNCTION_PASS("separate-const-offset-from-gep",
@@ -573,6 +572,12 @@ FUNCTION_PASS_WITH_PARAMS(
       return StackLifetimePrinterPass(dbgs(), Type);
     },
     parseStackLifetimeOptions, "may;must")
+FUNCTION_PASS_WITH_PARAMS(
+    "scalarizer", "ScalarizerPass",
+    [](ScalarizerPassOptions Opts) { return ScalarizerPass(Opts); },
+    parseScalarizerOptions,
+    "load-store;no-load-store;variable-insert-extract;"
+    "no-variable-insert-extract;min-bits=N;")
 FUNCTION_PASS_WITH_PARAMS(
     "separate-const-offset-from-gep", "SeparateConstOffsetFromGEPPass",
     [](bool LowerGEP) { return SeparateConstOffsetFromGEPPass(LowerGEP); },
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 036484c9c1c0c..0ae3b2c84a5a1 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -751,7 +751,8 @@ PrintIRInstrumentation::~PrintIRInstrumentation() {
 static SmallString<32> getIRFileDisplayName(Any IR) {
   SmallString<32> Result;
   raw_svector_ostream ResultStream(Result);
-  const Module *M = unwrapModule(IR);
+  const Module *M = unwrapModule(IR, /*Force=*/true);
+  assert(M && "should have unwrapped module");
   uint64_t NameHash = xxh3_64bits(M->getName());
   unsigned MaxHashWidth = sizeof(uint64_t) * 2;
   write_hex(ResultStream, NameHash, HexPrintStyle::Lower, MaxHashWidth);
@@ -1357,7 +1358,7 @@ void PreservedCFGCheckerInstrumentation::registerCallbacks(
   bool Registered = false;
   PIC.registerBeforeNonSkippedPassCallback([this, &MAM, Registered](
                                                StringRef P, Any IR) mutable {
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
     assert(&PassStack.emplace_back(P));
 #endif
     (void)this;
@@ -1386,7 +1387,7 @@ void PreservedCFGCheckerInstrumentation::registerCallbacks(
 
   PIC.registerAfterPassInvalidatedCallback(
       [this](StringRef P, const PreservedAnalyses &PassPA) {
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
         assert(PassStack.pop_back_val() == P &&
                "Before and After callbacks must correspond");
 #endif
@@ -1395,7 +1396,7 @@ void PreservedCFGCheckerInstrumentation::registerCallbacks(
 
   PIC.registerAfterPassCallback([this, &MAM](StringRef P, Any IR,
                                              const PreservedAnalyses &PassPA) {
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
     assert(PassStack.pop_back_val() == P &&
            "Before and After callbacks must correspond");
 #endif
@@ -2039,13 +2040,14 @@ DotCfgDiff::DotCfgDiff(StringRef Title, const FuncDataT<DCData> &Before,
     StringRef Colour = E.second;
 
     // Look for an edge from Source to Sink
-    if (EdgeLabels.count(SourceSink) == 0)
-      EdgeLabels.insert({SourceSink, colourize(Value.str(), Colour)});
+    auto [It, Inserted] = EdgeLabels.try_emplace(SourceSink);
+    if (Inserted)
+      It->getValue() = colourize(Value.str(), Colour);
     else {
-      StringRef V = EdgeLabels.find(SourceSink)->getValue();
+      StringRef V = It->getValue();
       std::string NV = colourize(V.str() + " " + Value.str(), Colour);
       Colour = CommonColour;
-      EdgeLabels[SourceSink] = NV;
+      It->getValue() = NV;
     }
     SourceNode.addEdge(SinkNode, Value, Colour);
   }
diff --git a/llvm/lib/ProfileData/MemProfReader.cpp b/llvm/lib/ProfileData/MemProfReader.cpp
index 5e2d76e24dab7..58622e5ed254e 100644
--- a/llvm/lib/ProfileData/MemProfReader.cpp
+++ b/llvm/lib/ProfileData/MemProfReader.cpp
@@ -193,14 +193,10 @@ CallStackMap readStackInfo(const char *Ptr) {
 // addresses.
 bool mergeStackMap(const CallStackMap &From, CallStackMap &To) {
   for (const auto &[Id, Stack] : From) {
-    auto I = To.find(Id);
-    if (I == To.end()) {
-      To[Id] = Stack;
-    } else {
-      // Check that the PCs are the same (in order).
-      if (Stack != I->second)
-        return true;
-    }
+    auto [It, Inserted] = To.try_emplace(Id, Stack);
+    // Check that the PCs are the same (in order).
+    if (!Inserted && Stack != It->second)
+      return true;
   }
   return false;
 }
diff --git a/llvm/lib/SandboxIR/Instruction.cpp b/llvm/lib/SandboxIR/Instruction.cpp
index 276c4f0872b10..78118613edb3c 100644
--- a/llvm/lib/SandboxIR/Instruction.cpp
+++ b/llvm/lib/SandboxIR/Instruction.cpp
@@ -286,14 +286,9 @@ void Instruction::dumpOS(raw_ostream &OS) const {
 }
 #endif // NDEBUG
 
-VAArgInst *VAArgInst::create(Value *List, Type *Ty, BBIterator WhereIt,
-                             BasicBlock *WhereBB, Context &Ctx,
-                             const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+VAArgInst *VAArgInst::create(Value *List, Type *Ty, InsertPosition Pos,
+                             Context &Ctx, const Twine &Name) {
+  auto &Builder = setInsertPos(Pos);
   auto *LLVMI =
       cast<llvm::VAArgInst>(Builder.CreateVAArg(List->Val, Ty->LLVMTy, Name));
   return Ctx.createVAArgInst(LLVMI);
@@ -303,26 +298,16 @@ Value *VAArgInst::getPointerOperand() {
   return Ctx.getValue(cast<llvm::VAArgInst>(Val)->getPointerOperand());
 }
 
-FreezeInst *FreezeInst::create(Value *V, BBIterator WhereIt,
-                               BasicBlock *WhereBB, Context &Ctx,
+FreezeInst *FreezeInst::create(Value *V, InsertPosition Pos, Context &Ctx,
                                const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  auto &Builder = setInsertPos(Pos);
   auto *LLVMI = cast<llvm::FreezeInst>(Builder.CreateFreeze(V->Val, Name));
   return Ctx.createFreezeInst(LLVMI);
 }
 
-FenceInst *FenceInst::create(AtomicOrdering Ordering, BBIterator WhereIt,
-                             BasicBlock *WhereBB, Context &Ctx,
-                             SyncScope::ID SSID) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+FenceInst *FenceInst::create(AtomicOrdering Ordering, InsertPosition Pos,
+                             Context &Ctx, SyncScope::ID SSID) {
+  auto &Builder = Instruction::setInsertPos(Pos);
   llvm::FenceInst *LLVMI = Builder.CreateFence(Ordering, SSID);
   return Ctx.createFenceInst(LLVMI);
 }
@@ -342,9 +327,9 @@ void FenceInst::setSyncScopeID(SyncScope::ID SSID) {
   cast<llvm::FenceInst>(Val)->setSyncScopeID(SSID);
 }
 
-Value *SelectInst::createCommon(Value *Cond, Value *True, Value *False,
-                                const Twine &Name, IRBuilder<> &Builder,
-                                Context &Ctx) {
+Value *SelectInst::create(Value *Cond, Value *True, Value *False,
+                          InsertPosition Pos, Context &Ctx, const Twine &Name) {
+  auto &Builder = Instruction::setInsertPos(Pos);
   llvm::Value *NewV =
       Builder.CreateSelect(Cond->Val, True->Val, False->Val, Name);
   if (auto *NewSI = dyn_cast<llvm::SelectInst>(NewV))
@@ -353,24 +338,6 @@ Value *SelectInst::createCommon(Value *Cond, Value *True, Value *False,
   return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
 }
 
-Value *SelectInst::create(Value *Cond, Value *True, Value *False,
-                          Instruction *InsertBefore, Context &Ctx,
-                          const Twine &Name) {
-  llvm::Instruction *BeforeIR = InsertBefore->getTopmostLLVMInstruction();
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(BeforeIR);
-  return createCommon(Cond, True, False, Name, Builder, Ctx);
-}
-
-Value *SelectInst::create(Value *Cond, Value *True, Value *False,
-                          BasicBlock *InsertAtEnd, Context &Ctx,
-                          const Twine &Name) {
-  auto *IRInsertAtEnd = cast<llvm::BasicBlock>(InsertAtEnd->Val);
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(IRInsertAtEnd);
-  return createCommon(Cond, True, False, Name, Builder, Ctx);
-}
-
 void SelectInst::swapValues() {
   Ctx.getTracker().emplaceIfTracking<UseSwap>(getOperandUse(1),
                                               getOperandUse(2));
@@ -381,42 +348,17 @@ bool SelectInst::classof(const Value *From) {
   return From->getSubclassID() == ClassID::Select;
 }
 
-BranchInst *BranchInst::create(BasicBlock *IfTrue, Instruction *InsertBefore,
-                               Context &Ctx) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  llvm::Instruction *LLVMBefore = InsertBefore->getTopmostLLVMInstruction();
-  Builder.SetInsertPoint(cast<llvm::Instruction>(LLVMBefore));
-  llvm::BranchInst *NewBr =
-      Builder.CreateBr(cast<llvm::BasicBlock>(IfTrue->Val));
-  return Ctx.createBranchInst(NewBr);
-}
-
-BranchInst *BranchInst::create(BasicBlock *IfTrue, BasicBlock *InsertAtEnd,
+BranchInst *BranchInst::create(BasicBlock *IfTrue, InsertPosition Pos,
                                Context &Ctx) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
+  auto &Builder = setInsertPos(Pos);
   llvm::BranchInst *NewBr =
       Builder.CreateBr(cast<llvm::BasicBlock>(IfTrue->Val));
   return Ctx.createBranchInst(NewBr);
 }
 
 BranchInst *BranchInst::create(BasicBlock *IfTrue, BasicBlock *IfFalse,
-                               Value *Cond, Instruction *InsertBefore,
-                               Context &Ctx) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  llvm::Instruction *LLVMBefore = InsertBefore->getTopmostLLVMInstruction();
-  Builder.SetInsertPoint(LLVMBefore);
-  llvm::BranchInst *NewBr =
-      Builder.CreateCondBr(Cond->Val, cast<llvm::BasicBlock>(IfTrue->Val),
-                           cast<llvm::BasicBlock>(IfFalse->Val));
-  return Ctx.createBranchInst(NewBr);
-}
-
-BranchInst *BranchInst::create(BasicBlock *IfTrue, BasicBlock *IfFalse,
-                               Value *Cond, BasicBlock *InsertAtEnd,
-                               Context &Ctx) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
+                               Value *Cond, InsertPosition Pos, Context &Ctx) {
+  auto &Builder = setInsertPos(Pos);
   llvm::BranchInst *NewBr =
       Builder.CreateCondBr(Cond->Val, cast<llvm::BasicBlock>(IfTrue->Val),
                            cast<llvm::BasicBlock>(IfFalse->Val));
@@ -460,34 +402,9 @@ void LoadInst::setVolatile(bool V) {
 }
 
 LoadInst *LoadInst::create(Type *Ty, Value *Ptr, MaybeAlign Align,
-                           Instruction *InsertBefore, Context &Ctx,
-                           const Twine &Name) {
-  return create(Ty, Ptr, Align, InsertBefore, /*IsVolatile=*/false, Ctx, Name);
-}
-
-LoadInst *LoadInst::create(Type *Ty, Value *Ptr, MaybeAlign Align,
-                           Instruction *InsertBefore, bool IsVolatile,
-                           Context &Ctx, const Twine &Name) {
-  llvm::Instruction *BeforeIR = InsertBefore->getTopmostLLVMInstruction();
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(BeforeIR);
-  auto *NewLI =
-      Builder.CreateAlignedLoad(Ty->LLVMTy, Ptr->Val, Align, IsVolatile, Name);
-  auto *NewSBI = Ctx.createLoadInst(NewLI);
-  return NewSBI;
-}
-
-LoadInst *LoadInst::create(Type *Ty, Value *Ptr, MaybeAlign Align,
-                           BasicBlock *InsertAtEnd, Context &Ctx,
+                           InsertPosition Pos, bool IsVolatile, Context &Ctx,
                            const Twine &Name) {
-  return create(Ty, Ptr, Align, InsertAtEnd, /*IsVolatile=*/false, Ctx, Name);
-}
-
-LoadInst *LoadInst::create(Type *Ty, Value *Ptr, MaybeAlign Align,
-                           BasicBlock *InsertAtEnd, bool IsVolatile,
-                           Context &Ctx, const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
+  auto &Builder = setInsertPos(Pos);
   auto *NewLI =
       Builder.CreateAlignedLoad(Ty->LLVMTy, Ptr->Val, Align, IsVolatile, Name);
   auto *NewSBI = Ctx.createLoadInst(NewLI);
@@ -510,32 +427,9 @@ void StoreInst::setVolatile(bool V) {
 }
 
 StoreInst *StoreInst::create(Value *V, Value *Ptr, MaybeAlign Align,
-                             Instruction *InsertBefore, Context &Ctx) {
-  return create(V, Ptr, Align, InsertBefore, /*IsVolatile=*/false, Ctx);
-}
-
-StoreInst *StoreInst::create(Value *V, Value *Ptr, MaybeAlign Align,
-                             Instruction *InsertBefore, bool IsVolatile,
-                             Context &Ctx) {
-  llvm::Instruction *BeforeIR = InsertBefore->getTopmostLLVMInstruction();
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(BeforeIR);
-  auto *NewSI = Builder.CreateAlignedStore(V->Val, Ptr->Val, Align, IsVolatile);
-  auto *NewSBI = Ctx.createStoreInst(NewSI);
-  return NewSBI;
-}
-
-StoreInst *StoreInst::create(Value *V, Value *Ptr, MaybeAlign Align,
-                             BasicBlock *InsertAtEnd, Context &Ctx) {
-  return create(V, Ptr, Align, InsertAtEnd, /*IsVolatile=*/false, Ctx);
-}
-
-StoreInst *StoreInst::create(Value *V, Value *Ptr, MaybeAlign Align,
-                             BasicBlock *InsertAtEnd, bool IsVolatile,
+                             InsertPosition Pos, bool IsVolatile,
                              Context &Ctx) {
-  auto *InsertAtEndIR = cast<llvm::BasicBlock>(InsertAtEnd->Val);
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(InsertAtEndIR);
+  auto &Builder = setInsertPos(Pos);
   auto *NewSI = Builder.CreateAlignedStore(V->Val, Ptr->Val, Align, IsVolatile);
   auto *NewSBI = Ctx.createStoreInst(NewSI);
   return NewSBI;
@@ -553,19 +447,8 @@ Value *StoreInst::getPointerOperand() const {
   return Ctx.getValue(cast<llvm::StoreInst>(Val)->getPointerOperand());
 }
 
-UnreachableInst *UnreachableInst::create(Instruction *InsertBefore,
-                                         Context &Ctx) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  llvm::Instruction *LLVMBefore = InsertBefore->getTopmostLLVMInstruction();
-  Builder.SetInsertPoint(LLVMBefore);
-  llvm::UnreachableInst *NewUI = Builder.CreateUnreachable();
-  return Ctx.createUnreachableInst(NewUI);
-}
-
-UnreachableInst *UnreachableInst::create(BasicBlock *InsertAtEnd,
-                                         Context &Ctx) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
+UnreachableInst *UnreachableInst::create(InsertPosition Pos, Context &Ctx) {
+  auto &Builder = setInsertPos(Pos);
   llvm::UnreachableInst *NewUI = Builder.CreateUnreachable();
   return Ctx.createUnreachableInst(NewUI);
 }
@@ -584,18 +467,9 @@ ReturnInst *ReturnInst::createCommon(Value *RetVal, IRBuilder<> &Builder,
   return Ctx.createReturnInst(NewRI);
 }
 
-ReturnInst *ReturnInst::create(Value *RetVal, Instruction *InsertBefore,
-                               Context &Ctx) {
-  llvm::Instruction *BeforeIR = InsertBefore->getTopmostLLVMInstruction();
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(BeforeIR);
-  return createCommon(RetVal, Builder, Ctx);
-}
-
-ReturnInst *ReturnInst::create(Value *RetVal, BasicBlock *InsertAtEnd,
+ReturnInst *ReturnInst::create(Value *RetVal, InsertPosition Pos,
                                Context &Ctx) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
+  auto &Builder = setInsertPos(Pos);
   return createCommon(RetVal, Builder, Ctx);
 }
 
@@ -640,14 +514,9 @@ void CallBase::setCalledFunction(Function *F) {
 }
 
 CallInst *CallInst::create(FunctionType *FTy, Value *Func,
-                           ArrayRef<Value *> Args, BasicBlock::iterator WhereIt,
-                           BasicBlock *WhereBB, Context &Ctx,
-                           const Twine &NameStr) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+                           ArrayRef<Value *> Args, InsertPosition Pos,
+                           Context &Ctx, const Twine &NameStr) {
+  auto &Builder = setInsertPos(Pos);
   SmallVector<llvm::Value *> LLVMArgs;
   LLVMArgs.reserve(Args.size());
   for (Value *Arg : Args)
@@ -657,30 +526,11 @@ CallInst *CallInst::create(FunctionType *FTy, Value *Func,
   return Ctx.createCallInst(NewCI);
 }
 
-CallInst *CallInst::create(FunctionType *FTy, Value *Func,
-                           ArrayRef<Value *> Args, Instruction *InsertBefore,
-                           Context &Ctx, const Twine &NameStr) {
-  return CallInst::create(FTy, Func, Args, InsertBefore->getIterator(),
-                          InsertBefore->getParent(), Ctx, NameStr);
-}
-
-CallInst *CallInst::create(FunctionType *FTy, Value *Func,
-                           ArrayRef<Value *> Args, BasicBlock *InsertAtEnd,
-                           Context &Ctx, const Twine &NameStr) {
-  return CallInst::create(FTy, Func, Args, InsertAtEnd->end(), InsertAtEnd, Ctx,
-                          NameStr);
-}
-
 InvokeInst *InvokeInst::create(FunctionType *FTy, Value *Func,
                                BasicBlock *IfNormal, BasicBlock *IfException,
-                               ArrayRef<Value *> Args, BBIterator WhereIt,
-                               BasicBlock *WhereBB, Context &Ctx,
-                               const Twine &NameStr) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+                               ArrayRef<Value *> Args, InsertPosition Pos,
+                               Context &Ctx, const Twine &NameStr) {
+  auto &Builder = setInsertPos(Pos);
   SmallVector<llvm::Value *> LLVMArgs;
   LLVMArgs.reserve(Args.size());
   for (Value *Arg : Args)
@@ -692,24 +542,6 @@ InvokeInst *InvokeInst::create(FunctionType *FTy, Value *Func,
   return Ctx.createInvokeInst(Invoke);
 }
 
-InvokeInst *InvokeInst::create(FunctionType *FTy, Value *Func,
-                               BasicBlock *IfNormal, BasicBlock *IfException,
-                               ArrayRef<Value *> Args,
-                               Instruction *InsertBefore, Context &Ctx,
-                               const Twine &NameStr) {
-  return create(FTy, Func, IfNormal, IfException, Args,
-                InsertBefore->getIterator(), InsertBefore->getParent(), Ctx,
-                NameStr);
-}
-
-InvokeInst *InvokeInst::create(FunctionType *FTy, Value *Func,
-                               BasicBlock *IfNormal, BasicBlock *IfException,
-                               ArrayRef<Value *> Args, BasicBlock *InsertAtEnd,
-                               Context &Ctx, const Twine &NameStr) {
-  return create(FTy, Func, IfNormal, IfException, Args, InsertAtEnd->end(),
-                InsertAtEnd, Ctx, NameStr);
-}
-
 BasicBlock *InvokeInst::getNormalDest() const {
   return cast<BasicBlock>(
       Ctx.getValue(cast<llvm::InvokeInst>(Val)->getNormalDest()));
@@ -739,15 +571,9 @@ BasicBlock *InvokeInst::getSuccessor(unsigned SuccIdx) const {
 CallBrInst *CallBrInst::create(FunctionType *FTy, Value *Func,
                                BasicBlock *DefaultDest,
                                ArrayRef<BasicBlock *> IndirectDests,
-                               ArrayRef<Value *> Args, BBIterator WhereIt,
-                               BasicBlock *WhereBB, Context &Ctx,
-                               const Twine &NameStr) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-
+                               ArrayRef<Value *> Args, InsertPosition Pos,
+                               Context &Ctx, const Twine &NameStr) {
+  auto &Builder = setInsertPos(Pos);
   SmallVector<llvm::BasicBlock *> LLVMIndirectDests;
   LLVMIndirectDests.reserve(IndirectDests.size());
   for (BasicBlock *IndDest : IndirectDests)
@@ -765,25 +591,6 @@ CallBrInst *CallBrInst::create(FunctionType *FTy, Value *Func,
   return Ctx.createCallBrInst(CallBr);
 }
 
-CallBrInst *CallBrInst::create(FunctionType *FTy, Value *Func,
-                               BasicBlock *DefaultDest,
-                               ArrayRef<BasicBlock *> IndirectDests,
-                               ArrayRef<Value *> Args,
-                               Instruction *InsertBefore, Context &Ctx,
-                               const Twine &NameStr) {
-  return create(FTy, Func, DefaultDest, IndirectDests, Args,
-                InsertBefore->getIterator(), InsertBefore->getParent(), Ctx,
-                NameStr);
-}
-CallBrInst *CallBrInst::create(FunctionType *FTy, Value *Func,
-                               BasicBlock *DefaultDest,
-                               ArrayRef<BasicBlock *> IndirectDests,
-                               ArrayRef<Value *> Args, BasicBlock *InsertAtEnd,
-                               Context &Ctx, const Twine &NameStr) {
-  return create(FTy, Func, DefaultDest, IndirectDests, Args, InsertAtEnd->end(),
-                InsertAtEnd, Ctx, NameStr);
-}
-
 Value *CallBrInst::getIndirectDestLabel(unsigned Idx) const {
   return Ctx.getValue(cast<llvm::CallBrInst>(Val)->getIndirectDestLabel(Idx));
 }
@@ -826,13 +633,9 @@ BasicBlock *CallBrInst::getSuccessor(unsigned Idx) const {
 }
 
 LandingPadInst *LandingPadInst::create(Type *RetTy, unsigned NumReservedClauses,
-                                       BBIterator WhereIt, BasicBlock *WhereBB,
-                                       Context &Ctx, const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+                                       InsertPosition Pos, Context &Ctx,
+                                       const Twine &Name) {
+  auto &Builder = setInsertPos(Pos);
   llvm::LandingPadInst *LLVMI =
       Builder.CreateLandingPad(RetTy->LLVMTy, NumReservedClauses, Name);
   return Ctx.createLandingPadInst(LLVMI);
@@ -879,13 +682,9 @@ CatchSwitchInst *CatchPadInst::getCatchSwitch() const {
 }
 
 CatchPadInst *CatchPadInst::create(Value *ParentPad, ArrayRef<Value *> Args,
-                                   BBIterator WhereIt, BasicBlock *WhereBB,
-                                   Context &Ctx, const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+                                   InsertPosition Pos, Context &Ctx,
+                                   const Twine &Name) {
+  auto &Builder = setInsertPos(Pos);
   SmallVector<llvm::Value *> LLVMArgs;
   LLVMArgs.reserve(Args.size());
   for (auto *Arg : Args)
@@ -896,13 +695,9 @@ CatchPadInst *CatchPadInst::create(Value *ParentPad, ArrayRef<Value *> Args,
 }
 
 CleanupPadInst *CleanupPadInst::create(Value *ParentPad, ArrayRef<Value *> Args,
-                                       BBIterator WhereIt, BasicBlock *WhereBB,
-                                       Context &Ctx, const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+                                       InsertPosition Pos, Context &Ctx,
+                                       const Twine &Name) {
+  auto &Builder = setInsertPos(Pos);
   SmallVector<llvm::Value *> LLVMArgs;
   LLVMArgs.reserve(Args.size());
   for (auto *Arg : Args)
@@ -1791,10 +1586,9 @@ void PossiblyNonNegInst::setNonNeg(bool B) {
 }
 
 Value *InsertElementInst::create(Value *Vec, Value *NewElt, Value *Idx,
-                                 Instruction *InsertBefore, Context &Ctx,
+                                 InsertPosition Pos, Context &Ctx,
                                  const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(InsertBefore->getTopmostLLVMInstruction());
+  auto &Builder = Instruction::setInsertPos(Pos);
   llvm::Value *NewV =
       Builder.CreateInsertElement(Vec->Val, NewElt->Val, Idx->Val, Name);
   if (auto *NewInsert = dyn_cast<llvm::InsertElementInst>(NewV))
@@ -1803,24 +1597,9 @@ Value *InsertElementInst::create(Value *Vec, Value *NewElt, Value *Idx,
   return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
 }
 
-Value *InsertElementInst::create(Value *Vec, Value *NewElt, Value *Idx,
-                                 BasicBlock *InsertAtEnd, Context &Ctx,
-                                 const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
-  llvm::Value *NewV =
-      Builder.CreateInsertElement(Vec->Val, NewElt->Val, Idx->Val, Name);
-  if (auto *NewInsert = dyn_cast<llvm::InsertElementInst>(NewV))
-    return Ctx.createInsertElementInst(NewInsert);
-  assert(isa<llvm::Constant>(NewV) && "Expected constant");
-  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
-}
-
-Value *ExtractElementInst::create(Value *Vec, Value *Idx,
-                                  Instruction *InsertBefore, Context &Ctx,
-                                  const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(InsertBefore->getTopmostLLVMInstruction());
+Value *ExtractElementInst::create(Value *Vec, Value *Idx, InsertPosition Pos,
+                                  Context &Ctx, const Twine &Name) {
+  auto &Builder = setInsertPos(Pos);
   llvm::Value *NewV = Builder.CreateExtractElement(Vec->Val, Idx->Val, Name);
   if (auto *NewExtract = dyn_cast<llvm::ExtractElementInst>(NewV))
     return Ctx.createExtractElementInst(NewExtract);
@@ -1828,36 +1607,10 @@ Value *ExtractElementInst::create(Value *Vec, Value *Idx,
   return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
 }
 
-Value *ExtractElementInst::create(Value *Vec, Value *Idx,
-                                  BasicBlock *InsertAtEnd, Context &Ctx,
-                                  const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
-  llvm::Value *NewV = Builder.CreateExtractElement(Vec->Val, Idx->Val, Name);
-  if (auto *NewExtract = dyn_cast<llvm::ExtractElementInst>(NewV))
-    return Ctx.createExtractElementInst(NewExtract);
-  assert(isa<llvm::Constant>(NewV) && "Expected constant");
-  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
-}
-
-Value *ShuffleVectorInst::create(Value *V1, Value *V2, Value *Mask,
-                                 Instruction *InsertBefore, Context &Ctx,
-                                 const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(InsertBefore->getTopmostLLVMInstruction());
-  llvm::Value *NewV =
-      Builder.CreateShuffleVector(V1->Val, V2->Val, Mask->Val, Name);
-  if (auto *NewShuffle = dyn_cast<llvm::ShuffleVectorInst>(NewV))
-    return Ctx.createShuffleVectorInst(NewShuffle);
-  assert(isa<llvm::Constant>(NewV) && "Expected constant");
-  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
-}
-
 Value *ShuffleVectorInst::create(Value *V1, Value *V2, Value *Mask,
-                                 BasicBlock *InsertAtEnd, Context &Ctx,
+                                 InsertPosition Pos, Context &Ctx,
                                  const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
+  auto &Builder = setInsertPos(Pos);
   llvm::Value *NewV =
       Builder.CreateShuffleVector(V1->Val, V2->Val, Mask->Val, Name);
   if (auto *NewShuffle = dyn_cast<llvm::ShuffleVectorInst>(NewV))
@@ -1867,22 +1620,9 @@ Value *ShuffleVectorInst::create(Value *V1, Value *V2, Value *Mask,
 }
 
 Value *ShuffleVectorInst::create(Value *V1, Value *V2, ArrayRef<int> Mask,
-                                 Instruction *InsertBefore, Context &Ctx,
+                                 InsertPosition Pos, Context &Ctx,
                                  const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(InsertBefore->getTopmostLLVMInstruction());
-  llvm::Value *NewV = Builder.CreateShuffleVector(V1->Val, V2->Val, Mask, Name);
-  if (auto *NewShuffle = dyn_cast<llvm::ShuffleVectorInst>(NewV))
-    return Ctx.createShuffleVectorInst(NewShuffle);
-  assert(isa<llvm::Constant>(NewV) && "Expected constant");
-  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
-}
-
-Value *ShuffleVectorInst::create(Value *V1, Value *V2, ArrayRef<int> Mask,
-                                 BasicBlock *InsertAtEnd, Context &Ctx,
-                                 const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
+  auto &Builder = setInsertPos(Pos);
   llvm::Value *NewV = Builder.CreateShuffleVector(V1->Val, V2->Val, Mask, Name);
   if (auto *NewShuffle = dyn_cast<llvm::ShuffleVectorInst>(NewV))
     return Ctx.createShuffleVectorInst(NewShuffle);
@@ -1924,13 +1664,9 @@ VectorType *ExtractElementInst::getVectorOperandType() const {
 }
 
 Value *ExtractValueInst::create(Value *Agg, ArrayRef<unsigned> Idxs,
-                                BBIterator WhereIt, BasicBlock *WhereBB,
-                                Context &Ctx, const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+                                InsertPosition Pos, Context &Ctx,
+                                const Twine &Name) {
+  auto &Builder = setInsertPos(Pos);
   llvm::Value *NewV = Builder.CreateExtractValue(Agg->Val, Idxs, Name);
   if (auto *NewExtractValueInst = dyn_cast<llvm::ExtractValueInst>(NewV))
     return Ctx.createExtractValueInst(NewExtractValueInst);
@@ -1944,13 +1680,9 @@ Type *ExtractValueInst::getIndexedType(Type *Agg, ArrayRef<unsigned> Idxs) {
 }
 
 Value *InsertValueInst::create(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs,
-                               BBIterator WhereIt, BasicBlock *WhereBB,
-                               Context &Ctx, const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+                               InsertPosition Pos, Context &Ctx,
+                               const Twine &Name) {
+  auto &Builder = setInsertPos(Pos);
   llvm::Value *NewV = Builder.CreateInsertValue(Agg->Val, Val->Val, Idxs, Name);
   if (auto *NewInsertValueInst = dyn_cast<llvm::InsertValueInst>(NewV))
     return Ctx.createInsertValueInst(NewInsertValueInst);
diff --git a/llvm/lib/SandboxIR/PassManager.cpp b/llvm/lib/SandboxIR/PassManager.cpp
index 4168420a01ce2..95bc5e56bb3ec 100644
--- a/llvm/lib/SandboxIR/PassManager.cpp
+++ b/llvm/lib/SandboxIR/PassManager.cpp
@@ -20,6 +20,16 @@ bool FunctionPassManager::runOnFunction(Function &F) {
   return Change;
 }
 
+bool RegionPassManager::runOnRegion(Region &R) {
+  bool Change = false;
+  for (RegionPass *Pass : Passes) {
+    Change |= Pass->runOnRegion(R);
+    // TODO: run the verifier.
+  }
+  // TODO: Check ChangeAll against hashes before/after.
+  return Change;
+}
+
 FunctionPassManager &
 PassRegistry::parseAndCreatePassPipeline(StringRef Pipeline) {
   static constexpr const char EndToken = '\0';
diff --git a/llvm/lib/Support/APFloat.cpp b/llvm/lib/Support/APFloat.cpp
index dee917fd56104..03413f6eb6fe7 100644
--- a/llvm/lib/Support/APFloat.cpp
+++ b/llvm/lib/Support/APFloat.cpp
@@ -119,6 +119,13 @@ struct fltSemantics {
   fltNonfiniteBehavior nonFiniteBehavior = fltNonfiniteBehavior::IEEE754;
 
   fltNanEncoding nanEncoding = fltNanEncoding::IEEE;
+
+  /* Whether this semantics has an encoding for Zero */
+  bool hasZero = true;
+
+  /* Whether this semantics can represent signed values */
+  bool hasSignedRepr = true;
+
   // Returns true if any number described by this semantics can be precisely
   // represented by the specified semantics. Does not take into account
   // the value of fltNonfiniteBehavior.
@@ -145,6 +152,10 @@ static constexpr fltSemantics semFloat8E4M3B11FNUZ = {
     4, -10, 4, 8, fltNonfiniteBehavior::NanOnly, fltNanEncoding::NegativeZero};
 static constexpr fltSemantics semFloat8E3M4 = {3, -2, 5, 8};
 static constexpr fltSemantics semFloatTF32 = {127, -126, 11, 19};
+static constexpr fltSemantics semFloat8E8M0FNU = {
+    127,   -127, 1, 8, fltNonfiniteBehavior::NanOnly, fltNanEncoding::AllOnes,
+    false, false};
+
 static constexpr fltSemantics semFloat6E3M2FN = {
     4, -2, 3, 6, fltNonfiniteBehavior::FiniteOnly};
 static constexpr fltSemantics semFloat6E2M3FN = {
@@ -222,6 +233,8 @@ const llvm::fltSemantics &APFloatBase::EnumToSemantics(Semantics S) {
     return Float8E3M4();
   case S_FloatTF32:
     return FloatTF32();
+  case S_Float8E8M0FNU:
+    return Float8E8M0FNU();
   case S_Float6E3M2FN:
     return Float6E3M2FN();
   case S_Float6E2M3FN:
@@ -264,6 +277,8 @@ APFloatBase::SemanticsToEnum(const llvm::fltSemantics &Sem) {
     return S_Float8E3M4;
   else if (&Sem == &llvm::APFloat::FloatTF32())
     return S_FloatTF32;
+  else if (&Sem == &llvm::APFloat::Float8E8M0FNU())
+    return S_Float8E8M0FNU;
   else if (&Sem == &llvm::APFloat::Float6E3M2FN())
     return S_Float6E3M2FN;
   else if (&Sem == &llvm::APFloat::Float6E2M3FN())
@@ -294,6 +309,7 @@ const fltSemantics &APFloatBase::Float8E4M3B11FNUZ() {
 }
 const fltSemantics &APFloatBase::Float8E3M4() { return semFloat8E3M4; }
 const fltSemantics &APFloatBase::FloatTF32() { return semFloatTF32; }
+const fltSemantics &APFloatBase::Float8E8M0FNU() { return semFloat8E8M0FNU; }
 const fltSemantics &APFloatBase::Float6E3M2FN() { return semFloat6E3M2FN; }
 const fltSemantics &APFloatBase::Float6E2M3FN() { return semFloat6E2M3FN; }
 const fltSemantics &APFloatBase::Float4E2M1FN() { return semFloat4E2M1FN; }
@@ -396,7 +412,8 @@ static inline Error createError(const Twine &Err) {
 }
 
 static constexpr inline unsigned int partCountForBits(unsigned int bits) {
-  return ((bits) + APFloatBase::integerPartWidth - 1) / APFloatBase::integerPartWidth;
+  return std::max(1u, (bits + APFloatBase::integerPartWidth - 1) /
+                          APFloatBase::integerPartWidth);
 }
 
 /* Returns 0U-9U.  Return values >= 10U are not digits.  */
@@ -918,6 +935,10 @@ void IEEEFloat::makeNaN(bool SNaN, bool Negative, const APInt *fill) {
   if (semantics->nonFiniteBehavior == fltNonfiniteBehavior::FiniteOnly)
     llvm_unreachable("This floating point format does not support NaN");
 
+  if (Negative && !semantics->hasSignedRepr)
+    llvm_unreachable(
+        "This floating point format does not support signed values");
+
   category = fcNaN;
   sign = Negative;
   exponent = exponentNaN();
@@ -955,7 +976,8 @@ void IEEEFloat::makeNaN(bool SNaN, bool Negative, const APInt *fill) {
       significand[part] = 0;
   }
 
-  unsigned QNaNBit = semantics->precision - 2;
+  unsigned QNaNBit =
+      (semantics->precision >= 2) ? (semantics->precision - 2) : 0;
 
   if (SNaN) {
     // We always have to clear the QNaN bit to make it an SNaN.
@@ -1025,6 +1047,19 @@ bool IEEEFloat::isSmallestNormalized() const {
          isSignificandAllZerosExceptMSB();
 }
 
+unsigned int IEEEFloat::getNumHighBits() const {
+  const unsigned int PartCount = partCountForBits(semantics->precision);
+  const unsigned int Bits = PartCount * integerPartWidth;
+
+  // Compute how many bits are used in the final word.
+  // When precision is just 1, it represents the 'Pth'
+  // Precision bit and not the actual significand bit.
+  const unsigned int NumHighBits = (semantics->precision > 1)
+                                       ? (Bits - semantics->precision + 1)
+                                       : (Bits - semantics->precision);
+  return NumHighBits;
+}
+
 bool IEEEFloat::isSignificandAllOnes() const {
   // Test if the significand excluding the integral bit is all ones. This allows
   // us to test for binade boundaries.
@@ -1035,13 +1070,12 @@ bool IEEEFloat::isSignificandAllOnes() const {
       return false;
 
   // Set the unused high bits to all ones when we compare.
-  const unsigned NumHighBits =
-    PartCount*integerPartWidth - semantics->precision + 1;
+  const unsigned NumHighBits = getNumHighBits();
   assert(NumHighBits <= integerPartWidth && NumHighBits > 0 &&
          "Can not have more high bits to fill than integerPartWidth");
   const integerPart HighBitFill =
     ~integerPart(0) << (integerPartWidth - NumHighBits);
-  if (~(Parts[PartCount - 1] | HighBitFill))
+  if ((semantics->precision <= 1) || (~(Parts[PartCount - 1] | HighBitFill)))
     return false;
 
   return true;
@@ -1062,8 +1096,7 @@ bool IEEEFloat::isSignificandAllOnesExceptLSB() const {
   }
 
   // Set the unused high bits to all ones when we compare.
-  const unsigned NumHighBits =
-      PartCount * integerPartWidth - semantics->precision + 1;
+  const unsigned NumHighBits = getNumHighBits();
   assert(NumHighBits <= integerPartWidth && NumHighBits > 0 &&
          "Can not have more high bits to fill than integerPartWidth");
   const integerPart HighBitFill = ~integerPart(0)
@@ -1085,13 +1118,12 @@ bool IEEEFloat::isSignificandAllZeros() const {
       return false;
 
   // Compute how many bits are used in the final word.
-  const unsigned NumHighBits =
-    PartCount*integerPartWidth - semantics->precision + 1;
+  const unsigned NumHighBits = getNumHighBits();
   assert(NumHighBits < integerPartWidth && "Can not have more high bits to "
          "clear than integerPartWidth");
   const integerPart HighBitMask = ~integerPart(0) >> NumHighBits;
 
-  if (Parts[PartCount - 1] & HighBitMask)
+  if ((semantics->precision > 1) && (Parts[PartCount - 1] & HighBitMask))
     return false;
 
   return true;
@@ -1106,25 +1138,26 @@ bool IEEEFloat::isSignificandAllZerosExceptMSB() const {
       return false;
   }
 
-  const unsigned NumHighBits =
-      PartCount * integerPartWidth - semantics->precision + 1;
-  return Parts[PartCount - 1] == integerPart(1)
-                                     << (integerPartWidth - NumHighBits);
+  const unsigned NumHighBits = getNumHighBits();
+  const integerPart MSBMask = integerPart(1)
+                              << (integerPartWidth - NumHighBits);
+  return ((semantics->precision <= 1) || (Parts[PartCount - 1] == MSBMask));
 }
 
 bool IEEEFloat::isLargest() const {
+  bool IsMaxExp = isFiniteNonZero() && exponent == semantics->maxExponent;
   if (semantics->nonFiniteBehavior == fltNonfiniteBehavior::NanOnly &&
       semantics->nanEncoding == fltNanEncoding::AllOnes) {
     // The largest number by magnitude in our format will be the floating point
     // number with maximum exponent and with significand that is all ones except
     // the LSB.
-    return isFiniteNonZero() && exponent == semantics->maxExponent &&
-           isSignificandAllOnesExceptLSB();
+    return (IsMaxExp && APFloat::hasSignificand(*semantics))
+               ? isSignificandAllOnesExceptLSB()
+               : IsMaxExp;
   } else {
     // The largest number by magnitude in our format will be the floating point
     // number with maximum exponent and with significand that is all ones.
-    return isFiniteNonZero() && exponent == semantics->maxExponent &&
-           isSignificandAllOnes();
+    return IsMaxExp && isSignificandAllOnes();
   }
 }
 
@@ -1165,7 +1198,13 @@ IEEEFloat::IEEEFloat(const fltSemantics &ourSemantics, integerPart value) {
 
 IEEEFloat::IEEEFloat(const fltSemantics &ourSemantics) {
   initialize(&ourSemantics);
-  makeZero(false);
+  // The Float8E8MOFNU format does not have a representation
+  // for zero. So, use the closest representation instead.
+  // Moreover, the all-zero encoding represents a valid
+  // normal value (which is the smallestNormalized here).
+  // Hence, we call makeSmallestNormalized (where category is
+  // 'fcNormal') instead of makeZero (where category is 'fcZero').
+  ourSemantics.hasZero ? makeZero(false) : makeSmallestNormalized(false);
 }
 
 // Delegate to the previous constructor, because later copy constructor may
@@ -1245,7 +1284,8 @@ IEEEFloat::integerPart IEEEFloat::subtractSignificand(const IEEEFloat &rhs,
    on to the full-precision result of the multiplication.  Returns the
    lost fraction.  */
 lostFraction IEEEFloat::multiplySignificand(const IEEEFloat &rhs,
-                                            IEEEFloat addend) {
+                                            IEEEFloat addend,
+                                            bool ignoreAddend) {
   unsigned int omsb;        // One, not zero, based MSB.
   unsigned int partsCount, newPartsCount, precision;
   integerPart *lhsSignificand;
@@ -1289,7 +1329,7 @@ lostFraction IEEEFloat::multiplySignificand(const IEEEFloat &rhs,
   // toward left by two bits, and adjust exponent accordingly.
   exponent += 2;
 
-  if (addend.isNonZero()) {
+  if (!ignoreAddend && addend.isNonZero()) {
     // The intermediate result of the multiplication has "2 * precision"
     // signicant bit; adjust the addend to be consistent with mul result.
     //
@@ -1377,7 +1417,12 @@ lostFraction IEEEFloat::multiplySignificand(const IEEEFloat &rhs,
 }
 
 lostFraction IEEEFloat::multiplySignificand(const IEEEFloat &rhs) {
-  return multiplySignificand(rhs, IEEEFloat(*semantics));
+  // When the given semantics has zero, the addend here is a zero.
+  // i.e . it belongs to the 'fcZero' category.
+  // But when the semantics does not support zero, we need to
+  // explicitly convey that this addend should be ignored
+  // for multiplication.
+  return multiplySignificand(rhs, IEEEFloat(*semantics), !semantics->hasZero);
 }
 
 /* Multiply the significands of LHS and RHS to DST.  */
@@ -1483,7 +1528,8 @@ lostFraction IEEEFloat::shiftSignificandRight(unsigned int bits) {
 
 /* Shift the significand left BITS bits, subtract BITS from its exponent.  */
 void IEEEFloat::shiftSignificandLeft(unsigned int bits) {
-  assert(bits < semantics->precision);
+  assert(bits < semantics->precision ||
+         (semantics->precision == 1 && bits <= 1));
 
   if (bits) {
     unsigned int partsCount = partCount();
@@ -1678,6 +1724,8 @@ IEEEFloat::opStatus IEEEFloat::normalize(roundingMode rounding_mode,
       category = fcZero;
       if (semantics->nanEncoding == fltNanEncoding::NegativeZero)
         sign = false;
+      if (!semantics->hasZero)
+        makeSmallestNormalized(false);
     }
 
     return opOK;
@@ -1729,6 +1777,11 @@ IEEEFloat::opStatus IEEEFloat::normalize(roundingMode rounding_mode,
     category = fcZero;
     if (semantics->nanEncoding == fltNanEncoding::NegativeZero)
       sign = false;
+    // This condition handles the case where the semantics
+    // does not have zero but uses the all-zero encoding
+    // to represent the smallest normal value.
+    if (!semantics->hasZero)
+      makeSmallestNormalized(false);
   }
 
   /* The fcZero case is a denormal that underflowed to zero.  */
@@ -1807,6 +1860,10 @@ lostFraction IEEEFloat::addOrSubtractSignificand(const IEEEFloat &rhs,
 
   /* Subtraction is more subtle than one might naively expect.  */
   if (subtract) {
+    if ((bits < 0) && !semantics->hasSignedRepr)
+      llvm_unreachable(
+          "This floating point format does not support signed values");
+
     IEEEFloat temp_rhs(rhs);
 
     if (bits == 0)
@@ -2252,6 +2309,17 @@ IEEEFloat::opStatus IEEEFloat::mod(const IEEEFloat &rhs) {
     V.sign = sign;
 
     fs = subtract(V, rmNearestTiesToEven);
+
+    // When the semantics supports zero, this loop's
+    // exit-condition is handled by the 'isFiniteNonZero'
+    // category check above. However, when the semantics
+    // does not have 'fcZero' and we have reached the
+    // minimum possible value, (and any further subtract
+    // will underflow to the same value) explicitly
+    // provide an exit-path here.
+    if (!semantics->hasZero && this->isSmallest())
+      break;
+
     assert(fs==opOK);
   }
   if (isZero()) {
@@ -2606,6 +2674,8 @@ IEEEFloat::opStatus IEEEFloat::convert(const fltSemantics &toSemantics,
     fs = opOK;
   }
 
+  if (category == fcZero && !semantics->hasZero)
+    makeSmallestNormalized(false);
   return fs;
 }
 
@@ -3070,6 +3140,8 @@ IEEEFloat::convertFromDecimalString(StringRef str, roundingMode rounding_mode) {
     fs = opOK;
     if (semantics->nanEncoding == fltNanEncoding::NegativeZero)
       sign = false;
+    if (!semantics->hasZero)
+      makeSmallestNormalized(false);
 
     /* Check whether the normalized exponent is high enough to overflow
        max during the log-rebasing in the max-exponent check below. */
@@ -3237,6 +3309,10 @@ IEEEFloat::convertFromString(StringRef str, roundingMode rounding_mode) {
   StringRef::iterator p = str.begin();
   size_t slen = str.size();
   sign = *p == '-' ? 1 : 0;
+  if (sign && !semantics->hasSignedRepr)
+    llvm_unreachable(
+        "This floating point format does not support signed values");
+
   if (*p == '-' || *p == '+') {
     p++;
     slen--;
@@ -3533,15 +3609,16 @@ APInt IEEEFloat::convertPPCDoubleDoubleAPFloatToAPInt() const {
 template <const fltSemantics &S>
 APInt IEEEFloat::convertIEEEFloatToAPInt() const {
   assert(semantics == &S);
-
-  constexpr int bias = -(S.minExponent - 1);
+  const int bias =
+      (semantics == &semFloat8E8M0FNU) ? -S.minExponent : -(S.minExponent - 1);
   constexpr unsigned int trailing_significand_bits = S.precision - 1;
   constexpr int integer_bit_part = trailing_significand_bits / integerPartWidth;
   constexpr integerPart integer_bit =
       integerPart{1} << (trailing_significand_bits % integerPartWidth);
   constexpr uint64_t significand_mask = integer_bit - 1;
   constexpr unsigned int exponent_bits =
-      S.sizeInBits - 1 - trailing_significand_bits;
+      trailing_significand_bits ? (S.sizeInBits - 1 - trailing_significand_bits)
+                                : S.sizeInBits;
   static_assert(exponent_bits < 64);
   constexpr uint64_t exponent_mask = (uint64_t{1} << exponent_bits) - 1;
 
@@ -3557,6 +3634,8 @@ APInt IEEEFloat::convertIEEEFloatToAPInt() const {
         !(significandParts()[integer_bit_part] & integer_bit))
       myexponent = 0; // denormal
   } else if (category == fcZero) {
+    if (!S.hasZero)
+      llvm_unreachable("semantics does not support zero!");
     myexponent = ::exponentZero(S) + bias;
     mysignificand.fill(0);
   } else if (category == fcInfinity) {
@@ -3659,6 +3738,11 @@ APInt IEEEFloat::convertFloatTF32APFloatToAPInt() const {
   return convertIEEEFloatToAPInt<semFloatTF32>();
 }
 
+APInt IEEEFloat::convertFloat8E8M0FNUAPFloatToAPInt() const {
+  assert(partCount() == 1);
+  return convertIEEEFloatToAPInt<semFloat8E8M0FNU>();
+}
+
 APInt IEEEFloat::convertFloat6E3M2FNAPFloatToAPInt() const {
   assert(partCount() == 1);
   return convertIEEEFloatToAPInt<semFloat6E3M2FN>();
@@ -3721,6 +3805,9 @@ APInt IEEEFloat::bitcastToAPInt() const {
   if (semantics == (const llvm::fltSemantics *)&semFloatTF32)
     return convertFloatTF32APFloatToAPInt();
 
+  if (semantics == (const llvm::fltSemantics *)&semFloat8E8M0FNU)
+    return convertFloat8E8M0FNUAPFloatToAPInt();
+
   if (semantics == (const llvm::fltSemantics *)&semFloat6E3M2FN)
     return convertFloat6E3M2FNAPFloatToAPInt();
 
@@ -3819,6 +3906,40 @@ void IEEEFloat::initFromPPCDoubleDoubleAPInt(const APInt &api) {
   }
 }
 
+// The E8M0 format has the following characteristics:
+// It is an 8-bit unsigned format with only exponents (no actual significand).
+// No encodings for {zero, infinities or denorms}.
+// NaN is represented by all 1's.
+// Bias is 127.
+void IEEEFloat::initFromFloat8E8M0FNUAPInt(const APInt &api) {
+  const uint64_t exponent_mask = 0xff;
+  uint64_t val = api.getRawData()[0];
+  uint64_t myexponent = (val & exponent_mask);
+
+  initialize(&semFloat8E8M0FNU);
+  assert(partCount() == 1);
+
+  // This format has unsigned representation only
+  sign = 0;
+
+  // Set the significand
+  // This format does not have any significand but the 'Pth' precision bit is
+  // always set to 1 for consistency in APFloat's internal representation.
+  uint64_t mysignificand = 1;
+  significandParts()[0] = mysignificand;
+
+  // This format can either have a NaN or fcNormal
+  // All 1's i.e. 255 is a NaN
+  if (val == exponent_mask) {
+    category = fcNaN;
+    exponent = exponentNaN();
+    return;
+  }
+  // Handle fcNormal...
+  category = fcNormal;
+  exponent = myexponent - 127; // 127 is bias
+  return;
+}
 template <const fltSemantics &S>
 void IEEEFloat::initFromIEEEAPInt(const APInt &api) {
   assert(api.getBitWidth() == S.sizeInBits);
@@ -3999,6 +4120,8 @@ void IEEEFloat::initFromAPInt(const fltSemantics *Sem, const APInt &api) {
     return initFromFloat8E3M4APInt(api);
   if (Sem == &semFloatTF32)
     return initFromFloatTF32APInt(api);
+  if (Sem == &semFloat8E8M0FNU)
+    return initFromFloat8E8M0FNUAPInt(api);
   if (Sem == &semFloat6E3M2FN)
     return initFromFloat6E3M2FNAPInt(api);
   if (Sem == &semFloat6E2M3FN)
@@ -4012,6 +4135,9 @@ void IEEEFloat::initFromAPInt(const fltSemantics *Sem, const APInt &api) {
 /// Make this number the largest magnitude normal number in the given
 /// semantics.
 void IEEEFloat::makeLargest(bool Negative) {
+  if (Negative && !semantics->hasSignedRepr)
+    llvm_unreachable(
+        "This floating point format does not support signed values");
   // We want (in interchange format):
   //   sign = {Negative}
   //   exponent = 1..10
@@ -4032,15 +4158,18 @@ void IEEEFloat::makeLargest(bool Negative) {
   significand[PartCount - 1] = (NumUnusedHighBits < integerPartWidth)
                                    ? (~integerPart(0) >> NumUnusedHighBits)
                                    : 0;
-
   if (semantics->nonFiniteBehavior == fltNonfiniteBehavior::NanOnly &&
-      semantics->nanEncoding == fltNanEncoding::AllOnes)
+      semantics->nanEncoding == fltNanEncoding::AllOnes &&
+      (semantics->precision > 1))
     significand[0] &= ~integerPart(1);
 }
 
 /// Make this number the smallest magnitude denormal number in the given
 /// semantics.
 void IEEEFloat::makeSmallest(bool Negative) {
+  if (Negative && !semantics->hasSignedRepr)
+    llvm_unreachable(
+        "This floating point format does not support signed values");
   // We want (in interchange format):
   //   sign = {Negative}
   //   exponent = 0..0
@@ -4052,6 +4181,9 @@ void IEEEFloat::makeSmallest(bool Negative) {
 }
 
 void IEEEFloat::makeSmallestNormalized(bool Negative) {
+  if (Negative && !semantics->hasSignedRepr)
+    llvm_unreachable(
+        "This floating point format does not support signed values");
   // We want (in interchange format):
   //   sign = {Negative}
   //   exponent = 0..0
@@ -4509,6 +4641,8 @@ IEEEFloat::opStatus IEEEFloat::next(bool nextDown) {
       exponent = 0;
       if (semantics->nanEncoding == fltNanEncoding::NegativeZero)
         sign = false;
+      if (!semantics->hasZero)
+        makeSmallestNormalized(false);
       break;
     }
 
@@ -4574,7 +4708,10 @@ IEEEFloat::opStatus IEEEFloat::next(bool nextDown) {
       // the integral bit to 1, and increment the exponent. If we have a
       // denormal always increment since moving denormals and the numbers in the
       // smallest normal binade have the same exponent in our representation.
-      bool WillCrossBinadeBoundary = !isDenormal() && isSignificandAllOnes();
+      // If there are only exponents, any increment always crosses the
+      // BinadeBoundary.
+      bool WillCrossBinadeBoundary = !APFloat::hasSignificand(*semantics) ||
+                                     (!isDenormal() && isSignificandAllOnes());
 
       if (WillCrossBinadeBoundary) {
         integerPart *Parts = significandParts();
@@ -4626,6 +4763,9 @@ void IEEEFloat::makeInf(bool Negative) {
 }
 
 void IEEEFloat::makeZero(bool Negative) {
+  if (!semantics->hasZero)
+    llvm_unreachable("This floating point format does not support Zero");
+
   category = fcZero;
   sign = Negative;
   if (semantics->nanEncoding == fltNanEncoding::NegativeZero) {
diff --git a/llvm/lib/Support/Caching.cpp b/llvm/lib/Support/Caching.cpp
index 1ef51db218e89..66e540efaca97 100644
--- a/llvm/lib/Support/Caching.cpp
+++ b/llvm/lib/Support/Caching.cpp
@@ -37,8 +37,8 @@ Expected<FileCache> llvm::localCache(const Twine &CacheNameRef,
   TempFilePrefixRef.toVector(TempFilePrefix);
   CacheDirectoryPathRef.toVector(CacheDirectoryPath);
 
-  return [=](unsigned Task, StringRef Key,
-             const Twine &ModuleName) -> Expected<AddStreamFn> {
+  auto Func = [=](unsigned Task, StringRef Key,
+                  const Twine &ModuleName) -> Expected<AddStreamFn> {
     // This choice of file name allows the cache to be pruned (see pruneCache()
     // in include/llvm/Support/CachePruning.h).
     SmallString<64> EntryPath;
@@ -167,4 +167,5 @@ Expected<FileCache> llvm::localCache(const Twine &CacheNameRef,
           Task);
     };
   };
+  return FileCache(Func, CacheDirectoryPathRef.str());
 }
diff --git a/llvm/lib/Support/KnownBits.cpp b/llvm/lib/Support/KnownBits.cpp
index 6863c5c0af5dc..89668af378070 100644
--- a/llvm/lib/Support/KnownBits.cpp
+++ b/llvm/lib/Support/KnownBits.cpp
@@ -18,6 +18,15 @@
 
 using namespace llvm;
 
+KnownBits KnownBits::flipSignBit(const KnownBits &Val) {
+  unsigned SignBitPosition = Val.getBitWidth() - 1;
+  APInt Zero = Val.Zero;
+  APInt One = Val.One;
+  Zero.setBitVal(SignBitPosition, Val.One[SignBitPosition]);
+  One.setBitVal(SignBitPosition, Val.Zero[SignBitPosition]);
+  return KnownBits(Zero, One);
+}
+
 static KnownBits computeForAddCarry(const KnownBits &LHS, const KnownBits &RHS,
                                     bool CarryZero, bool CarryOne) {
 
@@ -200,16 +209,7 @@ KnownBits KnownBits::umin(const KnownBits &LHS, const KnownBits &RHS) {
 }
 
 KnownBits KnownBits::smax(const KnownBits &LHS, const KnownBits &RHS) {
-  // Flip the range of values: [-0x80000000, 0x7FFFFFFF] <-> [0, 0xFFFFFFFF]
-  auto Flip = [](const KnownBits &Val) {
-    unsigned SignBitPosition = Val.getBitWidth() - 1;
-    APInt Zero = Val.Zero;
-    APInt One = Val.One;
-    Zero.setBitVal(SignBitPosition, Val.One[SignBitPosition]);
-    One.setBitVal(SignBitPosition, Val.Zero[SignBitPosition]);
-    return KnownBits(Zero, One);
-  };
-  return Flip(umax(Flip(LHS), Flip(RHS)));
+  return flipSignBit(umax(flipSignBit(LHS), flipSignBit(RHS)));
 }
 
 KnownBits KnownBits::smin(const KnownBits &LHS, const KnownBits &RHS) {
@@ -763,11 +763,10 @@ KnownBits KnownBits::usub_sat(const KnownBits &LHS, const KnownBits &RHS) {
   return computeForSatAddSub(/*Add*/ false, /*Signed*/ false, LHS, RHS);
 }
 
-static KnownBits avgCompute(KnownBits LHS, KnownBits RHS, bool IsCeil,
-                            bool IsSigned) {
+static KnownBits avgComputeU(KnownBits LHS, KnownBits RHS, bool IsCeil) {
   unsigned BitWidth = LHS.getBitWidth();
-  LHS = IsSigned ? LHS.sext(BitWidth + 1) : LHS.zext(BitWidth + 1);
-  RHS = IsSigned ? RHS.sext(BitWidth + 1) : RHS.zext(BitWidth + 1);
+  LHS = LHS.zext(BitWidth + 1);
+  RHS = RHS.zext(BitWidth + 1);
   LHS =
       computeForAddCarry(LHS, RHS, /*CarryZero*/ !IsCeil, /*CarryOne*/ IsCeil);
   LHS = LHS.extractBits(BitWidth, 1);
@@ -775,23 +774,19 @@ static KnownBits avgCompute(KnownBits LHS, KnownBits RHS, bool IsCeil,
 }
 
 KnownBits KnownBits::avgFloorS(const KnownBits &LHS, const KnownBits &RHS) {
-  return avgCompute(LHS, RHS, /* IsCeil */ false,
-                    /* IsSigned */ true);
+  return flipSignBit(avgFloorU(flipSignBit(LHS), flipSignBit(RHS)));
 }
 
 KnownBits KnownBits::avgFloorU(const KnownBits &LHS, const KnownBits &RHS) {
-  return avgCompute(LHS, RHS, /* IsCeil */ false,
-                    /* IsSigned */ false);
+  return avgComputeU(LHS, RHS, /*IsCeil=*/false);
 }
 
 KnownBits KnownBits::avgCeilS(const KnownBits &LHS, const KnownBits &RHS) {
-  return avgCompute(LHS, RHS, /* IsCeil */ true,
-                    /* IsSigned */ true);
+  return flipSignBit(avgCeilU(flipSignBit(LHS), flipSignBit(RHS)));
 }
 
 KnownBits KnownBits::avgCeilU(const KnownBits &LHS, const KnownBits &RHS) {
-  return avgCompute(LHS, RHS, /* IsCeil */ true,
-                    /* IsSigned */ false);
+  return avgComputeU(LHS, RHS, /*IsCeil=*/true);
 }
 
 KnownBits KnownBits::mul(const KnownBits &LHS, const KnownBits &RHS,
diff --git a/llvm/lib/Support/Unix/Threading.inc b/llvm/lib/Support/Unix/Threading.inc
index 0afd1d817f88e..a354984cabb1e 100644
--- a/llvm/lib/Support/Unix/Threading.inc
+++ b/llvm/lib/Support/Unix/Threading.inc
@@ -237,14 +237,14 @@ void llvm::get_thread_name(SmallVectorImpl<char> &Name) {
   char Buffer[len] = {'\0'}; // FIXME: working around MSan false positive.
   if (0 == ::pthread_getname_np(::pthread_self(), Buffer, len))
     Name.append(Buffer, Buffer + strlen(Buffer));
-#elif defined(HAVE_PTHREAD_GET_NAME_NP)
+#elif defined(HAVE_PTHREAD_GET_NAME_NP) && HAVE_PTHREAD_GET_NAME_NP
   constexpr uint32_t len = get_max_thread_name_length_impl();
   char buf[len];
   ::pthread_get_name_np(::pthread_self(), buf, len);
 
   Name.append(buf, buf + strlen(buf));
 
-#elif defined(HAVE_PTHREAD_GETNAME_NP)
+#elif defined(HAVE_PTHREAD_GETNAME_NP) && HAVE_PTHREAD_GETNAME_NP
   constexpr uint32_t len = get_max_thread_name_length_impl();
   char buf[len];
   ::pthread_getname_np(::pthread_self(), buf, len);
diff --git a/llvm/lib/TableGen/CMakeLists.txt b/llvm/lib/TableGen/CMakeLists.txt
index c550840f147d7..84815c7736997 100644
--- a/llvm/lib/TableGen/CMakeLists.txt
+++ b/llvm/lib/TableGen/CMakeLists.txt
@@ -11,6 +11,7 @@ add_llvm_component_library(LLVMTableGen
   TableGenBackendSkeleton.cpp
   TGLexer.cpp
   TGParser.cpp
+  TGTimer.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/TableGen
diff --git a/llvm/lib/TableGen/DetailedRecordsBackend.cpp b/llvm/lib/TableGen/DetailedRecordsBackend.cpp
index 7d17c4d68c3f1..61fd3634902cc 100644
--- a/llvm/lib/TableGen/DetailedRecordsBackend.cpp
+++ b/llvm/lib/TableGen/DetailedRecordsBackend.cpp
@@ -152,7 +152,8 @@ void DetailedRecordsEmitter::printTemplateArgs(const Record &Rec,
 // are enclosed in parentheses.
 void DetailedRecordsEmitter::printSuperclasses(const Record &Rec,
                                                raw_ostream &OS) {
-  ArrayRef<std::pair<Record *, SMRange>> Superclasses = Rec.getSuperClasses();
+  ArrayRef<std::pair<const Record *, SMRange>> Superclasses =
+      Rec.getSuperClasses();
   if (Superclasses.empty()) {
     OS << "  Superclasses: (none)\n";
     return;
diff --git a/llvm/lib/TableGen/Error.cpp b/llvm/lib/TableGen/Error.cpp
index e21f369833be6..45cdcc8ae2619 100644
--- a/llvm/lib/TableGen/Error.cpp
+++ b/llvm/lib/TableGen/Error.cpp
@@ -160,18 +160,22 @@ void PrintFatalError(const RecordVal *RecVal, const Twine &Msg) {
 
 // Check an assertion: Obtain the condition value and be sure it is true.
 // If not, print a nonfatal error along with the message.
-void CheckAssert(SMLoc Loc, Init *Condition, Init *Message) {
+bool CheckAssert(SMLoc Loc, Init *Condition, Init *Message) {
   auto *CondValue = dyn_cast_or_null<IntInit>(Condition->convertInitializerTo(
       IntRecTy::get(Condition->getRecordKeeper())));
-  if (!CondValue)
+  if (!CondValue) {
     PrintError(Loc, "assert condition must of type bit, bits, or int.");
-  else if (!CondValue->getValue()) {
+    return true;
+  }
+  if (!CondValue->getValue()) {
     PrintError(Loc, "assertion failed");
     if (auto *MessageInit = dyn_cast<StringInit>(Message))
       PrintNote(MessageInit->getValue());
     else
       PrintNote("(assert message is not a string)");
+    return true;
   }
+  return false;
 }
 
 // Dump a message to stderr.
diff --git a/llvm/lib/TableGen/Main.cpp b/llvm/lib/TableGen/Main.cpp
index a4c41223c0762..55a99cbfc58ac 100644
--- a/llvm/lib/TableGen/Main.cpp
+++ b/llvm/lib/TableGen/Main.cpp
@@ -29,6 +29,7 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TGTimer.h"
 #include "llvm/TableGen/TableGenBackend.h"
 #include <memory>
 #include <string>
@@ -98,13 +99,14 @@ static int createDependencyFile(const TGParser &Parser, const char *argv0) {
 int llvm::TableGenMain(const char *argv0,
                        std::function<TableGenMainFn> MainFn) {
   RecordKeeper Records;
+  TGTimer &Timer = Records.getTimer();
 
   if (TimePhases)
-    Records.startPhaseTiming();
+    Timer.startPhaseTiming();
 
   // Parse the input file.
 
-  Records.startTimer("Parse, build records");
+  Timer.startTimer("Parse, build records");
   ErrorOr<std::unique_ptr<MemoryBuffer>> FileOrErr =
       MemoryBuffer::getFileOrSTDIN(InputFilename, /*IsText=*/true);
   if (std::error_code EC = FileOrErr.getError())
@@ -124,10 +126,10 @@ int llvm::TableGenMain(const char *argv0,
 
   if (Parser.ParseFile())
     return 1;
-  Records.stopTimer();
+  Timer.stopTimer();
 
   // Write output to memory.
-  Records.startBackendTimer("Backend overall");
+  Timer.startBackendTimer("Backend overall");
   std::string OutString;
   raw_string_ostream Out(OutString);
   unsigned status = 0;
@@ -135,7 +137,7 @@ int llvm::TableGenMain(const char *argv0,
   // case, attempt to apply the MainFn.
   if (TableGen::Emitter::ApplyCallback(Records, Out))
     status = MainFn ? MainFn(Out, Records) : 1;
-  Records.stopBackendTimer();
+  Timer.stopBackendTimer();
   if (status)
     return 1;
 
@@ -148,7 +150,7 @@ int llvm::TableGenMain(const char *argv0,
       return Ret;
   }
 
-  Records.startTimer("Write output");
+  Timer.startTimer("Write output");
   bool WriteFile = true;
   if (WriteIfChanged) {
     // Only updates the real output file if there are any differences.
@@ -169,9 +171,9 @@ int llvm::TableGenMain(const char *argv0,
     if (ErrorsPrinted == 0)
       OutFile.keep();
   }
-  
-  Records.stopTimer();
-  Records.stopPhaseTiming();
+
+  Timer.stopTimer();
+  Timer.stopPhaseTiming();
 
   if (ErrorsPrinted > 0)
     return reportError(argv0, Twine(ErrorsPrinted) + " errors.\n");
diff --git a/llvm/lib/TableGen/Record.cpp b/llvm/lib/TableGen/Record.cpp
index 0a5d384239d5e..447ecb7d74d24 100644
--- a/llvm/lib/TableGen/Record.cpp
+++ b/llvm/lib/TableGen/Record.cpp
@@ -28,6 +28,7 @@
 #include "llvm/Support/SMLoc.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/TGTimer.h"
 #include <cassert>
 #include <cstdint>
 #include <map>
@@ -746,7 +747,7 @@ Init *ListInit::convertInitializerTo(const RecTy *Ty) const {
   return nullptr;
 }
 
-Record *ListInit::getElementAsRecord(unsigned i) const {
+const Record *ListInit::getElementAsRecord(unsigned i) const {
   assert(i < NumValues && "List element index out of range!");
   DefInit *DI = dyn_cast<DefInit>(getElement(i));
   if (!DI)
@@ -871,7 +872,7 @@ Init *UnOpInit::Fold(Record *CurRec, bool IsFinal) const {
 
     } else if (isa<RecordRecTy>(getType())) {
       if (StringInit *Name = dyn_cast<StringInit>(LHS)) {
-        Record *D = RK.getDef(Name->getValue());
+        const Record *D = RK.getDef(Name->getValue());
         if (!D && CurRec) {
           // Self-references are allowed, but their resolution is delayed until
           // the final resolve to ensure that we get the correct type for them.
@@ -1713,7 +1714,7 @@ Init *TernOpInit::Fold(Record *CurRec) const {
     StringInit *RHSs = dyn_cast<StringInit>(RHS);
 
     if (LHSd && MHSd && RHSd) {
-      Record *Val = RHSd->getDef();
+      const Record *Val = RHSd->getDef();
       if (LHSd->getAsString() == RHSd->getAsString())
         Val = MHSd->getDef();
       return Val->getDefInit();
@@ -2113,7 +2114,7 @@ Init *ExistsOpInit::Fold(Record *CurRec, bool IsFinal) const {
   if (StringInit *Name = dyn_cast<StringInit>(Expr)) {
 
     // Look up all defined records to see if we can find one.
-    Record *D = CheckType->getRecordKeeper().getDef(Name->getValue());
+    const Record *D = CheckType->getRecordKeeper().getDef(Name->getValue());
     if (D) {
       // Check if types are compatible.
       return IntInit::get(getRecordKeeper(),
@@ -2265,7 +2266,7 @@ Init *VarBitInit::resolveReferences(Resolver &R) const {
   return const_cast<VarBitInit*>(this);
 }
 
-DefInit::DefInit(Record *D)
+DefInit::DefInit(const Record *D)
     : TypedInit(IK_DefInit, D->getType()), Def(D) {}
 
 Init *DefInit::convertInitializerTo(const RecTy *Ty) const {
@@ -2292,11 +2293,12 @@ static void ProfileVarDefInit(FoldingSetNodeID &ID, Record *Class,
     ID.AddPointer(I);
 }
 
-VarDefInit::VarDefInit(Record *Class, unsigned N)
-    : TypedInit(IK_VarDefInit, RecordRecTy::get(Class)), Class(Class),
+VarDefInit::VarDefInit(SMLoc Loc, Record *Class, unsigned N)
+    : TypedInit(IK_VarDefInit, RecordRecTy::get(Class)), Loc(Loc), Class(Class),
       NumArgs(N) {}
 
-VarDefInit *VarDefInit::get(Record *Class, ArrayRef<ArgumentInit *> Args) {
+VarDefInit *VarDefInit::get(SMLoc Loc, Record *Class,
+                            ArrayRef<ArgumentInit *> Args) {
   FoldingSetNodeID ID;
   ProfileVarDefInit(ID, Class, Args);
 
@@ -2307,7 +2309,7 @@ VarDefInit *VarDefInit::get(Record *Class, ArrayRef<ArgumentInit *> Args) {
 
   void *Mem = RK.Allocator.Allocate(
       totalSizeToAlloc<ArgumentInit *>(Args.size()), alignof(VarDefInit));
-  VarDefInit *I = new (Mem) VarDefInit(Class, Args.size());
+  VarDefInit *I = new (Mem) VarDefInit(Loc, Class, Args.size());
   std::uninitialized_copy(Args.begin(), Args.end(),
                           I->getTrailingObjects<ArgumentInit *>());
   RK.TheVarDefInitPool.InsertNode(I, IP);
@@ -2323,9 +2325,8 @@ DefInit *VarDefInit::instantiate() {
     return Def;
 
   RecordKeeper &Records = Class->getRecords();
-  auto NewRecOwner =
-      std::make_unique<Record>(Records.getNewAnonymousName(), Class->getLoc(),
-                               Records, Record::RK_AnonymousDef);
+  auto NewRecOwner = std::make_unique<Record>(
+      Records.getNewAnonymousName(), Loc, Records, Record::RK_AnonymousDef);
   Record *NewRec = NewRecOwner.get();
 
   // Copy values from class to instance
@@ -2357,9 +2358,8 @@ DefInit *VarDefInit::instantiate() {
   NewRec->resolveReferences(R);
 
   // Add superclasses.
-  ArrayRef<std::pair<Record *, SMRange>> SCs = Class->getSuperClasses();
-  for (const auto &SCPair : SCs)
-    NewRec->addSuperClass(SCPair.first, SCPair.second);
+  for (const auto &[SC, Loc] : Class->getSuperClasses())
+    NewRec->addSuperClass(SC, Loc);
 
   NewRec->addSuperClass(
       Class, SMRange(Class->getLoc().back(), Class->getLoc().back()));
@@ -2390,7 +2390,7 @@ Init *VarDefInit::resolveReferences(Resolver &R) const {
   }
 
   if (Changed) {
-    auto New = VarDefInit::get(Class, NewArgs);
+    auto *New = VarDefInit::get(Loc, Class, NewArgs);
     if (!UR.foundUnresolved())
       return New->instantiate();
     return New;
@@ -2445,7 +2445,7 @@ Init *FieldInit::resolveReferences(Resolver &R) const {
 
 Init *FieldInit::Fold(Record *CurRec) const {
   if (DefInit *DI = dyn_cast<DefInit>(Rec)) {
-    Record *Def = DI->getDef();
+    const Record *Def = DI->getDef();
     if (Def == CurRec)
       PrintFatalError(CurRec->getLoc(),
                       Twine("Attempting to access field '") +
@@ -2656,7 +2656,7 @@ void DagInit::Profile(FoldingSetNodeID &ID) const {
                  ArrayRef(getTrailingObjects<StringInit *>(), NumArgNames));
 }
 
-Record *DagInit::getOperatorAsDef(ArrayRef<SMLoc> Loc) const {
+const Record *DagInit::getOperatorAsDef(ArrayRef<SMLoc> Loc) const {
   if (DefInit *DefI = dyn_cast<DefInit>(Val))
     return DefI->getDef();
   PrintFatalError(Loc, "Expected record as operator");
@@ -2837,8 +2837,8 @@ const RecordRecTy *Record::getType() const {
 
 DefInit *Record::getDefInit() const {
   if (!CorrespondingDefInit) {
-    CorrespondingDefInit = new (TrackedRecords.getImpl().Allocator)
-        DefInit(const_cast<Record *>(this));
+    CorrespondingDefInit =
+        new (TrackedRecords.getImpl().Allocator) DefInit(this);
   }
   return CorrespondingDefInit;
 }
@@ -2869,7 +2869,7 @@ void Record::setName(Init *NewName) {
 // so we can step through the direct superclasses in reverse order.
 
 bool Record::hasDirectSuperClass(const Record *Superclass) const {
-  ArrayRef<std::pair<Record *, SMRange>> SCs = getSuperClasses();
+  ArrayRef<std::pair<const Record *, SMRange>> SCs = getSuperClasses();
 
   for (int I = SCs.size() - 1; I >= 0; --I) {
     const Record *SC = SCs[I].first;
@@ -2883,10 +2883,10 @@ bool Record::hasDirectSuperClass(const Record *Superclass) const {
 
 void Record::getDirectSuperClasses(
     SmallVectorImpl<const Record *> &Classes) const {
-  ArrayRef<std::pair<Record *, SMRange>> SCs = getSuperClasses();
+  ArrayRef<std::pair<const Record *, SMRange>> SCs = getSuperClasses();
 
   while (!SCs.empty()) {
-    Record *SC = SCs.back().first;
+    const Record *SC = SCs.back().first;
     SCs = SCs.drop_back(1 + SC->getSuperClasses().size());
     Classes.push_back(SC);
   }
@@ -2965,11 +2965,11 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const Record &R) {
   }
 
   OS << " {";
-  ArrayRef<std::pair<Record *, SMRange>> SC = R.getSuperClasses();
+  ArrayRef<std::pair<const Record *, SMRange>> SC = R.getSuperClasses();
   if (!SC.empty()) {
     OS << "\t//";
-    for (const auto &SuperPair : SC)
-      OS << " " << SuperPair.first->getNameInitAsString();
+    for (const auto &[SC, _] : SC)
+      OS << " " << SC->getNameInitAsString();
   }
   OS << "\n";
 
@@ -3047,22 +3047,8 @@ ListInit *Record::getValueAsListInit(StringRef FieldName) const {
                                 "' exists but does not have a list value");
 }
 
-std::vector<Record*>
-Record::getValueAsListOfDefs(StringRef FieldName) const {
-  ListInit *List = getValueAsListInit(FieldName);
-  std::vector<Record*> Defs;
-  for (Init *I : List->getValues()) {
-    if (DefInit *DI = dyn_cast<DefInit>(I))
-      Defs.push_back(DI->getDef());
-    else
-      PrintFatalError(getLoc(), "Record `" + getName() + "', field `" +
-        FieldName + "' list is not entirely DefInit!");
-  }
-  return Defs;
-}
-
 std::vector<const Record *>
-Record::getValueAsListOfConstDefs(StringRef FieldName) const {
+Record::getValueAsListOfDefs(StringRef FieldName) const {
   ListInit *List = getValueAsListInit(FieldName);
   std::vector<const Record *> Defs;
   for (const Init *I : List->getValues()) {
@@ -3122,7 +3108,7 @@ Record::getValueAsListOfStrings(StringRef FieldName) const {
   return Strings;
 }
 
-Record *Record::getValueAsDef(StringRef FieldName) const {
+const Record *Record::getValueAsDef(StringRef FieldName) const {
   const RecordVal *R = getValue(FieldName);
   if (!R || !R->getValue())
     PrintFatalError(getLoc(), "Record `" + getName() +
@@ -3134,7 +3120,7 @@ Record *Record::getValueAsDef(StringRef FieldName) const {
     FieldName + "' does not have a def initializer!");
 }
 
-Record *Record::getValueAsOptionalDef(StringRef FieldName) const {
+const Record *Record::getValueAsOptionalDef(StringRef FieldName) const {
   const RecordVal *R = getValue(FieldName);
   if (!R || !R->getValue())
     PrintFatalError(getLoc(), "Record `" + getName() +
@@ -3148,7 +3134,6 @@ Record *Record::getValueAsOptionalDef(StringRef FieldName) const {
     FieldName + "' does not have either a def initializer or '?'!");
 }
 
-
 bool Record::getValueAsBit(StringRef FieldName) const {
   const RecordVal *R = getValue(FieldName);
   if (!R || !R->getValue())
@@ -3198,11 +3183,19 @@ void Record::checkRecordAssertions() {
   RecordResolver R(*this);
   R.setFinal(true);
 
+  bool AnyFailed = false;
   for (const auto &Assertion : getAssertions()) {
     Init *Condition = Assertion.Condition->resolveReferences(R);
     Init *Message = Assertion.Message->resolveReferences(R);
-    CheckAssert(Assertion.Loc, Condition, Message);
+    AnyFailed |= CheckAssert(Assertion.Loc, Condition, Message);
   }
+
+  if (!AnyFailed)
+    return;
+
+  // If any of the record assertions failed, print some context that will
+  // help see where the record that caused these assert failures is defined.
+  PrintError(this, "assertion failed in this record");
 }
 
 void Record::emitRecordDumps() {
@@ -3226,7 +3219,9 @@ void Record::checkUnusedTemplateArgs() {
 }
 
 RecordKeeper::RecordKeeper()
-    : Impl(std::make_unique<detail::RecordKeeperImpl>(*this)) {}
+    : Impl(std::make_unique<detail::RecordKeeperImpl>(*this)),
+      Timer(std::make_unique<TGTimer>()) {}
+
 RecordKeeper::~RecordKeeper() = default;
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -3250,65 +3245,20 @@ Init *RecordKeeper::getNewAnonymousName() {
   return AnonymousNameInit::get(*this, getImpl().AnonCounter++);
 }
 
-// These functions implement the phase timing facility. Starting a timer
-// when one is already running stops the running one.
-
-void RecordKeeper::startTimer(StringRef Name) const {
-  if (TimingGroup) {
-    if (LastTimer && LastTimer->isRunning()) {
-      LastTimer->stopTimer();
-      if (BackendTimer) {
-        LastTimer->clear();
-        BackendTimer = false;
-      }
-    }
-
-    LastTimer = new Timer("", Name, *TimingGroup);
-    LastTimer->startTimer();
-  }
-}
-
-void RecordKeeper::stopTimer() {
-  if (TimingGroup) {
-    assert(LastTimer && "No phase timer was started");
-    LastTimer->stopTimer();
-  }
-}
-
-void RecordKeeper::startBackendTimer(StringRef Name) {
-  if (TimingGroup) {
-    startTimer(Name);
-    BackendTimer = true;
-  }
-}
-
-void RecordKeeper::stopBackendTimer() {
-  if (TimingGroup) {
-    if (BackendTimer) {
-      stopTimer();
-      BackendTimer = false;
-    }
-  }
-}
-
-template <typename VecTy>
-const VecTy &RecordKeeper::getAllDerivedDefinitionsImpl(
-    StringRef ClassName, std::map<std::string, VecTy> &Cache) const {
+ArrayRef<const Record *>
+RecordKeeper::getAllDerivedDefinitions(StringRef ClassName) const {
   // We cache the record vectors for single classes. Many backends request
   // the same vectors multiple times.
-  auto Pair = Cache.try_emplace(ClassName.str());
-  if (Pair.second)
-    Pair.first->second =
-        getAllDerivedDefinitionsImpl<VecTy>(ArrayRef(ClassName));
-
-  return Pair.first->second;
+  auto [Iter, Inserted] = Cache.try_emplace(ClassName.str());
+  if (Inserted)
+    Iter->second = getAllDerivedDefinitions(ArrayRef(ClassName));
+  return Iter->second;
 }
 
-template <typename VecTy>
-VecTy RecordKeeper::getAllDerivedDefinitionsImpl(
-    ArrayRef<StringRef> ClassNames) const {
+std::vector<const Record *>
+RecordKeeper::getAllDerivedDefinitions(ArrayRef<StringRef> ClassNames) const {
   SmallVector<const Record *, 2> ClassRecs;
-  VecTy Defs;
+  std::vector<const Record *> Defs;
 
   assert(ClassNames.size() > 0 && "At least one class must be passed.");
   for (const auto &ClassName : ClassNames) {
@@ -3328,46 +3278,11 @@ VecTy RecordKeeper::getAllDerivedDefinitionsImpl(
   return Defs;
 }
 
-template <typename VecTy>
-const VecTy &RecordKeeper::getAllDerivedDefinitionsIfDefinedImpl(
-    StringRef ClassName, std::map<std::string, VecTy> &Cache) const {
-  return getClass(ClassName)
-             ? getAllDerivedDefinitionsImpl<VecTy>(ClassName, Cache)
-             : Cache[""];
-}
-
-ArrayRef<const Record *>
-RecordKeeper::getAllDerivedDefinitions(StringRef ClassName) const {
-  return getAllDerivedDefinitionsImpl<std::vector<const Record *>>(
-      ClassName, ClassRecordsMapConst);
-}
-
-const std::vector<Record *> &
-RecordKeeper::getAllDerivedDefinitions(StringRef ClassName) {
-  return getAllDerivedDefinitionsImpl<std::vector<Record *>>(ClassName,
-                                                             ClassRecordsMap);
-}
-
-std::vector<const Record *>
-RecordKeeper::getAllDerivedDefinitions(ArrayRef<StringRef> ClassNames) const {
-  return getAllDerivedDefinitionsImpl<std::vector<const Record *>>(ClassNames);
-}
-
-std::vector<Record *>
-RecordKeeper::getAllDerivedDefinitions(ArrayRef<StringRef> ClassNames) {
-  return getAllDerivedDefinitionsImpl<std::vector<Record *>>(ClassNames);
-}
-
 ArrayRef<const Record *>
 RecordKeeper::getAllDerivedDefinitionsIfDefined(StringRef ClassName) const {
-  return getAllDerivedDefinitionsIfDefinedImpl<std::vector<const Record *>>(
-      ClassName, ClassRecordsMapConst);
-}
-
-const std::vector<Record *> &
-RecordKeeper::getAllDerivedDefinitionsIfDefined(StringRef ClassName) {
-  return getAllDerivedDefinitionsIfDefinedImpl<std::vector<Record *>>(
-      ClassName, ClassRecordsMap);
+  if (getClass(ClassName))
+    return getAllDerivedDefinitions(ClassName);
+  return Cache[""];
 }
 
 void RecordKeeper::dumpAllocationStats(raw_ostream &OS) const {
diff --git a/llvm/lib/TableGen/SetTheory.cpp b/llvm/lib/TableGen/SetTheory.cpp
index edb99827f7c67..47718cc8b0e7f 100644
--- a/llvm/lib/TableGen/SetTheory.cpp
+++ b/llvm/lib/TableGen/SetTheory.cpp
@@ -222,7 +222,7 @@ struct SequenceOp : public SetTheory::Operator {
       std::string Name;
       raw_string_ostream OS(Name);
       OS << format(Format.c_str(), unsigned(From));
-      Record *Rec = Records.getDef(Name);
+      const Record *Rec = Records.getDef(Name);
       if (!Rec)
         PrintFatalError(Loc, "No def named '" + Name + "': " +
           Expr->getAsString());
diff --git a/llvm/lib/TableGen/TGParser.cpp b/llvm/lib/TableGen/TGParser.cpp
index 91a3617f8579e..aed4f3fe0e96b 100644
--- a/llvm/lib/TableGen/TGParser.cpp
+++ b/llvm/lib/TableGen/TGParser.cpp
@@ -328,12 +328,11 @@ bool TGParser::AddSubClass(Record *CurRec, SubClassReference &SubClass) {
 
   // Since everything went well, we can now set the "superclass" list for the
   // current record.
-  ArrayRef<std::pair<Record *, SMRange>> SCs = SC->getSuperClasses();
-  for (const auto &SCPair : SCs) {
-    if (CurRec->isSubClassOf(SCPair.first))
+  for (const auto &[SC, Loc] : SC->getSuperClasses()) {
+    if (CurRec->isSubClassOf(SC))
       return Error(SubClass.RefRange.Start,
-                   "Already subclass of '" + SCPair.first->getName() + "'!\n");
-    CurRec->addSuperClass(SCPair.first, SCPair.second);
+                   "Already subclass of '" + SC->getName() + "'!\n");
+    CurRec->addSuperClass(SC, Loc);
   }
 
   if (CurRec->isSubClassOf(SC))
@@ -542,7 +541,7 @@ bool TGParser::resolve(const std::vector<RecordsEntry> &Source,
 /// Resolve the record fully and add it to the record keeper.
 bool TGParser::addDefOne(std::unique_ptr<Record> Rec) {
   Init *NewName = nullptr;
-  if (Record *Prev = Records.getDef(Rec->getNameInitAsString())) {
+  if (const Record *Prev = Records.getDef(Rec->getNameInitAsString())) {
     if (!Rec->isAnonymous()) {
       PrintError(Rec->getLoc(),
                  "def already exists: " + Rec->getNameInitAsString());
@@ -1109,7 +1108,7 @@ const RecTy *TGParser::ParseType() {
       Lex.Lex();
       return I->second;
     }
-    if (Record *R = ParseClassID())
+    if (const Record *R = ParseClassID())
       return RecordRecTy::get(R);
     TokError("unknown class name");
     return nullptr;
@@ -2720,7 +2719,7 @@ Init *TGParser::ParseSimpleValue(Record *CurRec, const RecTy *ItemType,
 
     if (TrackReferenceLocs)
       Class->appendReferenceLoc(NameLoc);
-    return VarDefInit::get(Class, Args)->Fold();
+    return VarDefInit::get(NameLoc.Start, Class, Args)->Fold();
   }
   case tgtok::l_brace: {           // Value ::= '{' ValueList '}'
     SMLoc BraceLoc = Lex.getLoc();
@@ -3001,7 +3000,8 @@ Init *TGParser::ParseValue(Record *CurRec, const RecTy *ItemType,
       // Add a reference to this field if we know the record class.
       if (TrackReferenceLocs) {
         if (auto *DI = dyn_cast<DefInit>(Result)) {
-          DI->getDef()->getValue(FieldName)->addReferenceLoc(FieldNameLoc);
+          const RecordVal *V = DI->getDef()->getValue(FieldName);
+          const_cast<RecordVal *>(V)->addReferenceLoc(FieldNameLoc);
         } else if (auto *TI = dyn_cast<TypedInit>(Result)) {
           if (auto *RecTy = dyn_cast<RecordRecTy>(TI->getType())) {
             for (const Record *R : RecTy->getClasses())
diff --git a/llvm/lib/TableGen/TGTimer.cpp b/llvm/lib/TableGen/TGTimer.cpp
new file mode 100644
index 0000000000000..d95bd2e6be9da
--- /dev/null
+++ b/llvm/lib/TableGen/TGTimer.cpp
@@ -0,0 +1,54 @@
+//===- TGTimer.cpp - TableGen Timer implementation --------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implement the tablegen timer class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/TableGen/TGTimer.h"
+using namespace llvm;
+
+// These functions implement the phase timing facility. Starting a timer
+// when one is already running stops the running one.
+void TGTimer::startTimer(StringRef Name) {
+  if (!TimingGroup)
+    return;
+  if (LastTimer && LastTimer->isRunning()) {
+    LastTimer->stopTimer();
+    if (BackendTimer) {
+      LastTimer->clear();
+      BackendTimer = false;
+    }
+  }
+
+  LastTimer = std::make_unique<Timer>("", Name, *TimingGroup);
+  LastTimer->startTimer();
+}
+
+void TGTimer::stopTimer() {
+  if (!TimingGroup)
+    return;
+
+  assert(LastTimer && "No phase timer was started");
+  LastTimer->stopTimer();
+}
+
+void TGTimer::startBackendTimer(StringRef Name) {
+  if (!TimingGroup)
+    return;
+
+  startTimer(Name);
+  BackendTimer = true;
+}
+
+void TGTimer::stopBackendTimer() {
+  if (!TimingGroup || !BackendTimer)
+    return;
+  stopTimer();
+  BackendTimer = false;
+}
diff --git a/llvm/lib/TableGen/TableGenBackend.cpp b/llvm/lib/TableGen/TableGenBackend.cpp
index 210fff6545862..153ca39cfba08 100644
--- a/llvm/lib/TableGen/TableGenBackend.cpp
+++ b/llvm/lib/TableGen/TableGenBackend.cpp
@@ -60,7 +60,7 @@ Opt::Opt(StringRef Name, FnT CB, StringRef Desc, bool ByDefault) {
 
 /// Apply callback specified on the command line. Returns true if no callback
 /// was applied.
-bool llvm::TableGen::Emitter::ApplyCallback(RecordKeeper &Records,
+bool llvm::TableGen::Emitter::ApplyCallback(const RecordKeeper &Records,
                                             raw_ostream &OS) {
   FnT Fn = CallbackFunction->getValue();
   if (!Fn)
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index 69d6b02fefffe..97671bc59f6b9 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -501,7 +501,7 @@ def FeatureSSVE_FP8DOT2 : ExtensionWithMArch<"ssve-fp8dot2", "SSVE_FP8DOT2", "FE
   "Enable SVE2 FP8 2-way dot product instructions", [FeatureSSVE_FP8DOT4]>;
 
 def FeatureSME_LUTv2 : ExtensionWithMArch<"sme-lutv2", "SME_LUTv2", "FEAT_SME_LUTv2",
-  "Enable Scalable Matrix Extension (SME) LUTv2 instructions">;
+  "Enable Scalable Matrix Extension (SME) LUTv2 instructions", [FeatureSME2]>;
 
 def FeatureSMEF8F32 : ExtensionWithMArch<"sme-f8f32", "SMEF8F32", "FEAT_SME_F8F32",
   "Enable Scalable Matrix Extension (SME) F8F32 instructions", [FeatureSME2, FeatureFP8]>;
@@ -518,6 +518,11 @@ def FeaturePAuthLR : ExtensionWithMArch<"pauth-lr", "PAuthLR", "FEAT_PAuth_LR",
 def FeatureTLBIW : ExtensionWithMArch<"tlbiw", "TLBIW", "FEAT_TLBIW",
   "Enable Armv9.5-A TLBI VMALL for Dirty State">;
 
+//===----------------------------------------------------------------------===//
+//  Armv9.6 Architecture Extensions
+//===----------------------------------------------------------------------===//
+
+
 //===----------------------------------------------------------------------===//
 //  Other Features
 //===----------------------------------------------------------------------===//
@@ -827,6 +832,9 @@ def HasV9_4aOps : Architecture64<9, 4, "a", "v9.4a",
 def HasV9_5aOps : Architecture64<9, 5, "a", "v9.5a",
   [HasV9_4aOps, FeatureCPA],
   !listconcat(HasV9_4aOps.DefaultExts, [FeatureCPA,  FeatureLUT, FeatureFAMINMAX])>;
+def HasV9_6aOps : Architecture64<9, 6, "a", "v9.6a",
+  [HasV9_5aOps],
+  !listconcat(HasV9_5aOps.DefaultExts, [])>;
 def HasV8_0rOps : Architecture64<8, 0, "r", "v8r",
   [ //v8.1
     FeatureCRC, FeaturePAN, FeatureLSE, FeatureCONTEXTIDREL2,
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 3de09eca94233..48e1b96d841ef 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -4301,6 +4301,29 @@ static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) {
                      Op.getOperand(1));
 }
 
+// Converts SETCC (AND X Y) Z ULT -> SETCC (AND X (Y & ~(Z - 1)) 0 EQ when Y is
+// a power of 2. This is then lowered to ANDS X (Y & ~(Z - 1)) instead of SUBS
+// (AND X Y) Z which produces a better opt with EmitComparison
+static void simplifySetCCIntoEq(ISD::CondCode &CC, SDValue &LHS, SDValue &RHS,
+                                SelectionDAG &DAG, const SDLoc dl) {
+  if (CC == ISD::SETULT && LHS.getOpcode() == ISD::AND && LHS->hasOneUse()) {
+    ConstantSDNode *LHSConstOp = dyn_cast<ConstantSDNode>(LHS.getOperand(1));
+    ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
+    if (LHSConstOp && RHSConst) {
+      uint64_t LHSConstValue = LHSConstOp->getZExtValue();
+      uint64_t RHSConstant = RHSConst->getZExtValue();
+      if (isPowerOf2_64(RHSConstant)) {
+        uint64_t NewMaskValue = LHSConstValue & ~(RHSConstant - 1);
+        LHS =
+            DAG.getNode(ISD::AND, dl, LHS.getValueType(), LHS.getOperand(0),
+                        DAG.getConstant(NewMaskValue, dl, LHS.getValueType()));
+        RHS = DAG.getConstant(0, dl, RHS.getValueType());
+        CC = ISD::SETEQ;
+      }
+    }
+  }
+}
+
 SDValue AArch64TargetLowering::LowerFP_EXTEND(SDValue Op,
                                               SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
@@ -5549,10 +5572,17 @@ static SDValue getSVEPredicateBitCast(EVT VT, SDValue Op, SelectionDAG &DAG) {
          "Only expect to cast between legal scalable predicate types!");
 
   // Return the operand if the cast isn't changing type,
-  // e.g. <n x 16 x i1> -> <n x 16 x i1>
   if (InVT == VT)
     return Op;
 
+  // Look through casts to <vscale x 16 x i1> when their input has more lanes
+  // than VT. This will increase the chances of removing casts that introduce
+  // new lanes, which have to be explicitly zero'd.
+  if (Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
+      Op.getConstantOperandVal(0) == Intrinsic::aarch64_sve_convert_to_svbool &&
+      Op.getOperand(1).getValueType().bitsGT(VT))
+    Op = Op.getOperand(1);
+
   SDValue Reinterpret = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Op);
 
   // We only have to zero the lanes if new lanes are being defined, e.g. when
@@ -10589,6 +10619,9 @@ SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
   }
 
   if (LHS.getValueType().isInteger()) {
+
+    simplifySetCCIntoEq(CC, LHS, RHS, DAG, dl);
+
     SDValue CCVal;
     SDValue Cmp = getAArch64Cmp(
         LHS, RHS, ISD::getSetCCInverse(CC, LHS.getValueType()), CCVal, DAG, dl);
diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index 17b28e5bfe861..802797a14ee42 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -938,10 +938,11 @@ defm FAMAX_4Z4Z : sme2_fp_sve_destructive_vector_vg4_multi<"famax", 0b0010100>;
 defm FAMIN_4Z4Z : sme2_fp_sve_destructive_vector_vg4_multi<"famin", 0b0010101>;
 } //[HasSME2, HasFAMINMAX]
 
-let Predicates = [HasSME2, HasSME_LUTv2] in {
+
+let Predicates = [HasSME_LUTv2] in {
 defm MOVT_TIZ : sme2_movt_zt_to_zt<"movt",  0b0011111, int_aarch64_sme_write_lane_zt, int_aarch64_sme_write_zt>;
 def LUTI4_4ZZT2Z    : sme2_luti4_vector_vg4<0b00, 0b00,"luti4">;
-} //[HasSME2, HasSME_LUTv2]
+} //[HasSME_LUTv2]
 
 let Predicates = [HasSME2p1, HasSME_LUTv2] in {
 def LUTI4_S_4ZZT2Z  : sme2_luti4_vector_vg4_strided<0b00, 0b00, "luti4">;
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 7240f6a22a87b..76362768e0aa6 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -2425,8 +2425,8 @@ let Predicates = [HasBF16, HasSVEorSME] in {
   defm BFMLALT_ZZZ : sve2_fp_mla_long<0b101, "bfmlalt", nxv4f32, nxv8bf16, int_aarch64_sve_bfmlalt>;
   defm BFMLALB_ZZZI : sve2_fp_mla_long_by_indexed_elem<0b100, "bfmlalb", nxv4f32, nxv8bf16, int_aarch64_sve_bfmlalb_lane_v2>;
   defm BFMLALT_ZZZI : sve2_fp_mla_long_by_indexed_elem<0b101, "bfmlalt", nxv4f32, nxv8bf16, int_aarch64_sve_bfmlalt_lane_v2>;
-  defm BFCVT_ZPmZ   : sve_bfloat_convert<0b1, "bfcvt",   int_aarch64_sve_fcvt_bf16f32, AArch64fcvtr_mt>;
-  defm BFCVTNT_ZPmZ : sve_bfloat_convert<0b0, "bfcvtnt", int_aarch64_sve_fcvtnt_bf16f32>;
+  defm BFCVT_ZPmZ   : sve_bfloat_convert<0b1, "bfcvt",   int_aarch64_sve_fcvt_bf16f32_v2, AArch64fcvtr_mt>;
+  defm BFCVTNT_ZPmZ : sve_bfloat_convert<0b0, "bfcvtnt", int_aarch64_sve_fcvtnt_bf16f32_v2>;
 } // End HasBF16, HasSVEorSME
 
 let Predicates = [HasSVEorSME] in {
diff --git a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
index 72823fdcd858d..1d2b257377828 100644
--- a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
+++ b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
@@ -487,12 +487,11 @@ Instruction *AArch64StackTagging::insertBaseTaggedPointer(
       IRB.CreateCall(IRG_SP, {Constant::getNullValue(IRB.getInt64Ty())});
   Base->setName("basetag");
   auto TargetTriple = Triple(M.getTargetTriple());
-  // This is not a stable ABI for now, so only allow in dev builds with API
-  // level 10000.
+  // This ABI will make it into Android API level 35.
   // The ThreadLong format is the same as with HWASan, but the entries for
   // stack MTE take two slots (16 bytes).
   if (ClRecordStackHistory == instr && TargetTriple.isAndroid() &&
-      TargetTriple.isAArch64() && !TargetTriple.isAndroidVersionLT(10000) &&
+      TargetTriple.isAArch64() && !TargetTriple.isAndroidVersionLT(35) &&
       !AllocasToInstrument.empty()) {
     constexpr int StackMteSlot = -3;
     constexpr uint64_t TagMask = 0xFULL << 56;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 84aea83424e4d..80d5168ae961a 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -2157,7 +2157,7 @@ AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
   switch (IID) {
   default:
     break;
-  case Intrinsic::aarch64_sve_fcvt_bf16f32:
+  case Intrinsic::aarch64_sve_fcvt_bf16f32_v2:
   case Intrinsic::aarch64_sve_fcvt_f16f32:
   case Intrinsic::aarch64_sve_fcvt_f16f64:
   case Intrinsic::aarch64_sve_fcvt_f32f16:
@@ -2188,7 +2188,7 @@ AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
   case Intrinsic::aarch64_sve_ucvtf_f32i64:
   case Intrinsic::aarch64_sve_ucvtf_f64i32:
     return instCombineSVEAllOrNoActiveUnary(IC, II);
-  case Intrinsic::aarch64_sve_fcvtnt_bf16f32:
+  case Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2:
   case Intrinsic::aarch64_sve_fcvtnt_f16f32:
   case Intrinsic::aarch64_sve_fcvtnt_f32f64:
   case Intrinsic::aarch64_sve_fcvtxnt_f32f64:
diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index 6a4b94a216832..df69c20b1359f 100644
--- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -3771,6 +3771,8 @@ static void setRequiredFeatureString(FeatureBitset FBS, std::string &Str) {
     Str += "ARMv9.4a";
   else if (FBS[AArch64::HasV9_5aOps])
     Str += "ARMv9.5a";
+  else if (FBS[AArch64::HasV9_6aOps])
+    Str += "ARMv9.6a";
   else if (FBS[AArch64::HasV8_0rOps])
     Str += "ARMv8r";
   else {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 51aeee023f2e3..3d313ca00f125 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -275,7 +275,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .lowerFor({s16, v4s16, v8s16});
 
   getActionDefinitionsBuilder(G_FREM)
-      .libcallFor({s32, s64})
+      .libcallFor({s32, s64, s128})
       .minScalar(0, s32)
       .scalarize(0);
 
@@ -291,11 +291,11 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .scalarize(0)
       // Regardless of FP16 support, widen 16-bit elements to 32-bits.
       .minScalar(0, s32)
-      .libcallFor({s32, s64});
+      .libcallFor({s32, s64, s128});
   getActionDefinitionsBuilder(G_FPOWI)
       .scalarize(0)
       .minScalar(0, s32)
-      .libcallFor({{s32, s32}, {s64, s32}});
+      .libcallFor({{s32, s32}, {s64, s32}, {s128, s32}});
 
   getActionDefinitionsBuilder(G_INSERT)
       .legalIf(all(typeInSet(0, {s32, s64, p0}),
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 0bfac6465a1f3..13c2a90a963f8 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -8811,7 +8811,7 @@ multiclass sve_bfloat_convert<bit N, string asm, SDPatternOperator op,
                               SDPatternOperator ir_op = null_frag> {
   def NAME : sve_bfloat_convert<N, asm>;
 
-  def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i1, nxv4f32, !cast<Instruction>(NAME)>;
+  def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv4i1, nxv4f32, !cast<Instruction>(NAME)>;
   def : SVE_1_Op_Passthru_Round_Pat<nxv4bf16, ir_op, nxv4i1, nxv4f32, !cast<Instruction>(NAME)>;
   def : SVE_1_Op_Passthru_Round_Pat<nxv2bf16, ir_op, nxv2i1, nxv2f32, !cast<Instruction>(NAME)>;
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsanInstrumentation.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsanInstrumentation.cpp
index 4c8ddbd9aabd5..a6a8597f22a4a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsanInstrumentation.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsanInstrumentation.cpp
@@ -235,7 +235,7 @@ void getInterestingMemoryOperands(
     Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true,
                              XCHG->getCompareOperand()->getType(),
                              std::nullopt);
-  } else if (auto CI = dyn_cast<CallInst>(I)) {
+  } else if (auto *CI = dyn_cast<CallInst>(I)) {
     switch (CI->getIntrinsicID()) {
     case Intrinsic::masked_load:
     case Intrinsic::masked_store:
@@ -257,7 +257,7 @@ void getInterestingMemoryOperands(
     case Intrinsic::masked_compressstore: {
       bool IsWrite = CI->getIntrinsicID() == Intrinsic::masked_compressstore;
       unsigned OpOffset = IsWrite ? 1 : 0;
-      auto BasePtr = CI->getOperand(OpOffset);
+      auto *BasePtr = CI->getOperand(OpOffset);
       MaybeAlign Alignment = BasePtr->getPointerAlignment(DL);
       Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();
       IRBuilder<> IB(I);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index 4f6633d8027c7..93b6ba0595b70 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -332,7 +332,7 @@ void AMDGPUAsmPrinter::emitGlobalVariable(const GlobalVariable *GV) {
 
     emitVisibility(GVSym, GV->getVisibility(), !GV->isDeclaration());
     emitLinkage(GV, GVSym);
-    auto TS = getTargetStreamer();
+    auto *TS = getTargetStreamer();
     TS->emitAMDGPULDS(GVSym, Size, Alignment);
     return;
   }
@@ -358,6 +358,7 @@ bool AMDGPUAsmPrinter::doInitialization(Module &M) {
       report_fatal_error("Unexpected code object version");
     }
   }
+
   return AsmPrinter::doInitialization(M);
 }
 
@@ -511,6 +512,8 @@ bool AMDGPUAsmPrinter::doFinalization(Module &M) {
   for (Function &F : M.functions())
     validateMCResourceInfo(F);
 
+  RI.reset();
+
   return AsmPrinter::doFinalization(M);
 }
 
@@ -1235,8 +1238,8 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
     // return ((Dst & ~Mask) | (Value << Shift))
     auto SetBits = [&Ctx](const MCExpr *Dst, const MCExpr *Value, uint32_t Mask,
                           uint32_t Shift) {
-      auto Shft = MCConstantExpr::create(Shift, Ctx);
-      auto Msk = MCConstantExpr::create(Mask, Ctx);
+      const auto *Shft = MCConstantExpr::create(Shift, Ctx);
+      const auto *Msk = MCConstantExpr::create(Mask, Ctx);
       Dst = MCBinaryExpr::createAnd(Dst, MCUnaryExpr::createNot(Msk, Ctx), Ctx);
       Dst = MCBinaryExpr::createOr(
           Dst, MCBinaryExpr::createShl(Value, Shft, Ctx), Ctx);
@@ -1411,7 +1414,7 @@ void AMDGPUAsmPrinter::EmitPALMetadata(const MachineFunction &MF,
        const SIProgramInfo &CurrentProgramInfo) {
   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
   auto CC = MF.getFunction().getCallingConv();
-  auto MD = getTargetStreamer()->getPALMetadata();
+  auto *MD = getTargetStreamer()->getPALMetadata();
   auto &Ctx = MF.getContext();
 
   MD->setEntryPoint(CC, MF.getFunction().getName());
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
index 25e36dc4b3691..351e9f25e29cf 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
@@ -142,7 +142,7 @@ struct AMDGPUIncomingArgHandler : public CallLowering::IncomingValueHandler {
                             const CCValAssign &VA) override {
     MachineFunction &MF = MIRBuilder.getMF();
 
-    auto MMO = MF.getMachineMemOperand(
+    auto *MMO = MF.getMachineMemOperand(
         MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, MemTy,
         inferAlignFromPtrInfo(MF, MPO));
     MIRBuilder.buildLoad(ValVReg, Addr, *MMO);
@@ -244,7 +244,7 @@ struct AMDGPUOutgoingArgHandler : public AMDGPUOutgoingValueHandler {
     uint64_t LocMemOffset = VA.getLocMemOffset();
     const auto &ST = MF.getSubtarget<GCNSubtarget>();
 
-    auto MMO = MF.getMachineMemOperand(
+    auto *MMO = MF.getMachineMemOperand(
         MPO, MachineMemOperand::MOStore, MemTy,
         commonAlignment(ST.getStackAlignment(), LocMemOffset));
     MIRBuilder.buildStore(ValVReg, Addr, *MMO);
@@ -1007,7 +1007,7 @@ bool AMDGPUCallLowering::doCallerAndCalleePassArgsTheSameWay(
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
 
   // Make sure that the caller and callee preserve all of the same registers.
-  auto TRI = ST.getRegisterInfo();
+  const auto *TRI = ST.getRegisterInfo();
 
   const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
   const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
@@ -1142,6 +1142,9 @@ bool AMDGPUCallLowering::isEligibleForTailCallOptimization(
     return false;
   }
 
+  // FIXME: We need to check if any arguments passed in SGPR are uniform. If
+  // they are not, this cannot be a tail call. If they are uniform, but may be
+  // VGPR, we need to insert readfirstlanes.
   if (!areCalleeOutgoingArgsTailCallable(Info, MF, OutArgs))
     return false;
 
@@ -1219,7 +1222,7 @@ bool AMDGPUCallLowering::lowerTailCall(
     if (!ExecArg.Ty->isIntegerTy(ST.getWavefrontSize()))
       return false;
 
-    if (auto CI = dyn_cast<ConstantInt>(ExecArg.OrigValue)) {
+    if (const auto *CI = dyn_cast<ConstantInt>(ExecArg.OrigValue)) {
       MIB.addImm(CI->getSExtValue());
     } else {
       MIB.addReg(ExecArg.Regs[0]);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
index 26116bfa3c2fe..b67d78e450bb8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
@@ -163,8 +163,8 @@ std::string MetadataStreamerMsgPackV4::getTypeName(Type *Ty,
   case Type::DoubleTyID:
     return "double";
   case Type::FixedVectorTyID: {
-    auto VecTy = cast<FixedVectorType>(Ty);
-    auto ElTy = VecTy->getElementType();
+    auto *VecTy = cast<FixedVectorType>(Ty);
+    auto *ElTy = VecTy->getElementType();
     auto NumElements = VecTy->getNumElements();
     return (Twine(getTypeName(ElTy, Signed)) + Twine(NumElements)).str();
   }
@@ -199,7 +199,7 @@ void MetadataStreamerMsgPackV4::emitTargetID(
 }
 
 void MetadataStreamerMsgPackV4::emitPrintf(const Module &Mod) {
-  auto Node = Mod.getNamedMetadata("llvm.printf.fmts");
+  auto *Node = Mod.getNamedMetadata("llvm.printf.fmts");
   if (!Node)
     return;
 
@@ -214,10 +214,10 @@ void MetadataStreamerMsgPackV4::emitPrintf(const Module &Mod) {
 void MetadataStreamerMsgPackV4::emitKernelLanguage(const Function &Func,
                                                    msgpack::MapDocNode Kern) {
   // TODO: What about other languages?
-  auto Node = Func.getParent()->getNamedMetadata("opencl.ocl.version");
+  auto *Node = Func.getParent()->getNamedMetadata("opencl.ocl.version");
   if (!Node || !Node->getNumOperands())
     return;
-  auto Op0 = Node->getOperand(0);
+  auto *Op0 = Node->getOperand(0);
   if (Op0->getNumOperands() <= 1)
     return;
 
@@ -233,11 +233,11 @@ void MetadataStreamerMsgPackV4::emitKernelLanguage(const Function &Func,
 void MetadataStreamerMsgPackV4::emitKernelAttrs(const Function &Func,
                                                 msgpack::MapDocNode Kern) {
 
-  if (auto Node = Func.getMetadata("reqd_work_group_size"))
+  if (auto *Node = Func.getMetadata("reqd_work_group_size"))
     Kern[".reqd_workgroup_size"] = getWorkGroupDimensions(Node);
-  if (auto Node = Func.getMetadata("work_group_size_hint"))
+  if (auto *Node = Func.getMetadata("work_group_size_hint"))
     Kern[".workgroup_size_hint"] = getWorkGroupDimensions(Node);
-  if (auto Node = Func.getMetadata("vec_type_hint")) {
+  if (auto *Node = Func.getMetadata("vec_type_hint")) {
     Kern[".vec_type_hint"] = Kern.getDocument()->getNode(
         getTypeName(
             cast<ValueAsMetadata>(Node->getOperand(0))->getType(),
@@ -271,7 +271,7 @@ void MetadataStreamerMsgPackV4::emitKernelArgs(const MachineFunction &MF,
 void MetadataStreamerMsgPackV4::emitKernelArg(const Argument &Arg,
                                               unsigned &Offset,
                                               msgpack::ArrayDocNode Args) {
-  auto Func = Arg.getParent();
+  const auto *Func = Arg.getParent();
   auto ArgNo = Arg.getArgNo();
   const MDNode *Node;
 
@@ -317,7 +317,7 @@ void MetadataStreamerMsgPackV4::emitKernelArg(const Argument &Arg,
   Type *Ty = Arg.hasByRefAttr() ? Arg.getParamByRefType() : Arg.getType();
 
   // FIXME: Need to distinguish in memory alignment from pointer alignment.
-  if (auto PtrTy = dyn_cast<PointerType>(Ty)) {
+  if (auto *PtrTy = dyn_cast<PointerType>(Ty)) {
     if (PtrTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS)
       PointeeAlign = Arg.getParamAlign().valueOrOne();
   }
@@ -353,7 +353,7 @@ void MetadataStreamerMsgPackV4::emitKernelArg(
   if (PointeeAlign)
     Arg[".pointee_align"] = Arg.getDocument()->getNode(PointeeAlign->value());
 
-  if (auto PtrTy = dyn_cast<PointerType>(Ty))
+  if (auto *PtrTy = dyn_cast<PointerType>(Ty))
     if (auto Qualifier = getAddressSpaceQualifier(PtrTy->getAddressSpace()))
       // Limiting address space to emit only for a certain ValueKind.
       if (ValueKind == "global_buffer" || ValueKind == "dynamic_shared_pointer")
@@ -393,7 +393,7 @@ void MetadataStreamerMsgPackV4::emitHiddenKernelArgs(
 
   const Module *M = Func.getParent();
   auto &DL = M->getDataLayout();
-  auto Int64Ty = Type::getInt64Ty(Func.getContext());
+  auto *Int64Ty = Type::getInt64Ty(Func.getContext());
 
   Offset = alignTo(Offset, ST.getAlignmentForImplicitArgPtr());
 
@@ -407,7 +407,7 @@ void MetadataStreamerMsgPackV4::emitHiddenKernelArgs(
     emitKernelArg(DL, Int64Ty, Align(8), "hidden_global_offset_z", Offset,
                   Args);
 
-  auto Int8PtrTy =
+  auto *Int8PtrTy =
       PointerType::get(Func.getContext(), AMDGPUAS::GLOBAL_ADDRESS);
 
   if (HiddenArgNumBytes >= 32) {
@@ -592,9 +592,9 @@ void MetadataStreamerMsgPackV5::emitHiddenKernelArgs(
   auto &DL = M->getDataLayout();
   const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
 
-  auto Int64Ty = Type::getInt64Ty(Func.getContext());
-  auto Int32Ty = Type::getInt32Ty(Func.getContext());
-  auto Int16Ty = Type::getInt16Ty(Func.getContext());
+  auto *Int64Ty = Type::getInt64Ty(Func.getContext());
+  auto *Int32Ty = Type::getInt32Ty(Func.getContext());
+  auto *Int16Ty = Type::getInt16Ty(Func.getContext());
 
   Offset = alignTo(Offset, ST.getAlignmentForImplicitArgPtr());
   emitKernelArg(DL, Int32Ty, Align(4), "hidden_block_count_x", Offset, Args);
@@ -621,7 +621,7 @@ void MetadataStreamerMsgPackV5::emitHiddenKernelArgs(
   emitKernelArg(DL, Int16Ty, Align(2), "hidden_grid_dims", Offset, Args);
 
   Offset += 6; // Reserved.
-  auto Int8PtrTy =
+  auto *Int8PtrTy =
       PointerType::get(Func.getContext(), AMDGPUAS::GLOBAL_ADDRESS);
 
   if (M->getNamedMetadata("llvm.printf.fmts")) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 4e913d1b32e1f..a367db7062774 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -394,7 +394,7 @@ void PipelineSolver::reset() {
     for (auto &SG : SyncPipeline) {
       SmallVector<SUnit *, 32> TempCollection = SG.Collection;
       SG.Collection.clear();
-      auto SchedBarr = llvm::find_if(TempCollection, [](SUnit *SU) {
+      auto *SchedBarr = llvm::find_if(TempCollection, [](SUnit *SU) {
         return SU->getInstr()->getOpcode() == AMDGPU::SCHED_GROUP_BARRIER;
       });
       if (SchedBarr != TempCollection.end())
@@ -421,7 +421,7 @@ void PipelineSolver::convertSyncMapsToArrays() {
             std::pair(SUsToCandSGs.first, SUsToCandSGs.second));
         continue;
       }
-      auto SortPosition = PipelineInstrs[PipelineIDx].begin();
+      auto *SortPosition = PipelineInstrs[PipelineIDx].begin();
       // Insert them in sorted order -- this allows for good parsing order in
       // the greedy algorithm
       while (SortPosition != PipelineInstrs[PipelineIDx].end() &&
@@ -515,7 +515,7 @@ void PipelineSolver::removeEdges(
     SUnit *Pred = PredSuccPair.first;
     SUnit *Succ = PredSuccPair.second;
 
-    auto Match = llvm::find_if(
+    auto *Match = llvm::find_if(
         Succ->Preds, [&Pred](SDep &P) { return P.getSUnit() == Pred; });
     if (Match != Succ->Preds.end()) {
       assert(Match->isArtificial());
@@ -639,8 +639,8 @@ bool PipelineSolver::solveExact() {
              : populateReadyList(ReadyList, CurrSU.second.begin(),
                                  CurrSU.second.end());
 
-  auto I = ReadyList.begin();
-  auto E = ReadyList.end();
+  auto *I = ReadyList.begin();
+  auto *E = ReadyList.end();
   for (; I != E; ++I) {
     // If we are trying SGs in least cost order, and the current SG is cost
     // infeasible, then all subsequent SGs will also be cost infeasible, so we
@@ -942,7 +942,7 @@ class MFMAExpInterleaveOpt final : public IGLPStrategy {
     bool apply(const SUnit *SU, const ArrayRef<SUnit *> Collection,
                SmallVectorImpl<SchedGroup> &SyncPipe) override {
 
-      auto DAG = SyncPipe[0].DAG;
+      auto *DAG = SyncPipe[0].DAG;
 
       if (Cache->empty()) {
         auto I = DAG->SUnits.rbegin();
@@ -976,7 +976,7 @@ class MFMAExpInterleaveOpt final : public IGLPStrategy {
                SmallVectorImpl<SchedGroup> &SyncPipe) override {
       bool FoundTrans = false;
       unsigned Counter = 1;
-      auto DAG = SyncPipe[0].DAG;
+      auto *DAG = SyncPipe[0].DAG;
 
       if (Cache->empty()) {
         SmallVector<SUnit *, 8> Worklist;
@@ -1016,13 +1016,13 @@ class MFMAExpInterleaveOpt final : public IGLPStrategy {
   public:
     bool apply(const SUnit *SU, const ArrayRef<SUnit *> Collection,
                SmallVectorImpl<SchedGroup> &SyncPipe) override {
-      auto DAG = SyncPipe[0].DAG;
+      auto *DAG = SyncPipe[0].DAG;
 
       if (!SU || !TII->isMFMAorWMMA(*ChainSeed->getInstr()))
         return false;
 
       if (Cache->empty()) {
-        auto TempSU = ChainSeed;
+        auto *TempSU = ChainSeed;
         auto Depth = Number;
         while (Depth > 0) {
           --Depth;
@@ -1232,7 +1232,7 @@ class MFMAExpInterleaveOpt final : public IGLPStrategy {
       if (!OtherGroup->Collection.size())
         return true;
 
-      auto DAG = SyncPipe[0].DAG;
+      auto *DAG = SyncPipe[0].DAG;
 
       for (auto &OtherEle : OtherGroup->Collection)
         if (DAG->IsReachable(const_cast<SUnit *>(SU), OtherEle))
@@ -1275,7 +1275,7 @@ class MFMAExpInterleaveOpt final : public IGLPStrategy {
         return false;
 
       if (Cache->empty()) {
-        auto TempSU = ChainSeed;
+        auto *TempSU = ChainSeed;
         auto Depth = Number;
         while (Depth > 0) {
           --Depth;
@@ -1315,7 +1315,7 @@ class MFMAExpInterleaveOpt final : public IGLPStrategy {
                SmallVectorImpl<SchedGroup> &SyncPipe) override {
 
       SmallVector<SUnit *, 12> Worklist;
-      auto DAG = SyncPipe[0].DAG;
+      auto *DAG = SyncPipe[0].DAG;
       if (Cache->empty()) {
         for (auto &SU : DAG->SUnits)
           if (TII->isTRANS(SU.getInstr()->getOpcode())) {
@@ -1509,7 +1509,7 @@ bool MFMAExpInterleaveOpt::analyzeDAG(const SIInstrInfo *TII) {
                       return isBitPack(Opc);
                     });
 
-  auto PackPred =
+  auto *PackPred =
       std::find_if((*TempMFMA)->Preds.begin(), (*TempMFMA)->Preds.end(),
                    [&isBitPack](SDep &Pred) {
                      auto Opc = Pred.getSUnit()->getInstr()->getOpcode();
@@ -1868,7 +1868,7 @@ class MFMASmallGemmSingleWaveOpt final : public IGLPStrategy {
       }
 
       assert(Cache->size());
-      auto DAG = SyncPipe[0].DAG;
+      auto *DAG = SyncPipe[0].DAG;
       for (auto &Elt : *Cache) {
         if (DAG->IsReachable(Elt, const_cast<SUnit *>(SU)))
           return true;
@@ -1886,7 +1886,7 @@ class MFMASmallGemmSingleWaveOpt final : public IGLPStrategy {
   public:
     bool apply(const SUnit *SU, const ArrayRef<SUnit *> Collection,
                SmallVectorImpl<SchedGroup> &SyncPipe) override {
-      auto MI = SU->getInstr();
+      auto *MI = SU->getInstr();
       if (MI->getOpcode() != AMDGPU::V_PERM_B32_e64)
         return false;
 
@@ -1952,7 +1952,7 @@ class MFMASmallGemmSingleWaveOpt final : public IGLPStrategy {
   public:
     bool apply(const SUnit *SU, const ArrayRef<SUnit *> Collection,
                SmallVectorImpl<SchedGroup> &SyncPipe) override {
-      auto MI = SU->getInstr();
+      auto *MI = SU->getInstr();
       if (MI->getOpcode() == TargetOpcode::BUNDLE)
         return false;
       if (!Collection.size())
@@ -2023,7 +2023,7 @@ class MFMASmallGemmSingleWaveOpt final : public IGLPStrategy {
           return false;
       }
 
-      auto DAG = SyncPipe[0].DAG;
+      auto *DAG = SyncPipe[0].DAG;
       // Does the previous DS_WRITE share a V_PERM predecessor with this
       // VMEM_READ
       return llvm::any_of(*Cache, [&SU, &DAG](SUnit *Elt) {
@@ -2070,7 +2070,7 @@ bool MFMASmallGemmSingleWaveOpt::applyIGLPStrategy(
          "DSWCounters should be zero in pre-RA scheduling!");
   SmallVector<SUnit *, 6> DSWithPerms;
   for (auto &SU : DAG->SUnits) {
-    auto I = SU.getInstr();
+    auto *I = SU.getInstr();
     if (TII->isMFMAorWMMA(*I))
       ++MFMACount;
     else if (TII->isDS(*I)) {
@@ -2091,8 +2091,8 @@ bool MFMASmallGemmSingleWaveOpt::applyIGLPStrategy(
 
   if (IsInitial) {
     DSWWithPermCount = DSWithPerms.size();
-    auto I = DSWithPerms.begin();
-    auto E = DSWithPerms.end();
+    auto *I = DSWithPerms.begin();
+    auto *E = DSWithPerms.end();
 
     // Get the count of DS_WRITES with V_PERM predecessors which
     // have loop carried dependencies (WAR) on the same VMEM_READs.
@@ -2113,7 +2113,7 @@ bool MFMASmallGemmSingleWaveOpt::applyIGLPStrategy(
           break;
 
         for (auto &Succ : Pred.getSUnit()->Succs) {
-          auto MI = Succ.getSUnit()->getInstr();
+          auto *MI = Succ.getSUnit()->getInstr();
           if (!TII->isVMEM(*MI) || !MI->mayLoad())
             continue;
 
@@ -2123,13 +2123,13 @@ bool MFMASmallGemmSingleWaveOpt::applyIGLPStrategy(
             continue;
           }
 
-          if (!VMEMLookup.contains(MI)) {
+          auto [It, Inserted] = VMEMLookup.try_emplace(MI, *I);
+          if (Inserted) {
             MissedAny = true;
-            VMEMLookup[MI] = *I;
             continue;
           }
 
-          Cand = VMEMLookup[MI];
+          Cand = It->second;
           if (llvm::is_contained(Counted, Cand)) {
             MissedAny = true;
             break;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index ff8798edb3cc0..21fffba14287e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -1537,7 +1537,7 @@ static bool IsCopyFromSGPR(const SIRegisterInfo &TRI, SDValue Val) {
   auto Reg = cast<RegisterSDNode>(Val.getOperand(1))->getReg();
   if (!Reg.isPhysical())
     return false;
-  auto RC = TRI.getPhysRegBaseClass(Reg);
+  const auto *RC = TRI.getPhysRegBaseClass(Reg);
   return RC && TRI.isSGPRClass(RC);
 }
 
@@ -1855,13 +1855,13 @@ bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(SDNode *N,
 }
 
 static SDValue SelectSAddrFI(SelectionDAG *CurDAG, SDValue SAddr) {
-  if (auto FI = dyn_cast<FrameIndexSDNode>(SAddr)) {
+  if (auto *FI = dyn_cast<FrameIndexSDNode>(SAddr)) {
     SAddr = CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0));
   } else if (SAddr.getOpcode() == ISD::ADD &&
              isa<FrameIndexSDNode>(SAddr.getOperand(0))) {
     // Materialize this into a scalar move for scalar address to avoid
     // readfirstlane.
-    auto FI = cast<FrameIndexSDNode>(SAddr.getOperand(0));
+    auto *FI = cast<FrameIndexSDNode>(SAddr.getOperand(0));
     SDValue TFI = CurDAG->getTargetFrameIndex(FI->getIndex(),
                                               FI->getValueType(0));
     SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_I32, SDLoc(SAddr),
@@ -2391,7 +2391,7 @@ bool AMDGPUDAGToDAGISel::isCBranchSCC(const SDNode *N) const {
     return true;
 
   if (VT == MVT::i64) {
-    auto ST = static_cast<const GCNSubtarget *>(Subtarget);
+    const auto *ST = static_cast<const GCNSubtarget *>(Subtarget);
 
     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
     return (CC == ISD::SETEQ || CC == ISD::SETNE) && ST->hasScalarCompareEq64();
@@ -3600,7 +3600,7 @@ bool AMDGPUDAGToDAGISel::isVGPRImm(const SDNode * N) const {
 }
 
 bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode *N) const {
-  auto Ld = cast<LoadSDNode>(N);
+  const auto *Ld = cast<LoadSDNode>(N);
 
   const MachineMemOperand *MMO = Ld->getMemOperand();
   if (N->isDivergent() && !AMDGPUInstrInfo::isUniformMMO(MMO))
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 94fdf4effa10a..cceb89e23f129 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -4202,7 +4202,7 @@ SDValue AMDGPUTargetLowering::performTruncateCombine(
   // integer operation.
   // trunc (srl (bitcast (build_vector x, y))), 16 -> trunc (bitcast y)
   if (Src.getOpcode() == ISD::SRL && !VT.isVector()) {
-    if (auto K = isConstOrConstSplat(Src.getOperand(1))) {
+    if (auto *K = isConstOrConstSplat(Src.getOperand(1))) {
       if (2 * K->getZExtValue() == Src.getValueType().getScalarSizeInBits()) {
         SDValue BV = stripBitcast(Src.getOperand(0));
         if (BV.getOpcode() == ISD::BUILD_VECTOR &&
@@ -5779,7 +5779,7 @@ void AMDGPUTargetLowering::computeKnownBitsForTargetNode(
     break;
   }
   case AMDGPUISD::LDS: {
-    auto GA = cast<GlobalAddressSDNode>(Op.getOperand(0).getNode());
+    auto *GA = cast<GlobalAddressSDNode>(Op.getOperand(0).getNode());
     Align Alignment = GA->getGlobal()->getPointerAlignment(DAG.getDataLayout());
 
     Known.Zero.setHighBits(16);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp
index 88429e3f0e218..45207c06a788a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp
@@ -116,8 +116,8 @@ void addInstToMergeableList(
       Value *Arg = II->getArgOperand(I);
       if (I == ImageDimIntr->VAddrEnd - 1) {
         // Check FragId group.
-        auto FragIdList = cast<ConstantInt>(IIList.front()->getArgOperand(I));
-        auto FragId = cast<ConstantInt>(II->getArgOperand(I));
+        auto *FragIdList = cast<ConstantInt>(IIList.front()->getArgOperand(I));
+        auto *FragId = cast<ConstantInt>(II->getArgOperand(I));
         AllEqual = FragIdList->getValue().udiv(4) == FragId->getValue().udiv(4);
       } else {
         // Check all arguments except FragId.
@@ -219,7 +219,8 @@ bool optimizeSection(ArrayRef<SmallVector<IntrinsicInst *, 4>> MergeableInsts) {
       continue;
 
     const uint8_t FragIdIndex = ImageDimIntr->VAddrEnd - 1;
-    auto FragId = cast<ConstantInt>(IIList.front()->getArgOperand(FragIdIndex));
+    auto *FragId =
+        cast<ConstantInt>(IIList.front()->getArgOperand(FragIdIndex));
     const APInt &NewFragIdVal = FragId->getValue().udiv(4) * 4;
 
     // Create the new instructions.
@@ -251,7 +252,7 @@ bool optimizeSection(ArrayRef<SmallVector<IntrinsicInst *, 4>> MergeableInsts) {
     // Create the new extractelement instructions.
     for (auto &II : IIList) {
       Value *VecOp = nullptr;
-      auto Idx = cast<ConstantInt>(II->getArgOperand(FragIdIndex));
+      auto *Idx = cast<ConstantInt>(II->getArgOperand(FragIdIndex));
       B.SetCurrentDebugLocation(II->getDebugLoc());
       if (NumElts == 1) {
         VecOp = B.CreateExtractElement(NewCalls[0], Idx->getValue().urem(4));
@@ -275,7 +276,7 @@ bool optimizeSection(ArrayRef<SmallVector<IntrinsicInst *, 4>> MergeableInsts) {
     Modified = true;
   }
 
-  for (auto I : InstrsToErase)
+  for (auto *I : InstrsToErase)
     I->eraseFromParent();
 
   return Modified;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
index 7619a39bac9c1..3f2bb5df8836b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
@@ -30,7 +30,7 @@ class AMDGPUInsertDelayAlu : public MachineFunctionPass {
   const SIInstrInfo *SII;
   const TargetRegisterInfo *TRI;
 
-  TargetSchedModel SchedModel;
+  const TargetSchedModel *SchedModel;
 
   AMDGPUInsertDelayAlu() : MachineFunctionPass(ID) {}
 
@@ -387,7 +387,7 @@ class AMDGPUInsertDelayAlu : public MachineFunctionPass {
       if (Type != OTHER) {
         // TODO: Scan implicit defs too?
         for (const auto &Op : MI.defs()) {
-          unsigned Latency = SchedModel.computeOperandLatency(
+          unsigned Latency = SchedModel->computeOperandLatency(
               &MI, Op.getOperandNo(), nullptr, 0);
           for (MCRegUnit Unit : TRI->regunits(Op.getReg()))
             State[Unit] = DelayInfo(Type, Latency);
@@ -429,8 +429,7 @@ class AMDGPUInsertDelayAlu : public MachineFunctionPass {
 
     SII = ST.getInstrInfo();
     TRI = ST.getRegisterInfo();
-
-    SchedModel.init(&ST);
+    SchedModel = &SII->getSchedModel();
 
     // Calculate the delay state for each basic block, iterating until we reach
     // a fixed point.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index e8674c4c77595..ecb4d4fa5d5c3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -144,7 +144,7 @@ static std::optional<Instruction *> modifyIntrinsicCall(
 
   bool RemoveOldIntr = &OldIntr != &InstToReplace;
 
-  auto RetValue = IC.eraseInstFromFunction(InstToReplace);
+  auto *RetValue = IC.eraseInstFromFunction(InstToReplace);
   if (RemoveOldIntr)
     IC.eraseInstFromFunction(OldIntr);
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index febf0711c7d57..15bdd9ae293a1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -1474,8 +1474,8 @@ bool AMDGPUInstructionSelector::selectRelocConstant(MachineInstr &I) const {
   Module *M = MF->getFunction().getParent();
   const MDNode *Metadata = I.getOperand(2).getMetadata();
   auto SymbolName = cast<MDString>(Metadata->getOperand(0))->getString();
-  auto RelocSymbol = cast<GlobalVariable>(
-    M->getOrInsertGlobal(SymbolName, Type::getInt32Ty(M->getContext())));
+  auto *RelocSymbol = cast<GlobalVariable>(
+      M->getOrInsertGlobal(SymbolName, Type::getInt32Ty(M->getContext())));
 
   MachineBasicBlock *BB = I.getParent();
   BuildMI(*BB, &I, I.getDebugLoc(),
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
index 3767a6b379f81..77350dbb6167c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
@@ -111,7 +111,7 @@ class LiveRegOptimizer {
     if (!VTy)
       return false;
 
-    auto TLI = ST->getTargetLowering();
+    const auto *TLI = ST->getTargetLowering();
 
     Type *EltTy = VTy->getElementType();
     // If the element size is not less than the convert to scalar size, then we
@@ -454,7 +454,7 @@ bool AMDGPULateCodeGenPrepare::visitLoadInst(LoadInst &LI) {
   IRB.SetCurrentDebugLocation(LI.getDebugLoc());
 
   unsigned LdBits = DL->getTypeStoreSizeInBits(LI.getType());
-  auto IntNTy = Type::getIntNTy(LI.getContext(), LdBits);
+  auto *IntNTy = Type::getIntNTy(LI.getContext(), LdBits);
 
   auto *NewPtr = IRB.CreateConstGEP1_64(
       IRB.getInt8Ty(),
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 271c8d45fd4a2..121035028649f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -494,6 +494,8 @@ static bool loadStoreBitcastWorkaround(const LLT Ty) {
     return false;
 
   const unsigned Size = Ty.getSizeInBits();
+  if (Ty.isPointerVector())
+    return true;
   if (Size <= 64)
     return false;
   // Address space 8 pointers get their own workaround.
@@ -502,9 +504,6 @@ static bool loadStoreBitcastWorkaround(const LLT Ty) {
   if (!Ty.isVector())
     return true;
 
-  if (Ty.isPointerVector())
-    return true;
-
   unsigned EltSize = Ty.getScalarSizeInBits();
   return EltSize != 32 && EltSize != 64;
 }
@@ -4802,7 +4801,7 @@ bool AMDGPULegalizerInfo::legalizeFastUnsafeFDIV(MachineInstr &MI,
   bool AllowInaccurateRcp = MI.getFlag(MachineInstr::FmAfn) ||
                             MF.getTarget().Options.UnsafeFPMath;
 
-  if (auto CLHS = getConstantFPVRegVal(LHS, MRI)) {
+  if (const auto *CLHS = getConstantFPVRegVal(LHS, MRI)) {
     if (!AllowInaccurateRcp && ResTy != LLT::scalar(16))
       return false;
 
@@ -5794,8 +5793,9 @@ Register AMDGPULegalizerInfo::handleD16VData(MachineIRBuilder &B,
   return Reg;
 }
 
-Register AMDGPULegalizerInfo::fixStoreSourceType(
-  MachineIRBuilder &B, Register VData, bool IsFormat) const {
+Register AMDGPULegalizerInfo::fixStoreSourceType(MachineIRBuilder &B,
+                                                 Register VData, LLT MemTy,
+                                                 bool IsFormat) const {
   MachineRegisterInfo *MRI = B.getMRI();
   LLT Ty = MRI->getType(VData);
 
@@ -5805,6 +5805,10 @@ Register AMDGPULegalizerInfo::fixStoreSourceType(
   if (hasBufferRsrcWorkaround(Ty))
     return castBufferRsrcToV4I32(VData, B);
 
+  if (shouldBitcastLoadStoreType(ST, Ty, MemTy)) {
+    Ty = getBitcastRegisterType(Ty);
+    VData = B.buildBitcast(Ty, VData).getReg(0);
+  }
   // Fixup illegal register types for i8 stores.
   if (Ty == LLT::scalar(8) || Ty == S16) {
     Register AnyExt = B.buildAnyExt(LLT::scalar(32), VData).getReg(0);
@@ -5822,22 +5826,26 @@ Register AMDGPULegalizerInfo::fixStoreSourceType(
 }
 
 bool AMDGPULegalizerInfo::legalizeBufferStore(MachineInstr &MI,
-                                              MachineRegisterInfo &MRI,
-                                              MachineIRBuilder &B,
+                                              LegalizerHelper &Helper,
                                               bool IsTyped,
                                               bool IsFormat) const {
+  MachineIRBuilder &B = Helper.MIRBuilder;
+  MachineRegisterInfo &MRI = *B.getMRI();
+
   Register VData = MI.getOperand(1).getReg();
   LLT Ty = MRI.getType(VData);
   LLT EltTy = Ty.getScalarType();
   const bool IsD16 = IsFormat && (EltTy.getSizeInBits() == 16);
   const LLT S32 = LLT::scalar(32);
 
-  VData = fixStoreSourceType(B, VData, IsFormat);
-  castBufferRsrcArgToV4I32(MI, B, 2);
-  Register RSrc = MI.getOperand(2).getReg();
-
   MachineMemOperand *MMO = *MI.memoperands_begin();
   const int MemSize = MMO->getSize().getValue();
+  LLT MemTy = MMO->getMemoryType();
+
+  VData = fixStoreSourceType(B, VData, MemTy, IsFormat);
+
+  castBufferRsrcArgToV4I32(MI, B, 2);
+  Register RSrc = MI.getOperand(2).getReg();
 
   unsigned ImmOffset;
 
@@ -5930,10 +5938,13 @@ static void buildBufferLoad(unsigned Opc, Register LoadDstReg, Register RSrc,
 }
 
 bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
-                                             MachineRegisterInfo &MRI,
-                                             MachineIRBuilder &B,
+                                             LegalizerHelper &Helper,
                                              bool IsFormat,
                                              bool IsTyped) const {
+  MachineIRBuilder &B = Helper.MIRBuilder;
+  MachineRegisterInfo &MRI = *B.getMRI();
+  GISelChangeObserver &Observer = Helper.Observer;
+
   // FIXME: Verifier should enforce 1 MMO for these intrinsics.
   MachineMemOperand *MMO = *MI.memoperands_begin();
   const LLT MemTy = MMO->getMemoryType();
@@ -5982,9 +5993,21 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
   // Make addrspace 8 pointers loads into 4xs32 loads here, so the rest of the
   // logic doesn't have to handle that case.
   if (hasBufferRsrcWorkaround(Ty)) {
+    Observer.changingInstr(MI);
     Ty = castBufferRsrcFromV4I32(MI, B, MRI, 0);
+    Observer.changedInstr(MI);
     Dst = MI.getOperand(0).getReg();
+    B.setInsertPt(B.getMBB(), MI);
   }
+  if (shouldBitcastLoadStoreType(ST, Ty, MemTy)) {
+    Ty = getBitcastRegisterType(Ty);
+    Observer.changingInstr(MI);
+    Helper.bitcastDst(MI, Ty, 0);
+    Observer.changedInstr(MI);
+    Dst = MI.getOperand(0).getReg();
+    B.setInsertPt(B.getMBB(), MI);
+  }
+
   LLT EltTy = Ty.getScalarType();
   const bool IsD16 = IsFormat && (EltTy.getSizeInBits() == 16);
   const bool Unpacked = ST.hasUnpackedD16VMem();
@@ -7364,17 +7387,17 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
   case Intrinsic::amdgcn_raw_ptr_buffer_store:
   case Intrinsic::amdgcn_struct_buffer_store:
   case Intrinsic::amdgcn_struct_ptr_buffer_store:
-    return legalizeBufferStore(MI, MRI, B, false, false);
+    return legalizeBufferStore(MI, Helper, false, false);
   case Intrinsic::amdgcn_raw_buffer_store_format:
   case Intrinsic::amdgcn_raw_ptr_buffer_store_format:
   case Intrinsic::amdgcn_struct_buffer_store_format:
   case Intrinsic::amdgcn_struct_ptr_buffer_store_format:
-    return legalizeBufferStore(MI, MRI, B, false, true);
+    return legalizeBufferStore(MI, Helper, false, true);
   case Intrinsic::amdgcn_raw_tbuffer_store:
   case Intrinsic::amdgcn_raw_ptr_tbuffer_store:
   case Intrinsic::amdgcn_struct_tbuffer_store:
   case Intrinsic::amdgcn_struct_ptr_tbuffer_store:
-    return legalizeBufferStore(MI, MRI, B, true, true);
+    return legalizeBufferStore(MI, Helper, true, true);
   case Intrinsic::amdgcn_raw_buffer_load:
   case Intrinsic::amdgcn_raw_ptr_buffer_load:
   case Intrinsic::amdgcn_raw_atomic_buffer_load:
@@ -7383,17 +7406,17 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
   case Intrinsic::amdgcn_struct_ptr_buffer_load:
   case Intrinsic::amdgcn_struct_atomic_buffer_load:
   case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load:
-    return legalizeBufferLoad(MI, MRI, B, false, false);
+    return legalizeBufferLoad(MI, Helper, false, false);
   case Intrinsic::amdgcn_raw_buffer_load_format:
   case Intrinsic::amdgcn_raw_ptr_buffer_load_format:
   case Intrinsic::amdgcn_struct_buffer_load_format:
   case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
-    return legalizeBufferLoad(MI, MRI, B, true, false);
+    return legalizeBufferLoad(MI, Helper, true, false);
   case Intrinsic::amdgcn_raw_tbuffer_load:
   case Intrinsic::amdgcn_raw_ptr_tbuffer_load:
   case Intrinsic::amdgcn_struct_tbuffer_load:
   case Intrinsic::amdgcn_struct_ptr_tbuffer_load:
-    return legalizeBufferLoad(MI, MRI, B, true, true);
+    return legalizeBufferLoad(MI, Helper, true, true);
   case Intrinsic::amdgcn_raw_buffer_atomic_swap:
   case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
   case Intrinsic::amdgcn_struct_buffer_atomic_swap:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
index 84470dc75b60e..86c15197805d2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
@@ -195,15 +195,13 @@ class AMDGPULegalizerInfo final : public LegalizerInfo {
 
   Register handleD16VData(MachineIRBuilder &B, MachineRegisterInfo &MRI,
                           Register Reg, bool ImageStore = false) const;
-  Register fixStoreSourceType(MachineIRBuilder &B, Register VData,
+  Register fixStoreSourceType(MachineIRBuilder &B, Register VData, LLT MemTy,
                               bool IsFormat) const;
 
-  bool legalizeBufferStore(MachineInstr &MI, MachineRegisterInfo &MRI,
-                           MachineIRBuilder &B, bool IsTyped,
-                           bool IsFormat) const;
-  bool legalizeBufferLoad(MachineInstr &MI, MachineRegisterInfo &MRI,
-                          MachineIRBuilder &B, bool IsFormat,
-                          bool IsTyped) const;
+  bool legalizeBufferStore(MachineInstr &MI, LegalizerHelper &Helper,
+                           bool IsTyped, bool IsFormat) const;
+  bool legalizeBufferLoad(MachineInstr &MI, LegalizerHelper &Helper,
+                          bool IsFormat, bool IsTyped) const;
   bool legalizeBufferAtomic(MachineInstr &MI, MachineIRBuilder &B,
                             Intrinsic::ID IID) const;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelAttributes.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelAttributes.cpp
index e724c978c44b6..7d66d07c9d0fb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelAttributes.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelAttributes.cpp
@@ -87,7 +87,7 @@ Function *getBasePtrIntrinsic(Module &M, bool IsV5OrAbove) {
 static bool processUse(CallInst *CI, bool IsV5OrAbove) {
   Function *F = CI->getParent()->getParent();
 
-  auto MD = F->getMetadata("reqd_work_group_size");
+  auto *MD = F->getMetadata("reqd_work_group_size");
   const bool HasReqdWorkGroupSize = MD && MD->getNumOperands() == 3;
 
   const bool HasUniformWorkGroupSize =
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
index a166087a5cdc7..51a5b7702c009 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
@@ -324,7 +324,7 @@ class AMDGPULowerModuleLDS {
     for (GlobalVariable *GV : Variables) {
       auto ConstantGepIt = LDSVarsToConstantGEP.find(GV);
       if (ConstantGepIt != LDSVarsToConstantGEP.end()) {
-        auto elt = ConstantExpr::getPtrToInt(ConstantGepIt->second, I32);
+        auto *elt = ConstantExpr::getPtrToInt(ConstantGepIt->second, I32);
         Elements.push_back(elt);
       } else {
         Elements.push_back(PoisonValue::get(I32));
@@ -850,7 +850,7 @@ class AMDGPULowerModuleLDS {
     }
 
     assert(func->hasName()); // Checked by caller
-    auto emptyCharArray = ArrayType::get(Type::getInt8Ty(Ctx), 0);
+    auto *emptyCharArray = ArrayType::get(Type::getInt8Ty(Ctx), 0);
     GlobalVariable *N = new GlobalVariable(
         M, emptyCharArray, false, GlobalValue::ExternalLinkage, nullptr,
         Twine("llvm.amdgcn." + func->getName() + ".dynlds"), nullptr, GlobalValue::NotThreadLocal, AMDGPUAS::LOCAL_ADDRESS,
@@ -890,8 +890,8 @@ class AMDGPULowerModuleLDS {
 
           markUsedByKernel(func, N);
 
-          auto emptyCharArray = ArrayType::get(Type::getInt8Ty(Ctx), 0);
-          auto GEP = ConstantExpr::getGetElementPtr(
+          auto *emptyCharArray = ArrayType::get(Type::getInt8Ty(Ctx), 0);
+          auto *GEP = ConstantExpr::getGetElementPtr(
               emptyCharArray, N, ConstantInt::get(I32, 0), true);
           newDynamicLDS.push_back(ConstantExpr::getPtrToInt(GEP, I32));
         } else {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.h b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.h
index 5c656f158e714..7176cc5d3439b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.h
@@ -55,7 +55,7 @@ static inline const MCExpr *lowerAddrSpaceCast(const TargetMachine &TM,
   // Clang generates addrspacecast for null pointers in private and local
   // address space, which needs to be lowered.
   if (CE && CE->getOpcode() == Instruction::AddrSpaceCast) {
-    auto Op = CE->getOperand(0);
+    auto *Op = CE->getOperand(0);
     auto SrcAddr = Op->getType()->getPointerAddressSpace();
     if (Op->isNullValue() && AT.getNullPointerValue(SrcAddr) == 0) {
       auto DstAddr = CE->getType()->getPointerAddressSpace();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.cpp
index f608a9a4f470f..da0397fa20bd1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.cpp
@@ -71,6 +71,8 @@ void MCResourceInfo::assignMaxRegs(MCContext &OutContext) {
   assignMaxRegSym(MaxSGPRSym, MaxSGPR);
 }
 
+void MCResourceInfo::reset() { *this = MCResourceInfo(); }
+
 void MCResourceInfo::finalize(MCContext &OutContext) {
   assert(!Finalized && "Cannot finalize ResourceInfo again.");
   Finalized = true;
@@ -157,8 +159,12 @@ void MCResourceInfo::gatherResourceInfo(
       ArgExprs.push_back(
           MCConstantExpr::create(FRI.CalleeSegmentSize, OutContext));
 
-    if (!FRI.HasIndirectCall) {
-      for (const Function *Callee : FRI.Callees) {
+    SmallPtrSet<const Function *, 8> Seen;
+    Seen.insert(&MF.getFunction());
+    for (const Function *Callee : FRI.Callees) {
+      if (!Seen.insert(Callee).second)
+        continue;
+      if (!Callee->isDeclaration()) {
         MCSymbol *calleeValSym =
             getSymbol(Callee->getName(), RIK_PrivateSegSize, OutContext);
         ArgExprs.push_back(MCSymbolRefExpr::create(calleeValSym, OutContext));
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.h
index 08c0c106d5aa9..9dc34100e644e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.h
@@ -75,6 +75,8 @@ class MCResourceInfo {
   const MCExpr *getSymRefExpr(StringRef FuncName, ResourceInfoKind RIK,
                               MCContext &Ctx);
 
+  void reset();
+
   // Resolves the final symbols that requires the inter-function resource info
   // to be resolved.
   void finalize(MCContext &OutContext);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
index 99b4fca20bb2d..1d83d0c4c9337 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
@@ -31,7 +31,7 @@ getKernelDynLDSGlobalFromFunction(const Function &F) {
 static bool hasLDSKernelArgument(const Function &F) {
   for (const Argument &Arg : F.args()) {
     Type *ArgTy = Arg.getType();
-    if (auto PtrTy = dyn_cast<PointerType>(ArgTy)) {
+    if (auto *PtrTy = dyn_cast<PointerType>(ArgTy)) {
       if (PtrTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS)
         return true;
     }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp
index 040e931b82af2..3a3751892c8b6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp
@@ -118,15 +118,15 @@ struct AMDGPUPerfHint {
 
 static std::pair<const Value *, const Type *> getMemoryInstrPtrAndType(
     const Instruction *Inst) {
-  if (auto LI = dyn_cast<LoadInst>(Inst))
+  if (const auto *LI = dyn_cast<LoadInst>(Inst))
     return {LI->getPointerOperand(), LI->getType()};
-  if (auto SI = dyn_cast<StoreInst>(Inst))
+  if (const auto *SI = dyn_cast<StoreInst>(Inst))
     return {SI->getPointerOperand(), SI->getValueOperand()->getType()};
-  if (auto AI = dyn_cast<AtomicCmpXchgInst>(Inst))
+  if (const auto *AI = dyn_cast<AtomicCmpXchgInst>(Inst))
     return {AI->getPointerOperand(), AI->getCompareOperand()->getType()};
-  if (auto AI = dyn_cast<AtomicRMWInst>(Inst))
+  if (const auto *AI = dyn_cast<AtomicRMWInst>(Inst))
     return {AI->getPointerOperand(), AI->getValOperand()->getType()};
-  if (auto MI = dyn_cast<AnyMemIntrinsic>(Inst))
+  if (const auto *MI = dyn_cast<AnyMemIntrinsic>(Inst))
     return {MI->getRawDest(), Type::getInt8Ty(MI->getContext())};
 
   return {nullptr, nullptr};
@@ -148,8 +148,8 @@ bool AMDGPUPerfHint::isIndirectAccess(const Instruction *Inst) const {
       continue;
     LLVM_DEBUG(dbgs() << "  check: " << *V << '\n');
 
-    if (auto LD = dyn_cast<LoadInst>(V)) {
-      auto M = LD->getPointerOperand();
+    if (const auto *LD = dyn_cast<LoadInst>(V)) {
+      const auto *M = LD->getPointerOperand();
       if (isGlobalAddr(M)) {
         LLVM_DEBUG(dbgs() << "    is IA\n");
         return true;
@@ -157,32 +157,32 @@ bool AMDGPUPerfHint::isIndirectAccess(const Instruction *Inst) const {
       continue;
     }
 
-    if (auto GEP = dyn_cast<GetElementPtrInst>(V)) {
-      auto P = GEP->getPointerOperand();
+    if (const auto *GEP = dyn_cast<GetElementPtrInst>(V)) {
+      const auto *P = GEP->getPointerOperand();
       WorkSet.insert(P);
       for (unsigned I = 1, E = GEP->getNumIndices() + 1; I != E; ++I)
         WorkSet.insert(GEP->getOperand(I));
       continue;
     }
 
-    if (auto U = dyn_cast<UnaryInstruction>(V)) {
+    if (const auto *U = dyn_cast<UnaryInstruction>(V)) {
       WorkSet.insert(U->getOperand(0));
       continue;
     }
 
-    if (auto BO = dyn_cast<BinaryOperator>(V)) {
+    if (const auto *BO = dyn_cast<BinaryOperator>(V)) {
       WorkSet.insert(BO->getOperand(0));
       WorkSet.insert(BO->getOperand(1));
       continue;
     }
 
-    if (auto S = dyn_cast<SelectInst>(V)) {
+    if (const auto *S = dyn_cast<SelectInst>(V)) {
       WorkSet.insert(S->getFalseValue());
       WorkSet.insert(S->getTrueValue());
       continue;
     }
 
-    if (auto E = dyn_cast<ExtractElementInst>(V)) {
+    if (const auto *E = dyn_cast<ExtractElementInst>(V)) {
       WorkSet.insert(E->getVectorOperand());
       continue;
     }
@@ -331,7 +331,7 @@ bool AMDGPUPerfHint::needLimitWave(const AMDGPUPerfHintAnalysis::FuncInfo &FI) {
 }
 
 bool AMDGPUPerfHint::isGlobalAddr(const Value *V) const {
-  if (auto PT = dyn_cast<PointerType>(V->getType())) {
+  if (auto *PT = dyn_cast<PointerType>(V->getType())) {
     unsigned As = PT->getAddressSpace();
     // Flat likely points to global too.
     return As == AMDGPUAS::GLOBAL_ADDRESS || As == AMDGPUAS::FLAT_ADDRESS;
@@ -340,7 +340,7 @@ bool AMDGPUPerfHint::isGlobalAddr(const Value *V) const {
 }
 
 bool AMDGPUPerfHint::isLocalAddr(const Value *V) const {
-  if (auto PT = dyn_cast<PointerType>(V->getType()))
+  if (auto *PT = dyn_cast<PointerType>(V->getType()))
     return PT->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
   return false;
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp
index 02b0d436451a3..a899805dc46b1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp
@@ -436,7 +436,7 @@ bool AMDGPUPrintfRuntimeBindingImpl::run(Module &M) {
   if (TT.getArch() == Triple::r600)
     return false;
 
-  auto PrintfFunction = M.getFunction("printf");
+  auto *PrintfFunction = M.getFunction("printf");
   if (!PrintfFunction || !PrintfFunction->isDeclaration() ||
       M.getModuleFlag("openmp"))
     return false;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index bc771d4ef6c08..f597c1ae68a17 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -2357,8 +2357,9 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
 
     setRegsToType(MRI, DefRegs, HalfTy);
 
-    B.buildSelect(DefRegs[0], CondRegs[0], Src1Regs[0], Src2Regs[0]);
-    B.buildSelect(DefRegs[1], CondRegs[0], Src1Regs[1], Src2Regs[1]);
+    auto Flags = MI.getFlags();
+    B.buildSelect(DefRegs[0], CondRegs[0], Src1Regs[0], Src2Regs[0], Flags);
+    B.buildSelect(DefRegs[1], CondRegs[0], Src1Regs[1], Src2Regs[1], Flags);
 
     MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank);
     MI.eraseFromParent();
@@ -2440,8 +2441,9 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
 
     setRegsToType(MRI, DefRegs, HalfTy);
 
-    B.buildInstr(Opc, {DefRegs[0]}, {Src0Regs[0], Src1Regs[0]});
-    B.buildInstr(Opc, {DefRegs[1]}, {Src0Regs[1], Src1Regs[1]});
+    auto Flags = MI.getFlags();
+    B.buildInstr(Opc, {DefRegs[0]}, {Src0Regs[0], Src1Regs[0]}, Flags);
+    B.buildInstr(Opc, {DefRegs[1]}, {Src0Regs[1], Src1Regs[1]}, Flags);
 
     MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank);
     MI.eraseFromParent();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index 67d8715d3f1c2..9809a289df093 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -190,7 +190,7 @@ std::pair<unsigned, unsigned> AMDGPUSubtarget::getWavesPerEU(
 }
 
 static unsigned getReqdWorkGroupSize(const Function &Kernel, unsigned Dim) {
-  auto Node = Kernel.getMetadata("reqd_work_group_size");
+  auto *Node = Kernel.getMetadata("reqd_work_group_size");
   if (Node && Node->getNumOperands() == 3)
     return mdconst::extract<ConstantInt>(Node->getOperand(Dim))->getZExtValue();
   return std::numeric_limits<unsigned>::max();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSwLowerLDS.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSwLowerLDS.cpp
index e75b70b00c3ef..4669bb45473cb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSwLowerLDS.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSwLowerLDS.cpp
@@ -368,7 +368,7 @@ void AMDGPUSwLowerLDS::buildSwDynLDSGlobal(Function *Func) {
       LDSParams.IndirectAccess.DynamicLDSGlobals.empty())
     return;
   // Create new global pointer variable
-  auto emptyCharArray = ArrayType::get(IRB.getInt8Ty(), 0);
+  auto *emptyCharArray = ArrayType::get(IRB.getInt8Ty(), 0);
   LDSParams.SwDynLDS = new GlobalVariable(
       M, emptyCharArray, false, GlobalValue::ExternalLinkage, nullptr,
       "llvm.amdgcn." + Func->getName() + ".dynlds", nullptr,
@@ -952,8 +952,7 @@ Constant *AMDGPUSwLowerLDS::getAddressesOfVariablesInKernel(
       ArrayType::get(IRB.getPtrTy(AMDGPUAS::GLOBAL_ADDRESS), Variables.size());
 
   SmallVector<Constant *> Elements;
-  for (size_t i = 0; i < Variables.size(); i++) {
-    GlobalVariable *GV = Variables[i];
+  for (auto *GV : Variables) {
     if (!LDSParams.LDSToReplacementIndicesMap.contains(GV)) {
       Elements.push_back(
           PoisonValue::get(IRB.getPtrTy(AMDGPUAS::GLOBAL_ADDRESS)));
@@ -1070,7 +1069,8 @@ void AMDGPUSwLowerLDS::lowerNonKernelLDSAccesses(
       IRB.CreateLoad(IRB.getPtrTy(AMDGPUAS::GLOBAL_ADDRESS), BaseLoad);
 
   for (GlobalVariable *GV : LDSGlobals) {
-    auto GVIt = std::find(OrdereLDSGlobals.begin(), OrdereLDSGlobals.end(), GV);
+    const auto *GVIt =
+        std::find(OrdereLDSGlobals.begin(), OrdereLDSGlobals.end(), GV);
     assert(GVIt != OrdereLDSGlobals.end());
     uint32_t GVOffset = std::distance(OrdereLDSGlobals.begin(), GVIt);
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 1f2148c2922de..2c84cdac76d02 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -556,8 +556,8 @@ createGCNMaxILPMachineScheduler(MachineSchedContext *C) {
 static ScheduleDAGInstrs *
 createIterativeGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
   const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
-  auto DAG = new GCNIterativeScheduler(C,
-    GCNIterativeScheduler::SCHEDULE_LEGACYMAXOCCUPANCY);
+  auto *DAG = new GCNIterativeScheduler(
+      C, GCNIterativeScheduler::SCHEDULE_LEGACYMAXOCCUPANCY);
   DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
   if (ST.shouldClusterStores())
     DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
@@ -572,8 +572,7 @@ static ScheduleDAGInstrs *createMinRegScheduler(MachineSchedContext *C) {
 static ScheduleDAGInstrs *
 createIterativeILPMachineScheduler(MachineSchedContext *C) {
   const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
-  auto DAG = new GCNIterativeScheduler(C,
-    GCNIterativeScheduler::SCHEDULE_ILP);
+  auto *DAG = new GCNIterativeScheduler(C, GCNIterativeScheduler::SCHEDULE_ILP);
   DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
   if (ST.shouldClusterStores())
     DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
@@ -1533,6 +1532,11 @@ bool GCNPassConfig::addRegAssignAndRewriteOptimized() {
   // since FastRegAlloc does the replacements itself.
   addPass(createVirtRegRewriter(false));
 
+  // At this point, the sgpr-regalloc has been done and it is good to have the
+  // stack slot coloring to try to optimize the SGPR spill stack indices before
+  // attempting the custom SGPR spill lowering.
+  addPass(&StackSlotColoringID);
+
   // Equivalent of PEI for SGPRs.
   addPass(&SILowerSGPRSpillsLegacyID);
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 4cf7733a260ff..d348166c2d9a0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -784,7 +784,7 @@ InstructionCost GCNTTIImpl::getCFInstrCost(unsigned Opcode,
   switch (Opcode) {
   case Instruction::Br: {
     // Branch instruction takes about 4 slots on gfx900.
-    auto BI = dyn_cast_or_null<BranchInst>(I);
+    const auto *BI = dyn_cast_or_null<BranchInst>(I);
     if (BI && BI->isUnconditional())
       return SCost ? 1 : 4;
     // Suppose conditional branch takes additional 3 exec manipulations
@@ -792,7 +792,7 @@ InstructionCost GCNTTIImpl::getCFInstrCost(unsigned Opcode,
     return CBrCost;
   }
   case Instruction::Switch: {
-    auto SI = dyn_cast_or_null<SwitchInst>(I);
+    const auto *SI = dyn_cast_or_null<SwitchInst>(I);
     // Each case (including default) takes 1 cmp + 1 cbr instructions in
     // average.
     return (SI ? (SI->getNumCases() + 1) : 4) * (CBrCost + 1);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp b/llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp
index 327666c2a0f28..cd7866b86d55b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp
@@ -53,16 +53,16 @@ namespace {
     /// Keep the largest version as the sole operand if PickFirst is false.
     /// Otherwise pick it from the first value, representing kernel module.
     bool unifyVersionMD(Module &M, StringRef Name, bool PickFirst) {
-      auto NamedMD = M.getNamedMetadata(Name);
+      auto *NamedMD = M.getNamedMetadata(Name);
       if (!NamedMD || NamedMD->getNumOperands() <= 1)
         return false;
       MDNode *MaxMD = nullptr;
       auto MaxVer = 0U;
       for (auto *VersionMD : NamedMD->operands()) {
         assert(VersionMD->getNumOperands() == 2);
-        auto CMajor = mdconst::extract<ConstantInt>(VersionMD->getOperand(0));
+        auto *CMajor = mdconst::extract<ConstantInt>(VersionMD->getOperand(0));
         auto VersionMajor = CMajor->getZExtValue();
-        auto CMinor = mdconst::extract<ConstantInt>(VersionMD->getOperand(1));
+        auto *CMinor = mdconst::extract<ConstantInt>(VersionMD->getOperand(1));
         auto VersionMinor = CMinor->getZExtValue();
         auto Ver = (VersionMajor * 100) + (VersionMinor * 10);
         if (Ver > MaxVer) {
@@ -86,7 +86,7 @@ namespace {
   /// !n2 = !{!"cl_khr_image"}
   /// Combine it into a single list with unique operands.
   bool unifyExtensionMD(Module &M, StringRef Name) {
-    auto NamedMD = M.getNamedMetadata(Name);
+    auto *NamedMD = M.getNamedMetadata(Name);
     if (!NamedMD || NamedMD->getNumOperands() == 1)
       return false;
 
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index e12db4ab058ed..758f864fd20e6 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -6118,7 +6118,7 @@ bool AMDGPUAsmParser::ParseDirectivePALMetadataBegin() {
                           AMDGPU::PALMD::AssemblerDirectiveEnd, String))
     return true;
 
-  auto PALMetadata = getTargetStreamer().getPALMetadata();
+  auto *PALMetadata = getTargetStreamer().getPALMetadata();
   if (!PALMetadata->setFromString(String))
     return Error(getLoc(), "invalid PAL metadata");
   return false;
@@ -6132,7 +6132,7 @@ bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
                  "not available on non-amdpal OSes")).str());
   }
 
-  auto PALMetadata = getTargetStreamer().getPALMetadata();
+  auto *PALMetadata = getTargetStreamer().getPALMetadata();
   PALMetadata->setLegacy();
   for (;;) {
     uint32_t Key, Value;
@@ -7959,7 +7959,7 @@ AMDGPUAsmParser::parseStructuredOpFields(ArrayRef<StructuredOpField *> Fields) {
         !skipToken(AsmToken::Colon, "colon expected"))
       return ParseStatus::Failure;
 
-    auto I =
+    const auto *I =
         find_if(Fields, [Id](StructuredOpField *F) { return F->Id == Id; });
     if (I == Fields.end())
       return Error(IdLoc, "unknown field");
@@ -8798,7 +8798,7 @@ void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands,
   // we don't allow modifiers for this operand in assembler so src2_modifiers
   // should be 0.
   if (isMAC(Opc)) {
-    auto it = Inst.begin();
+    auto *it = Inst.begin();
     std::advance(it, AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2_modifiers));
     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
     ++it;
@@ -9627,7 +9627,7 @@ void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
   // it has src2 register operand that is tied to dst operand
   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
       Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
-    auto it = Inst.begin();
+    auto *it = Inst.begin();
     std::advance(
       it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
index 9eedcc636fd94..fdef9865b82c0 100644
--- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
+++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
@@ -94,7 +94,7 @@ static int insertNamedMCOperand(MCInst &MI, const MCOperand &Op,
                                 uint16_t NameIdx) {
   int OpIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), NameIdx);
   if (OpIdx != -1) {
-    auto I = MI.begin();
+    auto *I = MI.begin();
     std::advance(I, OpIdx);
     MI.insert(I, Op);
   }
@@ -104,7 +104,7 @@ static int insertNamedMCOperand(MCInst &MI, const MCOperand &Op,
 static DecodeStatus decodeSOPPBrTarget(MCInst &Inst, unsigned Imm,
                                        uint64_t Addr,
                                        const MCDisassembler *Decoder) {
-  auto DAsm = static_cast<const AMDGPUDisassembler*>(Decoder);
+  const auto *DAsm = static_cast<const AMDGPUDisassembler *>(Decoder);
 
   // Our branches take a simm16.
   int64_t Offset = SignExtend64<16>(Imm) * 4 + 4 + Addr;
@@ -116,7 +116,7 @@ static DecodeStatus decodeSOPPBrTarget(MCInst &Inst, unsigned Imm,
 
 static DecodeStatus decodeSMEMOffset(MCInst &Inst, unsigned Imm, uint64_t Addr,
                                      const MCDisassembler *Decoder) {
-  auto DAsm = static_cast<const AMDGPUDisassembler*>(Decoder);
+  const auto *DAsm = static_cast<const AMDGPUDisassembler *>(Decoder);
   int64_t Offset;
   if (DAsm->isGFX12Plus()) { // GFX12 supports 24-bit signed offsets.
     Offset = SignExtend64<24>(Imm);
@@ -130,20 +130,20 @@ static DecodeStatus decodeSMEMOffset(MCInst &Inst, unsigned Imm, uint64_t Addr,
 
 static DecodeStatus decodeBoolReg(MCInst &Inst, unsigned Val, uint64_t Addr,
                                   const MCDisassembler *Decoder) {
-  auto DAsm = static_cast<const AMDGPUDisassembler*>(Decoder);
+  const auto *DAsm = static_cast<const AMDGPUDisassembler *>(Decoder);
   return addOperand(Inst, DAsm->decodeBoolReg(Val));
 }
 
 static DecodeStatus decodeSplitBarrier(MCInst &Inst, unsigned Val,
                                        uint64_t Addr,
                                        const MCDisassembler *Decoder) {
-  auto DAsm = static_cast<const AMDGPUDisassembler *>(Decoder);
+  const auto *DAsm = static_cast<const AMDGPUDisassembler *>(Decoder);
   return addOperand(Inst, DAsm->decodeSplitBarrier(Val));
 }
 
 static DecodeStatus decodeDpp8FI(MCInst &Inst, unsigned Val, uint64_t Addr,
                                  const MCDisassembler *Decoder) {
-  auto DAsm = static_cast<const AMDGPUDisassembler *>(Decoder);
+  const auto *DAsm = static_cast<const AMDGPUDisassembler *>(Decoder);
   return addOperand(Inst, DAsm->decodeDpp8FI(Val));
 }
 
@@ -185,7 +185,7 @@ static DecodeStatus decodeSrcOp(MCInst &Inst, unsigned EncSize,
                                 AMDGPU::OperandSemantics Sema,
                                 const MCDisassembler *Decoder) {
   assert(Imm < (1U << EncSize) && "Operand doesn't fit encoding!");
-  auto DAsm = static_cast<const AMDGPUDisassembler *>(Decoder);
+  const auto *DAsm = static_cast<const AMDGPUDisassembler *>(Decoder);
   return addOperand(Inst, DAsm->decodeSrcOp(OpWidth, EncImm, MandatoryLiteral,
                                             ImmWidth, Sema));
 }
@@ -312,7 +312,7 @@ static DecodeStatus DecodeVGPR_16RegisterClass(MCInst &Inst, unsigned Imm,
 
   bool IsHi = Imm & (1 << 9);
   unsigned RegIdx = Imm & 0xff;
-  auto DAsm = static_cast<const AMDGPUDisassembler *>(Decoder);
+  const auto *DAsm = static_cast<const AMDGPUDisassembler *>(Decoder);
   return addOperand(Inst, DAsm->createVGPR16Operand(RegIdx, IsHi));
 }
 
@@ -323,7 +323,7 @@ DecodeVGPR_16_Lo128RegisterClass(MCInst &Inst, unsigned Imm, uint64_t /*Addr*/,
 
   bool IsHi = Imm & (1 << 7);
   unsigned RegIdx = Imm & 0x7f;
-  auto DAsm = static_cast<const AMDGPUDisassembler *>(Decoder);
+  const auto *DAsm = static_cast<const AMDGPUDisassembler *>(Decoder);
   return addOperand(Inst, DAsm->createVGPR16Operand(RegIdx, IsHi));
 }
 
@@ -393,7 +393,7 @@ static bool IsAGPROperand(const MCInst &Inst, int OpIdx,
 static DecodeStatus decodeAVLdSt(MCInst &Inst, unsigned Imm,
                                  AMDGPUDisassembler::OpWidthTy Opw,
                                  const MCDisassembler *Decoder) {
-  auto DAsm = static_cast<const AMDGPUDisassembler*>(Decoder);
+  const auto *DAsm = static_cast<const AMDGPUDisassembler *>(Decoder);
   if (!DAsm->isGFX90A()) {
     Imm &= 511;
   } else {
@@ -435,7 +435,7 @@ static DecodeStatus decodeOperand_VSrc_f64(MCInst &Inst, unsigned Imm,
                                            uint64_t Addr,
                                            const MCDisassembler *Decoder) {
   assert(Imm < (1 << 9) && "9-bit encoding");
-  auto DAsm = static_cast<const AMDGPUDisassembler *>(Decoder);
+  const auto *DAsm = static_cast<const AMDGPUDisassembler *>(Decoder);
   return addOperand(Inst,
                     DAsm->decodeSrcOp(AMDGPUDisassembler::OPW64, Imm, false, 64,
                                       AMDGPU::OperandSemantics::FP64));
@@ -451,7 +451,7 @@ DECODE_SDWA(VopcDst)
 static DecodeStatus decodeVersionImm(MCInst &Inst, unsigned Imm,
                                      uint64_t /* Addr */,
                                      const MCDisassembler *Decoder) {
-  auto DAsm = static_cast<const AMDGPUDisassembler *>(Decoder);
+  const auto *DAsm = static_cast<const AMDGPUDisassembler *>(Decoder);
   return addOperand(Inst, DAsm->decodeVersionImm(Imm));
 }
 
@@ -675,7 +675,7 @@ DecodeStatus AMDGPUDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
     int TFEOpIdx =
         AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::tfe);
     if (TFEOpIdx != -1) {
-      auto TFEIter = MI.begin();
+      auto *TFEIter = MI.begin();
       std::advance(TFEIter, TFEOpIdx);
       MI.insert(TFEIter, MCOperand::createImm(0));
     }
@@ -686,7 +686,7 @@ DecodeStatus AMDGPUDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
     int SWZOpIdx =
         AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::swz);
     if (SWZOpIdx != -1) {
-      auto SWZIter = MI.begin();
+      auto *SWZIter = MI.begin();
       std::advance(SWZIter, SWZOpIdx);
       MI.insert(SWZIter, MCOperand::createImm(0));
     }
@@ -1774,10 +1774,10 @@ MCOperand AMDGPUDisassembler::decodeVersionImm(unsigned Imm) const {
     return MCOperand::createImm(Imm);
 
   const auto &Versions = AMDGPU::UCVersion::getGFXVersions();
-  auto I = find_if(Versions,
-                   [Version = Version](const AMDGPU::UCVersion::GFXVersion &V) {
-                     return V.Code == Version;
-                   });
+  const auto *I = find_if(
+      Versions, [Version = Version](const AMDGPU::UCVersion::GFXVersion &V) {
+        return V.Code == Version;
+      });
   MCContext &Ctx = getContext();
   const MCExpr *E;
   if (I == Versions.end())
diff --git a/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp b/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp
index 833cd6dfa4ba2..cc802b5fbb67c 100644
--- a/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp
@@ -501,7 +501,7 @@ MachineInstr *GCNDPPCombine::createDPPInst(
       return nullptr;
     }
     CombOldVGPR = getRegSubRegPair(*Src1);
-    auto MovDst = TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst);
+    auto *MovDst = TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst);
     const TargetRegisterClass *RC = MRI->getRegClass(MovDst->getReg());
     if (!isOfRegClass(CombOldVGPR, *RC, *MRI)) {
       LLVM_DEBUG(dbgs() << "  failed: src1 has wrong register class\n");
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index cc39fd1740683..44afccb0690d0 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -59,10 +59,10 @@ static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
 GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF)
     : IsHazardRecognizerMode(false), CurrCycleInstr(nullptr), MF(MF),
       ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
-      TRI(TII.getRegisterInfo()), UseVALUReadHazardExhaustiveSearch(false),
+      TRI(TII.getRegisterInfo()), TSchedModel(TII.getSchedModel()),
+      UseVALUReadHazardExhaustiveSearch(false),
       ClauseUses(TRI.getNumRegUnits()), ClauseDefs(TRI.getNumRegUnits()) {
   MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5;
-  TSchedModel.init(&ST);
   RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST);
 }
 
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
index e840e2445188f..adb2278c48eeb 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
@@ -46,7 +46,7 @@ class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
   const GCNSubtarget &ST;
   const SIInstrInfo &TII;
   const SIRegisterInfo &TRI;
-  TargetSchedModel TSchedModel;
+  const TargetSchedModel &TSchedModel;
   bool RunLdsBranchVmemWARHazardFixup;
   BitVector VALUReadHazardSGPRs;
   bool UseVALUReadHazardExhaustiveSearch;
diff --git a/llvm/lib/Target/AMDGPU/GCNILPSched.cpp b/llvm/lib/Target/AMDGPU/GCNILPSched.cpp
index 8f15cc1b2b537..79b4a68c6a44e 100644
--- a/llvm/lib/Target/AMDGPU/GCNILPSched.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNILPSched.cpp
@@ -240,7 +240,7 @@ GCNILPScheduler::Candidate* GCNILPScheduler::pickCandidate() {
     return nullptr;
   auto Best = AvailQueue.begin();
   for (auto I = std::next(AvailQueue.begin()), E = AvailQueue.end(); I != E; ++I) {
-    auto NewBestSU = pickBest(Best->SU, I->SU);
+    const auto *NewBestSU = pickBest(Best->SU, I->SU);
     if (NewBestSU != Best->SU) {
       assert(NewBestSU == I->SU);
       Best = I;
@@ -272,7 +272,7 @@ void GCNILPScheduler::advanceToCycle(unsigned NextCycle) {
 
 void GCNILPScheduler::releasePredecessors(const SUnit* SU) {
   for (const auto &PredEdge : SU->Preds) {
-    auto PredSU = PredEdge.getSUnit();
+    auto *PredSU = PredEdge.getSUnit();
     if (PredEdge.isWeak())
       continue;
     assert(PredSU->isBoundaryNode() || PredSU->NumSuccsLeft > 0);
@@ -311,7 +311,7 @@ GCNILPScheduler::schedule(ArrayRef<const SUnit*> BotRoots,
   Schedule.reserve(SUnits.size());
   while (true) {
     if (AvailQueue.empty() && !PendingQueue.empty()) {
-      auto EarliestSU =
+      auto *EarliestSU =
           llvm::min_element(PendingQueue, [=](const Candidate &C1,
                                               const Candidate &C2) {
             return C1.SU->getHeight() < C2.SU->getHeight();
@@ -328,10 +328,10 @@ GCNILPScheduler::schedule(ArrayRef<const SUnit*> BotRoots,
                << ' ' << C.SU->NodeNum;
                dbgs() << '\n';);
 
-    auto C = pickCandidate();
+    auto *C = pickCandidate();
     assert(C);
     AvailQueue.remove(*C);
-    auto SU = C->SU;
+    auto *SU = C->SU;
     LLVM_DEBUG(dbgs() << "Selected "; DAG.dumpNode(*SU));
 
     advanceToCycle(SU->getHeight());
diff --git a/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp b/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp
index 061b0515031b1..13504508e2fb2 100644
--- a/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp
@@ -47,7 +47,7 @@ static void printRegion(raw_ostream &OS,
                         const LiveIntervals *LIS,
                         unsigned MaxInstNum =
                           std::numeric_limits<unsigned>::max()) {
-  auto BB = Begin->getParent();
+  auto *BB = Begin->getParent();
   OS << BB->getParent()->getName() << ":" << printMBBReference(*BB) << ' '
      << BB->getName() << ":\n";
   auto I = Begin;
@@ -76,7 +76,7 @@ static void printLivenessInfo(raw_ostream &OS,
                               MachineBasicBlock::iterator Begin,
                               MachineBasicBlock::iterator End,
                               const LiveIntervals *LIS) {
-  const auto BB = Begin->getParent();
+  auto *const BB = Begin->getParent();
   const auto &MRI = BB->getParent()->getRegInfo();
 
   const auto LiveIns = getLiveRegsBefore(*Begin, *LIS);
@@ -90,7 +90,7 @@ static void printLivenessInfo(raw_ostream &OS,
 LLVM_DUMP_METHOD
 void GCNIterativeScheduler::printRegions(raw_ostream &OS) const {
   const auto &ST = MF.getSubtarget<GCNSubtarget>();
-  for (const auto R : Regions) {
+  for (auto *const R : Regions) {
     OS << "Region to schedule ";
     printRegion(OS, R->Begin, R->End, LIS, 1);
     printLivenessInfo(OS, R->Begin, R->End, LIS);
@@ -127,7 +127,7 @@ class GCNIterativeScheduler::BuildDAG {
 public:
   BuildDAG(const Region &R, GCNIterativeScheduler &_Sch)
     : Sch(_Sch) {
-    auto BB = R.Begin->getParent();
+    auto *BB = R.Begin->getParent();
     Sch.BaseClass::startBlock(BB);
     Sch.BaseClass::enterRegion(BB, R.Begin, R.End, R.NumRegionInstrs);
 
@@ -165,7 +165,7 @@ class GCNIterativeScheduler::OverrideLegacyStrategy {
     , SaveSchedImpl(std::move(_Sch.SchedImpl))
     , SaveMaxRP(R.MaxPressure) {
     Sch.SchedImpl.reset(&OverrideStrategy);
-    auto BB = R.Begin->getParent();
+    auto *BB = R.Begin->getParent();
     Sch.BaseClass::startBlock(BB);
     Sch.BaseClass::enterRegion(BB, R.Begin, R.End, R.NumRegionInstrs);
   }
@@ -355,7 +355,7 @@ void GCNIterativeScheduler::scheduleRegion(Region &R, Range &&Schedule,
 #ifndef NDEBUG
   const auto SchedMaxRP = getSchedulePressure(R, Schedule);
 #endif
-  auto BB = R.Begin->getParent();
+  auto *BB = R.Begin->getParent();
   auto Top = R.Begin;
   for (const auto &I : Schedule) {
     auto MI = getMachineInstr(I);
diff --git a/llvm/lib/Target/AMDGPU/GCNMinRegStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNMinRegStrategy.cpp
index 04a6f2a7f4fd5..4154f946de8ff 100644
--- a/llvm/lib/Target/AMDGPU/GCNMinRegStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNMinRegStrategy.cpp
@@ -88,7 +88,7 @@ int GCNMinRegScheduler::getReadySuccessors(const SUnit *SU) const {
   for (auto SDep : SU->Succs) {
     bool wouldBeScheduled = true;
     for (auto PDep : SDep.getSUnit()->Preds) {
-      auto PSU = PDep.getSUnit();
+      auto *PSU = PDep.getSUnit();
       assert(!PSU->isBoundaryNode());
       if (PSU != SU && !isScheduled(PSU)) {
         wouldBeScheduled = false;
@@ -143,7 +143,7 @@ GCNMinRegScheduler::Candidate* GCNMinRegScheduler::pickCandidate() {
     LLVM_DEBUG(dbgs() << "\nSelecting min non-ready producing candidate among "
                       << Num << '\n');
     Num = findMax(Num, [=](const Candidate &C) {
-      auto SU = C.SU;
+      const auto *SU = C.SU;
       int Res = getNotReadySuccessors(SU);
       LLVM_DEBUG(dbgs() << "SU(" << SU->NodeNum << ") would left non-ready "
                         << Res << " successors, metric = " << -Res << '\n');
@@ -154,7 +154,7 @@ GCNMinRegScheduler::Candidate* GCNMinRegScheduler::pickCandidate() {
     LLVM_DEBUG(dbgs() << "\nSelecting most producing candidate among " << Num
                       << '\n');
     Num = findMax(Num, [=](const Candidate &C) {
-      auto SU = C.SU;
+      const auto *SU = C.SU;
       auto Res = getReadySuccessors(SU);
       LLVM_DEBUG(dbgs() << "SU(" << SU->NodeNum << ") would make ready " << Res
                         << " successors, metric = " << Res << '\n');
@@ -181,7 +181,7 @@ void GCNMinRegScheduler::bumpPredsPriority(const SUnit *SchedSU, int Priority) {
         S.getKind() != SDep::Data)
       continue;
     for (const auto &P : S.getSUnit()->Preds) {
-      auto PSU = P.getSUnit();
+      auto *PSU = P.getSUnit();
       assert(!PSU->isBoundaryNode());
       if (PSU != SchedSU && !isScheduled(PSU)) {
         Set.insert(PSU);
@@ -190,7 +190,7 @@ void GCNMinRegScheduler::bumpPredsPriority(const SUnit *SchedSU, int Priority) {
   }
   SmallVector<const SUnit*, 32> Worklist(Set.begin(), Set.end());
   while (!Worklist.empty()) {
-    auto SU = Worklist.pop_back_val();
+    const auto *SU = Worklist.pop_back_val();
     assert(!SU->isBoundaryNode());
     for (const auto &P : SU->Preds) {
       if (!P.getSUnit()->isBoundaryNode() && !isScheduled(P.getSUnit()) &&
@@ -212,7 +212,7 @@ void GCNMinRegScheduler::bumpPredsPriority(const SUnit *SchedSU, int Priority) {
 
 void GCNMinRegScheduler::releaseSuccessors(const SUnit* SU, int Priority) {
   for (const auto &S : SU->Succs) {
-    auto SuccSU = S.getSUnit();
+    auto *SuccSU = S.getSUnit();
     if (S.isWeak())
       continue;
     assert(SuccSU->isBoundaryNode() || getNumPreds(SuccSU) > 0);
@@ -246,10 +246,10 @@ GCNMinRegScheduler::schedule(ArrayRef<const SUnit*> TopRoots,
                << ' ' << C.SU->NodeNum << "(P" << C.Priority << ')';
                dbgs() << '\n';);
 
-    auto C = pickCandidate();
+    auto *C = pickCandidate();
     assert(C);
     RQ.remove(*C);
-    auto SU = C->SU;
+    const auto *SU = C->SU;
     LLVM_DEBUG(dbgs() << "Selected "; DAG.dumpNode(*SU));
 
     releaseSuccessors(SU, StepNo);
diff --git a/llvm/lib/Target/AMDGPU/GCNNSAReassign.cpp b/llvm/lib/Target/AMDGPU/GCNNSAReassign.cpp
index d6395fd75924d..18ffd8820f95e 100644
--- a/llvm/lib/Target/AMDGPU/GCNNSAReassign.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNNSAReassign.cpp
@@ -328,12 +328,14 @@ bool GCNNSAReassign::runOnMachineFunction(MachineFunction &MF) {
         continue;
     } else {
       // Check we did not make it worse for other instructions.
-      auto I = std::lower_bound(Candidates.begin(), &C, MinInd,
-                                [this](const Candidate &C, SlotIndex I) {
-                                  return LIS->getInstructionIndex(*C.first) < I;
-                                });
-      for (auto E = Candidates.end(); Success && I != E &&
-              LIS->getInstructionIndex(*I->first) < MaxInd; ++I) {
+      auto *I =
+          std::lower_bound(Candidates.begin(), &C, MinInd,
+                           [this](const Candidate &C, SlotIndex I) {
+                             return LIS->getInstructionIndex(*C.first) < I;
+                           });
+      for (auto *E = Candidates.end();
+           Success && I != E && LIS->getInstructionIndex(*I->first) < MaxInd;
+           ++I) {
         if (I->second && CheckNSA(*I->first, true) < NSA_Status::CONTIGUOUS) {
           Success = false;
           LLVM_DEBUG(dbgs() << "\tNSA conversion conflict with " << *I->first);
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index c83af729f501f..cb0624f11592d 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -38,8 +38,9 @@ bool llvm::isEqual(const GCNRPTracker::LiveRegSet &S1,
 unsigned GCNRegPressure::getRegKind(Register Reg,
                                     const MachineRegisterInfo &MRI) {
   assert(Reg.isVirtual());
-  const auto RC = MRI.getRegClass(Reg);
-  auto STI = static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
+  const auto *const RC = MRI.getRegClass(Reg);
+  const auto *STI =
+      static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
   return STI->isSGPRClass(RC)
              ? (STI->getRegSizeInBits(*RC) == 32 ? SGPR32 : SGPR_TUPLE)
          : STI->isAGPRClass(RC)
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp
index 4fbd7d0f88945..d1212ec76f986 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp
@@ -77,7 +77,7 @@ void AMDGPUMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
     OS << "occupancy(";
     break;
   }
-  for (auto It = Args.begin(); It != Args.end(); ++It) {
+  for (const auto *It = Args.begin(); It != Args.end(); ++It) {
     (*It)->print(OS, MAI, /*InParens=*/false);
     if ((It + 1) != Args.end())
       OS << ", ";
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp
index 77e7e30ff5281..14b3cdf37650c 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp
@@ -82,8 +82,8 @@ MCKernelDescriptor::getDefaultAmdhsaKernelDescriptor(const MCSubtargetInfo *STI,
 void MCKernelDescriptor::bits_set(const MCExpr *&Dst, const MCExpr *Value,
                                   uint32_t Shift, uint32_t Mask,
                                   MCContext &Ctx) {
-  auto Sft = MCConstantExpr::create(Shift, Ctx);
-  auto Msk = MCConstantExpr::create(Mask, Ctx);
+  const auto *Sft = MCConstantExpr::create(Shift, Ctx);
+  const auto *Msk = MCConstantExpr::create(Mask, Ctx);
   Dst = MCBinaryExpr::createAnd(Dst, MCUnaryExpr::createNot(Msk, Ctx), Ctx);
   Dst = MCBinaryExpr::createOr(Dst, MCBinaryExpr::createShl(Value, Sft, Ctx),
                                Ctx);
@@ -91,8 +91,8 @@ void MCKernelDescriptor::bits_set(const MCExpr *&Dst, const MCExpr *Value,
 
 const MCExpr *MCKernelDescriptor::bits_get(const MCExpr *Src, uint32_t Shift,
                                            uint32_t Mask, MCContext &Ctx) {
-  auto Sft = MCConstantExpr::create(Shift, Ctx);
-  auto Msk = MCConstantExpr::create(Mask, Ctx);
+  const auto *Sft = MCConstantExpr::create(Shift, Ctx);
+  const auto *Msk = MCConstantExpr::create(Mask, Ctx);
   return MCBinaryExpr::createLShr(MCBinaryExpr::createAnd(Src, Msk, Ctx), Sft,
                                   Ctx);
 }
diff --git a/llvm/lib/Target/AMDGPU/R600OpenCLImageTypeLoweringPass.cpp b/llvm/lib/Target/AMDGPU/R600OpenCLImageTypeLoweringPass.cpp
index 604a4cb1bf881..02fa4cd581b93 100644
--- a/llvm/lib/Target/AMDGPU/R600OpenCLImageTypeLoweringPass.cpp
+++ b/llvm/lib/Target/AMDGPU/R600OpenCLImageTypeLoweringPass.cpp
@@ -83,7 +83,7 @@ GetFunctionFromMDNode(MDNode *Node) {
   if (NumOps != NumKernelArgMDNodes + 1)
     return nullptr;
 
-  auto F = mdconst::dyn_extract<Function>(Node->getOperand(0));
+  auto *F = mdconst::dyn_extract<Function>(Node->getOperand(0));
   if (!F)
     return nullptr;
 
@@ -153,7 +153,7 @@ class R600OpenCLImageTypeLoweringPass : public ModulePass {
     bool Modified = false;
 
     for (auto &Use : ImageArg.uses()) {
-      auto Inst = dyn_cast<CallInst>(Use.getUser());
+      auto *Inst = dyn_cast<CallInst>(Use.getUser());
       if (!Inst) {
         continue;
       }
@@ -186,7 +186,7 @@ class R600OpenCLImageTypeLoweringPass : public ModulePass {
     bool Modified = false;
 
     for (const auto &Use : SamplerArg.uses()) {
-      auto Inst = dyn_cast<CallInst>(Use.getUser());
+      auto *Inst = dyn_cast<CallInst>(Use.getUser());
       if (!Inst) {
         continue;
       }
@@ -218,7 +218,7 @@ class R600OpenCLImageTypeLoweringPass : public ModulePass {
 
     bool Modified = false;
     InstsToErase.clear();
-    for (auto ArgI = F->arg_begin(); ArgI != F->arg_end(); ++ArgI) {
+    for (auto *ArgI = F->arg_begin(); ArgI != F->arg_end(); ++ArgI) {
       Argument &Arg = *ArgI;
       StringRef Type = ArgTypeFromMD(KernelMDNode, Arg.getArgNo());
 
@@ -287,10 +287,10 @@ class R600OpenCLImageTypeLoweringPass : public ModulePass {
     }
 
     // Create function with new signature and clone the old body into it.
-    auto NewFT = FunctionType::get(FT->getReturnType(), ArgTypes, false);
-    auto NewF = Function::Create(NewFT, F->getLinkage(), F->getName());
+    auto *NewFT = FunctionType::get(FT->getReturnType(), ArgTypes, false);
+    auto *NewF = Function::Create(NewFT, F->getLinkage(), F->getName());
     ValueToValueMapTy VMap;
-    auto NewFArgIt = NewF->arg_begin();
+    auto *NewFArgIt = NewF->arg_begin();
     for (auto &Arg: F->args()) {
       auto ArgName = Arg.getName();
       NewFArgIt->setName(ArgName);
diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index 46f5097c679fb..654ae412f39c1 100644
--- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -587,7 +587,7 @@ static bool hoistAndMergeSGPRInits(unsigned Reg,
   for (auto &Init : Inits) {
     auto &Defs = Init.second;
     for (auto *MI : Defs) {
-      auto MBB = MI->getParent();
+      auto *MBB = MI->getParent();
       MachineInstr &BoundaryMI = *getFirstNonPrologue(MBB, TII);
       MachineBasicBlock::reverse_iterator B(BoundaryMI);
       // Check if B should actually be a boundary. If not set the previous
@@ -769,7 +769,7 @@ bool SIFixSGPRCopies::run(MachineFunction &MF) {
   lowerVGPR2SGPRCopies(MF);
   // Postprocessing
   fixSCCCopies(MF);
-  for (auto MI : S2VCopies) {
+  for (auto *MI : S2VCopies) {
     // Check if it is still valid
     if (MI->isCopy()) {
       const TargetRegisterClass *SrcRC, *DstRC;
@@ -778,12 +778,12 @@ bool SIFixSGPRCopies::run(MachineFunction &MF) {
         tryChangeVGPRtoSGPRinCopy(*MI, TRI, TII);
     }
   }
-  for (auto MI : RegSequences) {
+  for (auto *MI : RegSequences) {
     // Check if it is still valid
     if (MI->isRegSequence())
       foldVGPRCopyIntoRegSequence(*MI, TRI, TII, *MRI);
   }
-  for (auto MI : PHINodes) {
+  for (auto *MI : PHINodes) {
     processPHINode(*MI);
   }
   if (MF.getTarget().getOptLevel() > CodeGenOptLevel::None && EnableM0Merge)
@@ -968,7 +968,7 @@ void SIFixSGPRCopies::analyzeVGPRToSGPRCopy(MachineInstr* MI) {
         for (auto &U : MRI->use_instructions(Reg))
           Users.push_back(&U);
     }
-    for (auto U : Users) {
+    for (auto *U : Users) {
       if (TII->isSALU(*U))
         Info.SChain.insert(U);
       AnalysisWorklist.push_back(U);
@@ -996,7 +996,7 @@ bool SIFixSGPRCopies::needToBeConvertedToVALU(V2SCopyInfo *Info) {
   // of the same register.
   SmallSet<std::pair<Register, unsigned>, 4> SrcRegs;
   for (auto J : Info->Siblings) {
-    auto InfoIt = V2SCopies.find(J);
+    auto *InfoIt = V2SCopies.find(J);
     if (InfoIt != V2SCopies.end()) {
       MachineInstr *SiblingCopy = InfoIt->second.Copy;
       if (SiblingCopy->isImplicitDef())
@@ -1031,12 +1031,12 @@ void SIFixSGPRCopies::lowerVGPR2SGPRCopies(MachineFunction &MF) {
 
   while (!LoweringWorklist.empty()) {
     unsigned CurID = LoweringWorklist.pop_back_val();
-    auto CurInfoIt = V2SCopies.find(CurID);
+    auto *CurInfoIt = V2SCopies.find(CurID);
     if (CurInfoIt != V2SCopies.end()) {
       V2SCopyInfo C = CurInfoIt->second;
       LLVM_DEBUG(dbgs() << "Processing ...\n"; C.dump());
       for (auto S : C.Siblings) {
-        auto SibInfoIt = V2SCopies.find(S);
+        auto *SibInfoIt = V2SCopies.find(S);
         if (SibInfoIt != V2SCopies.end()) {
           V2SCopyInfo &SI = SibInfoIt->second;
           LLVM_DEBUG(dbgs() << "Sibling:\n"; SI.dump());
diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index 3d1657392884f..bc162b0953a7b 100644
--- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -749,11 +749,11 @@ void SIFrameLowering::emitEntryFunctionScratchRsrcRegSetup(
     // at offset 0 (or offset 16 for a compute shader).
     MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
     const MCInstrDesc &LoadDwordX4 = TII->get(AMDGPU::S_LOAD_DWORDX4_IMM);
-    auto MMO = MF.getMachineMemOperand(PtrInfo,
-                                       MachineMemOperand::MOLoad |
-                                           MachineMemOperand::MOInvariant |
-                                           MachineMemOperand::MODereferenceable,
-                                       16, Align(4));
+    auto *MMO = MF.getMachineMemOperand(
+        PtrInfo,
+        MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
+            MachineMemOperand::MODereferenceable,
+        16, Align(4));
     unsigned Offset = Fn.getCallingConv() == CallingConv::AMDGPU_CS ? 16 : 0;
     const GCNSubtarget &Subtarget = MF.getSubtarget<GCNSubtarget>();
     unsigned EncodedOffset = AMDGPU::convertSMRDOffsetUnits(Subtarget, Offset);
@@ -800,7 +800,7 @@ void SIFrameLowering::emitEntryFunctionScratchRsrcRegSetup(
         const MCInstrDesc &LoadDwordX2 = TII->get(AMDGPU::S_LOAD_DWORDX2_IMM);
 
         MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
-        auto MMO = MF.getMachineMemOperand(
+        auto *MMO = MF.getMachineMemOperand(
             PtrInfo,
             MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
                 MachineMemOperand::MODereferenceable,
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 5e4cf705cc9e4..f34325e1a5afa 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -3592,6 +3592,8 @@ bool SITargetLowering::isEligibleForTailCallOptimization(
   SmallVector<CCValAssign, 16> ArgLocs;
   CCState CCInfo(CalleeCC, IsVarArg, MF, ArgLocs, Ctx);
 
+  // FIXME: We are not allocating special input registers, so we will be
+  // deciding based on incorrect register assignments.
   CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, IsVarArg));
 
   const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
@@ -3601,6 +3603,21 @@ bool SITargetLowering::isEligibleForTailCallOptimization(
   if (CCInfo.getStackSize() > FuncInfo->getBytesInStackArgArea())
     return false;
 
+  for (const auto &[CCVA, ArgVal] : zip_equal(ArgLocs, OutVals)) {
+    // FIXME: What about inreg arguments that end up passed in memory?
+    if (!CCVA.isRegLoc())
+      continue;
+
+    // If we are passing an argument in an SGPR, and the value is divergent,
+    // this call requires a waterfall loop.
+    if (ArgVal->isDivergent() && TRI->isSGPRPhysReg(CCVA.getLocReg())) {
+      LLVM_DEBUG(
+          dbgs() << "Cannot tail call due to divergent outgoing argument in "
+                 << printReg(CCVA.getLocReg(), TRI) << '\n');
+      return false;
+    }
+  }
+
   const MachineRegisterInfo &MRI = MF.getRegInfo();
   return parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals);
 }
@@ -3733,6 +3750,7 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
   // arguments to begin at SP+0. Completely unused for non-tail calls.
   int32_t FPDiff = 0;
   MachineFrameInfo &MFI = MF.getFrameInfo();
+  auto *TRI = static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo());
 
   // Adjust the stack pointer for the new arguments...
   // These operations are automatically eliminated by the prolog/epilog pass
@@ -3755,6 +3773,8 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
     }
   }
 
+  const unsigned NumSpecialInputs = RegsToPass.size();
+
   MVT PtrVT = MVT::i32;
 
   // Walk the register/memloc assignments, inserting copies/loads.
@@ -3856,16 +3876,40 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
   if (!MemOpChains.empty())
     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
 
+  SDValue ReadFirstLaneID =
+      DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, DL, MVT::i32);
+
+  SDValue TokenGlue;
+  if (CLI.ConvergenceControlToken) {
+    TokenGlue = DAG.getNode(ISD::CONVERGENCECTRL_GLUE, DL, MVT::Glue,
+                            CLI.ConvergenceControlToken);
+  }
+
   // Build a sequence of copy-to-reg nodes chained together with token chain
   // and flag operands which copy the outgoing args into the appropriate regs.
   SDValue InGlue;
-  for (auto &RegToPass : RegsToPass) {
-    Chain = DAG.getCopyToReg(Chain, DL, RegToPass.first,
-                             RegToPass.second, InGlue);
+
+  unsigned ArgIdx = 0;
+  for (auto [Reg, Val] : RegsToPass) {
+    if (ArgIdx++ >= NumSpecialInputs && !Val->isDivergent() &&
+        TRI->isSGPRPhysReg(Reg)) {
+      // Speculatively insert a readfirstlane in case this is a uniform value in
+      // a VGPR.
+      //
+      // FIXME: We need to execute this in a waterfall loop if it is a divergent
+      // value, so let that continue to produce invalid code.
+
+      SmallVector<SDValue, 3> ReadfirstlaneArgs({ReadFirstLaneID, Val});
+      if (TokenGlue)
+        ReadfirstlaneArgs.push_back(TokenGlue);
+      Val = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Val.getValueType(),
+                        ReadfirstlaneArgs);
+    }
+
+    Chain = DAG.getCopyToReg(Chain, DL, Reg, Val, InGlue);
     InGlue = Chain.getValue(1);
   }
 
-
   // We don't usually want to end the call-sequence here because we would tidy
   // the frame up *after* the call, however in the ABI-changing tail-call case
   // we've carefully laid out the parameters so that when sp is reset they'll be
@@ -3875,15 +3919,33 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
     InGlue = Chain.getValue(1);
   }
 
-  std::vector<SDValue> Ops;
-  Ops.push_back(Chain);
-  Ops.push_back(Callee);
+  std::vector<SDValue> Ops({Chain});
+
   // Add a redundant copy of the callee global which will not be legalized, as
   // we need direct access to the callee later.
   if (GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(Callee)) {
     const GlobalValue *GV = GSD->getGlobal();
+    Ops.push_back(Callee);
     Ops.push_back(DAG.getTargetGlobalAddress(GV, DL, MVT::i64));
   } else {
+    if (IsTailCall) {
+      assert(!Callee->isDivergent() &&
+             "cannot tail call a divergent call target");
+
+      // isEligibleForTailCallOptimization considered whether the call target is
+      // divergent, but we may still end up with a uniform value in a VGPR.
+      // Insert a readfirstlane just in case.
+      SDValue ReadFirstLaneID =
+          DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, DL, MVT::i32);
+
+      SmallVector<SDValue, 3> ReadfirstlaneArgs({ReadFirstLaneID, Callee});
+      if (TokenGlue)
+        ReadfirstlaneArgs.push_back(TokenGlue); // Wire up convergence token.
+      Callee = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Callee.getValueType(),
+                           ReadfirstlaneArgs);
+    }
+
+    Ops.push_back(Callee);
     Ops.push_back(DAG.getTargetConstant(0, DL, MVT::i64));
   }
 
@@ -3905,7 +3967,6 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
   }
 
   // Add a register mask operand representing the call-preserved registers.
-  auto *TRI = static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo());
   const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
   assert(Mask && "Missing call preserved mask for calling convention");
   Ops.push_back(DAG.getRegisterMask(Mask));
@@ -7287,7 +7348,7 @@ SDValue SITargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
 
   // Specially handle the case of v4i16 with static indexing.
   unsigned NumElts = VecVT.getVectorNumElements();
-  auto KIdx = dyn_cast<ConstantSDNode>(Idx);
+  auto *KIdx = dyn_cast<ConstantSDNode>(Idx);
   if (NumElts == 4 && EltSize == 16 && KIdx) {
     SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Vec);
 
@@ -7879,7 +7940,7 @@ static SDValue constructRetValue(SelectionDAG &DAG, MachineSDNode *Result,
 
 static bool parseTexFail(SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE,
                          SDValue *LWE, bool &IsTexFail) {
-  auto TexFailCtrlConst = cast<ConstantSDNode>(TexFailCtrl.getNode());
+  auto *TexFailCtrlConst = cast<ConstantSDNode>(TexFailCtrl.getNode());
 
   uint64_t Value = TexFailCtrlConst->getZExtValue();
   if (Value) {
@@ -8284,7 +8345,7 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
     return Op;
 
   MachineSDNode *NewNode = DAG.getMachineNode(Opcode, DL, ResultTypes, Ops);
-  if (auto MemOp = dyn_cast<MemSDNode>(Op)) {
+  if (auto *MemOp = dyn_cast<MemSDNode>(Op)) {
     MachineMemOperand *MemRef = MemOp->getMemOperand();
     DAG.setNodeMemRefs(NewNode, {MemRef});
   }
@@ -8435,7 +8496,7 @@ SDValue SITargetLowering::lowerWorkitemID(SelectionDAG &DAG, SDValue Op,
 SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
                                                   SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
-  auto MFI = MF.getInfo<SIMachineFunctionInfo>();
+  auto *MFI = MF.getInfo<SIMachineFunctionInfo>();
 
   EVT VT = Op.getValueType();
   SDLoc DL(Op);
@@ -8755,7 +8816,7 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     Module *M = const_cast<Module *>(MF.getFunction().getParent());
     const MDNode *Metadata = cast<MDNodeSDNode>(Op.getOperand(1))->getMD();
     auto SymbolName = cast<MDString>(Metadata->getOperand(0))->getString();
-    auto RelocSymbol = cast<GlobalVariable>(
+    auto *RelocSymbol = cast<GlobalVariable>(
         M->getOrInsertGlobal(SymbolName, Type::getInt32Ty(M->getContext())));
     SDValue GA = DAG.getTargetGlobalAddress(RelocSymbol, DL, MVT::i32, 0,
                                             SIInstrInfo::MO_ABS32_LO);
@@ -9382,7 +9443,7 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
       Ops.push_back(M0Val.getValue(0));
     }
 
-    auto NewMI = DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops);
+    auto *NewMI = DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops);
     return SDValue(NewMI, 0);
   }
   default:
@@ -9829,7 +9890,7 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
         StorePtrI, F | MachineMemOperand::MOStore, sizeof(int32_t),
         LoadMMO->getBaseAlign(), LoadMMO->getAAInfo());
 
-    auto Load = DAG.getMachineNode(Opc, DL, M->getVTList(), Ops);
+    auto *Load = DAG.getMachineNode(Opc, DL, M->getVTList(), Ops);
     DAG.setNodeMemRefs(Load, {LoadMMO, StoreMMO});
 
     return SDValue(Load, 0);
@@ -9907,7 +9968,7 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
         StorePtrI, F | MachineMemOperand::MOStore, sizeof(int32_t), Align(4),
         LoadMMO->getAAInfo());
 
-    auto Load = DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops);
+    auto *Load = DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops);
     DAG.setNodeMemRefs(Load, {LoadMMO, StoreMMO});
 
     return SDValue(Load, 0);
@@ -9984,7 +10045,7 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
       Ops.push_back(copyToM0(DAG, Chain, DL, BarOp).getValue(0));
     }
 
-    auto NewMI = DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops);
+    auto *NewMI = DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops);
     return SDValue(NewMI, 0);
   }
   case Intrinsic::amdgcn_s_prefetch_data: {
@@ -11815,7 +11876,7 @@ calculateSrcByte(const SDValue Op, uint64_t DestByte, uint64_t SrcIndex = 0,
 
   case ISD::SRA:
   case ISD::SRL: {
-    auto ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
+    auto *ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
     if (!ShiftOp)
       return std::nullopt;
 
@@ -11885,7 +11946,7 @@ calculateByteProvider(const SDValue &Op, unsigned Index, unsigned Depth,
     if (IsVec)
       return std::nullopt;
 
-    auto BitMaskOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
+    auto *BitMaskOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
     if (!BitMaskOp)
       return std::nullopt;
 
@@ -11909,7 +11970,7 @@ calculateByteProvider(const SDValue &Op, unsigned Index, unsigned Depth,
       return std::nullopt;
 
     // fshr(X,Y,Z): (X << (BW - (Z % BW))) | (Y >> (Z % BW))
-    auto ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(2));
+    auto *ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(2));
     if (!ShiftOp || Op.getValueType().isVector())
       return std::nullopt;
 
@@ -11936,7 +11997,7 @@ calculateByteProvider(const SDValue &Op, unsigned Index, unsigned Depth,
     if (IsVec)
       return std::nullopt;
 
-    auto ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
+    auto *ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
     if (!ShiftOp)
       return std::nullopt;
 
@@ -11964,7 +12025,7 @@ calculateByteProvider(const SDValue &Op, unsigned Index, unsigned Depth,
     if (IsVec)
       return std::nullopt;
 
-    auto ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
+    auto *ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
     if (!ShiftOp)
       return std::nullopt;
 
@@ -12033,7 +12094,7 @@ calculateByteProvider(const SDValue &Op, unsigned Index, unsigned Depth,
   }
 
   case ISD::LOAD: {
-    auto L = cast<LoadSDNode>(Op.getNode());
+    auto *L = cast<LoadSDNode>(Op.getNode());
 
     unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
     if (NarrowBitWidth % 8 != 0)
@@ -12066,7 +12127,7 @@ calculateByteProvider(const SDValue &Op, unsigned Index, unsigned Depth,
   }
 
   case ISD::EXTRACT_VECTOR_ELT: {
-    auto IdxOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
+    auto *IdxOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
     if (!IdxOp)
       return std::nullopt;
     auto VecIdx = IdxOp->getZExtValue();
@@ -12081,7 +12142,7 @@ calculateByteProvider(const SDValue &Op, unsigned Index, unsigned Depth,
     if (IsVec)
       return std::nullopt;
 
-    auto PermMask = dyn_cast<ConstantSDNode>(Op->getOperand(2));
+    auto *PermMask = dyn_cast<ConstantSDNode>(Op->getOperand(2));
     if (!PermMask)
       return std::nullopt;
 
@@ -12380,7 +12441,7 @@ SDValue SITargetLowering::performOrCombine(SDNode *N,
         return true;
 
       // If we have any non-vectorized use, then it is a candidate for v_perm
-      for (auto VUse : OrUse->uses()) {
+      for (auto *VUse : OrUse->uses()) {
         if (!VUse->getValueType(0).isVector())
           return true;
 
@@ -13902,7 +13963,7 @@ static void placeSources(ByteProvider<SDValue> &Src0,
                (IterElt.DWordOffset == (BPP.first.SrcOffset / 4));
       };
 
-      auto Match = llvm::find_if(Srcs, MatchesFirst);
+      auto *Match = llvm::find_if(Srcs, MatchesFirst);
       if (Match != Srcs.end()) {
         Match->PermMask = addPermMasks(FirstMask, Match->PermMask);
         FirstGroup = I;
@@ -13915,7 +13976,7 @@ static void placeSources(ByteProvider<SDValue> &Src0,
         return IterElt.SrcOp == *BPP.second.Src &&
                (IterElt.DWordOffset == (BPP.second.SrcOffset / 4));
       };
-      auto Match = llvm::find_if(Srcs, MatchesSecond);
+      auto *Match = llvm::find_if(Srcs, MatchesSecond);
       if (Match != Srcs.end()) {
         Match->PermMask = addPermMasks(SecondMask, Match->PermMask);
       } else
@@ -13948,7 +14009,7 @@ static SDValue resolveSources(SelectionDAG &DAG, SDLoc SL,
 
   // If we just have one source, just permute it accordingly.
   if (Srcs.size() == 1) {
-    auto Elt = Srcs.begin();
+    auto *Elt = Srcs.begin();
     auto EltOp = getDWordFromOffset(DAG, SL, Elt->SrcOp, Elt->DWordOffset);
 
     // v_perm will produce the original value
@@ -13959,8 +14020,8 @@ static SDValue resolveSources(SelectionDAG &DAG, SDLoc SL,
                        DAG.getConstant(Elt->PermMask, SL, MVT::i32));
   }
 
-  auto FirstElt = Srcs.begin();
-  auto SecondElt = std::next(FirstElt);
+  auto *FirstElt = Srcs.begin();
+  auto *SecondElt = std::next(FirstElt);
 
   SmallVector<SDValue, 2> Perms;
 
@@ -14214,11 +14275,11 @@ SDValue SITargetLowering::performAddCombine(SDNode *N,
       if (UniqueEntries) {
         UseOriginalSrc = true;
 
-        auto FirstElt = Src0s.begin();
+        auto *FirstElt = Src0s.begin();
         auto FirstEltOp =
             getDWordFromOffset(DAG, SL, FirstElt->SrcOp, FirstElt->DWordOffset);
 
-        auto SecondElt = Src1s.begin();
+        auto *SecondElt = Src1s.begin();
         auto SecondEltOp = getDWordFromOffset(DAG, SL, SecondElt->SrcOp,
                                               SecondElt->DWordOffset);
 
@@ -14561,7 +14622,7 @@ SDValue SITargetLowering::performSetCCCombine(SDNode *N,
   EVT VT = LHS.getValueType();
   ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
 
-  auto CRHS = dyn_cast<ConstantSDNode>(RHS);
+  auto *CRHS = dyn_cast<ConstantSDNode>(RHS);
   if (!CRHS) {
     CRHS = dyn_cast<ConstantSDNode>(LHS);
     if (CRHS) {
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index e64b35d230d48..15f4114826e40 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -2409,7 +2409,7 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
   MLI = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
   PDT = &getAnalysis<MachinePostDominatorTreeWrapperPass>().getPostDomTree();
-  if (auto AAR = getAnalysisIfAvailable<AAResultsWrapperPass>())
+  if (auto *AAR = getAnalysisIfAvailable<AAResultsWrapperPass>())
     AA = &AAR->getAAResults();
 
   AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST->getCPU());
@@ -2538,7 +2538,7 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
       if (Brackets->hasPendingEvent()) {
         BlockInfo *MoveBracketsToSucc = nullptr;
         for (MachineBasicBlock *Succ : MBB->successors()) {
-          auto SuccBII = BlockInfos.find(Succ);
+          auto *SuccBII = BlockInfos.find(Succ);
           BlockInfo &SuccBI = SuccBII->second;
           if (!SuccBI.Incoming) {
             SuccBI.Dirty = true;
@@ -2620,6 +2620,7 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
     Modified = true;
   }
   ReleaseVGPRInsts.clear();
+  PreheadersToFlush.clear();
   SLoadAddresses.clear();
 
   return Modified;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 5c39b2a4fc96a..b379447dd8d4c 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -528,13 +528,13 @@ static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
   if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
     return false;
 
-  auto MO1 = *MI1.memoperands_begin();
-  auto MO2 = *MI2.memoperands_begin();
+  auto *MO1 = *MI1.memoperands_begin();
+  auto *MO2 = *MI2.memoperands_begin();
   if (MO1->getAddrSpace() != MO2->getAddrSpace())
     return false;
 
-  auto Base1 = MO1->getValue();
-  auto Base2 = MO2->getValue();
+  const auto *Base1 = MO1->getValue();
+  const auto *Base2 = MO2->getValue();
   if (!Base1 || !Base2)
     return false;
   Base1 = getUnderlyingObject(Base1);
@@ -2008,7 +2008,7 @@ void SIInstrInfo::insertNoops(MachineBasicBlock &MBB,
 }
 
 void SIInstrInfo::insertReturn(MachineBasicBlock &MBB) const {
-  auto MF = MBB.getParent();
+  auto *MF = MBB.getParent();
   SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
 
   assert(Info->isEntryFunction());
@@ -3819,7 +3819,7 @@ MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI,
         SlotIndex NewIndex = LIS->getInstructionIndex(*MIB).getRegSlot(true);
         auto &LI = LIS->getInterval(Def.getReg());
         auto UpdateDefIndex = [&](LiveRange &LR) {
-          auto S = LR.find(OldIndex);
+          auto *S = LR.find(OldIndex);
           if (S != LR.end() && S->start == OldIndex) {
             assert(S->valno && S->valno->def == OldIndex);
             S->start = NewIndex;
@@ -5061,7 +5061,7 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
   }
 
   if (isSOPK(MI)) {
-    auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16);
+    const auto *Op = getNamedOperand(MI, AMDGPU::OpName::simm16);
     if (Desc.isBranch()) {
       if (!Op->isMBB()) {
         ErrInfo = "invalid branch target for SOPK instruction";
@@ -6402,7 +6402,7 @@ static void emitLoadScalarOpsFromVGPRLoop(
         }
       } // End for loop.
 
-      auto SScalarOpRC =
+      const auto *SScalarOpRC =
           TRI->getEquivalentSGPRClass(MRI.getRegClass(VScalarOp));
       Register SScalarOp = MRI.createVirtualRegister(SScalarOpRC);
 
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index f7554906a9c98..7041b59964645 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -52,12 +52,12 @@ struct SIInstrWorklist {
   void insert(MachineInstr *MI);
 
   MachineInstr *top() const {
-    auto iter = InstrList.begin();
+    const auto *iter = InstrList.begin();
     return *iter;
   }
 
   void erase_top() {
-    auto iter = InstrList.begin();
+    const auto *iter = InstrList.begin();
     InstrList.erase(iter);
   }
 
diff --git a/llvm/lib/Target/AMDGPU/SILateBranchLowering.cpp b/llvm/lib/Target/AMDGPU/SILateBranchLowering.cpp
index afc6353ec8116..62cda9e7ef2f3 100644
--- a/llvm/lib/Target/AMDGPU/SILateBranchLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SILateBranchLowering.cpp
@@ -226,7 +226,7 @@ bool SILateBranchLowering::runOnMachineFunction(MachineFunction &MF) {
     }
 
     for (auto *MI : EpilogInstrs) {
-      auto MBB = MI->getParent();
+      auto *MBB = MI->getParent();
       if (MBB == &MF.back() && MI == &MBB->back())
         continue;
 
diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index 23d04fae42015..904321a344db1 100644
--- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -2056,7 +2056,7 @@ Register SILoadStoreOptimizer::computeBase(MachineInstr &MI,
 void SILoadStoreOptimizer::updateBaseAndOffset(MachineInstr &MI,
                                                Register NewBase,
                                                int32_t NewOffset) const {
-  auto Base = TII->getNamedOperand(MI, AMDGPU::OpName::vaddr);
+  auto *Base = TII->getNamedOperand(MI, AMDGPU::OpName::vaddr);
   Base->setReg(NewBase);
   Base->setIsKill(false);
   TII->getNamedOperand(MI, AMDGPU::OpName::offset)->setImm(NewOffset);
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
index 8be9a082a7fd0..e59dd724b94f8 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
@@ -347,7 +347,7 @@ void SIMachineFunctionInfo::shiftWwmVGPRsToLowestRange(
 
     // Replace the register in SpillPhysVGPRs. This is needed to look for free
     // lanes while spilling special SGPRs like FP, BP, etc. during PEI.
-    auto RegItr = std::find(SpillPhysVGPRs.begin(), SpillPhysVGPRs.end(), Reg);
+    auto *RegItr = std::find(SpillPhysVGPRs.begin(), SpillPhysVGPRs.end(), Reg);
     if (RegItr != SpillPhysVGPRs.end()) {
       unsigned Idx = std::distance(SpillPhysVGPRs.begin(), RegItr);
       SpillPhysVGPRs[Idx] = NewReg;
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
index 669f98dd865d6..c8c305e24c710 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
@@ -639,15 +639,17 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunction,
   // Check if an entry created for \p Reg in PrologEpilogSGPRSpills. Return true
   // on success and false otherwise.
   bool hasPrologEpilogSGPRSpillEntry(Register Reg) const {
-    auto I = find_if(PrologEpilogSGPRSpills,
-                     [&Reg](const auto &Spill) { return Spill.first == Reg; });
+    const auto *I = find_if(PrologEpilogSGPRSpills, [&Reg](const auto &Spill) {
+      return Spill.first == Reg;
+    });
     return I != PrologEpilogSGPRSpills.end();
   }
 
   // Get the scratch SGPR if allocated to save/restore \p Reg.
   Register getScratchSGPRCopyDstReg(Register Reg) const {
-    auto I = find_if(PrologEpilogSGPRSpills,
-                     [&Reg](const auto &Spill) { return Spill.first == Reg; });
+    const auto *I = find_if(PrologEpilogSGPRSpills, [&Reg](const auto &Spill) {
+      return Spill.first == Reg;
+    });
     if (I != PrologEpilogSGPRSpills.end() &&
         I->second.getKind() == SGPRSaveKind::COPY_TO_SCRATCH_SGPR)
       return I->second.getReg();
@@ -676,8 +678,9 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunction,
 
   const PrologEpilogSGPRSaveRestoreInfo &
   getPrologEpilogSGPRSaveRestoreInfo(Register Reg) const {
-    auto I = find_if(PrologEpilogSGPRSpills,
-                     [&Reg](const auto &Spill) { return Spill.first == Reg; });
+    const auto *I = find_if(PrologEpilogSGPRSpills, [&Reg](const auto &Spill) {
+      return Spill.first == Reg;
+    });
     assert(I != PrologEpilogSGPRSpills.end());
 
     return I->second;
@@ -888,7 +891,7 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunction,
   }
 
   MCRegister getPreloadedReg(AMDGPUFunctionArgInfo::PreloadedValue Value) const {
-    auto Arg = std::get<0>(ArgInfo.getPreloadedValue(Value));
+    const auto *Arg = std::get<0>(ArgInfo.getPreloadedValue(Value));
     return Arg ? Arg->getRegister() : MCRegister();
   }
 
diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp
index 494b2341f0e0e..31f65d82a4d2b 100644
--- a/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp
+++ b/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp
@@ -392,7 +392,7 @@ bool SIOptimizeExecMaskingPreRA::runOnMachineFunction(MachineFunction &MF) {
       SmallVector<MachineBasicBlock*, 4> Blocks({&MBB});
 
       while (!Blocks.empty()) {
-        auto CurBB = Blocks.pop_back_val();
+        auto *CurBB = Blocks.pop_back_val();
         auto I = CurBB->rbegin(), E = CurBB->rend();
         if (I != E) {
           if (I->isUnconditionalBranch() || I->getOpcode() == AMDGPU::S_ENDPGM)
diff --git a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
index 86cb0e6944ed7..467f042892ceb 100644
--- a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
@@ -710,7 +710,7 @@ SIPeepholeSDWA::matchSDWAOperand(MachineInstr &MI) {
 
     MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
     MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
-    auto ValSrc = Src1;
+    auto *ValSrc = Src1;
     auto Imm = foldToImm(*Src0);
 
     if (!Imm) {
@@ -1151,7 +1151,7 @@ bool SIPeepholeSDWA::convertToSDWA(MachineInstr &MI,
   }
 
   // Check for a preserved register that needs to be copied.
-  auto DstUnused = TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused);
+  auto *DstUnused = TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused);
   if (DstUnused &&
       DstUnused->getImm() == AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) {
     // We expect, if we are here, that the instruction was already in it's SDWA form,
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 207cf7bc46b21..a8cdbbd8a3c5b 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -1959,8 +1959,13 @@ bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI, int Index,
 
   if (SpillToVGPR) {
 
-    assert(SB.NumSubRegs == VGPRSpills.size() &&
-           "Num of VGPR lanes should be equal to num of SGPRs spilled");
+    // Since stack slot coloring pass is trying to optimize SGPR spills,
+    // VGPR lanes (mapped from spill stack slot) may be shared for SGPR
+    // spills of different sizes. This accounts for number of VGPR lanes alloted
+    // equal to the largest SGPR being spilled in them.
+    assert(SB.NumSubRegs <= VGPRSpills.size() &&
+           "Num of SGPRs spilled should be less than or equal to num of "
+           "the VGPR lanes.");
 
     for (unsigned i = 0, e = SB.NumSubRegs; i < e; ++i) {
       Register SubReg =
@@ -2270,7 +2275,7 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
   assert(MF->getRegInfo().isReserved(MFI->getScratchRSrcReg()) &&
          "unreserved scratch RSRC register");
 
-  MachineOperand &FIOp = MI->getOperand(FIOperandNum);
+  MachineOperand *FIOp = &MI->getOperand(FIOperandNum);
   int Index = MI->getOperand(FIOperandNum).getIndex();
 
   Register FrameReg = FrameInfo.isFixedObjectIndex(Index) && hasBasePointer(*MF)
@@ -2452,6 +2457,211 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
       MI->eraseFromParent();
       return true;
     }
+    case AMDGPU::V_ADD_U32_e32:
+    case AMDGPU::V_ADD_U32_e64:
+    case AMDGPU::V_ADD_CO_U32_e32:
+    case AMDGPU::V_ADD_CO_U32_e64: {
+      // TODO: Handle sub, and, or.
+      unsigned NumDefs = MI->getNumExplicitDefs();
+      unsigned Src0Idx = NumDefs;
+
+      bool HasClamp = false;
+      MachineOperand *VCCOp = nullptr;
+
+      switch (MI->getOpcode()) {
+      case AMDGPU::V_ADD_U32_e32:
+        break;
+      case AMDGPU::V_ADD_U32_e64:
+        HasClamp = MI->getOperand(3).getImm();
+        break;
+      case AMDGPU::V_ADD_CO_U32_e32:
+        VCCOp = &MI->getOperand(3);
+        break;
+      case AMDGPU::V_ADD_CO_U32_e64:
+        VCCOp = &MI->getOperand(1);
+        HasClamp = MI->getOperand(4).getImm();
+        break;
+      default:
+        break;
+      }
+      bool DeadVCC = !VCCOp || VCCOp->isDead();
+      MachineOperand &DstOp = MI->getOperand(0);
+      Register DstReg = DstOp.getReg();
+
+      unsigned OtherOpIdx =
+          FIOperandNum == Src0Idx ? FIOperandNum + 1 : Src0Idx;
+      MachineOperand *OtherOp = &MI->getOperand(OtherOpIdx);
+
+      unsigned Src1Idx = Src0Idx + 1;
+      Register MaterializedReg = FrameReg;
+      Register ScavengedVGPR;
+
+      if (FrameReg && !ST.enableFlatScratch()) {
+        // We should just do an in-place update of the result register. However,
+        // the value there may also be used by the add, in which case we need a
+        // temporary register.
+        //
+        // FIXME: The scavenger is not finding the result register in the
+        // common case where the add does not read the register.
+
+        ScavengedVGPR = RS->scavengeRegisterBackwards(
+            AMDGPU::VGPR_32RegClass, MI, /*RestoreAfter=*/false, /*SPAdj=*/0);
+
+        // TODO: If we have a free SGPR, it's sometimes better to use a scalar
+        // shift.
+        BuildMI(*MBB, *MI, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64))
+            .addDef(ScavengedVGPR, RegState::Renamable)
+            .addImm(ST.getWavefrontSizeLog2())
+            .addReg(FrameReg);
+        MaterializedReg = ScavengedVGPR;
+      }
+
+      int64_t Offset = FrameInfo.getObjectOffset(Index);
+      // For the non-immediate case, we could fall through to the default
+      // handling, but we do an in-place update of the result register here to
+      // avoid scavenging another register.
+      if (OtherOp->isImm()) {
+        OtherOp->setImm(OtherOp->getImm() + Offset);
+        Offset = 0;
+      }
+
+      if ((!OtherOp->isImm() || OtherOp->getImm() != 0) && MaterializedReg) {
+        if (ST.enableFlatScratch() &&
+            !TII->isOperandLegal(*MI, Src1Idx, OtherOp)) {
+          // We didn't need the shift above, so we have an SGPR for the frame
+          // register, but may have a VGPR only operand.
+          //
+          // TODO: On gfx10+, we can easily change the opcode to the e64 version
+          // and use the higher constant bus restriction to avoid this copy.
+
+          if (!ScavengedVGPR) {
+            ScavengedVGPR = RS->scavengeRegisterBackwards(
+                AMDGPU::VGPR_32RegClass, MI, /*RestoreAfter=*/false,
+                /*SPAdj=*/0);
+          }
+
+          assert(ScavengedVGPR != DstReg);
+
+          BuildMI(*MBB, *MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), ScavengedVGPR)
+              .addReg(MaterializedReg,
+                      MaterializedReg != FrameReg ? RegState::Kill : 0);
+          MaterializedReg = ScavengedVGPR;
+        }
+
+        // TODO: In the flat scratch case, if this is an add of an SGPR, and SCC
+        // is not live, we could use a scalar add + vector add instead of 2
+        // vector adds.
+        auto AddI32 = BuildMI(*MBB, *MI, DL, TII->get(MI->getOpcode()))
+                          .addDef(DstReg, RegState::Renamable);
+        if (NumDefs == 2)
+          AddI32.add(MI->getOperand(1));
+
+        unsigned MaterializedRegFlags =
+            MaterializedReg != FrameReg ? RegState::Kill : 0;
+
+        if (isVGPRClass(getPhysRegBaseClass(MaterializedReg))) {
+          // If we know we have a VGPR already, it's more likely the other
+          // operand is a legal vsrc0.
+          AddI32
+            .add(*OtherOp)
+            .addReg(MaterializedReg, MaterializedRegFlags);
+        } else {
+          // Commute operands to avoid violating VOP2 restrictions. This will
+          // typically happen when using scratch.
+          AddI32
+            .addReg(MaterializedReg, MaterializedRegFlags)
+            .add(*OtherOp);
+        }
+
+        if (MI->getOpcode() == AMDGPU::V_ADD_CO_U32_e64 ||
+            MI->getOpcode() == AMDGPU::V_ADD_U32_e64)
+          AddI32.addImm(0); // clamp
+
+        if (MI->getOpcode() == AMDGPU::V_ADD_CO_U32_e32)
+          AddI32.setOperandDead(3); // Dead vcc
+
+        MaterializedReg = DstReg;
+
+        OtherOp->ChangeToRegister(MaterializedReg, false);
+        OtherOp->setIsKill(true);
+        FIOp->ChangeToImmediate(Offset);
+        Offset = 0;
+      } else if (Offset != 0) {
+        assert(!MaterializedReg);
+        FIOp->ChangeToImmediate(Offset);
+        Offset = 0;
+      } else {
+        if (DeadVCC && !HasClamp) {
+          assert(Offset == 0);
+
+          // TODO: Losing kills and implicit operands. Just mutate to copy and
+          // let lowerCopy deal with it?
+          if (OtherOp->isReg() && OtherOp->getReg() == DstReg) {
+            // Folded to an identity copy.
+            MI->eraseFromParent();
+            return true;
+          }
+
+          // The immediate value should be in OtherOp
+          MI->setDesc(TII->get(AMDGPU::V_MOV_B32_e32));
+          MI->removeOperand(FIOperandNum);
+
+          unsigned NumOps = MI->getNumOperands();
+          for (unsigned I = NumOps - 2; I >= 2; --I)
+            MI->removeOperand(I);
+
+          if (NumDefs == 2)
+            MI->removeOperand(1);
+
+          // The code below can't deal with a mov.
+          return true;
+        }
+
+        // This folded to a constant, but we have to keep the add around for
+        // pointless implicit defs or clamp modifier.
+        FIOp->ChangeToImmediate(0);
+      }
+
+      // Try to improve legality by commuting.
+      if (!TII->isOperandLegal(*MI, Src1Idx) && TII->commuteInstruction(*MI)) {
+        std::swap(FIOp, OtherOp);
+        std::swap(FIOperandNum, OtherOpIdx);
+      }
+
+      for (unsigned SrcIdx : {Src1Idx, Src0Idx}) {
+        // Depending on operand constraints we may need to insert another copy.
+        if (!TII->isOperandLegal(*MI, SrcIdx)) {
+          // If commuting didn't make the operands legal, we need to materialize
+          // in a register.
+          // TODO: Can use SGPR on gfx10+ in some cases.
+          if (!ScavengedVGPR) {
+            ScavengedVGPR = RS->scavengeRegisterBackwards(
+                AMDGPU::VGPR_32RegClass, MI, /*RestoreAfter=*/false,
+                /*SPAdj=*/0);
+          }
+
+          assert(ScavengedVGPR != DstReg);
+
+          MachineOperand &Src = MI->getOperand(SrcIdx);
+          BuildMI(*MBB, *MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), ScavengedVGPR)
+              .add(Src);
+
+          Src.ChangeToRegister(ScavengedVGPR, false);
+          Src.setIsKill(true);
+        }
+      }
+
+      // Fold out add of 0 case that can appear in kernels.
+      if (FIOp->isImm() && FIOp->getImm() == 0 && DeadVCC && !HasClamp) {
+        if (OtherOp->isReg() && OtherOp->getReg() != DstReg) {
+          BuildMI(*MBB, *MI, DL, TII->get(AMDGPU::COPY), DstReg).add(*OtherOp);
+        }
+
+        MI->eraseFromParent();
+      }
+
+      return true;
+    }
     case AMDGPU::S_ADD_I32: {
       // TODO: Handle s_or_b32, s_and_b32.
       unsigned OtherOpIdx = FIOperandNum == 1 ? 2 : 1;
@@ -2492,9 +2702,9 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
         Offset = 0;
 
         if (MaterializedReg)
-          FIOp.ChangeToRegister(MaterializedReg, false);
+          FIOp->ChangeToRegister(MaterializedReg, false);
         else
-          FIOp.ChangeToImmediate(0);
+          FIOp->ChangeToImmediate(0);
       } else if (MaterializedReg) {
         // If we can't fold the other operand, do another increment.
         Register DstReg = DstOp.getReg();
@@ -2517,19 +2727,19 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
         OtherOp.ChangeToRegister(MaterializedReg, false);
         OtherOp.setIsKill(true);
         OtherOp.setIsRenamable(true);
-        FIOp.ChangeToImmediate(Offset);
+        FIOp->ChangeToImmediate(Offset);
       } else {
         // If we don't have any other offset to apply, we can just directly
         // interpret the frame index as the offset.
-        FIOp.ChangeToImmediate(Offset);
+        FIOp->ChangeToImmediate(Offset);
       }
 
       if (DeadSCC && OtherOp.isImm() && OtherOp.getImm() == 0) {
         assert(Offset == 0);
         MI->removeOperand(3);
         MI->removeOperand(OtherOpIdx);
-        MI->setDesc(TII->get(FIOp.isReg() ? AMDGPU::COPY : AMDGPU::S_MOV_B32));
-      } else if (DeadSCC && FIOp.isImm() && FIOp.getImm() == 0) {
+        MI->setDesc(TII->get(FIOp->isReg() ? AMDGPU::COPY : AMDGPU::S_MOV_B32));
+      } else if (DeadSCC && FIOp->isImm() && FIOp->getImm() == 0) {
         assert(Offset == 0);
         MI->removeOperand(3);
         MI->removeOperand(FIOperandNum);
@@ -2537,7 +2747,7 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
             TII->get(OtherOp.isReg() ? AMDGPU::COPY : AMDGPU::S_MOV_B32));
       }
 
-      assert(!FIOp.isFI());
+      assert(!FIOp->isFI());
       return true;
     }
     default: {
@@ -2553,7 +2763,7 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
 
           // The offset is always swizzled, just replace it
           if (FrameReg)
-            FIOp.ChangeToRegister(FrameReg, false);
+            FIOp->ChangeToRegister(FrameReg, false);
 
           MachineOperand *OffsetOp =
             TII->getNamedOperand(*MI, AMDGPU::OpName::offset);
@@ -2606,18 +2816,18 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
         }
 
         if (!FrameReg) {
-          FIOp.ChangeToImmediate(Offset);
-          if (TII->isImmOperandLegal(*MI, FIOperandNum, FIOp))
+          FIOp->ChangeToImmediate(Offset);
+          if (TII->isImmOperandLegal(*MI, FIOperandNum, *FIOp))
             return false;
         }
 
         // We need to use register here. Check if we can use an SGPR or need
         // a VGPR.
-        FIOp.ChangeToRegister(AMDGPU::M0, false);
-        bool UseSGPR = TII->isOperandLegal(*MI, FIOperandNum, &FIOp);
+        FIOp->ChangeToRegister(AMDGPU::M0, false);
+        bool UseSGPR = TII->isOperandLegal(*MI, FIOperandNum, FIOp);
 
         if (!Offset && FrameReg && UseSGPR) {
-          FIOp.setReg(FrameReg);
+          FIOp->setReg(FrameReg);
           return false;
         }
 
@@ -2626,8 +2836,8 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
 
         Register TmpReg =
             RS->scavengeRegisterBackwards(*RC, MI, false, 0, !UseSGPR);
-        FIOp.setReg(TmpReg);
-        FIOp.setIsKill();
+        FIOp->setReg(TmpReg);
+        FIOp->setIsKill();
 
         if ((!FrameReg || !Offset) && TmpReg) {
           unsigned Opc = UseSGPR ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
@@ -2656,8 +2866,8 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
         if (!TmpSReg) {
           // Use frame register and restore it after.
           TmpSReg = FrameReg;
-          FIOp.setReg(FrameReg);
-          FIOp.setIsKill(false);
+          FIOp->setReg(FrameReg);
+          FIOp->setIsKill(false);
         }
 
         if (NeedSaveSCC) {
@@ -2905,7 +3115,7 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
           MI->eraseFromParent();
           return true;
         }
-        FIOp.ChangeToRegister(ResultReg, false, false, true);
+        FIOp->ChangeToRegister(ResultReg, false, false, true);
         return false;
       }
 
@@ -2936,13 +3146,13 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
       // If the offset is simply too big, don't convert to a scratch wave offset
       // relative index.
 
-      FIOp.ChangeToImmediate(Offset);
-      if (!TII->isImmOperandLegal(*MI, FIOperandNum, FIOp)) {
+      FIOp->ChangeToImmediate(Offset);
+      if (!TII->isImmOperandLegal(*MI, FIOperandNum, *FIOp)) {
         Register TmpReg = RS->scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass,
                                                         MI, false, 0);
         BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpReg)
           .addImm(Offset);
-        FIOp.ChangeToRegister(TmpReg, false, false, true);
+        FIOp->ChangeToRegister(TmpReg, false, false, true);
       }
     }
   }
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index 409e5418abc8e..99fa632c0300b 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -210,6 +210,9 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
   }
 
   bool isSGPRReg(const MachineRegisterInfo &MRI, Register Reg) const;
+  bool isSGPRPhysReg(Register Reg) const {
+    return isSGPRClass(getPhysRegBaseClass(Reg));
+  }
 
   /// \returns true if this class contains only VGPR registers
   static bool isVGPRClass(const TargetRegisterClass *RC) {
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
index ef9adde13348f..902feacede83f 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
@@ -590,7 +590,7 @@ class RegisterTypes<list<ValueType> reg_types> {
 
 def Reg16Types : RegisterTypes<[i16, f16, bf16]>;
 def Reg32Types : RegisterTypes<[i32, f32, v2i16, v2f16, v2bf16, p2, p3, p5, p6]>;
-def Reg64Types : RegisterTypes<[i64, f64, v2i32, v2f32, p0, v4i16, v4f16, v4bf16]>;
+def Reg64Types : RegisterTypes<[i64, f64, v2i32, v2f32, p0, p1, p4, v4i16, v4f16, v4bf16]>;
 def Reg96Types : RegisterTypes<[v3i32, v3f32]>;
 def Reg128Types : RegisterTypes<[v4i32, v4f32, v2i64, v2f64, v8i16, v8f16, v8bf16]>;
 
diff --git a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
index 9967a65244413..78267d402b6c9 100644
--- a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
+++ b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
@@ -752,13 +752,13 @@ MachineInstr *SIShrinkInstructions::matchSwap(MachineInstr &MovT) const {
     MachineBasicBlock &MBB = *MovT.getParent();
     SmallVector<MachineInstr *, 4> Swaps;
     if (Size == 2) {
-      auto MIB = BuildMI(MBB, MovX->getIterator(), MovT.getDebugLoc(),
-                         TII->get(AMDGPU::V_SWAP_B16))
-                     .addDef(X)
-                     .addDef(Y)
-                     .addReg(Y)
-                     .addReg(X)
-                     .getInstr();
+      auto *MIB = BuildMI(MBB, MovX->getIterator(), MovT.getDebugLoc(),
+                          TII->get(AMDGPU::V_SWAP_B16))
+                      .addDef(X)
+                      .addDef(Y)
+                      .addReg(Y)
+                      .addReg(X)
+                      .getInstr();
       Swaps.push_back(MIB);
     } else {
       assert(Size > 0 && Size % 4 == 0);
@@ -766,13 +766,13 @@ MachineInstr *SIShrinkInstructions::matchSwap(MachineInstr &MovT) const {
         TargetInstrInfo::RegSubRegPair X1, Y1;
         X1 = getSubRegForIndex(X, Xsub, I);
         Y1 = getSubRegForIndex(Y, Ysub, I);
-        auto MIB = BuildMI(MBB, MovX->getIterator(), MovT.getDebugLoc(),
-                           TII->get(AMDGPU::V_SWAP_B32))
-                       .addDef(X1.Reg, 0, X1.SubReg)
-                       .addDef(Y1.Reg, 0, Y1.SubReg)
-                       .addReg(Y1.Reg, 0, Y1.SubReg)
-                       .addReg(X1.Reg, 0, X1.SubReg)
-                       .getInstr();
+        auto *MIB = BuildMI(MBB, MovX->getIterator(), MovT.getDebugLoc(),
+                            TII->get(AMDGPU::V_SWAP_B32))
+                        .addDef(X1.Reg, 0, X1.SubReg)
+                        .addDef(Y1.Reg, 0, Y1.SubReg)
+                        .addReg(Y1.Reg, 0, Y1.SubReg)
+                        .addReg(X1.Reg, 0, X1.SubReg)
+                        .getInstr();
         Swaps.push_back(MIB);
       }
     }
diff --git a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
index ef6c92dfa9b9f..fba1d0c026958 100644
--- a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
+++ b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
@@ -366,8 +366,8 @@ void SIWholeQuadMode::markDefs(const MachineInstr &UseMI, LiveRange &LR,
 
       // Find next predecessor to process
       unsigned Idx = NextPredIdx;
-      auto PI = MBB->pred_begin() + Idx;
-      auto PE = MBB->pred_end();
+      const auto *PI = MBB->pred_begin() + Idx;
+      const auto *PE = MBB->pred_end();
       for (; PI != PE && !NextValue; ++PI, ++Idx) {
         if (const VNInfo *VN = LR.getVNInfoBefore(LIS->getMBBEndIdx(*PI))) {
           if (!Visited.count(VisitKey(VN, DefinedLanes)))
@@ -1036,7 +1036,7 @@ MachineInstr *SIWholeQuadMode::lowerKillI1(MachineBasicBlock &MBB,
 // This can only happen once all the live mask registers have been created
 // and the execute state (WQM/StrictWWM/Exact) of instructions is known.
 void SIWholeQuadMode::lowerBlock(MachineBasicBlock &MBB) {
-  auto BII = Blocks.find(&MBB);
+  auto *BII = Blocks.find(&MBB);
   if (BII == Blocks.end())
     return;
 
@@ -1261,7 +1261,7 @@ void SIWholeQuadMode::fromStrictMode(MachineBasicBlock &MBB,
 }
 
 void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, bool IsEntry) {
-  auto BII = Blocks.find(&MBB);
+  auto *BII = Blocks.find(&MBB);
   if (BII == Blocks.end())
     return;
 
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index feec7b47ae294..20a81a3135f0b 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -171,7 +171,7 @@ bool isHsaAbi(const MCSubtargetInfo &STI) {
 }
 
 unsigned getAMDHSACodeObjectVersion(const Module &M) {
-  if (auto Ver = mdconst::extract_or_null<ConstantInt>(
+  if (auto *Ver = mdconst::extract_or_null<ConstantInt>(
           M.getModuleFlag("amdhsa_code_object_version"))) {
     return (unsigned)Ver->getZExtValue() / 100;
   }
@@ -649,8 +649,8 @@ int getVOPDFull(unsigned OpX, unsigned OpY, unsigned EncodingFamily) {
 std::pair<unsigned, unsigned> getVOPDComponents(unsigned VOPDOpcode) {
   const VOPDInfo *Info = getVOPDOpcodeHelper(VOPDOpcode);
   assert(Info);
-  auto OpX = getVOPDBaseFromComponent(Info->OpX);
-  auto OpY = getVOPDBaseFromComponent(Info->OpY);
+  const auto *OpX = getVOPDBaseFromComponent(Info->OpX);
+  const auto *OpY = getVOPDBaseFromComponent(Info->OpY);
   assert(OpX && OpY);
   return {OpX->BaseVOP, OpY->BaseVOP};
 }
@@ -1789,7 +1789,7 @@ static StringLiteral const *getNfmtLookupTable(const MCSubtargetInfo &STI) {
 }
 
 int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) {
-  auto lookupTable = getNfmtLookupTable(STI);
+  const auto *lookupTable = getNfmtLookupTable(STI);
   for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) {
     if (Name == lookupTable[Id])
       return Id;
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
index 92d09b3afa77d..4ad26ee895c7d 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
@@ -29,14 +29,14 @@ using namespace llvm::AMDGPU;
 
 // Read the PAL metadata from IR metadata, where it was put by the frontend.
 void AMDGPUPALMetadata::readFromIR(Module &M) {
-  auto NamedMD = M.getNamedMetadata("amdgpu.pal.metadata.msgpack");
+  auto *NamedMD = M.getNamedMetadata("amdgpu.pal.metadata.msgpack");
   if (NamedMD && NamedMD->getNumOperands()) {
     // This is the new msgpack format for metadata. It is a NamedMD containing
     // an MDTuple containing an MDString containing the msgpack data.
     BlobType = ELF::NT_AMDGPU_METADATA;
-    auto MDN = dyn_cast<MDTuple>(NamedMD->getOperand(0));
+    auto *MDN = dyn_cast<MDTuple>(NamedMD->getOperand(0));
     if (MDN && MDN->getNumOperands()) {
-      if (auto MDS = dyn_cast<MDString>(MDN->getOperand(0)))
+      if (auto *MDS = dyn_cast<MDString>(MDN->getOperand(0)))
         setFromMsgPackBlob(MDS->getString());
     }
     return;
@@ -52,12 +52,12 @@ void AMDGPUPALMetadata::readFromIR(Module &M) {
   // containing an MDTuple containing a number of MDNodes each of which is an
   // integer value, and each two integer values forms a key=value pair that we
   // store as Registers[key]=value in the map.
-  auto Tuple = dyn_cast<MDTuple>(NamedMD->getOperand(0));
+  auto *Tuple = dyn_cast<MDTuple>(NamedMD->getOperand(0));
   if (!Tuple)
     return;
   for (unsigned I = 0, E = Tuple->getNumOperands() & -2; I != E; I += 2) {
-    auto Key = mdconst::dyn_extract<ConstantInt>(Tuple->getOperand(I));
-    auto Val = mdconst::dyn_extract<ConstantInt>(Tuple->getOperand(I + 1));
+    auto *Key = mdconst::dyn_extract<ConstantInt>(Tuple->getOperand(I));
+    auto *Val = mdconst::dyn_extract<ConstantInt>(Tuple->getOperand(I + 1));
     if (!Key || !Val)
       continue;
     setRegister(Key->getZExtValue(), Val->getZExtValue());
@@ -76,7 +76,7 @@ bool AMDGPUPALMetadata::setFromBlob(unsigned Type, StringRef Blob) {
 
 // Set PAL metadata from legacy (array of key=value pairs) blob.
 bool AMDGPUPALMetadata::setFromLegacyBlob(StringRef Blob) {
-  auto Data = reinterpret_cast<const uint32_t *>(Blob.data());
+  const auto *Data = reinterpret_cast<const uint32_t *>(Blob.data());
   for (unsigned I = 0; I != Blob.size() / sizeof(uint32_t) / 2; ++I)
     setRegister(Data[I * 2], Data[I * 2 + 1]);
   return true;
@@ -751,7 +751,7 @@ static const char *getRegisterName(unsigned RegNum) {
       {0x2c75, "SPI_SHADER_USER_ACCUM_VS_3"},
 
       {0, nullptr}};
-  auto Entry = RegInfoTable;
+  const auto *Entry = RegInfoTable;
   for (; Entry->Num && Entry->Num != RegNum; ++Entry)
     ;
   return Entry->Name;
@@ -1070,13 +1070,13 @@ msgpack::DocNode *AMDGPUPALMetadata::refComputeRegister(StringRef field) {
 }
 
 bool AMDGPUPALMetadata::checkComputeRegisters(StringRef field, unsigned Val) {
-  if (auto N = refComputeRegister(field))
+  if (auto *N = refComputeRegister(field))
     return N->getUInt() == Val;
   return false;
 }
 
 bool AMDGPUPALMetadata::checkComputeRegisters(StringRef field, bool Val) {
-  if (auto N = refComputeRegister(field))
+  if (auto *N = refComputeRegister(field))
     return N->getBool() == Val;
   return false;
 }
diff --git a/llvm/lib/Target/ARM/ARMArchitectures.td b/llvm/lib/Target/ARM/ARMArchitectures.td
index e1e90cdae188a..301ed5bf3e3fc 100644
--- a/llvm/lib/Target/ARM/ARMArchitectures.td
+++ b/llvm/lib/Target/ARM/ARMArchitectures.td
@@ -285,7 +285,18 @@ def ARMv95a   : Architecture<"armv9.5-a", "ARMv95a",  [HasV9_5aOps,
                                                        FeatureCRC,
                                                        FeatureRAS,
                                                        FeatureDotProd]>;
-
+def ARMv96a   : Architecture<"armv9.6-a", "ARMv96a",  [HasV9_6aOps,
+                                                       FeatureAClass,
+                                                       FeatureDB,
+                                                       FeatureFPARMv8,
+                                                       FeatureNEON,
+                                                       FeatureDSP,
+                                                       FeatureTrustZone,
+                                                       FeatureMP,
+                                                       FeatureVirtualization,
+                                                       FeatureCRC,
+                                                       FeatureRAS,
+                                                       FeatureDotProd]>;
 def ARMv8r    : Architecture<"armv8-r",   "ARMv8r",   [HasV8Ops,
                                                        FeatureRClass,
                                                        FeatureDB,
diff --git a/llvm/lib/Target/ARM/ARMFeatures.td b/llvm/lib/Target/ARM/ARMFeatures.td
index c1449adc34dc7..3a2188adbec33 100644
--- a/llvm/lib/Target/ARM/ARMFeatures.td
+++ b/llvm/lib/Target/ARM/ARMFeatures.td
@@ -706,6 +706,11 @@ def HasV9_5aOps   : SubtargetFeature<"v9.5a", "HasV9_5aOps", "true",
                                    "Support ARM v9.5a instructions",
                                    [HasV9_4aOps]>;
 
+// Armv9.6-A is a v9-only architecture.
+def HasV9_6aOps   : SubtargetFeature<"v9.6a", "HasV9_6aOps", "true",
+                                   "Support ARM v9.6a instructions",
+                                   [HasV9_5aOps]>;
+
 def HasV8_1MMainlineOps : SubtargetFeature<
                "v8.1m.main", "HasV8_1MMainlineOps", "true",
                "Support ARM v8-1M Mainline instructions",
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
index e7587145b46f2..5c7ba897f5e5a 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
@@ -894,6 +894,7 @@ void ARMTargetELFStreamer::emitArchDefaultAttributes() {
   case ARM::ArchKind::ARMV9_3A:
   case ARM::ArchKind::ARMV9_4A:
   case ARM::ArchKind::ARMV9_5A:
+  case ARM::ArchKind::ARMV9_6A:
     S.setAttributeItem(CPU_arch_profile, ApplicationProfile, false);
     S.setAttributeItem(ARM_ISA_use, Allowed, false);
     S.setAttributeItem(THUMB_ISA_use, AllowThumb32, false);
diff --git a/llvm/lib/Target/BPF/BPFISelLowering.h b/llvm/lib/Target/BPF/BPFISelLowering.h
index 42707949e864c..d59098f9f569b 100644
--- a/llvm/lib/Target/BPF/BPFISelLowering.h
+++ b/llvm/lib/Target/BPF/BPFISelLowering.h
@@ -118,7 +118,9 @@ class BPFTargetLowering : public TargetLowering {
     return Op.size() >= 8 ? MVT::i64 : MVT::i32;
   }
 
-  bool isIntDivCheap(EVT VT, AttributeList Attr) const override { return true; }
+  bool isIntDivCheap(EVT VT, AttributeList Attr) const override {
+    return false;
+  }
 
   bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
                                          Type *Ty) const override {
diff --git a/llvm/lib/Target/BPF/BPFMIPeephole.cpp b/llvm/lib/Target/BPF/BPFMIPeephole.cpp
index c41eab319dbb9..2b17f2aaefe2b 100644
--- a/llvm/lib/Target/BPF/BPFMIPeephole.cpp
+++ b/llvm/lib/Target/BPF/BPFMIPeephole.cpp
@@ -618,7 +618,9 @@ static void collectBPFFastCalls(const TargetRegisterInfo *TRI,
     if (MI.isCall()) {
       unsigned LiveCallerSavedRegs = 0;
       for (MCRegister R : CallerSavedRegs) {
-        bool DoSpillFill = !MI.definesRegister(R, TRI) && LiveRegs.contains(R);
+        bool DoSpillFill = false;
+        for (MCPhysReg SR : TRI->subregs(R))
+          DoSpillFill |= !MI.definesRegister(SR, TRI) && LiveRegs.contains(SR);
         if (!DoSpillFill)
           continue;
         LiveCallerSavedRegs |= 1 << R;
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.cpp
index fc5ddde07f62f..acee2916de17c 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.cpp
@@ -278,16 +278,18 @@ CSKYMCCodeEmitter::getRegisterSeqOpValue(const MCInst &MI, unsigned Op,
 unsigned CSKYMCCodeEmitter::getImmJMPIX(const MCInst &MI, unsigned Idx,
                                         SmallVectorImpl<MCFixup> &Fixups,
                                         const MCSubtargetInfo &STI) const {
-  if (MI.getOperand(Idx).getImm() == 16)
+  switch (MI.getOperand(Idx).getImm()) {
+  default:
+    llvm_unreachable("Unhandled jmpix imm!");
+  case 16:
     return 0;
-  else if (MI.getOperand(Idx).getImm() == 24)
+  case 24:
     return 1;
-  else if (MI.getOperand(Idx).getImm() == 32)
+  case 32:
     return 2;
-  else if (MI.getOperand(Idx).getImm() == 40)
+  case 40:
     return 3;
-  else
-    assert(0);
+  }
 }
 
 MCFixupKind CSKYMCCodeEmitter::getTargetFixup(const MCExpr *Expr) const {
diff --git a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
index 926cbe97f24fd..39e73a11d9d7e 100644
--- a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
+++ b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
@@ -33,6 +33,16 @@
 
 using namespace llvm;
 
+class DXILIntrinsicExpansionLegacy : public ModulePass {
+
+public:
+  bool runOnModule(Module &M) override;
+  DXILIntrinsicExpansionLegacy() : ModulePass(ID) {}
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+  static char ID; // Pass identification.
+};
+
 static bool isIntrinsicExpansion(Function &F) {
   switch (F.getIntrinsicID()) {
   case Intrinsic::abs:
@@ -44,6 +54,7 @@ static bool isIntrinsicExpansion(Function &F) {
   case Intrinsic::dx_all:
   case Intrinsic::dx_any:
   case Intrinsic::dx_clamp:
+  case Intrinsic::dx_cross:
   case Intrinsic::dx_uclamp:
   case Intrinsic::dx_lerp:
   case Intrinsic::dx_length:
@@ -74,6 +85,42 @@ static Value *expandAbs(CallInst *Orig) {
                                  "dx.max");
 }
 
+static Value *expandCrossIntrinsic(CallInst *Orig) {
+
+  VectorType *VT = cast<VectorType>(Orig->getType());
+  if (cast<FixedVectorType>(VT)->getNumElements() != 3)
+    report_fatal_error(Twine("return vector must have exactly 3 elements"),
+                       /* gen_crash_diag=*/false);
+
+  Value *op0 = Orig->getOperand(0);
+  Value *op1 = Orig->getOperand(1);
+  IRBuilder<> Builder(Orig);
+
+  Value *op0_x = Builder.CreateExtractElement(op0, (uint64_t)0, "x0");
+  Value *op0_y = Builder.CreateExtractElement(op0, 1, "x1");
+  Value *op0_z = Builder.CreateExtractElement(op0, 2, "x2");
+
+  Value *op1_x = Builder.CreateExtractElement(op1, (uint64_t)0, "y0");
+  Value *op1_y = Builder.CreateExtractElement(op1, 1, "y1");
+  Value *op1_z = Builder.CreateExtractElement(op1, 2, "y2");
+
+  auto MulSub = [&](Value *x0, Value *y0, Value *x1, Value *y1) -> Value * {
+    Value *xy = Builder.CreateFMul(x0, y1);
+    Value *yx = Builder.CreateFMul(y0, x1);
+    return Builder.CreateFSub(xy, yx, Orig->getName());
+  };
+
+  Value *yz_zy = MulSub(op0_y, op0_z, op1_y, op1_z);
+  Value *zx_xz = MulSub(op0_z, op0_x, op1_z, op1_x);
+  Value *xy_yx = MulSub(op0_x, op0_y, op1_x, op1_y);
+
+  Value *cross = UndefValue::get(VT);
+  cross = Builder.CreateInsertElement(cross, yz_zy, (uint64_t)0);
+  cross = Builder.CreateInsertElement(cross, zx_xz, 1);
+  cross = Builder.CreateInsertElement(cross, xy_yx, 2);
+  return cross;
+}
+
 // Create appropriate DXIL float dot intrinsic for the given A and B operands
 // The appropriate opcode will be determined by the size of the operands
 // The dot product is placed in the position indicated by Orig
@@ -486,6 +533,9 @@ static bool expandIntrinsic(Function &F, CallInst *Orig) {
   case Intrinsic::dx_any:
     Result = expandAnyOrAllIntrinsic(Orig, IntrinsicId);
     break;
+  case Intrinsic::dx_cross:
+    Result = expandCrossIntrinsic(Orig);
+    break;
   case Intrinsic::dx_uclamp:
   case Intrinsic::dx_clamp:
     Result = expandClampIntrinsic(Orig, IntrinsicId);
diff --git a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.h b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.h
index c8ee4b1b934b2..c283386c6e3df 100644
--- a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.h
+++ b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.h
@@ -19,16 +19,6 @@ class DXILIntrinsicExpansion : public PassInfoMixin<DXILIntrinsicExpansion> {
 public:
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &);
 };
-
-class DXILIntrinsicExpansionLegacy : public ModulePass {
-
-public:
-  bool runOnModule(Module &M) override;
-  DXILIntrinsicExpansionLegacy() : ModulePass(ID) {}
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override;
-  static char ID; // Pass identification.
-};
 } // namespace llvm
 
 #endif // LLVM_TARGET_DIRECTX_DXILINTRINSICEXPANSION_H
diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
index 3ee3ee05563c2..c0f8d433833ee 100644
--- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
@@ -525,7 +525,6 @@ class DXILOpLoweringLegacy : public ModulePass {
 
   static char ID; // Pass identification.
   void getAnalysisUsage(llvm::AnalysisUsage &AU) const override {
-    AU.addRequired<DXILIntrinsicExpansionLegacy>();
     AU.addRequired<DXILResourceWrapperPass>();
     AU.addPreserved<DXILResourceWrapperPass>();
   }
diff --git a/llvm/lib/Target/DirectX/DirectXPassRegistry.def b/llvm/lib/Target/DirectX/DirectXPassRegistry.def
index a3e051b173d89..ae729a1082b86 100644
--- a/llvm/lib/Target/DirectX/DirectXPassRegistry.def
+++ b/llvm/lib/Target/DirectX/DirectXPassRegistry.def
@@ -23,6 +23,7 @@ MODULE_ANALYSIS("dxil-resource-md", DXILResourceMDAnalysis())
 #ifndef MODULE_PASS
 #define MODULE_PASS(NAME, CREATE_PASS)
 #endif
+MODULE_PASS("dxil-data-scalarization", DXILDataScalarization())
 MODULE_PASS("dxil-intrinsic-expansion", DXILIntrinsicExpansion())
 MODULE_PASS("dxil-op-lower", DXILOpLowering())
 MODULE_PASS("dxil-pretty-printer", DXILPrettyPrinterPass(dbgs()))
diff --git a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
index f358215ecf373..18251ea3bd01d 100644
--- a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
+++ b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "DirectXTargetMachine.h"
+#include "DXILDataScalarization.h"
 #include "DXILIntrinsicExpansion.h"
 #include "DXILOpLowering.h"
 #include "DXILPrettyPrinter.h"
diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
index 09d84d41a2294..12f6161cbf61b 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
@@ -964,8 +964,8 @@ bool NVPTXAsmPrinter::doFinalization(Module &M) {
   // Close the last emitted section
   if (hasDebugInfo()) {
     TS->closeLastSection();
-    // Emit empty .debug_loc section for better support of the empty files.
-    OutStreamer->emitRawText("\t.section\t.debug_loc\t{\t}");
+    // Emit empty .debug_macinfo section for better support of the empty files.
+    OutStreamer->emitRawText("\t.section\t.debug_macinfo\t{\t}");
   }
 
   // Output last DWARF .file directives, if any.
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index ec3d3dbc8f6aa..33ad30351c51c 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -800,13 +800,19 @@ InstructionCost PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
   // PPCTargetLowering can't compute the cost appropriately. So here we
   // explicitly check this case. There are also corresponding store
   // instructions.
-  unsigned MemBytes = Src->getPrimitiveSizeInBits();
-  if (ST->hasVSX() && IsAltivecType &&
-      (MemBytes == 64 || (ST->hasP8Vector() && MemBytes == 32)))
-    return 1;
+  unsigned MemBits = Src->getPrimitiveSizeInBits();
+  unsigned SrcBytes = LT.second.getStoreSize();
+  if (ST->hasVSX() && IsAltivecType) {
+    if (MemBits == 64 || (ST->hasP8Vector() && MemBits == 32))
+      return 1;
+
+    // Use lfiwax/xxspltw
+    Align AlignBytes = Alignment ? *Alignment : Align(1);
+    if (Opcode == Instruction::Load && MemBits == 32 && AlignBytes < SrcBytes)
+      return 2;
+  }
 
   // Aligned loads and stores are easy.
-  unsigned SrcBytes = LT.second.getStoreSize();
   if (!SrcBytes || !Alignment || *Alignment >= SrcBytes)
     return Cost;
 
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index fbad7d5d02db6..78d6d7587160a 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -485,8 +485,14 @@ struct RISCVOperand final : public MCParsedAsmOperand {
            RISCVMCRegisterClasses[RISCV::GPRF16RegClassID].contains(Reg.RegNum);
   }
 
+  bool isGPRF32() const {
+    return Kind == KindTy::Register &&
+           RISCVMCRegisterClasses[RISCV::GPRF32RegClassID].contains(Reg.RegNum);
+  }
+
   bool isGPRAsFPR() const { return isGPR() && Reg.IsGPRAsFPR; }
   bool isGPRAsFPR16() const { return isGPRF16() && Reg.IsGPRAsFPR; }
+  bool isGPRAsFPR32() const { return isGPRF32() && Reg.IsGPRAsFPR; }
   bool isGPRPairAsFPR() const { return isGPRPair() && Reg.IsGPRAsFPR; }
 
   bool isGPRPair() const {
@@ -1352,6 +1358,10 @@ unsigned RISCVAsmParser::validateTargetOperandClass(MCParsedAsmOperand &AsmOp,
     Op.Reg.RegNum = Reg - RISCV::X0 + RISCV::X0_H;
     return Match_Success;
   }
+  if (Kind == MCK_GPRAsFPR32 && Op.isGPRAsFPR()) {
+    Op.Reg.RegNum = Reg - RISCV::X0 + RISCV::X0_W;
+    return Match_Success;
+  }
 
   // There are some GPRF64AsFPR instructions that have no RV32 equivalent. We
   // reject them at parsing thinking we should match as GPRPairAsFPR for RV32.
diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
index c2659a51b0209..7c8206cb44dec 100644
--- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
+++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
@@ -94,6 +94,19 @@ static DecodeStatus DecodeGPRF16RegisterClass(MCInst &Inst, uint32_t RegNo,
   return MCDisassembler::Success;
 }
 
+static DecodeStatus DecodeGPRF32RegisterClass(MCInst &Inst, uint32_t RegNo,
+                                              uint64_t Address,
+                                              const MCDisassembler *Decoder) {
+  bool IsRVE = Decoder->getSubtargetInfo().hasFeature(RISCV::FeatureStdExtE);
+
+  if (RegNo >= 32 || (IsRVE && RegNo >= 16))
+    return MCDisassembler::Fail;
+
+  MCRegister Reg = RISCV::X0_W + RegNo;
+  Inst.addOperand(MCOperand::createReg(Reg));
+  return MCDisassembler::Success;
+}
+
 static DecodeStatus DecodeGPRX1X5RegisterClass(MCInst &Inst, uint32_t RegNo,
                                                uint64_t Address,
                                                const MCDisassembler *Decoder) {
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
index 92d00c26bd219..cfe8644b89229 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
@@ -43,14 +43,22 @@ class RISCVInstructionSelector : public InstructionSelector {
                            const RISCVRegisterBankInfo &RBI);
 
   bool select(MachineInstr &MI) override;
+
+  void setupMF(MachineFunction &MF, GISelKnownBits *KB,
+               CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
+               BlockFrequencyInfo *BFI) override {
+    InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI);
+    MRI = &MF.getRegInfo();
+  }
+
   static const char *getName() { return DEBUG_TYPE; }
 
 private:
   const TargetRegisterClass *
   getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB) const;
 
-  bool isRegInGprb(Register Reg, MachineRegisterInfo &MRI) const;
-  bool isRegInFprb(Register Reg, MachineRegisterInfo &MRI) const;
+  bool isRegInGprb(Register Reg) const;
+  bool isRegInFprb(Register Reg) const;
 
   // tblgen-erated 'select' implementation, used as the initial selector for
   // the patterns that don't require complex C++.
@@ -58,31 +66,23 @@ class RISCVInstructionSelector : public InstructionSelector {
 
   // A lowering phase that runs before any selection attempts.
   // Returns true if the instruction was modified.
-  void preISelLower(MachineInstr &MI, MachineIRBuilder &MIB,
-                    MachineRegisterInfo &MRI);
+  void preISelLower(MachineInstr &MI, MachineIRBuilder &MIB);
 
-  bool replacePtrWithInt(MachineOperand &Op, MachineIRBuilder &MIB,
-                         MachineRegisterInfo &MRI);
+  bool replacePtrWithInt(MachineOperand &Op, MachineIRBuilder &MIB);
 
   // Custom selection methods
-  bool selectCopy(MachineInstr &MI, MachineRegisterInfo &MRI) const;
-  bool selectImplicitDef(MachineInstr &MI, MachineIRBuilder &MIB,
-                         MachineRegisterInfo &MRI) const;
+  bool selectCopy(MachineInstr &MI) const;
+  bool selectImplicitDef(MachineInstr &MI, MachineIRBuilder &MIB) const;
   bool materializeImm(Register Reg, int64_t Imm, MachineIRBuilder &MIB) const;
-  bool selectAddr(MachineInstr &MI, MachineIRBuilder &MIB,
-                  MachineRegisterInfo &MRI, bool IsLocal = true,
+  bool selectAddr(MachineInstr &MI, MachineIRBuilder &MIB, bool IsLocal = true,
                   bool IsExternWeak = false) const;
   bool selectSExtInreg(MachineInstr &MI, MachineIRBuilder &MIB) const;
-  bool selectSelect(MachineInstr &MI, MachineIRBuilder &MIB,
-                    MachineRegisterInfo &MRI) const;
-  bool selectFPCompare(MachineInstr &MI, MachineIRBuilder &MIB,
-                       MachineRegisterInfo &MRI) const;
+  bool selectSelect(MachineInstr &MI, MachineIRBuilder &MIB) const;
+  bool selectFPCompare(MachineInstr &MI, MachineIRBuilder &MIB) const;
   void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
                  MachineIRBuilder &MIB) const;
-  bool selectMergeValues(MachineInstr &MI, MachineIRBuilder &MIB,
-                         MachineRegisterInfo &MRI) const;
-  bool selectUnmergeValues(MachineInstr &MI, MachineIRBuilder &MIB,
-                           MachineRegisterInfo &MRI) const;
+  bool selectMergeValues(MachineInstr &MI, MachineIRBuilder &MIB) const;
+  bool selectUnmergeValues(MachineInstr &MI, MachineIRBuilder &MIB) const;
 
   ComplexRendererFns selectShiftMask(MachineOperand &Root) const;
   ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
@@ -100,6 +100,8 @@ class RISCVInstructionSelector : public InstructionSelector {
     return selectSHXADD_UWOp(Root, ShAmt);
   }
 
+  ComplexRendererFns renderVLOp(MachineOperand &Root) const;
+
   // Custom renderers for tablegen
   void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
                     int OpIdx) const;
@@ -121,6 +123,8 @@ class RISCVInstructionSelector : public InstructionSelector {
   const RISCVRegisterBankInfo &RBI;
   const RISCVTargetMachine &TM;
 
+  MachineRegisterInfo *MRI = nullptr;
+
   // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel
   // uses "STI." in the code generated by TableGen. We need to unify the name of
   // Subtarget variable.
@@ -162,16 +166,15 @@ RISCVInstructionSelector::selectShiftMask(MachineOperand &Root) const {
     return std::nullopt;
 
   using namespace llvm::MIPatternMatch;
-  MachineRegisterInfo &MRI = MF->getRegInfo();
 
   Register RootReg = Root.getReg();
   Register ShAmtReg = RootReg;
-  const LLT ShiftLLT = MRI.getType(RootReg);
+  const LLT ShiftLLT = MRI->getType(RootReg);
   unsigned ShiftWidth = ShiftLLT.getSizeInBits();
   assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
   // Peek through zext.
   Register ZExtSrcReg;
-  if (mi_match(ShAmtReg, MRI, m_GZExt(m_Reg(ZExtSrcReg)))) {
+  if (mi_match(ShAmtReg, *MRI, m_GZExt(m_Reg(ZExtSrcReg)))) {
     ShAmtReg = ZExtSrcReg;
   }
 
@@ -191,7 +194,7 @@ RISCVInstructionSelector::selectShiftMask(MachineOperand &Root) const {
   //
   // 1. the lowest log2(XLEN) bits of the and mask are all set
   // 2. the bits of the register being masked are already unset (zero set)
-  if (mi_match(ShAmtReg, MRI, m_GAnd(m_Reg(AndSrcReg), m_ICst(AndMask)))) {
+  if (mi_match(ShAmtReg, *MRI, m_GAnd(m_Reg(AndSrcReg), m_ICst(AndMask)))) {
     APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
     if (ShMask.isSubsetOf(AndMask)) {
       ShAmtReg = AndSrcReg;
@@ -206,16 +209,16 @@ RISCVInstructionSelector::selectShiftMask(MachineOperand &Root) const {
 
   APInt Imm;
   Register Reg;
-  if (mi_match(ShAmtReg, MRI, m_GAdd(m_Reg(Reg), m_ICst(Imm)))) {
+  if (mi_match(ShAmtReg, *MRI, m_GAdd(m_Reg(Reg), m_ICst(Imm)))) {
     if (Imm != 0 && Imm.urem(ShiftWidth) == 0)
       // If we are shifting by X+N where N == 0 mod Size, then just shift by X
       // to avoid the ADD.
       ShAmtReg = Reg;
-  } else if (mi_match(ShAmtReg, MRI, m_GSub(m_ICst(Imm), m_Reg(Reg)))) {
+  } else if (mi_match(ShAmtReg, *MRI, m_GSub(m_ICst(Imm), m_Reg(Reg)))) {
     if (Imm != 0 && Imm.urem(ShiftWidth) == 0) {
       // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
       // to generate a NEG instead of a SUB of a constant.
-      ShAmtReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
+      ShAmtReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
       unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;
       return {{[=](MachineInstrBuilder &MIB) {
         MachineIRBuilder(*MIB.getInstr())
@@ -226,7 +229,7 @@ RISCVInstructionSelector::selectShiftMask(MachineOperand &Root) const {
     if (Imm.urem(ShiftWidth) == ShiftWidth - 1) {
       // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
       // to generate a NOT instead of a SUB of a constant.
-      ShAmtReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
+      ShAmtReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
       return {{[=](MachineInstrBuilder &MIB) {
         MachineIRBuilder(*MIB.getInstr())
             .buildInstr(RISCV::XORI, {ShAmtReg}, {Reg})
@@ -243,8 +246,6 @@ InstructionSelector::ComplexRendererFns
 RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
                                          unsigned ShAmt) const {
   using namespace llvm::MIPatternMatch;
-  MachineFunction &MF = *Root.getParent()->getParent()->getParent();
-  MachineRegisterInfo &MRI = MF.getRegInfo();
 
   if (!Root.isReg())
     return std::nullopt;
@@ -255,11 +256,11 @@ RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
   Register RegY;
   std::optional<bool> LeftShift;
   // (and (shl y, c2), mask)
-  if (mi_match(RootReg, MRI,
+  if (mi_match(RootReg, *MRI,
                m_GAnd(m_GShl(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
     LeftShift = true;
   // (and (lshr y, c2), mask)
-  else if (mi_match(RootReg, MRI,
+  else if (mi_match(RootReg, *MRI,
                     m_GAnd(m_GLShr(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
     LeftShift = false;
 
@@ -275,7 +276,7 @@ RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
       // Given (and (shl y, c2), mask) in which mask has no leading zeros and
       // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
       if (*LeftShift && Leading == 0 && C2.ult(Trailing) && Trailing == ShAmt) {
-        Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
+        Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
         return {{[=](MachineInstrBuilder &MIB) {
           MachineIRBuilder(*MIB.getInstr())
               .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
@@ -287,7 +288,7 @@ RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
       // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
       // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
       if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
-        Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
+        Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
         return {{[=](MachineInstrBuilder &MIB) {
           MachineIRBuilder(*MIB.getInstr())
               .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
@@ -301,12 +302,12 @@ RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
   LeftShift.reset();
 
   // (shl (and y, mask), c2)
-  if (mi_match(RootReg, MRI,
+  if (mi_match(RootReg, *MRI,
                m_GShl(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),
                       m_ICst(C2))))
     LeftShift = true;
   // (lshr (and y, mask), c2)
-  else if (mi_match(RootReg, MRI,
+  else if (mi_match(RootReg, *MRI,
                     m_GLShr(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),
                             m_ICst(C2))))
     LeftShift = false;
@@ -326,7 +327,7 @@ RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
              (Trailing - C2.getLimitedValue()) == ShAmt;
 
     if (Cond) {
-      Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
+      Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
       return {{[=](MachineInstrBuilder &MIB) {
         MachineIRBuilder(*MIB.getInstr())
             .buildInstr(RISCV::SRLIW, {DstReg}, {RegY})
@@ -343,8 +344,6 @@ InstructionSelector::ComplexRendererFns
 RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
                                             unsigned ShAmt) const {
   using namespace llvm::MIPatternMatch;
-  MachineFunction &MF = *Root.getParent()->getParent()->getParent();
-  MachineRegisterInfo &MRI = MF.getRegInfo();
 
   if (!Root.isReg())
     return std::nullopt;
@@ -356,7 +355,7 @@ RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
   APInt Mask, C2;
   Register RegX;
   if (mi_match(
-          RootReg, MRI,
+          RootReg, *MRI,
           m_OneNonDBGUse(m_GAnd(m_OneNonDBGUse(m_GShl(m_Reg(RegX), m_ICst(C2))),
                                 m_ICst(Mask))))) {
     Mask &= maskTrailingZeros<uint64_t>(C2.getLimitedValue());
@@ -365,7 +364,7 @@ RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
       unsigned Leading = Mask.countl_zero();
       unsigned Trailing = Mask.countr_zero();
       if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {
-        Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
+        Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
         return {{[=](MachineInstrBuilder &MIB) {
           MachineIRBuilder(*MIB.getInstr())
               .buildInstr(RISCV::SLLI, {DstReg}, {RegX})
@@ -380,14 +379,34 @@ RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
 }
 
 InstructionSelector::ComplexRendererFns
-RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
-  MachineFunction &MF = *Root.getParent()->getParent()->getParent();
-  MachineRegisterInfo &MRI = MF.getRegInfo();
+RISCVInstructionSelector::renderVLOp(MachineOperand &Root) const {
+  assert(Root.isReg() && "Expected operand to be a Register");
+  MachineInstr *RootDef = MRI->getVRegDef(Root.getReg());
+
+  if (RootDef->getOpcode() == TargetOpcode::G_CONSTANT) {
+    auto C = RootDef->getOperand(1).getCImm();
+    if (C->getValue().isAllOnes())
+      // If the operand is a G_CONSTANT with value of all ones it is larger than
+      // VLMAX. We convert it to an immediate with value VLMaxSentinel. This is
+      // recognized specially by the vsetvli insertion pass.
+      return {{[=](MachineInstrBuilder &MIB) {
+        MIB.addImm(RISCV::VLMaxSentinel);
+      }}};
 
+    if (isUInt<5>(C->getZExtValue())) {
+      uint64_t ZExtC = C->getZExtValue();
+      return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(ZExtC); }}};
+    }
+  }
+  return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); }}};
+}
+
+InstructionSelector::ComplexRendererFns
+RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
   if (!Root.isReg())
     return std::nullopt;
 
-  MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
+  MachineInstr *RootDef = MRI->getVRegDef(Root.getReg());
   if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
     return {{
         [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
@@ -395,11 +414,11 @@ RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
     }};
   }
 
-  if (isBaseWithConstantOffset(Root, MRI)) {
+  if (isBaseWithConstantOffset(Root, *MRI)) {
     MachineOperand &LHS = RootDef->getOperand(1);
     MachineOperand &RHS = RootDef->getOperand(2);
-    MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
-    MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
+    MachineInstr *LHSDef = MRI->getVRegDef(LHS.getReg());
+    MachineInstr *RHSDef = MRI->getVRegDef(RHS.getReg());
 
     int64_t RHSC = RHSDef->getOperand(1).getCImm()->getSExtValue();
     if (isInt<12>(RHSC)) {
@@ -441,9 +460,9 @@ static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
   }
 }
 
-static void getOperandsForBranch(Register CondReg, MachineRegisterInfo &MRI,
-                                 RISCVCC::CondCode &CC, Register &LHS,
-                                 Register &RHS) {
+static void getOperandsForBranch(Register CondReg, RISCVCC::CondCode &CC,
+                                 Register &LHS, Register &RHS,
+                                 MachineRegisterInfo &MRI) {
   // Try to fold an ICmp. If that fails, use a NE compare with X0.
   CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
   if (!mi_match(CondReg, MRI, m_GICmp(m_Pred(Pred), m_Reg(LHS), m_Reg(RHS)))) {
@@ -509,19 +528,18 @@ static void getOperandsForBranch(Register CondReg, MachineRegisterInfo &MRI,
 bool RISCVInstructionSelector::select(MachineInstr &MI) {
   MachineBasicBlock &MBB = *MI.getParent();
   MachineFunction &MF = *MBB.getParent();
-  MachineRegisterInfo &MRI = MF.getRegInfo();
   MachineIRBuilder MIB(MI);
 
-  preISelLower(MI, MIB, MRI);
+  preISelLower(MI, MIB);
   const unsigned Opc = MI.getOpcode();
 
   if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {
     if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {
       const Register DefReg = MI.getOperand(0).getReg();
-      const LLT DefTy = MRI.getType(DefReg);
+      const LLT DefTy = MRI->getType(DefReg);
 
       const RegClassOrRegBank &RegClassOrBank =
-          MRI.getRegClassOrRegBank(DefReg);
+          MRI->getRegClassOrRegBank(DefReg);
 
       const TargetRegisterClass *DefRC =
           RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
@@ -540,12 +558,12 @@ bool RISCVInstructionSelector::select(MachineInstr &MI) {
       }
 
       MI.setDesc(TII.get(TargetOpcode::PHI));
-      return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
+      return RBI.constrainGenericRegister(DefReg, *DefRC, *MRI);
     }
 
     // Certain non-generic instructions also need some special handling.
     if (MI.isCopy())
-      return selectCopy(MI, MRI);
+      return selectCopy(MI);
 
     return true;
   }
@@ -559,7 +577,7 @@ bool RISCVInstructionSelector::select(MachineInstr &MI) {
   case TargetOpcode::G_INTTOPTR:
   case TargetOpcode::G_TRUNC:
   case TargetOpcode::G_FREEZE:
-    return selectCopy(MI, MRI);
+    return selectCopy(MI);
   case TargetOpcode::G_CONSTANT: {
     Register DstReg = MI.getOperand(0).getReg();
     int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue();
@@ -576,9 +594,9 @@ bool RISCVInstructionSelector::select(MachineInstr &MI) {
     Register DstReg = MI.getOperand(0).getReg();
     const APFloat &FPimm = MI.getOperand(1).getFPImm()->getValueAPF();
     APInt Imm = FPimm.bitcastToAPInt();
-    unsigned Size = MRI.getType(DstReg).getSizeInBits();
+    unsigned Size = MRI->getType(DstReg).getSizeInBits();
     if (Size == 16 || Size == 32 || (Size == 64 && Subtarget->is64Bit())) {
-      Register GPRReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
+      Register GPRReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
       if (!materializeImm(GPRReg, Imm.getSExtValue(), MIB))
         return false;
 
@@ -592,8 +610,8 @@ bool RISCVInstructionSelector::select(MachineInstr &MI) {
       assert(Size == 64 && !Subtarget->is64Bit() &&
              "Unexpected size or subtarget");
       // Split into two pieces and build through the stack.
-      Register GPRRegHigh = MRI.createVirtualRegister(&RISCV::GPRRegClass);
-      Register GPRRegLow = MRI.createVirtualRegister(&RISCV::GPRRegClass);
+      Register GPRRegHigh = MRI->createVirtualRegister(&RISCV::GPRRegClass);
+      Register GPRRegLow = MRI->createVirtualRegister(&RISCV::GPRRegClass);
       if (!materializeImm(GPRRegHigh, Imm.extractBits(32, 32).getSExtValue(),
                           MIB))
         return false;
@@ -615,8 +633,7 @@ bool RISCVInstructionSelector::select(MachineInstr &MI) {
       return false;
     }
 
-    return selectAddr(MI, MIB, MRI, GV->isDSOLocal(),
-                      GV->hasExternalWeakLinkage());
+    return selectAddr(MI, MIB, GV->isDSOLocal(), GV->hasExternalWeakLinkage());
   }
   case TargetOpcode::G_JUMP_TABLE:
   case TargetOpcode::G_CONSTANT_POOL:
@@ -624,7 +641,7 @@ bool RISCVInstructionSelector::select(MachineInstr &MI) {
   case TargetOpcode::G_BRCOND: {
     Register LHS, RHS;
     RISCVCC::CondCode CC;
-    getOperandsForBranch(MI.getOperand(0).getReg(), MRI, CC, LHS, RHS);
+    getOperandsForBranch(MI.getOperand(0).getReg(), CC, LHS, RHS, *MRI);
 
     auto Bcc = MIB.buildInstr(RISCVCC::getBrCond(CC), {}, {LHS, RHS})
                    .addMBB(MI.getOperand(1).getMBB());
@@ -698,9 +715,9 @@ bool RISCVInstructionSelector::select(MachineInstr &MI) {
     return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
   }
   case TargetOpcode::G_SELECT:
-    return selectSelect(MI, MIB, MRI);
+    return selectSelect(MI, MIB);
   case TargetOpcode::G_FCMP:
-    return selectFPCompare(MI, MIB, MRI);
+    return selectFPCompare(MI, MIB);
   case TargetOpcode::G_FENCE: {
     AtomicOrdering FenceOrdering =
         static_cast<AtomicOrdering>(MI.getOperand(0).getImm());
@@ -711,18 +728,18 @@ bool RISCVInstructionSelector::select(MachineInstr &MI) {
     return true;
   }
   case TargetOpcode::G_IMPLICIT_DEF:
-    return selectImplicitDef(MI, MIB, MRI);
+    return selectImplicitDef(MI, MIB);
   case TargetOpcode::G_MERGE_VALUES:
-    return selectMergeValues(MI, MIB, MRI);
+    return selectMergeValues(MI, MIB);
   case TargetOpcode::G_UNMERGE_VALUES:
-    return selectUnmergeValues(MI, MIB, MRI);
+    return selectUnmergeValues(MI, MIB);
   default:
     return false;
   }
 }
 
-bool RISCVInstructionSelector::selectMergeValues(
-    MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) const {
+bool RISCVInstructionSelector::selectMergeValues(MachineInstr &MI,
+                                                 MachineIRBuilder &MIB) const {
   assert(MI.getOpcode() == TargetOpcode::G_MERGE_VALUES);
 
   // Build a F64 Pair from operands
@@ -731,14 +748,14 @@ bool RISCVInstructionSelector::selectMergeValues(
   Register Dst = MI.getOperand(0).getReg();
   Register Lo = MI.getOperand(1).getReg();
   Register Hi = MI.getOperand(2).getReg();
-  if (!isRegInFprb(Dst, MRI) || !isRegInGprb(Lo, MRI) || !isRegInGprb(Hi, MRI))
+  if (!isRegInFprb(Dst) || !isRegInGprb(Lo) || !isRegInGprb(Hi))
     return false;
   MI.setDesc(TII.get(RISCV::BuildPairF64Pseudo));
   return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
 }
 
 bool RISCVInstructionSelector::selectUnmergeValues(
-    MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) const {
+    MachineInstr &MI, MachineIRBuilder &MIB) const {
   assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
 
   // Split F64 Src into two s32 parts
@@ -747,44 +764,42 @@ bool RISCVInstructionSelector::selectUnmergeValues(
   Register Src = MI.getOperand(2).getReg();
   Register Lo = MI.getOperand(0).getReg();
   Register Hi = MI.getOperand(1).getReg();
-  if (!isRegInFprb(Src, MRI) || !isRegInGprb(Lo, MRI) || !isRegInGprb(Hi, MRI))
+  if (!isRegInFprb(Src) || !isRegInGprb(Lo) || !isRegInGprb(Hi))
     return false;
   MI.setDesc(TII.get(RISCV::SplitF64Pseudo));
   return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
 }
 
 bool RISCVInstructionSelector::replacePtrWithInt(MachineOperand &Op,
-                                                 MachineIRBuilder &MIB,
-                                                 MachineRegisterInfo &MRI) {
+                                                 MachineIRBuilder &MIB) {
   Register PtrReg = Op.getReg();
-  assert(MRI.getType(PtrReg).isPointer() && "Operand is not a pointer!");
+  assert(MRI->getType(PtrReg).isPointer() && "Operand is not a pointer!");
 
   const LLT sXLen = LLT::scalar(STI.getXLen());
   auto PtrToInt = MIB.buildPtrToInt(sXLen, PtrReg);
-  MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(RISCV::GPRBRegBankID));
+  MRI->setRegBank(PtrToInt.getReg(0), RBI.getRegBank(RISCV::GPRBRegBankID));
   Op.setReg(PtrToInt.getReg(0));
   return select(*PtrToInt);
 }
 
 void RISCVInstructionSelector::preISelLower(MachineInstr &MI,
-                                            MachineIRBuilder &MIB,
-                                            MachineRegisterInfo &MRI) {
+                                            MachineIRBuilder &MIB) {
   switch (MI.getOpcode()) {
   case TargetOpcode::G_PTR_ADD: {
     Register DstReg = MI.getOperand(0).getReg();
     const LLT sXLen = LLT::scalar(STI.getXLen());
 
-    replacePtrWithInt(MI.getOperand(1), MIB, MRI);
+    replacePtrWithInt(MI.getOperand(1), MIB);
     MI.setDesc(TII.get(TargetOpcode::G_ADD));
-    MRI.setType(DstReg, sXLen);
+    MRI->setType(DstReg, sXLen);
     break;
   }
   case TargetOpcode::G_PTRMASK: {
     Register DstReg = MI.getOperand(0).getReg();
     const LLT sXLen = LLT::scalar(STI.getXLen());
-    replacePtrWithInt(MI.getOperand(1), MIB, MRI);
+    replacePtrWithInt(MI.getOperand(1), MIB);
     MI.setDesc(TII.get(TargetOpcode::G_AND));
-    MRI.setType(DstReg, sXLen);
+    MRI->setType(DstReg, sXLen);
   }
   }
 }
@@ -876,32 +891,29 @@ const TargetRegisterClass *RISCVInstructionSelector::getRegClassForTypeOnBank(
   return nullptr;
 }
 
-bool RISCVInstructionSelector::isRegInGprb(Register Reg,
-                                           MachineRegisterInfo &MRI) const {
-  return RBI.getRegBank(Reg, MRI, TRI)->getID() == RISCV::GPRBRegBankID;
+bool RISCVInstructionSelector::isRegInGprb(Register Reg) const {
+  return RBI.getRegBank(Reg, *MRI, TRI)->getID() == RISCV::GPRBRegBankID;
 }
 
-bool RISCVInstructionSelector::isRegInFprb(Register Reg,
-                                           MachineRegisterInfo &MRI) const {
-  return RBI.getRegBank(Reg, MRI, TRI)->getID() == RISCV::FPRBRegBankID;
+bool RISCVInstructionSelector::isRegInFprb(Register Reg) const {
+  return RBI.getRegBank(Reg, *MRI, TRI)->getID() == RISCV::FPRBRegBankID;
 }
 
-bool RISCVInstructionSelector::selectCopy(MachineInstr &MI,
-                                          MachineRegisterInfo &MRI) const {
+bool RISCVInstructionSelector::selectCopy(MachineInstr &MI) const {
   Register DstReg = MI.getOperand(0).getReg();
 
   if (DstReg.isPhysical())
     return true;
 
   const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
-      MRI.getType(DstReg), *RBI.getRegBank(DstReg, MRI, TRI));
+      MRI->getType(DstReg), *RBI.getRegBank(DstReg, *MRI, TRI));
   assert(DstRC &&
          "Register class not available for LLT, register bank combination");
 
   // No need to constrain SrcReg. It will get constrained when
   // we hit another of its uses or its defs.
   // Copies do not have constraints.
-  if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
+  if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) {
     LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
                       << " operand\n");
     return false;
@@ -911,18 +923,18 @@ bool RISCVInstructionSelector::selectCopy(MachineInstr &MI,
   return true;
 }
 
-bool RISCVInstructionSelector::selectImplicitDef(
-    MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) const {
+bool RISCVInstructionSelector::selectImplicitDef(MachineInstr &MI,
+                                                 MachineIRBuilder &MIB) const {
   assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
 
   const Register DstReg = MI.getOperand(0).getReg();
   const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
-      MRI.getType(DstReg), *RBI.getRegBank(DstReg, MRI, TRI));
+      MRI->getType(DstReg), *RBI.getRegBank(DstReg, *MRI, TRI));
 
   assert(DstRC &&
          "Register class not available for LLT, register bank combination");
 
-  if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
+  if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) {
     LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
                       << " operand\n");
   }
@@ -932,11 +944,9 @@ bool RISCVInstructionSelector::selectImplicitDef(
 
 bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
                                               MachineIRBuilder &MIB) const {
-  MachineRegisterInfo &MRI = *MIB.getMRI();
-
   if (Imm == 0) {
     MIB.buildCopy(DstReg, Register(RISCV::X0));
-    RBI.constrainGenericRegister(DstReg, RISCV::GPRRegClass, MRI);
+    RBI.constrainGenericRegister(DstReg, RISCV::GPRRegClass, *MRI);
     return true;
   }
 
@@ -946,7 +956,7 @@ bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
 
   for (unsigned i = 0; i < NumInsts; i++) {
     Register TmpReg = i < NumInsts - 1
-                          ? MRI.createVirtualRegister(&RISCV::GPRRegClass)
+                          ? MRI->createVirtualRegister(&RISCV::GPRRegClass)
                           : DstReg;
     const RISCVMatInt::Inst &I = Seq[i];
     MachineInstr *Result;
@@ -981,9 +991,7 @@ bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
 }
 
 bool RISCVInstructionSelector::selectAddr(MachineInstr &MI,
-                                          MachineIRBuilder &MIB,
-                                          MachineRegisterInfo &MRI,
-                                          bool IsLocal,
+                                          MachineIRBuilder &MIB, bool IsLocal,
                                           bool IsExternWeak) const {
   assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
           MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||
@@ -993,7 +1001,7 @@ bool RISCVInstructionSelector::selectAddr(MachineInstr &MI,
   const MachineOperand &DispMO = MI.getOperand(1);
 
   Register DefReg = MI.getOperand(0).getReg();
-  const LLT DefTy = MRI.getType(DefReg);
+  const LLT DefTy = MRI->getType(DefReg);
 
   // When HWASAN is used and tagging of global variables is enabled
   // they should be accessed via the GOT, since the tagged address of a global
@@ -1040,7 +1048,7 @@ bool RISCVInstructionSelector::selectAddr(MachineInstr &MI,
     // Must lie within a single 2 GiB address range and must lie between
     // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi
     // (lui %hi(sym)) %lo(sym)).
-    Register AddrHiDest = MRI.createVirtualRegister(&RISCV::GPRRegClass);
+    Register AddrHiDest = MRI->createVirtualRegister(&RISCV::GPRRegClass);
     MachineInstr *AddrHi = MIB.buildInstr(RISCV::LUI, {AddrHiDest}, {})
                                .addDisp(DispMO, 0, RISCVII::MO_HI);
 
@@ -1120,19 +1128,18 @@ bool RISCVInstructionSelector::selectSExtInreg(MachineInstr &MI,
 }
 
 bool RISCVInstructionSelector::selectSelect(MachineInstr &MI,
-                                            MachineIRBuilder &MIB,
-                                            MachineRegisterInfo &MRI) const {
+                                            MachineIRBuilder &MIB) const {
   auto &SelectMI = cast<GSelect>(MI);
 
   Register LHS, RHS;
   RISCVCC::CondCode CC;
-  getOperandsForBranch(SelectMI.getCondReg(), MRI, CC, LHS, RHS);
+  getOperandsForBranch(SelectMI.getCondReg(), CC, LHS, RHS, *MRI);
 
   Register DstReg = SelectMI.getReg(0);
 
   unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;
-  if (RBI.getRegBank(DstReg, MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
-    unsigned Size = MRI.getType(DstReg).getSizeInBits();
+  if (RBI.getRegBank(DstReg, *MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
+    unsigned Size = MRI->getType(DstReg).getSizeInBits();
     Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR
                      : RISCV::Select_FPR64_Using_CC_GPR;
   }
@@ -1201,8 +1208,7 @@ static bool legalizeFCmpPredicate(Register &LHS, Register &RHS,
 // the result in DstReg.
 // FIXME: Maybe we should expand this earlier.
 bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI,
-                                               MachineIRBuilder &MIB,
-                                               MachineRegisterInfo &MRI) const {
+                                               MachineIRBuilder &MIB) const {
   auto &CmpMI = cast<GFCmp>(MI);
   CmpInst::Predicate Pred = CmpMI.getCond();
 
@@ -1210,7 +1216,7 @@ bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI,
   Register LHS = CmpMI.getLHSReg();
   Register RHS = CmpMI.getRHSReg();
 
-  unsigned Size = MRI.getType(LHS).getSizeInBits();
+  unsigned Size = MRI->getType(LHS).getSizeInBits();
   assert((Size == 16 || Size == 32 || Size == 64) && "Unexpected size");
 
   Register TmpReg = DstReg;
@@ -1219,7 +1225,7 @@ bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI,
   // First try swapping operands or inverting.
   if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {
     if (NeedInvert)
-      TmpReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
+      TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
     auto Cmp = MIB.buildInstr(getFCmpOpcode(Pred, Size), {TmpReg}, {LHS, RHS});
     if (!Cmp.constrainAllUses(TII, TRI, RBI))
       return false;
@@ -1235,7 +1241,7 @@ bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI,
     if (!Cmp2.constrainAllUses(TII, TRI, RBI))
       return false;
     if (NeedInvert)
-      TmpReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
+      TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
     auto Or =
         MIB.buildInstr(RISCV::OR, {TmpReg}, {Cmp1.getReg(0), Cmp2.getReg(0)});
     if (!Or.constrainAllUses(TII, TRI, RBI))
@@ -1253,7 +1259,7 @@ bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI,
     if (!Cmp2.constrainAllUses(TII, TRI, RBI))
       return false;
     if (NeedInvert)
-      TmpReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
+      TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
     auto And =
         MIB.buildInstr(RISCV::AND, {TmpReg}, {Cmp1.getReg(0), Cmp2.getReg(0)});
     if (!And.constrainAllUses(TII, TRI, RBI))
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 055193bcc2c8d..d1449f751b40a 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -158,14 +158,12 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
       .lower();
 
   auto &ShiftActions = getActionDefinitionsBuilder({G_ASHR, G_LSHR, G_SHL});
-  if (ST.is64Bit())
-    ShiftActions.customFor({{s32, s32}});
-  ShiftActions.legalFor({{s32, s32}, {s32, sXLen}, {sXLen, sXLen}})
+  ShiftActions.legalFor({{s32, s32}, {sXLen, sXLen}})
       .widenScalarToNextPow2(0)
       .clampScalar(1, s32, sXLen)
       .clampScalar(0, s32, sXLen)
       .minScalarSameAs(1, 0)
-      .widenScalarToNextPow2(1);
+      .maxScalarSameAs(1, 0);
 
   auto &ExtActions =
       getActionDefinitionsBuilder({G_ZEXT, G_SEXT, G_ANYEXT})
@@ -204,13 +202,8 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
   getActionDefinitionsBuilder({G_FSHL, G_FSHR}).lower();
 
   auto &RotateActions = getActionDefinitionsBuilder({G_ROTL, G_ROTR});
-  if (ST.hasStdExtZbb() || ST.hasStdExtZbkb()) {
-    RotateActions.legalFor({{s32, sXLen}, {sXLen, sXLen}});
-    // Widen s32 rotate amount to s64 so SDAG patterns will match.
-    if (ST.is64Bit())
-      RotateActions.widenScalarIf(all(typeIs(0, s32), typeIs(1, s32)),
-                                  changeTo(1, sXLen));
-  }
+  if (ST.hasStdExtZbb() || ST.hasStdExtZbkb())
+    RotateActions.legalFor({{s32, s32}, {sXLen, sXLen}});
   RotateActions.lower();
 
   getActionDefinitionsBuilder(G_BITREVERSE).maxScalar(0, sXLen).lower();
@@ -597,6 +590,31 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
 
   SplatActions.clampScalar(1, sXLen, sXLen);
 
+  LegalityPredicate ExtractSubvecBitcastPred = [=](const LegalityQuery &Query) {
+    LLT DstTy = Query.Types[0];
+    LLT SrcTy = Query.Types[1];
+    return DstTy.getElementType() == LLT::scalar(1) &&
+           DstTy.getElementCount().getKnownMinValue() >= 8 &&
+           SrcTy.getElementCount().getKnownMinValue() >= 8;
+  };
+  getActionDefinitionsBuilder(G_EXTRACT_SUBVECTOR)
+      // We don't have the ability to slide mask vectors down indexed by their
+      // i1 elements; the smallest we can do is i8. Often we are able to bitcast
+      // to equivalent i8 vectors.
+      .bitcastIf(
+          all(typeIsLegalBoolVec(0, BoolVecTys, ST),
+              typeIsLegalBoolVec(1, BoolVecTys, ST), ExtractSubvecBitcastPred),
+          [=](const LegalityQuery &Query) {
+            LLT CastTy = LLT::vector(
+                Query.Types[0].getElementCount().divideCoefficientBy(8), 8);
+            return std::pair(0, CastTy);
+          })
+      .customIf(LegalityPredicates::any(
+          all(typeIsLegalBoolVec(0, BoolVecTys, ST),
+              typeIsLegalBoolVec(1, BoolVecTys, ST)),
+          all(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST),
+              typeIsLegalIntOrFPVec(1, IntOrFPVecTys, ST))));
+
   getLegacyLegalizerInfo().computeTables();
 }
 
@@ -636,30 +654,6 @@ bool RISCVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
   }
 }
 
-bool RISCVLegalizerInfo::legalizeShlAshrLshr(
-    MachineInstr &MI, MachineIRBuilder &MIRBuilder,
-    GISelChangeObserver &Observer) const {
-  assert(MI.getOpcode() == TargetOpcode::G_ASHR ||
-         MI.getOpcode() == TargetOpcode::G_LSHR ||
-         MI.getOpcode() == TargetOpcode::G_SHL);
-  MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
-  // If the shift amount is a G_CONSTANT, promote it to a 64 bit type so the
-  // imported patterns can select it later. Either way, it will be legal.
-  Register AmtReg = MI.getOperand(2).getReg();
-  auto VRegAndVal = getIConstantVRegValWithLookThrough(AmtReg, MRI);
-  if (!VRegAndVal)
-    return true;
-  // Check the shift amount is in range for an immediate form.
-  uint64_t Amount = VRegAndVal->Value.getZExtValue();
-  if (Amount > 31)
-    return true; // This will have to remain a register variant.
-  auto ExtCst = MIRBuilder.buildConstant(LLT::scalar(64), Amount);
-  Observer.changingInstr(MI);
-  MI.getOperand(2).setReg(ExtCst.getReg(0));
-  Observer.changedInstr(MI);
-  return true;
-}
-
 bool RISCVLegalizerInfo::legalizeVAStart(MachineInstr &MI,
                                          MachineIRBuilder &MIRBuilder) const {
   // Stores the address of the VarArgsFrameIndex slot into the memory location
@@ -839,19 +833,21 @@ static MachineInstrBuilder buildAllOnesMask(LLT VecTy, const SrcOp &VL,
 
 /// Gets the two common "VL" operands: an all-ones mask and the vector length.
 /// VecTy is a scalable vector type.
-static std::pair<MachineInstrBuilder, Register>
+static std::pair<MachineInstrBuilder, MachineInstrBuilder>
 buildDefaultVLOps(const DstOp &Dst, MachineIRBuilder &MIB,
                   MachineRegisterInfo &MRI) {
   LLT VecTy = Dst.getLLTTy(MRI);
   assert(VecTy.isScalableVector() && "Expecting scalable container type");
-  Register VL(RISCV::X0);
-  MachineInstrBuilder Mask = buildAllOnesMask(VecTy, VL, MIB, MRI);
+  const RISCVSubtarget &STI = MIB.getMF().getSubtarget<RISCVSubtarget>();
+  LLT XLenTy(STI.getXLenVT());
+  auto VL = MIB.buildConstant(XLenTy, -1);
+  auto Mask = buildAllOnesMask(VecTy, VL, MIB, MRI);
   return {Mask, VL};
 }
 
 static MachineInstrBuilder
 buildSplatPartsS64WithVL(const DstOp &Dst, const SrcOp &Passthru, Register Lo,
-                         Register Hi, Register VL, MachineIRBuilder &MIB,
+                         Register Hi, const SrcOp &VL, MachineIRBuilder &MIB,
                          MachineRegisterInfo &MRI) {
   // TODO: If the Hi bits of the splat are undefined, then it's fine to just
   // splat Lo even if it might be sign extended. I don't think we have
@@ -867,7 +863,7 @@ buildSplatPartsS64WithVL(const DstOp &Dst, const SrcOp &Passthru, Register Lo,
 
 static MachineInstrBuilder
 buildSplatSplitS64WithVL(const DstOp &Dst, const SrcOp &Passthru,
-                         const SrcOp &Scalar, Register VL,
+                         const SrcOp &Scalar, const SrcOp &VL,
                          MachineIRBuilder &MIB, MachineRegisterInfo &MRI) {
   assert(Scalar.getLLTTy(MRI) == LLT::scalar(64) && "Unexpected VecTy!");
   auto Unmerge = MIB.buildUnmerge(LLT::scalar(32), Scalar);
@@ -931,12 +927,109 @@ bool RISCVLegalizerInfo::legalizeSplatVector(MachineInstr &MI,
   return true;
 }
 
+static LLT getLMUL1Ty(LLT VecTy) {
+  assert(VecTy.getElementType().getSizeInBits() <= 64 &&
+         "Unexpected vector LLT");
+  return LLT::scalable_vector(RISCV::RVVBitsPerBlock /
+                                  VecTy.getElementType().getSizeInBits(),
+                              VecTy.getElementType());
+}
+
+bool RISCVLegalizerInfo::legalizeExtractSubvector(MachineInstr &MI,
+                                                  MachineIRBuilder &MIB) const {
+  GExtractSubvector &ES = cast<GExtractSubvector>(MI);
+
+  MachineRegisterInfo &MRI = *MIB.getMRI();
+
+  Register Dst = ES.getReg(0);
+  Register Src = ES.getSrcVec();
+  uint64_t Idx = ES.getIndexImm();
+
+  // With an index of 0 this is a cast-like subvector, which can be performed
+  // with subregister operations.
+  if (Idx == 0)
+    return true;
+
+  LLT LitTy = MRI.getType(Dst);
+  LLT BigTy = MRI.getType(Src);
+
+  if (LitTy.getElementType() == LLT::scalar(1)) {
+    // We can't slide this mask vector up indexed by its i1 elements.
+    // This poses a problem when we wish to insert a scalable vector which
+    // can't be re-expressed as a larger type. Just choose the slow path and
+    // extend to a larger type, then truncate back down.
+    LLT ExtBigTy = BigTy.changeElementType(LLT::scalar(8));
+    LLT ExtLitTy = LitTy.changeElementType(LLT::scalar(8));
+    auto BigZExt = MIB.buildZExt(ExtBigTy, Src);
+    auto ExtractZExt = MIB.buildExtractSubvector(ExtLitTy, BigZExt, Idx);
+    auto SplatZero = MIB.buildSplatVector(
+        ExtLitTy, MIB.buildConstant(ExtLitTy.getElementType(), 0));
+    MIB.buildICmp(CmpInst::Predicate::ICMP_NE, Dst, ExtractZExt, SplatZero);
+    MI.eraseFromParent();
+    return true;
+  }
+
+  // extract_subvector scales the index by vscale if the subvector is scalable,
+  // and decomposeSubvectorInsertExtractToSubRegs takes this into account.
+  const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
+  MVT LitTyMVT = getMVTForLLT(LitTy);
+  auto Decompose =
+      RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
+          getMVTForLLT(BigTy), LitTyMVT, Idx, TRI);
+  unsigned RemIdx = Decompose.second;
+
+  // If the Idx has been completely eliminated then this is a subvector extract
+  // which naturally aligns to a vector register. These can easily be handled
+  // using subregister manipulation.
+  if (RemIdx == 0)
+    return true;
+
+  // Else LitTy is M1 or smaller and may need to be slid down: if LitTy
+  // was > M1 then the index would need to be a multiple of VLMAX, and so would
+  // divide exactly.
+  assert(
+      RISCVVType::decodeVLMUL(RISCVTargetLowering::getLMUL(LitTyMVT)).second ||
+      RISCVTargetLowering::getLMUL(LitTyMVT) == RISCVII::VLMUL::LMUL_1);
+
+  // If the vector type is an LMUL-group type, extract a subvector equal to the
+  // nearest full vector register type.
+  LLT InterLitTy = BigTy;
+  Register Vec = Src;
+  if (TypeSize::isKnownGT(BigTy.getSizeInBits(),
+                          getLMUL1Ty(BigTy).getSizeInBits())) {
+    // If BigTy has an LMUL > 1, then LitTy should have a smaller LMUL, and
+    // we should have successfully decomposed the extract into a subregister.
+    assert(Decompose.first != RISCV::NoSubRegister);
+    InterLitTy = getLMUL1Ty(BigTy);
+    // SDAG builds a TargetExtractSubreg. We cannot create a a Copy with SubReg
+    // specified on the source Register (the equivalent) since generic virtual
+    // register does not allow subregister index.
+    Vec = MIB.buildExtractSubvector(InterLitTy, Src, Idx - RemIdx).getReg(0);
+  }
+
+  // Slide this vector register down by the desired number of elements in order
+  // to place the desired subvector starting at element 0.
+  const LLT XLenTy(STI.getXLenVT());
+  auto SlidedownAmt = MIB.buildVScale(XLenTy, RemIdx);
+  auto [Mask, VL] = buildDefaultVLOps(LitTy, MIB, MRI);
+  uint64_t Policy = RISCVII::TAIL_AGNOSTIC | RISCVII::MASK_AGNOSTIC;
+  auto Slidedown = MIB.buildInstr(
+      RISCV::G_VSLIDEDOWN_VL, {InterLitTy},
+      {MIB.buildUndef(InterLitTy), Vec, SlidedownAmt, Mask, VL, Policy});
+
+  // Now the vector is in the right position, extract our final subvector. This
+  // should resolve to a COPY.
+  MIB.buildExtractSubvector(Dst, Slidedown, 0);
+
+  MI.eraseFromParent();
+  return true;
+}
+
 bool RISCVLegalizerInfo::legalizeCustom(
     LegalizerHelper &Helper, MachineInstr &MI,
     LostDebugLocObserver &LocObserver) const {
   MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
   MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
-  GISelChangeObserver &Observer = Helper.Observer;
   MachineFunction &MF = *MI.getParent()->getParent();
   switch (MI.getOpcode()) {
   default:
@@ -955,10 +1048,6 @@ bool RISCVLegalizerInfo::legalizeCustom(
       return true;
     return Helper.lowerConstant(MI);
   }
-  case TargetOpcode::G_SHL:
-  case TargetOpcode::G_ASHR:
-  case TargetOpcode::G_LSHR:
-    return legalizeShlAshrLshr(MI, MIRBuilder, Observer);
   case TargetOpcode::G_SEXT_INREG: {
     LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
     int64_t SizeInBits = MI.getOperand(2).getImm();
@@ -1001,6 +1090,8 @@ bool RISCVLegalizerInfo::legalizeCustom(
     return legalizeExt(MI, MIRBuilder);
   case TargetOpcode::G_SPLAT_VECTOR:
     return legalizeSplatVector(MI, MIRBuilder);
+  case TargetOpcode::G_EXTRACT_SUBVECTOR:
+    return legalizeExtractSubvector(MI, MIRBuilder);
   case TargetOpcode::G_LOAD:
   case TargetOpcode::G_STORE:
     return legalizeLoadStore(MI, Helper, MIRBuilder);
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
index 2fc28615e7630..d2afb175ae42b 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
@@ -46,6 +46,7 @@ class RISCVLegalizerInfo : public LegalizerInfo {
   bool legalizeVScale(MachineInstr &MI, MachineIRBuilder &MIB) const;
   bool legalizeExt(MachineInstr &MI, MachineIRBuilder &MIRBuilder) const;
   bool legalizeSplatVector(MachineInstr &MI, MachineIRBuilder &MIB) const;
+  bool legalizeExtractSubvector(MachineInstr &MI, MachineIRBuilder &MIB) const;
   bool legalizeLoadStore(MachineInstr &MI, LegalizerHelper &Helper,
                          MachineIRBuilder &MIB) const;
 };
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
index 5369be24f0e7c..aa06cb4d5327c 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
@@ -478,6 +478,21 @@ RISCVRegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     }
     break;
   }
+  case TargetOpcode::G_SPLAT_VECTOR: {
+    OpdsMapping[0] = getVRBValueMapping(MRI.getType(MI.getOperand(0).getReg())
+                                            .getSizeInBits()
+                                            .getKnownMinValue());
+
+    LLT ScalarTy = MRI.getType(MI.getOperand(1).getReg());
+    MachineInstr *DefMI = MRI.getVRegDef(MI.getOperand(1).getReg());
+    if ((GPRSize == 32 && ScalarTy.getSizeInBits() == 64) ||
+        onlyDefinesFP(*DefMI, MRI, TRI)) {
+      assert(MF.getSubtarget<RISCVSubtarget>().hasStdExtD());
+      OpdsMapping[1] = getFPValueMapping(ScalarTy.getSizeInBits());
+    } else
+      OpdsMapping[1] = GPRValueMapping;
+    break;
+  }
   default:
     // By default map all scalars to GPR.
     for (unsigned Idx = 0; Idx < NumOperands; ++Idx) {
diff --git a/llvm/lib/Target/RISCV/RISCVCallingConv.cpp b/llvm/lib/Target/RISCV/RISCVCallingConv.cpp
index d610f0b956027..d3bfbb0943766 100644
--- a/llvm/lib/Target/RISCV/RISCVCallingConv.cpp
+++ b/llvm/lib/Target/RISCV/RISCVCallingConv.cpp
@@ -156,6 +156,23 @@ static ArrayRef<MCPhysReg> getArgGPR16s(const RISCVABI::ABI ABI) {
   return ArrayRef(ArgIGPRs);
 }
 
+static ArrayRef<MCPhysReg> getArgGPR32s(const RISCVABI::ABI ABI) {
+  // The GPRs used for passing arguments in the ILP32* and LP64* ABIs, except
+  // the ILP32E ABI.
+  static const MCPhysReg ArgIGPRs[] = {RISCV::X10_W, RISCV::X11_W, RISCV::X12_W,
+                                       RISCV::X13_W, RISCV::X14_W, RISCV::X15_W,
+                                       RISCV::X16_W, RISCV::X17_W};
+  // The GPRs used for passing arguments in the ILP32E/LP64E ABI.
+  static const MCPhysReg ArgEGPRs[] = {RISCV::X10_W, RISCV::X11_W,
+                                       RISCV::X12_W, RISCV::X13_W,
+                                       RISCV::X14_W, RISCV::X15_W};
+
+  if (ABI == RISCVABI::ABI_ILP32E || ABI == RISCVABI::ABI_LP64E)
+    return ArrayRef(ArgEGPRs);
+
+  return ArrayRef(ArgIGPRs);
+}
+
 static ArrayRef<MCPhysReg> getFastCCArgGPRs(const RISCVABI::ABI ABI) {
   // The GPRs used for passing arguments in the FastCC, X5 and X6 might be used
   // for save-restore libcall, so we don't use them.
@@ -194,6 +211,26 @@ static ArrayRef<MCPhysReg> getFastCCArgGPRF16s(const RISCVABI::ABI ABI) {
   return ArrayRef(FastCCIGPRs);
 }
 
+static ArrayRef<MCPhysReg> getFastCCArgGPRF32s(const RISCVABI::ABI ABI) {
+  // The GPRs used for passing arguments in the FastCC, X5 and X6 might be used
+  // for save-restore libcall, so we don't use them.
+  // Don't use X7 for fastcc, since Zicfilp uses X7 as the label register.
+  static const MCPhysReg FastCCIGPRs[] = {
+      RISCV::X10_W, RISCV::X11_W, RISCV::X12_W, RISCV::X13_W,
+      RISCV::X14_W, RISCV::X15_W, RISCV::X16_W, RISCV::X17_W,
+      RISCV::X28_W, RISCV::X29_W, RISCV::X30_W, RISCV::X31_W};
+
+  // The GPRs used for passing arguments in the FastCC when using ILP32E/LP64E.
+  static const MCPhysReg FastCCEGPRs[] = {RISCV::X10_W, RISCV::X11_W,
+                                          RISCV::X12_W, RISCV::X13_W,
+                                          RISCV::X14_W, RISCV::X15_W};
+
+  if (ABI == RISCVABI::ABI_ILP32E || ABI == RISCVABI::ABI_LP64E)
+    return ArrayRef(FastCCEGPRs);
+
+  return ArrayRef(FastCCIGPRs);
+}
+
 // Pass a 2*XLEN argument that has been split into two XLEN values through
 // registers or the stack as necessary.
 static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1,
@@ -364,11 +401,17 @@ bool llvm::CC_RISCV(unsigned ValNo, MVT ValVT, MVT LocVT,
     }
   }
 
+  if (ValVT == MVT::f32 && Subtarget.hasStdExtZfinx()) {
+    if (MCRegister Reg = State.AllocateReg(getArgGPR32s(ABI))) {
+      State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
+      return false;
+    }
+  }
+
   ArrayRef<MCPhysReg> ArgGPRs = RISCV::getArgGPRs(ABI);
 
-  // Zfinx/Zdinx use GPR without a bitcast when possible.
-  if ((LocVT == MVT::f32 && XLen == 32 && Subtarget.hasStdExtZfinx()) ||
-      (LocVT == MVT::f64 && XLen == 64 && Subtarget.hasStdExtZdinx())) {
+  // Zdinx use GPR without a bitcast when possible.
+  if (LocVT == MVT::f64 && XLen == 64 && Subtarget.hasStdExtZdinx()) {
     if (MCRegister Reg = State.AllocateReg(ArgGPRs)) {
       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
       return false;
@@ -616,10 +659,16 @@ bool llvm::CC_RISCV_FastCC(unsigned ValNo, MVT ValVT, MVT LocVT,
     }
   }
 
+  // Check if there is an available GPRF32 before hitting the stack.
+  if (LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) {
+    if (MCRegister Reg = State.AllocateReg(getFastCCArgGPRF32s(ABI))) {
+      State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
+      return false;
+    }
+  }
+
   // Check if there is an available GPR before hitting the stack.
-  if ((LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
-      (LocVT == MVT::f64 && Subtarget.is64Bit() &&
-       Subtarget.hasStdExtZdinx())) {
+  if (LocVT == MVT::f64 && Subtarget.is64Bit() && Subtarget.hasStdExtZdinx()) {
     if (MCRegister Reg = State.AllocateReg(getFastCCArgGPRs(ABI))) {
       if (LocVT.getSizeInBits() != Subtarget.getXLen()) {
         LocVT = XLenVT;
@@ -723,9 +772,18 @@ bool llvm::CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
     }
   }
 
-  if ((LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
-      (LocVT == MVT::f64 && Subtarget.hasStdExtZdinx() &&
-       Subtarget.is64Bit())) {
+  if (LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) {
+    static const MCPhysReg GPR32List[] = {
+        RISCV::X9_W,  RISCV::X18_W, RISCV::X19_W, RISCV::X20_W,
+        RISCV::X21_W, RISCV::X22_W, RISCV::X23_W, RISCV::X24_W,
+        RISCV::X25_W, RISCV::X26_W, RISCV::X27_W};
+    if (MCRegister Reg = State.AllocateReg(GPR32List)) {
+      State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
+      return false;
+    }
+  }
+
+  if (LocVT == MVT::f64 && Subtarget.hasStdExtZdinx() && Subtarget.is64Bit()) {
     if (MCRegister Reg = State.AllocateReg(GPRList)) {
       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
       return false;
diff --git a/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp b/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp
index 713c7a0661def..d913c0b201a20 100644
--- a/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp
+++ b/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp
@@ -97,6 +97,8 @@ bool RISCVDeadRegisterDefinitions::runOnMachineFunction(MachineFunction &MF) {
         const TargetRegisterClass *RC = TII->getRegClass(Desc, I, TRI, MF);
         if (RC && RC->contains(RISCV::X0)) {
           X0Reg = RISCV::X0;
+        } else if (RC && RC->contains(RISCV::X0_W)) {
+          X0Reg = RISCV::X0_W;
         } else if (RC && RC->contains(RISCV::X0_H)) {
           X0Reg = RISCV::X0_H;
         } else {
diff --git a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
index 2501256ca6adf..5dcec078856ea 100644
--- a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
@@ -50,6 +50,8 @@ class RISCVExpandPseudo : public MachineFunctionPass {
                          MachineBasicBlock::iterator MBBI, unsigned Opcode);
   bool expandMV_FPR16INX(MachineBasicBlock &MBB,
                          MachineBasicBlock::iterator MBBI);
+  bool expandMV_FPR32INX(MachineBasicBlock &MBB,
+                         MachineBasicBlock::iterator MBBI);
   bool expandRV32ZdinxStore(MachineBasicBlock &MBB,
                             MachineBasicBlock::iterator MBBI);
   bool expandRV32ZdinxLoad(MachineBasicBlock &MBB,
@@ -108,6 +110,8 @@ bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB,
   switch (MBBI->getOpcode()) {
   case RISCV::PseudoMV_FPR16INX:
     return expandMV_FPR16INX(MBB, MBBI);
+  case RISCV::PseudoMV_FPR32INX:
+    return expandMV_FPR32INX(MBB, MBBI);
   case RISCV::PseudoRV32ZdinxSD:
     return expandRV32ZdinxStore(MBB, MBBI);
   case RISCV::PseudoRV32ZdinxLD:
@@ -287,6 +291,23 @@ bool RISCVExpandPseudo::expandMV_FPR16INX(MachineBasicBlock &MBB,
   return true;
 }
 
+bool RISCVExpandPseudo::expandMV_FPR32INX(MachineBasicBlock &MBB,
+                                          MachineBasicBlock::iterator MBBI) {
+  DebugLoc DL = MBBI->getDebugLoc();
+  const TargetRegisterInfo *TRI = STI->getRegisterInfo();
+  Register DstReg = TRI->getMatchingSuperReg(
+      MBBI->getOperand(0).getReg(), RISCV::sub_32, &RISCV::GPRRegClass);
+  Register SrcReg = TRI->getMatchingSuperReg(
+      MBBI->getOperand(1).getReg(), RISCV::sub_32, &RISCV::GPRRegClass);
+
+  BuildMI(MBB, MBBI, DL, TII->get(RISCV::ADDI), DstReg)
+      .addReg(SrcReg, getKillRegState(MBBI->getOperand(1).isKill()))
+      .addImm(0);
+
+  MBBI->eraseFromParent(); // The pseudo instruction is gone now.
+  return true;
+}
+
 // This function expands the PseudoRV32ZdinxSD for storing a double-precision
 // floating-point value into memory by generating an equivalent instruction
 // sequence for RV32.
diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index b0c525ea8c299..f388376c12c94 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -755,6 +755,19 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
   }
 }
 
+void RISCVFrameLowering::deallocateStack(MachineFunction &MF,
+                                         MachineBasicBlock &MBB,
+                                         MachineBasicBlock::iterator MBBI,
+                                         const DebugLoc &DL, uint64_t StackSize,
+                                         int64_t CFAOffset) const {
+  const RISCVRegisterInfo *RI = STI.getRegisterInfo();
+
+  Register SPReg = getSPReg(STI);
+
+  RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, StackOffset::getFixed(StackSize),
+                MachineInstr::FrameDestroy, getStackAlign());
+}
+
 void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
                                       MachineBasicBlock &MBB) const {
   const RISCVRegisterInfo *RI = STI.getRegisterInfo();
@@ -786,20 +799,49 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
       --MBBI;
   }
 
-  const auto &CSI = getUnmanagedCSI(MF, MFI.getCalleeSavedInfo());
+  const auto &CSI = MFI.getCalleeSavedInfo();
 
   // Skip to before the restores of scalar callee-saved registers
   // FIXME: assumes exactly one instruction is used to restore each
   // callee-saved register.
-  auto LastFrameDestroy = MBBI;
-  if (!CSI.empty())
-    LastFrameDestroy = std::prev(MBBI, CSI.size());
+  auto LastFrameDestroy = std::prev(MBBI, getUnmanagedCSI(MF, CSI).size());
 
-  uint64_t RealStackSize = getStackSizeWithRVVPadding(MF);
-  uint64_t StackSize = RealStackSize - RVFI->getReservedSpillsSize();
-  uint64_t FPOffset = RealStackSize - RVFI->getVarArgsSaveSize();
+  uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
+  uint64_t RealStackSize = FirstSPAdjustAmount ? FirstSPAdjustAmount
+                                               : getStackSizeWithRVVPadding(MF);
+  uint64_t StackSize = FirstSPAdjustAmount ? FirstSPAdjustAmount
+                                           : getStackSizeWithRVVPadding(MF) -
+                                                 RVFI->getReservedSpillsSize();
+  uint64_t FPOffset = FirstSPAdjustAmount ? FirstSPAdjustAmount
+                                          : getStackSizeWithRVVPadding(MF) -
+                                                RVFI->getVarArgsSaveSize();
   uint64_t RVVStackSize = RVFI->getRVVStackSize();
 
+  bool RestoreFP = RI->hasStackRealignment(MF) || MFI.hasVarSizedObjects() ||
+                   !hasReservedCallFrame(MF);
+
+  if (RVVStackSize) {
+    // If RestoreFP the stack pointer will be restored using the frame pointer
+    // value.
+    if (!RestoreFP)
+      adjustStackForRVV(MF, MBB, LastFrameDestroy, DL, RVVStackSize,
+                        MachineInstr::FrameDestroy);
+  }
+
+  if (FirstSPAdjustAmount) {
+    uint64_t SecondSPAdjustAmount =
+        getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount;
+    assert(SecondSPAdjustAmount > 0 &&
+           "SecondSPAdjustAmount should be greater than zero");
+
+    // If RestoreFP the stack pointer will be restored using the frame pointer
+    // value.
+    if (!RestoreFP)
+      RI->adjustReg(MBB, LastFrameDestroy, DL, SPReg, SPReg,
+                    StackOffset::getFixed(SecondSPAdjustAmount),
+                    MachineInstr::FrameDestroy, getStackAlign());
+  }
+
   // Restore the stack pointer using the value of the frame pointer. Only
   // necessary if the stack pointer was modified, meaning the stack size is
   // unknown.
@@ -810,35 +852,17 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
   // normally it's just checking the variable sized object is present or not
   // is enough, but we also don't preserve that at prologue/epilogue when
   // have vector objects in stack.
-  if (RI->hasStackRealignment(MF) || MFI.hasVarSizedObjects() ||
-      !hasReservedCallFrame(MF)) {
+  if (RestoreFP) {
     assert(hasFP(MF) && "frame pointer should not have been eliminated");
-    RI->adjustReg(MBB, LastFrameDestroy, DL, SPReg, FPReg,
-                  StackOffset::getFixed(-FPOffset),
-                  MachineInstr::FrameDestroy, getStackAlign());
-  } else {
-    if (RVVStackSize)
-      adjustStackForRVV(MF, MBB, LastFrameDestroy, DL, RVVStackSize,
-                        MachineInstr::FrameDestroy);
-  }
 
-  uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
-  if (FirstSPAdjustAmount) {
-    uint64_t SecondSPAdjustAmount =
-        getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount;
-    assert(SecondSPAdjustAmount > 0 &&
-           "SecondSPAdjustAmount should be greater than zero");
-
-    RI->adjustReg(MBB, LastFrameDestroy, DL, SPReg, SPReg,
-                  StackOffset::getFixed(SecondSPAdjustAmount),
-                  MachineInstr::FrameDestroy, getStackAlign());
+    RI->adjustReg(MBB, LastFrameDestroy, DL, SPReg, FPReg,
+                  StackOffset::getFixed(-FPOffset), MachineInstr::FrameDestroy,
+                  getStackAlign());
   }
 
-  if (FirstSPAdjustAmount)
-    StackSize = FirstSPAdjustAmount;
-
-  if (RVFI->isPushable(MF) && MBBI != MBB.end() &&
-      MBBI->getOpcode() == RISCV::CM_POP) {
+  bool ApplyPop = RVFI->isPushable(MF) && MBBI != MBB.end() &&
+                  MBBI->getOpcode() == RISCV::CM_POP;
+  if (ApplyPop) {
     // Use available stack adjustment in pop instruction to deallocate stack
     // space. Align the stack size down to a multiple of 16. This is needed for
     // RVE.
@@ -846,14 +870,19 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
     uint64_t Spimm = std::min(alignDown(StackSize, 16), (uint64_t)48);
     MBBI->getOperand(1).setImm(Spimm);
     StackSize -= Spimm;
-  }
 
-  // Deallocate stack
-  if (StackSize != 0) {
-    RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, StackOffset::getFixed(StackSize),
-                  MachineInstr::FrameDestroy, getStackAlign());
+    if (StackSize != 0)
+      deallocateStack(MF, MBB, MBBI, DL, StackSize,
+                      /*stack_adj of cm.pop instr*/ RealStackSize - StackSize);
+
+    ++MBBI;
   }
 
+  // Deallocate stack if we didn't already do it during cm.pop handling and
+  // StackSize isn't a zero
+  if (StackSize != 0 && !ApplyPop)
+    deallocateStack(MF, MBB, MBBI, DL, StackSize, 0);
+
   // Emit epilogue for shadow call stack.
   emitSCSEpilogue(MF, MBB, MBBI, DL);
 }
diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.h b/llvm/lib/Target/RISCV/RISCVFrameLowering.h
index 28ab4aff3b9d5..d660f3ad67c96 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.h
@@ -91,6 +91,10 @@ class RISCVFrameLowering : public TargetFrameLowering {
   void emitCalleeSavedRVVPrologCFI(MachineBasicBlock &MBB,
                                    MachineBasicBlock::iterator MI,
                                    bool HasFP) const;
+  void deallocateStack(MachineFunction &MF, MachineBasicBlock &MBB,
+                       MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
+                       uint64_t StackSize, int64_t CFAOffset) const;
+
   std::pair<int64_t, Align>
   assignRVVStackObjectOffsets(MachineFunction &MF) const;
 };
diff --git a/llvm/lib/Target/RISCV/RISCVGISel.td b/llvm/lib/Target/RISCV/RISCVGISel.td
index e14de46673aba..0656928ca41f8 100644
--- a/llvm/lib/Target/RISCV/RISCVGISel.td
+++ b/llvm/lib/Target/RISCV/RISCVGISel.td
@@ -50,6 +50,14 @@ def GIAddrRegImm :
   GIComplexOperandMatcher<s32, "selectAddrRegImm">,
   GIComplexPatternEquiv<AddrRegImm>;
 
+// FIXME: This is labelled as handling 's32', however the ComplexPattern it
+// refers to handles both i32 and i64 based on the HwMode. Currently this LLT
+// parameter appears to be ignored so this pattern works for both, however we
+// should add a LowLevelTypeByHwMode, and use that to define our XLenLLT instead
+// here.
+def GIVLOp : GIComplexOperandMatcher<s32, "renderVLOp">,
+             GIComplexPatternEquiv<VLOp>;
+
 // Convert from i32 immediate to i64 target immediate to make SelectionDAG type
 // checking happy so we can use ADDIW which expects an XLen immediate.
 def as_i64imm : SDNodeXForm<imm, [{
@@ -165,6 +173,7 @@ def : StPat<store, SD, GPR, PtrVT>;
 //===----------------------------------------------------------------------===//
 
 def simm12i32 : ImmLeaf<i32, [{return isInt<12>(Imm);}]>;
+def uimm5i32 : ImmLeaf<i32, [{return isUInt<5>(Imm);}]>;
 
 def zext_is_sext : PatFrag<(ops node:$src), (zext node:$src), [{
   KnownBits Known = CurDAG->computeKnownBits(N->getOperand(0), 0);
@@ -204,9 +213,12 @@ def : Pat<(i32 (or GPR:$rs1, simm12i32:$imm)),
 def : Pat<(i32 (xor GPR:$rs1, simm12i32:$imm)),
           (XORI GPR:$rs1, (i64 (as_i64imm $imm)))>;
 
-def : PatGprImm<shl, SLLIW, uimm5, i32>;
-def : PatGprImm<srl, SRLIW, uimm5, i32>;
-def : PatGprImm<sra, SRAIW, uimm5, i32>;
+def : Pat<(i32 (shl GPR:$rs1, uimm5i32:$imm)),
+          (SLLIW GPR:$rs1, (i64 (as_i64imm $imm)))>;
+def : Pat<(i32 (srl GPR:$rs1, uimm5i32:$imm)),
+          (SRLIW GPR:$rs1, (i64 (as_i64imm $imm)))>;
+def : Pat<(i32 (sra GPR:$rs1, uimm5i32:$imm)),
+          (SRAIW GPR:$rs1, (i64 (as_i64imm $imm)))>;
 
 def : Pat<(i32 (and GPR:$rs, TrailingOnesMask:$mask)),
           (SRLI (i32 (SLLI $rs, (i64 (XLenSubTrailingOnes $mask)))),
@@ -240,79 +252,10 @@ def : PatGprGpr<srem, REMW, i32, i32>;
 def : PatGprGpr<urem, REMUW, i32, i32>;
 }
 
-//===----------------------------------------------------------------------===//
-// Atomic RV64 i32 patterns not used by SelectionDAG
-//===----------------------------------------------------------------------===//
-
-class PatGprGprA<SDPatternOperator OpNode, RVInst Inst, ValueType vt>
-    : Pat<(vt (OpNode (XLenVT GPR:$rs1), (vt GPR:$rs2))), (Inst GPR:$rs1, GPR:$rs2)>;
-
-multiclass AMOPat2<string AtomicOp, string BaseInst, ValueType vt = XLenVT,
-                   list<Predicate> ExtraPreds = []> {
-let Predicates = !listconcat([HasStdExtA, NotHasStdExtZtso], ExtraPreds) in {
-  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_monotonic"),
-                   !cast<RVInst>(BaseInst), vt>;
-  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_acquire"),
-                   !cast<RVInst>(BaseInst#"_AQ"), vt>;
-  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_release"),
-                   !cast<RVInst>(BaseInst#"_RL"), vt>;
-  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_acq_rel"),
-                   !cast<RVInst>(BaseInst#"_AQ_RL"), vt>;
-  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_seq_cst"),
-                   !cast<RVInst>(BaseInst#"_AQ_RL"), vt>;
-}
-let Predicates = !listconcat([HasStdExtA, HasStdExtZtso], ExtraPreds) in {
-  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_monotonic"),
-                   !cast<RVInst>(BaseInst), vt>;
-  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_acquire"),
-                   !cast<RVInst>(BaseInst), vt>;
-  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_release"),
-                   !cast<RVInst>(BaseInst), vt>;
-  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_acq_rel"),
-                   !cast<RVInst>(BaseInst), vt>;
-  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_seq_cst"),
-                   !cast<RVInst>(BaseInst), vt>;
-}
-}
-
-defm : AMOPat2<"atomic_swap_i32", "AMOSWAP_W", i32>;
-defm : AMOPat2<"atomic_load_add_i32", "AMOADD_W", i32>;
-defm : AMOPat2<"atomic_load_and_i32", "AMOAND_W", i32>;
-defm : AMOPat2<"atomic_load_or_i32", "AMOOR_W", i32>;
-defm : AMOPat2<"atomic_load_xor_i32", "AMOXOR_W", i32>;
-defm : AMOPat2<"atomic_load_max_i32", "AMOMAX_W", i32>;
-defm : AMOPat2<"atomic_load_min_i32", "AMOMIN_W", i32>;
-defm : AMOPat2<"atomic_load_umax_i32", "AMOMAXU_W", i32>;
-defm : AMOPat2<"atomic_load_umin_i32", "AMOMINU_W", i32>;
-
-let Predicates = [HasStdExtA, IsRV64] in
-defm : PseudoCmpXchgPat<"atomic_cmp_swap_i32", PseudoCmpXchg32, i32>;
-
-let Predicates = [HasAtomicLdSt] in {
-  def : LdPat<atomic_load_8,  LB, i32>;
-  def : LdPat<atomic_load_16, LH, i32>;
-  def : LdPat<atomic_load_32, LW, i32>;
-
-  def : StPat<atomic_store_8,  SB, GPR, i32>;
-  def : StPat<atomic_store_16, SH, GPR, i32>;
-  def : StPat<atomic_store_32, SW, GPR, i32>;
-}
-
-
 //===----------------------------------------------------------------------===//
 // Zb* RV64 i32 patterns not used by SelectionDAG.
 //===----------------------------------------------------------------------===//
 
-def zexti16i32 : ComplexPattern<i32, 1, "selectZExtBits<16>">;
-def zexti8i32 : ComplexPattern<i32, 1, "selectZExtBits<8>">;
-
-def BCLRMaski32 : ImmLeaf<i32, [{
-  return !isInt<12>(Imm) && isPowerOf2_32(~Imm);
-}]>;
-def SingleBitSetMaski32 : ImmLeaf<i32, [{
-  return !isInt<12>(Imm) && isPowerOf2_32(Imm);
-}]>;
-
 let Predicates = [HasStdExtZbb, IsRV64] in {
 def : PatGpr<ctlz, CLZW, i32>;
 def : PatGpr<cttz, CTZW, i32>;
@@ -324,38 +267,20 @@ def : Pat<(i32 (sext_inreg GPR:$rs1, i16)), (SEXT_H GPR:$rs1)>;
 def : Pat<(i32 (and GPR:$rs, 0xFFFF)), (ZEXT_H_RV64 GPR:$rs)>;
 } // Predicates = [HasStdExtZbb, IsRV64]
 
-let Predicates = [HasStdExtZbkb, NoStdExtZbb, IsRV64] in {
-def : Pat<(i32 (and GPR:$rs, 0xFFFF)), (PACKW GPR:$rs, (XLenVT X0))>;
-}
-
 let Predicates = [HasStdExtZbbOrZbkb, IsRV64] in {
 def : Pat<(i32 (and GPR:$rs1, (not GPR:$rs2))), (ANDN GPR:$rs1, GPR:$rs2)>;
 def : Pat<(i32 (or  GPR:$rs1, (not GPR:$rs2))), (ORN  GPR:$rs1, GPR:$rs2)>;
 def : Pat<(i32 (xor GPR:$rs1, (not GPR:$rs2))), (XNOR GPR:$rs1, GPR:$rs2)>;
 
-def : PatGprGpr<shiftopw<rotl>, ROLW, i32, i64>;
-def : PatGprGpr<shiftopw<rotr>, RORW, i32, i64>;
-def : PatGprImm<rotr, RORIW, uimm5, i32>;
+def : PatGprGpr<rotl, ROLW, i32, i32>;
+def : PatGprGpr<rotr, RORW, i32, i32>;
+def : Pat<(i32 (rotl GPR:$rs1, uimm5i32:$imm)),
+          (RORIW GPR:$rs1, (i64 (as_i64imm $imm)))>;
 
-def : Pat<(i32 (rotl GPR:$rs1, uimm5:$rs2)),
-          (RORIW GPR:$rs1, (ImmSubFrom32 uimm5:$rs2))>;
+def : Pat<(i32 (rotl GPR:$rs1, uimm5i32:$rs2)),
+          (RORIW GPR:$rs1, (ImmSubFrom32 uimm5i32:$rs2))>;
 } // Predicates = [HasStdExtZbbOrZbkb, IsRV64]
 
-let Predicates = [HasStdExtZbkb, IsRV64] in {
-def : Pat<(or (and (shl GPR:$rs2, (i64 8)), 0xFFFF),
-              (zexti8i32 (i32 GPR:$rs1))),
-          (PACKH GPR:$rs1, GPR:$rs2)>;
-def : Pat<(or (shl (zexti8i32 (i32 GPR:$rs2)), (i64 8)),
-              (zexti8i32 (i32 GPR:$rs1))),
-          (PACKH GPR:$rs1, GPR:$rs2)>;
-def : Pat<(and (anyext (or (shl GPR:$rs2, (XLenVT 8)),
-                           (zexti8i32 (i32 GPR:$rs1)))), 0xFFFF),
-          (PACKH GPR:$rs1, GPR:$rs2)>;
-
-def : Pat<(i32 (or (shl GPR:$rs2, (i64 16)), (zexti16i32 (i32 GPR:$rs1)))),
-          (PACKW GPR:$rs1, GPR:$rs2)>;
-} // Predicates = [HasStdExtZbkb, IsRV64]
-
 let Predicates = [HasStdExtZba, IsRV64] in {
 def : Pat<(shl (i64 (zext i32:$rs1)), uimm5:$shamt),
           (SLLI_UW GPR:$rs1, uimm5:$shamt)>;
@@ -366,98 +291,7 @@ def : Pat<(zext GPR:$src), (ADD_UW GPR:$src, (XLenVT X0))>;
 
 foreach i = {1,2,3} in {
   defvar shxadd = !cast<Instruction>("SH"#i#"ADD");
-  def : Pat<(i32 (add_like_non_imm12 (shl GPR:$rs1, (i64 i)), GPR:$rs2)),
+  def : Pat<(i32 (add_like_non_imm12 (shl GPR:$rs1, (i32 i)), GPR:$rs2)),
             (shxadd GPR:$rs1, GPR:$rs2)>;
-  def : Pat<(i32 (riscv_shl_add GPR:$rs1, (i32 i), GPR:$rs2)),
-            (shxadd GPR:$rs1, GPR:$rs2)>;
-}
 }
-
-let Predicates = [HasStdExtZbs, IsRV64] in {
-def : Pat<(i32 (and (not (shiftop<shl> 1, (i64 GPR:$rs2))), GPR:$rs1)),
-          (BCLR GPR:$rs1, GPR:$rs2)>;
-def : Pat<(i32 (and (rotl -2, (i64 GPR:$rs2)), GPR:$rs1)),
-          (BCLR GPR:$rs1, GPR:$rs2)>;
-def : Pat<(i32 (or (shiftop<shl> 1, (i64 GPR:$rs2)), GPR:$rs1)),
-          (BSET GPR:$rs1, GPR:$rs2)>;
-def : Pat<(i32 (xor (shiftop<shl> 1, (i64 GPR:$rs2)), GPR:$rs1)),
-          (BINV GPR:$rs1, GPR:$rs2)>;
-def : Pat<(i32 (and (shiftop<srl> GPR:$rs1, (i64 GPR:$rs2)), 1)),
-          (BEXT GPR:$rs1, GPR:$rs2)>;
-def : Pat<(i64 (and (anyext (i32 (shiftop<srl> GPR:$rs1, (i64 GPR:$rs2)))), 1)),
-          (BEXT GPR:$rs1, GPR:$rs2)>;
-
-def : Pat<(i32 (shiftop<shl> 1, (i64 GPR:$rs2))),
-          (BSET (XLenVT X0), GPR:$rs2)>;
-def : Pat<(i32 (not (shiftop<shl> -1, (i64 GPR:$rs2)))),
-          (ADDI (i32 (BSET (XLenVT X0), GPR:$rs2)), -1)>;
-
-def : Pat<(i32 (and (srl GPR:$rs1, uimm5:$shamt), (i32 1))),
-          (BEXTI GPR:$rs1, uimm5:$shamt)>;
-
-def : Pat<(i32 (and GPR:$rs1, BCLRMaski32:$mask)),
-          (BCLRI GPR:$rs1, (i64 (BCLRXForm $mask)))>;
-def : Pat<(i32 (or GPR:$rs1, SingleBitSetMaski32:$mask)),
-          (BSETI GPR:$rs1, (i64 (SingleBitSetMaskToIndex $mask)))>;
-def : Pat<(i32 (xor GPR:$rs1, SingleBitSetMaski32:$mask)),
-          (BINVI GPR:$rs1, (i64 (SingleBitSetMaskToIndex $mask)))>;
-} // Predicates = [HasStdExtZbs, IsRV64]
-
-//===----------------------------------------------------------------------===//
-// XTHead RV64 i32 patterns not used by SelectionDAG.
-//===----------------------------------------------------------------------===//
-
-def sexti16i32 : ComplexPattern<i32, 1, "selectSExtBits<16>">;
-
-let Predicates = [HasVendorXTHeadMemIdx, IsRV64] in {
-defm : StoreUpdatePat<post_truncsti8, TH_SBIA, i32>;
-defm : StoreUpdatePat<pre_truncsti8, TH_SBIB, i32>;
-defm : StoreUpdatePat<post_truncsti16, TH_SHIA, i32>;
-defm : StoreUpdatePat<pre_truncsti16, TH_SHIB, i32>;
-
-defm : StIdxPat<truncstorei8, TH_SRB, GPR, i32>;
-defm : StIdxPat<truncstorei16, TH_SRH, GPR, i32>;
-
-defm : StZextIdxPat<truncstorei8, TH_SURB, GPR, i32>;
-defm : StZextIdxPat<truncstorei16, TH_SURH, GPR, i32>;
-defm : StZextIdxPat<store, TH_SURW, GPR, i32>;
-}
-
-let Predicates = [HasVendorXTHeadCondMov, IsRV64] in {
-def : Pat<(select (XLenVT GPR:$cond), (i32 GPR:$a), (i32 GPR:$b)),
-          (TH_MVEQZ GPR:$a, GPR:$b, GPR:$cond)>;
-def : Pat<(select (XLenVT GPR:$cond), (i32 GPR:$a), (i32 0)),
-          (TH_MVEQZ GPR:$a, (XLenVT X0), GPR:$cond)>;
-def : Pat<(select (XLenVT GPR:$cond), (i32 0), (i32 GPR:$b)),
-          (TH_MVNEZ GPR:$b, (XLenVT X0), GPR:$cond)>;
-
-def : Pat<(select (riscv_seteq (XLenVT GPR:$cond)), (i32 GPR:$a), (i32 GPR:$b)),
-          (TH_MVNEZ GPR:$a, GPR:$b, GPR:$cond)>;
-def : Pat<(select (riscv_setne (XLenVT GPR:$cond)), (i32 GPR:$a), (i32 GPR:$b)),
-          (TH_MVEQZ GPR:$a, GPR:$b, GPR:$cond)>;
-def : Pat<(select (riscv_seteq (XLenVT GPR:$cond)), (i32 GPR:$a), (i32 0)),
-          (TH_MVNEZ GPR:$a, (XLenVT X0), GPR:$cond)>;
-def : Pat<(select (riscv_setne (XLenVT GPR:$cond)), (i32 GPR:$a), (i32 0)),
-          (TH_MVEQZ GPR:$a, (XLenVT X0), GPR:$cond)>;
-def : Pat<(select (riscv_seteq (XLenVT GPR:$cond)), (i32 0), (i32 GPR:$b)),
-          (TH_MVEQZ GPR:$b, (XLenVT X0), GPR:$cond)>;
-def : Pat<(select (riscv_setne (XLenVT GPR:$cond)),  (i32 0), (i32 GPR:$b)),
-          (TH_MVNEZ GPR:$b, (XLenVT X0), GPR:$cond)>;
-} // Predicates = [HasVendorXTHeadCondMov]
-
-let Predicates = [HasVendorXTHeadMac, IsRV64] in {
-// mulaw, mulsw are available only in RV64.
-def : Pat<(i32 (add GPR:$rd, (mul GPR:$rs1, GPR:$rs2))),
-          (TH_MULAW GPR:$rd, GPR:$rs1, GPR:$rs2)>;
-def : Pat<(i32 (sub GPR:$rd, (mul GPR:$rs1, GPR:$rs2))),
-          (TH_MULSW GPR:$rd, GPR:$rs1, GPR:$rs2)>;
-// mulah, mulsh produce a sign-extended result.
-def : Pat<(i32 (add GPR:$rd,
-                    (mul (sexti16i32 (i32 GPR:$rs1)),
-                         (sexti16i32 (i32 GPR:$rs2))))),
-          (TH_MULAH GPR:$rd, GPR:$rs1, GPR:$rs2)>;
-def : Pat<(i32 (sub GPR:$rd,
-                    (mul (sexti16i32 (i32 GPR:$rs1)),
-                         (sexti16i32 (i32 GPR:$rs2))))),
-          (TH_MULSH GPR:$rd, GPR:$rs1, GPR:$rs2)>;
 }
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 23479c2edf1d9..dc3f8254cb4e0 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -931,6 +931,9 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
     if (VT.SimpleTy == MVT::f16 && Opc == RISCV::COPY) {
       Res =
           CurDAG->getTargetExtractSubreg(RISCV::sub_16, DL, VT, Imm).getNode();
+    } else if (VT.SimpleTy == MVT::f32 && Opc == RISCV::COPY) {
+      Res =
+          CurDAG->getTargetExtractSubreg(RISCV::sub_32, DL, VT, Imm).getNode();
     } else if (Opc == RISCV::FCVT_D_W_IN32X || Opc == RISCV::FCVT_D_W)
       Res = CurDAG->getMachineNode(
           Opc, DL, VT, Imm,
@@ -2935,6 +2938,14 @@ bool RISCVDAGToDAGISel::selectSETCC(SDValue N, ISD::CondCode ExpectedCCVal,
                     0);
       return true;
     }
+    if (isPowerOf2_64(CVal) && Subtarget->hasStdExtZbs()) {
+      Val = SDValue(
+          CurDAG->getMachineNode(
+              RISCV::BINVI, DL, N->getValueType(0), LHS,
+              CurDAG->getTargetConstant(Log2_64(CVal), DL, N->getValueType(0))),
+          0);
+      return true;
+    }
   }
 
   // If nothing else we can XOR the LHS and RHS to produce zero if they are
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index e77f8783f1727..463887b8b55e6 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -957,8 +957,6 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
         ISD::VP_FMUL,
         ISD::VP_FDIV,
         ISD::VP_FMA,
-        ISD::VP_REDUCE_FADD,
-        ISD::VP_REDUCE_SEQ_FADD,
         ISD::VP_REDUCE_FMIN,
         ISD::VP_REDUCE_FMAX,
         ISD::VP_SQRT,
@@ -1076,9 +1074,10 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
       setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::VP_SINT_TO_FP,
                           ISD::VP_UINT_TO_FP},
                          VT, Custom);
-      setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
-                          ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_INTERLEAVE,
-                          ISD::VECTOR_DEINTERLEAVE},
+      setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::CONCAT_VECTORS,
+                          ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
+                          ISD::VECTOR_DEINTERLEAVE, ISD::VECTOR_INTERLEAVE,
+                          ISD::VECTOR_REVERSE},
                          VT, Custom);
       MVT EltVT = VT.getVectorElementType();
       if (isTypeLegal(EltVT))
@@ -1490,6 +1489,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
                        ISD::INTRINSIC_WO_CHAIN, ISD::ADD, ISD::SUB, ISD::MUL,
                        ISD::AND, ISD::OR, ISD::XOR, ISD::SETCC, ISD::SELECT});
   setTargetDAGCombine(ISD::SRA);
+  setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
 
   if (Subtarget.hasStdExtFOrZfinx())
     setTargetDAGCombine({ISD::FADD, ISD::FMAXNUM, ISD::FMINNUM, ISD::FMUL});
@@ -1503,8 +1503,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
 
   if (Subtarget.hasStdExtZbkb())
     setTargetDAGCombine(ISD::BITREVERSE);
-  if (Subtarget.hasStdExtZfhminOrZhinxmin())
-    setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
+
   if (Subtarget.hasStdExtFOrZfinx())
     setTargetDAGCombine({ISD::ZERO_EXTEND, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
                          ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT});
@@ -8756,8 +8755,10 @@ SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
                                                     SelectionDAG &DAG) const {
   SDLoc DL(Op);
   MVT VecVT = Op.getSimpleValueType();
+  MVT XLenVT = Subtarget.getXLenVT();
   SDValue Vec = Op.getOperand(0);
   SDValue Val = Op.getOperand(1);
+  MVT ValVT = Val.getSimpleValueType();
   SDValue Idx = Op.getOperand(2);
 
   if (VecVT.getVectorElementType() == MVT::i1) {
@@ -8769,6 +8770,16 @@ SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
     return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Vec);
   }
 
+  if ((ValVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
+      ValVT == MVT::bf16) {
+    // If we don't have vfmv.s.f for f16/bf16, use fmv.x.h first.
+    MVT IntVT = VecVT.changeTypeToInteger();
+    SDValue IntInsert = DAG.getNode(
+        ISD::INSERT_VECTOR_ELT, DL, IntVT, DAG.getBitcast(IntVT, Vec),
+        DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Val), Idx);
+    return DAG.getBitcast(VecVT, IntInsert);
+  }
+
   MVT ContainerVT = VecVT;
   // If the operand is a fixed-length vector, convert to a scalable one.
   if (VecVT.isFixedLengthVector()) {
@@ -8812,8 +8823,6 @@ SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
                         AlignedIdx);
   }
 
-  MVT XLenVT = Subtarget.getXLenVT();
-
   bool IsLegalInsert = Subtarget.is64Bit() || Val.getValueType() != MVT::i64;
   // Even i64-element vectors on RV32 can be lowered without scalar
   // legalization if the most-significant 32 bits of the value are not affected
@@ -14447,15 +14456,23 @@ performSIGN_EXTEND_INREGCombine(SDNode *N, SelectionDAG &DAG,
                                 const RISCVSubtarget &Subtarget) {
   SDValue Src = N->getOperand(0);
   EVT VT = N->getValueType(0);
+  EVT SrcVT = cast<VTSDNode>(N->getOperand(1))->getVT();
+  unsigned Opc = Src.getOpcode();
 
   // Fold (sext_inreg (fmv_x_anyexth X), i16) -> (fmv_x_signexth X)
   // Don't do this with Zhinx. We need to explicitly sign extend the GPR.
-  if (Src.getOpcode() == RISCVISD::FMV_X_ANYEXTH &&
-      cast<VTSDNode>(N->getOperand(1))->getVT().bitsGE(MVT::i16) &&
+  if (Opc == RISCVISD::FMV_X_ANYEXTH && SrcVT.bitsGE(MVT::i16) &&
       Subtarget.hasStdExtZfhmin())
     return DAG.getNode(RISCVISD::FMV_X_SIGNEXTH, SDLoc(N), VT,
                        Src.getOperand(0));
 
+  // Fold (sext_inreg (shl X, Y), i32) -> (sllw X, Y) iff Y u< 32
+  if (Opc == ISD::SHL && Subtarget.is64Bit() && SrcVT == MVT::i32 &&
+      VT == MVT::i64 && !isa<ConstantSDNode>(Src.getOperand(1)) &&
+      DAG.computeKnownBits(Src.getOperand(1)).countMaxActiveBits() <= 5)
+    return DAG.getNode(RISCVISD::SLLW, SDLoc(N), VT, Src.getOperand(0),
+                       Src.getOperand(1));
+
   return SDValue();
 }
 
@@ -21293,7 +21310,7 @@ bool RISCVTargetLowering::preferScalarizeSplat(SDNode *N) const {
 }
 
 static Value *useTpOffset(IRBuilderBase &IRB, unsigned Offset) {
-  Module *M = IRB.GetInsertBlock()->getParent()->getParent();
+  Module *M = IRB.GetInsertBlock()->getModule();
   Function *ThreadPointerFunc =
       Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
   return IRB.CreateConstGEP1_32(IRB.getInt8Ty(),
@@ -21312,6 +21329,14 @@ Value *RISCVTargetLowering::getIRStackGuard(IRBuilderBase &IRB) const {
   if (Subtarget.isTargetAndroid())
     return useTpOffset(IRB, -0x18);
 
+  Module *M = IRB.GetInsertBlock()->getModule();
+
+  if (M->getStackProtectorGuard() == "tls") {
+    // Users must specify the offset explicitly
+    int Offset = M->getStackProtectorGuardOffset();
+    return useTpOffset(IRB, Offset);
+  }
+
   return TargetLowering::getIRStackGuard(IRB);
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrGISel.td b/llvm/lib/Target/RISCV/RISCVInstrGISel.td
index ba40662c49c1d..f6bf74c565ab3 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrGISel.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrGISel.td
@@ -57,3 +57,13 @@ def G_SPLAT_VECTOR_SPLIT_I64_VL : RISCVGenericInstruction {
   let InOperandList = (ins type0:$passthru, type1:$hi, type1:$lo, type2:$vl);
   let hasSideEffects = false;
 }
+
+// Pseudo equivalent to a RISCVISD::VSLIDEDOWN_VL
+def G_VSLIDEDOWN_VL : RISCVGenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$merge, type0:$vec, type1:$idx, type2:$mask,
+                       type1:$vl, type1:$policy);
+  let hasSideEffects = false;
+}
+def : GINodeEquiv<G_VSLIDEDOWN_VL, riscv_slidedown_vl>;
+
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 91503bd7f41f3..b8539a5d1add1 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -110,6 +110,7 @@ Register RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
     MemBytes = 2;
     break;
   case RISCV::LW:
+  case RISCV::LW_INX:
   case RISCV::FLW:
   case RISCV::LWU:
     MemBytes = 4;
@@ -150,6 +151,7 @@ Register RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
     MemBytes = 2;
     break;
   case RISCV::SW:
+  case RISCV::SW_INX:
   case RISCV::FSW:
     MemBytes = 4;
     break;
@@ -471,6 +473,13 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
     return;
   }
 
+  if (RISCV::GPRF32RegClass.contains(DstReg, SrcReg)) {
+    BuildMI(MBB, MBBI, DL, get(RISCV::PseudoMV_FPR32INX), DstReg)
+        .addReg(SrcReg,
+                getKillRegState(KillSrc) | getRenamableRegState(RenamableSrc));
+    return;
+  }
+
   if (RISCV::GPRPairRegClass.contains(DstReg, SrcReg)) {
     // Emit an ADDI for both parts of GPRPair.
     BuildMI(MBB, MBBI, DL, get(RISCV::ADDI),
@@ -595,6 +604,9 @@ void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
   } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
     Opcode = RISCV::SH_INX;
     IsScalableVector = false;
+  } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {
+    Opcode = RISCV::SW_INX;
+    IsScalableVector = false;
   } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
     Opcode = RISCV::PseudoRV32ZdinxSD;
     IsScalableVector = false;
@@ -681,6 +693,9 @@ void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
   } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
     Opcode = RISCV::LH_INX;
     IsScalableVector = false;
+  } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {
+    Opcode = RISCV::LW_INX;
+    IsScalableVector = false;
   } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
     Opcode = RISCV::PseudoRV32ZdinxLD;
     IsScalableVector = false;
@@ -1554,6 +1569,7 @@ unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
 
   switch (Opcode) {
   case RISCV::PseudoMV_FPR16INX:
+  case RISCV::PseudoMV_FPR32INX:
     // MV is always compressible to either c.mv or c.li rd, 0.
     return STI.hasStdExtCOrZca() ? 2 : 4;
   case TargetOpcode::STACKMAP:
@@ -2614,6 +2630,7 @@ bool RISCVInstrInfo::canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg,
   case RISCV::LH_INX:
   case RISCV::LHU:
   case RISCV::LW:
+  case RISCV::LW_INX:
   case RISCV::LWU:
   case RISCV::LD:
   case RISCV::FLH:
@@ -2623,6 +2640,7 @@ bool RISCVInstrInfo::canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg,
   case RISCV::SH:
   case RISCV::SH_INX:
   case RISCV::SW:
+  case RISCV::SW_INX:
   case RISCV::SD:
   case RISCV::FSH:
   case RISCV::FSW:
@@ -2692,9 +2710,11 @@ bool RISCVInstrInfo::getMemOperandsWithOffsetWidth(
   case RISCV::SH_INX:
   case RISCV::FSH:
   case RISCV::LW:
+  case RISCV::LW_INX:
   case RISCV::LWU:
   case RISCV::FLW:
   case RISCV::SW:
+  case RISCV::SW_INX:
   case RISCV::FSW:
   case RISCV::LD:
   case RISCV::FLD:
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index a2f1e3ded18fe..a2c92154f00dd 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -549,8 +549,8 @@ class HStore_rr<bits<7> funct7, string opcodestr>
 }
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-class ALU_ri<bits<3> funct3, string opcodestr, DAGOperand rty = GPR>
-    : RVInstI<funct3, OPC_OP_IMM, (outs rty:$rd), (ins rty:$rs1, simm12:$imm12),
+class ALU_ri<bits<3> funct3, string opcodestr>
+    : RVInstI<funct3, OPC_OP_IMM, (outs GPR:$rd), (ins GPR:$rs1, simm12:$imm12),
               opcodestr, "$rd, $rs1, $imm12">,
       Sched<[WriteIALU, ReadIALU]>;
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoC.td b/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
index 3f279b7a58ca6..7d742322b4296 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
@@ -331,6 +331,15 @@ def C_LW : CLoad_ri<0b010, "c.lw", GPRC, uimm7_lsb00>,
   let Inst{5} = imm{6};
 }
 
+let isCodeGenOnly = 1 in
+def C_LW_INX : CLoad_ri<0b010, "c.lw", GPRF32C, uimm7_lsb00>,
+               Sched<[WriteLDW, ReadMemBase]> {
+  bits<7> imm;
+  let Inst{12-10} = imm{5-3};
+  let Inst{6} = imm{2};
+  let Inst{5} = imm{6};
+}
+
 let DecoderNamespace = "RISCV32Only_",
     Predicates = [HasStdExtCOrZcfOrZce, HasStdExtF, IsRV32] in
 def C_FLW  : CLoad_ri<0b011, "c.flw", FPR32C, uimm7_lsb00>,
@@ -365,6 +374,15 @@ def C_SW : CStore_rri<0b110, "c.sw", GPRC, uimm7_lsb00>,
   let Inst{5} = imm{6};
 }
 
+let isCodeGenOnly = 1 in
+def C_SW_INX : CStore_rri<0b110, "c.sw", GPRF32C, uimm7_lsb00>,
+               Sched<[WriteSTW, ReadStoreData, ReadMemBase]> {
+  bits<7> imm;
+  let Inst{12-10} = imm{5-3};
+  let Inst{6} = imm{2};
+  let Inst{5} = imm{6};
+}
+
 let DecoderNamespace = "RISCV32Only_",
     Predicates = [HasStdExtCOrZcfOrZce, HasStdExtF, IsRV32]  in
 def C_FSW  : CStore_rri<0b111, "c.fsw", FPR32C, uimm7_lsb00>,
@@ -517,6 +535,13 @@ def C_LWSP : CStackLoad<0b010, "c.lwsp", GPRNoX0, uimm8_lsb00>,
   let Inst{3-2} = imm{7-6};
 }
 
+let isCodeGenOnly = 1 in
+def C_LWSP_INX : CStackLoad<0b010, "c.lwsp", GPRF32NoX0, uimm8_lsb00>,
+                 Sched<[WriteLDW, ReadMemBase]> {
+  let Inst{6-4} = imm{4-2};
+  let Inst{3-2} = imm{7-6};
+}
+
 let DecoderNamespace = "RISCV32Only_",
     Predicates = [HasStdExtCOrZcfOrZce, HasStdExtF, IsRV32] in
 def C_FLWSP  : CStackLoad<0b011, "c.flwsp", FPR32, uimm8_lsb00>,
@@ -575,6 +600,13 @@ def C_SWSP : CStackStore<0b110, "c.swsp", GPR, uimm8_lsb00>,
   let Inst{8-7}  = imm{7-6};
 }
 
+let isCodeGenOnly = 1 in
+def C_SWSP_INX : CStackStore<0b110, "c.swsp", GPRF32, uimm8_lsb00>,
+                 Sched<[WriteSTW, ReadStoreData, ReadMemBase]> {
+  let Inst{12-9} = imm{5-2};
+  let Inst{8-7}  = imm{7-6};
+}
+
 let DecoderNamespace = "RISCV32Only_",
     Predicates = [HasStdExtCOrZcfOrZce, HasStdExtF, IsRV32] in
 def C_FSWSP  : CStackStore<0b111, "c.fswsp", FPR32, uimm8_lsb00>,
@@ -869,6 +901,10 @@ def : CompressPat<(FLD FPR64C:$rd, GPRCMem:$rs1, uimm8_lsb000:$imm),
 let Predicates = [HasStdExtCOrZca] in {
 def : CompressPat<(LW GPRC:$rd, GPRCMem:$rs1, uimm7_lsb00:$imm),
                   (C_LW GPRC:$rd, GPRCMem:$rs1, uimm7_lsb00:$imm)>;
+
+let isCompressOnly = true in
+def : CompressPat<(LW_INX GPRF32C:$rd, GPRCMem:$rs1, uimm7_lsb00:$imm),
+                  (C_LW_INX GPRF32C:$rd, GPRCMem:$rs1, uimm7_lsb00:$imm)>;
 } // Predicates = [HasStdExtCOrZca]
 
 let Predicates = [HasStdExtCOrZcfOrZce, HasStdExtF, IsRV32] in {
@@ -889,6 +925,10 @@ def : CompressPat<(FSD FPR64C:$rs2, GPRCMem:$rs1, uimm8_lsb000:$imm),
 let Predicates = [HasStdExtCOrZca] in {
 def : CompressPat<(SW GPRC:$rs2, GPRCMem:$rs1, uimm7_lsb00:$imm),
                   (C_SW GPRC:$rs2, GPRCMem:$rs1, uimm7_lsb00:$imm)>;
+
+let isCompressOnly = true in
+def : CompressPat<(SW_INX GPRF32C:$rs2, GPRCMem:$rs1, uimm7_lsb00:$imm),
+                  (C_SW_INX GPRF32C:$rs2, GPRCMem:$rs1, uimm7_lsb00:$imm)>;
 } // Predicates = [HasStdExtCOrZca]
 
 let Predicates = [HasStdExtCOrZcfOrZce, HasStdExtF, IsRV32] in {
@@ -992,6 +1032,10 @@ def : CompressPat<(FLD FPR64:$rd, SPMem:$rs1, uimm9_lsb000:$imm),
 let Predicates = [HasStdExtCOrZca] in {
 def : CompressPat<(LW GPRNoX0:$rd, SPMem:$rs1,  uimm8_lsb00:$imm),
                   (C_LWSP GPRNoX0:$rd, SPMem:$rs1, uimm8_lsb00:$imm)>;
+
+let isCompressOnly = true in
+def : CompressPat<(LW_INX GPRF32NoX0:$rd, SPMem:$rs1,  uimm8_lsb00:$imm),
+                  (C_LWSP_INX GPRF32NoX0:$rd, SPMem:$rs1, uimm8_lsb00:$imm)>;
 } // Predicates = [HasStdExtCOrZca]
 
 let Predicates = [HasStdExtCOrZcfOrZce, HasStdExtF, IsRV32] in {
@@ -1034,6 +1078,10 @@ def : CompressPat<(FSD FPR64:$rs2, SPMem:$rs1, uimm9_lsb000:$imm),
 let Predicates = [HasStdExtCOrZca] in {
 def : CompressPat<(SW GPR:$rs2, SPMem:$rs1, uimm8_lsb00:$imm),
                   (C_SWSP GPR:$rs2, SPMem:$rs1, uimm8_lsb00:$imm)>;
+
+let isCompressOnly = true in
+def : CompressPat<(SW_INX GPRF32:$rs2, SPMem:$rs1, uimm8_lsb00:$imm),
+                  (C_SWSP_INX GPRF32:$rs2, SPMem:$rs1, uimm8_lsb00:$imm)>;
 } // Predicates = [HasStdExtCOrZca]
 
 let Predicates = [HasStdExtCOrZcfOrZce, HasStdExtF, IsRV32] in {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
index a00acb372dc2a..000b7cfedb0f9 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
@@ -83,15 +83,14 @@ def any_fma_nsz : PatFrag<(ops node:$rs1, node:$rs2, node:$rs3),
 
 // Zfinx
 
-def GPRAsFPR : AsmOperandClass {
-  let Name = "GPRAsFPR";
+def GPRAsFPR32 : AsmOperandClass {
+  let Name = "GPRAsFPR32";
   let ParserMethod = "parseGPRAsFPR";
   let RenderMethod = "addRegOperands";
 }
 
 def FPR32INX : RegisterOperand<GPRF32> {
-  let ParserMatchClass = GPRAsFPR;
-  let DecoderMethod = "DecodeGPRRegisterClass";
+  let ParserMatchClass = GPRAsFPR32;
 }
 
 // Describes a combination of predicates from F/D/Zfh/Zfhmin or
@@ -306,6 +305,19 @@ def FLW : FPLoad_r<0b010, "flw", FPR32, WriteFLD32>;
 def FSW : FPStore_r<0b010, "fsw", FPR32, WriteFST32>;
 } // Predicates = [HasStdExtF]
 
+let Predicates = [HasStdExtZfinx], isCodeGenOnly = 1 in {
+def LW_INX : Load_ri<0b010, "lw", GPRF32>, Sched<[WriteLDW, ReadMemBase]>;
+def SW_INX : Store_rri<0b010, "sw", GPRF32>,
+             Sched<[WriteSTW, ReadStoreData, ReadMemBase]>;
+
+// ADDI with GPRF32 register class to use for copy. This should not be used as
+// general ADDI, so the immediate should always be zero.
+let isReMaterializable = 1, isAsCheapAsAMove = 1, isMoveReg = 1,
+    hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+def PseudoMV_FPR32INX : Pseudo<(outs GPRF32:$rd), (ins GPRF32:$rs), []>,
+                        Sched<[WriteIALU, ReadIALU]>;
+}
+
 foreach Ext = FExts in {
   let SchedRW = [WriteFMA32, ReadFMA32, ReadFMA32, ReadFMA32Addend] in {
     defm FMADD_S  : FPFMA_rrr_frm_m<OPC_MADD,  0b00, "fmadd.s",  Ext>;
@@ -685,12 +697,10 @@ defm Select_FPR32INX : SelectCC_GPR_rrirr<FPR32INX, f32>;
 def PseudoFROUND_S_INX : PseudoFROUND<FPR32INX, f32>;
 
 /// Loads
-def : Pat<(f32 (load (AddrRegImm (XLenVT GPR:$rs1), simm12:$imm12))),
-          (COPY_TO_REGCLASS (LW GPR:$rs1, simm12:$imm12), GPRF32)>;
+def : LdPat<load, LW_INX, f32>;
 
 /// Stores
-def : Pat<(store (f32 FPR32INX:$rs2), (AddrRegImm (XLenVT GPR:$rs1), simm12:$imm12)),
-          (SW (COPY_TO_REGCLASS FPR32INX:$rs2, GPR), GPR:$rs1, simm12:$imm12)>;
+def : StPat<store, SW_INX, GPRF32, f32>;
 } // Predicates = [HasStdExtZfinx]
 
 let Predicates = [HasStdExtF] in {
@@ -701,8 +711,8 @@ def : Pat<(i32 (bitconvert FPR32:$rs1)), (FMV_X_W FPR32:$rs1)>;
 
 let Predicates = [HasStdExtZfinx] in {
 // Moves (no conversion)
-def : Pat<(f32 (bitconvert (i32 GPR:$rs1))), (COPY_TO_REGCLASS GPR:$rs1, GPRF32)>;
-def : Pat<(i32 (bitconvert FPR32INX:$rs1)), (COPY_TO_REGCLASS FPR32INX:$rs1, GPR)>;
+def : Pat<(f32 (bitconvert (i32 GPR:$rs1))), (EXTRACT_SUBREG GPR:$rs1, sub_32)>;
+def : Pat<(i32 (bitconvert FPR32INX:$rs1)), (INSERT_SUBREG (XLenVT (IMPLICIT_DEF)), FPR32INX:$rs1, sub_32)>;
 } // Predicates = [HasStdExtZfinx]
 
 let Predicates = [HasStdExtF] in {
@@ -781,8 +791,8 @@ def : Pat<(any_uint_to_fp (i64 GPR:$rs1)), (FCVT_S_LU $rs1, FRM_DYN)>;
 
 let Predicates = [HasStdExtZfinx, IsRV64] in {
 // Moves (no conversion)
-def : Pat<(riscv_fmv_w_x_rv64 GPR:$src), (COPY_TO_REGCLASS GPR:$src, GPRF32)>;
-def : Pat<(riscv_fmv_x_anyextw_rv64 GPRF32:$src), (COPY_TO_REGCLASS GPRF32:$src, GPR)>;
+def : Pat<(riscv_fmv_w_x_rv64 GPR:$src), (EXTRACT_SUBREG GPR:$src, sub_32)>;
+def : Pat<(riscv_fmv_x_anyextw_rv64 GPRF32:$src), (INSERT_SUBREG (XLenVT (IMPLICIT_DEF)), FPR32INX:$src, sub_32)>;
 
 // Use target specific isd nodes to help us remember the result is sign
 // extended. Matching sext_inreg+fptoui/fptosi may cause the conversion to be
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index a27c3a416816e..18749f00a10a5 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -2952,7 +2952,7 @@ foreach vti = NoGroupFloatVectors in {
   }
 }
 
-foreach vti = AllFloatVectors in {
+foreach vti = !listconcat(AllFloatVectors, AllBFloatVectors) in {
   defvar ivti = GetIntVTypeInfo<vti>.Vti;
   let Predicates = GetVTypePredicates<ivti>.Predicates in {
     def : Pat<(vti.Vector
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
index 8e6f281027695..3eb1fc6869403 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
@@ -599,15 +599,26 @@ def : Pat<(or (shl (zexti8 (XLenVT GPR:$rs2)), (XLenVT 8)),
 def : Pat<(and (or (shl GPR:$rs2, (XLenVT 8)),
                    (zexti8 (XLenVT GPR:$rs1))), 0xFFFF),
           (PACKH GPR:$rs1, GPR:$rs2)>;
+def : Pat<(or (shl (zexti8 (XLenVT GPR:$rs2)), (XLenVT 24)),
+              (shl (zexti8 (XLenVT GPR:$rs1)), (XLenVT 16))),
+          (SLLI (PACKH GPR:$rs1, GPR:$rs2), (XLenVT 16))>;
 
 def : Pat<(binop_allhusers<or> (shl GPR:$rs2, (XLenVT 8)),
                                (zexti8 (XLenVT GPR:$rs1))),
           (PACKH GPR:$rs1, GPR:$rs2)>;
 } // Predicates = [HasStdExtZbkb]
 
-let Predicates = [HasStdExtZbkb, IsRV32] in
+let Predicates = [HasStdExtZbkb, IsRV32] in {
 def : Pat<(i32 (or (zexti16 (i32 GPR:$rs1)), (shl GPR:$rs2, (i32 16)))),
           (PACK GPR:$rs1, GPR:$rs2)>;
+def : Pat<(or (or
+                  (shl (zexti8 (XLenVT GPR:$op1rs2)), (XLenVT 24)),
+                  (shl (zexti8 (XLenVT GPR:$op1rs1)), (XLenVT 16))),
+              (or
+                  (shl (zexti8 (XLenVT GPR:$op0rs2)), (XLenVT 8)),
+                  (zexti8 (XLenVT GPR:$op0rs1)))),
+          (PACK (PACKH GPR:$op0rs1, GPR:$op0rs2), (PACKH GPR:$op1rs1, GPR:$op1rs2))>;
+}
 
 let Predicates = [HasStdExtZbkb, IsRV64] in {
 def : Pat<(i64 (or (zexti32 (i64 GPR:$rs1)), (shl GPR:$rs2, (i64 32)))),
diff --git a/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp b/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp
index 5973e5bf2e525..df5501e37f831 100644
--- a/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp
+++ b/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp
@@ -109,7 +109,9 @@ static unsigned log2LdstWidth(unsigned Opcode) {
   case RISCV::SH_INX:
     return 1;
   case RISCV::LW:
+  case RISCV::LW_INX:
   case RISCV::SW:
+  case RISCV::SW_INX:
   case RISCV::FLW:
   case RISCV::FSW:
     return 2;
@@ -136,7 +138,9 @@ static unsigned offsetMask(unsigned Opcode) {
   case RISCV::SH_INX:
     return maskTrailingOnes<unsigned>(1U);
   case RISCV::LW:
+  case RISCV::LW_INX:
   case RISCV::SW:
+  case RISCV::SW_INX:
   case RISCV::FLW:
   case RISCV::FSW:
   case RISCV::LD:
@@ -178,6 +182,7 @@ static int64_t getBaseAdjustForCompression(int64_t Offset, unsigned Opcode) {
 static bool isCompressedReg(Register Reg) {
   return RISCV::GPRCRegClass.contains(Reg) ||
          RISCV::GPRF16CRegClass.contains(Reg) ||
+         RISCV::GPRF32CRegClass.contains(Reg) ||
          RISCV::FPR32CRegClass.contains(Reg) ||
          RISCV::FPR64CRegClass.contains(Reg);
 }
@@ -195,6 +200,7 @@ static bool isCompressibleLoad(const MachineInstr &MI) {
   case RISCV::LHU:
     return STI.hasStdExtZcb();
   case RISCV::LW:
+  case RISCV::LW_INX:
   case RISCV::LD:
     return STI.hasStdExtCOrZca();
   case RISCV::FLW:
@@ -216,6 +222,7 @@ static bool isCompressibleStore(const MachineInstr &MI) {
   case RISCV::SH_INX:
     return STI.hasStdExtZcb();
   case RISCV::SW:
+  case RISCV::SW_INX:
   case RISCV::SD:
     return STI.hasStdExtCOrZca();
   case RISCV::FSW:
@@ -329,6 +336,8 @@ static Register analyzeCompressibleUses(MachineInstr &FirstMI,
     RCToScavenge = &RISCV::GPRCRegClass;
   else if (RISCV::GPRF16RegClass.contains(RegImm.Reg))
     RCToScavenge = &RISCV::GPRF16CRegClass;
+  else if (RISCV::GPRF32RegClass.contains(RegImm.Reg))
+    RCToScavenge = &RISCV::GPRF32CRegClass;
   else if (RISCV::FPR32RegClass.contains(RegImm.Reg))
     RCToScavenge = &RISCV::FPR32CRegClass;
   else if (RISCV::FPR64RegClass.contains(RegImm.Reg))
@@ -424,6 +433,11 @@ bool RISCVMakeCompressibleOpt::runOnMachineFunction(MachineFunction &Fn) {
         BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(RISCV::PseudoMV_FPR16INX),
                 NewReg)
             .addReg(RegImm.Reg);
+      } else if (RISCV::GPRF32RegClass.contains(RegImm.Reg)) {
+        assert(RegImm.Imm == 0);
+        BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(RISCV::PseudoMV_FPR32INX),
+                NewReg)
+            .addReg(RegImm.Reg);
       } else {
         // If we are looking at replacing an FPR register we don't expect to
         // have any offset. The only compressible FP instructions with an offset
diff --git a/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp b/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp
index b3a2877edde4e..a324deb4e48f5 100644
--- a/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp
+++ b/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp
@@ -387,6 +387,7 @@ bool RISCVMergeBaseOffsetOpt::foldIntoMemoryOps(MachineInstr &Hi,
     case RISCV::LH:
     case RISCV::LH_INX:
     case RISCV::LW:
+    case RISCV::LW_INX:
     case RISCV::LBU:
     case RISCV::LHU:
     case RISCV::LWU:
@@ -398,6 +399,7 @@ bool RISCVMergeBaseOffsetOpt::foldIntoMemoryOps(MachineInstr &Hi,
     case RISCV::SH:
     case RISCV::SH_INX:
     case RISCV::SW:
+    case RISCV::SW_INX:
     case RISCV::SD:
     case RISCV::FSH:
     case RISCV::FSW:
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
index e3c9ac52d66a3..33363aa8b7183 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
@@ -124,41 +124,81 @@ let RegAltNameIndices = [ABIRegAltName] in {
 
   let SubRegIndices = [sub_16] in {
   let isConstant = true in
-  def X0  : RISCVRegWithSubRegs<0, "x0", [X0_H], ["zero"]>, DwarfRegNum<[0]>;
+  def X0_W  : RISCVRegWithSubRegs<0, "x0", [X0_H], ["zero"]>;
   let CostPerUse = [0, 1] in {
-  def X1  : RISCVRegWithSubRegs<1, "x1", [X1_H], ["ra"]>, DwarfRegNum<[1]>;
-  def X2  : RISCVRegWithSubRegs<2, "x2", [X2_H], ["sp"]>, DwarfRegNum<[2]>;
-  def X3  : RISCVRegWithSubRegs<3, "x3", [X3_H], ["gp"]>, DwarfRegNum<[3]>;
-  def X4  : RISCVRegWithSubRegs<4, "x4", [X4_H], ["tp"]>, DwarfRegNum<[4]>;
-  def X5  : RISCVRegWithSubRegs<5, "x5", [X5_H], ["t0"]>, DwarfRegNum<[5]>;
-  def X6  : RISCVRegWithSubRegs<6, "x6", [X6_H], ["t1"]>, DwarfRegNum<[6]>;
-  def X7  : RISCVRegWithSubRegs<7, "x7", [X7_H], ["t2"]>, DwarfRegNum<[7]>;
+  def X1_W  : RISCVRegWithSubRegs<1, "x1", [X1_H], ["ra"]>;
+  def X2_W  : RISCVRegWithSubRegs<2, "x2", [X2_H], ["sp"]>;
+  def X3_W  : RISCVRegWithSubRegs<3, "x3", [X3_H], ["gp"]>;
+  def X4_W  : RISCVRegWithSubRegs<4, "x4", [X4_H], ["tp"]>;
+  def X5_W  : RISCVRegWithSubRegs<5, "x5", [X5_H], ["t0"]>;
+  def X6_W  : RISCVRegWithSubRegs<6, "x6", [X6_H], ["t1"]>;
+  def X7_W  : RISCVRegWithSubRegs<7, "x7", [X7_H], ["t2"]>;
   }
-  def X8  : RISCVRegWithSubRegs<8, "x8", [X8_H], ["s0", "fp"]>, DwarfRegNum<[8]>;
-  def X9  : RISCVRegWithSubRegs<9, "x9", [X9_H], ["s1"]>, DwarfRegNum<[9]>;
-  def X10 : RISCVRegWithSubRegs<10,"x10", [X10_H], ["a0"]>, DwarfRegNum<[10]>;
-  def X11 : RISCVRegWithSubRegs<11,"x11", [X11_H], ["a1"]>, DwarfRegNum<[11]>;
-  def X12 : RISCVRegWithSubRegs<12,"x12", [X12_H], ["a2"]>, DwarfRegNum<[12]>;
-  def X13 : RISCVRegWithSubRegs<13,"x13", [X13_H], ["a3"]>, DwarfRegNum<[13]>;
-  def X14 : RISCVRegWithSubRegs<14,"x14", [X14_H], ["a4"]>, DwarfRegNum<[14]>;
-  def X15 : RISCVRegWithSubRegs<15,"x15", [X15_H], ["a5"]>, DwarfRegNum<[15]>;
+  def X8_W  : RISCVRegWithSubRegs<8, "x8", [X8_H], ["s0", "fp"]>;
+  def X9_W  : RISCVRegWithSubRegs<9, "x9", [X9_H], ["s1"]>;
+  def X10_W : RISCVRegWithSubRegs<10,"x10", [X10_H], ["a0"]>;
+  def X11_W : RISCVRegWithSubRegs<11,"x11", [X11_H], ["a1"]>;
+  def X12_W : RISCVRegWithSubRegs<12,"x12", [X12_H], ["a2"]>;
+  def X13_W : RISCVRegWithSubRegs<13,"x13", [X13_H], ["a3"]>;
+  def X14_W : RISCVRegWithSubRegs<14,"x14", [X14_H], ["a4"]>;
+  def X15_W : RISCVRegWithSubRegs<15,"x15", [X15_H], ["a5"]>;
   let CostPerUse = [0, 1] in {
-  def X16 : RISCVRegWithSubRegs<16,"x16", [X16_H], ["a6"]>, DwarfRegNum<[16]>;
-  def X17 : RISCVRegWithSubRegs<17,"x17", [X17_H], ["a7"]>, DwarfRegNum<[17]>;
-  def X18 : RISCVRegWithSubRegs<18,"x18", [X18_H], ["s2"]>, DwarfRegNum<[18]>;
-  def X19 : RISCVRegWithSubRegs<19,"x19", [X19_H], ["s3"]>, DwarfRegNum<[19]>;
-  def X20 : RISCVRegWithSubRegs<20,"x20", [X20_H], ["s4"]>, DwarfRegNum<[20]>;
-  def X21 : RISCVRegWithSubRegs<21,"x21", [X21_H], ["s5"]>, DwarfRegNum<[21]>;
-  def X22 : RISCVRegWithSubRegs<22,"x22", [X22_H], ["s6"]>, DwarfRegNum<[22]>;
-  def X23 : RISCVRegWithSubRegs<23,"x23", [X23_H], ["s7"]>, DwarfRegNum<[23]>;
-  def X24 : RISCVRegWithSubRegs<24,"x24", [X24_H], ["s8"]>, DwarfRegNum<[24]>;
-  def X25 : RISCVRegWithSubRegs<25,"x25", [X25_H], ["s9"]>, DwarfRegNum<[25]>;
-  def X26 : RISCVRegWithSubRegs<26,"x26", [X26_H], ["s10"]>, DwarfRegNum<[26]>;
-  def X27 : RISCVRegWithSubRegs<27,"x27", [X27_H], ["s11"]>, DwarfRegNum<[27]>;
-  def X28 : RISCVRegWithSubRegs<28,"x28", [X28_H], ["t3"]>, DwarfRegNum<[28]>;
-  def X29 : RISCVRegWithSubRegs<29,"x29", [X29_H], ["t4"]>, DwarfRegNum<[29]>;
-  def X30 : RISCVRegWithSubRegs<30,"x30", [X30_H], ["t5"]>, DwarfRegNum<[30]>;
-  def X31 : RISCVRegWithSubRegs<31,"x31", [X31_H], ["t6"]>, DwarfRegNum<[31]>;
+  def X16_W : RISCVRegWithSubRegs<16,"x16", [X16_H], ["a6"]>;
+  def X17_W : RISCVRegWithSubRegs<17,"x17", [X17_H], ["a7"]>;
+  def X18_W : RISCVRegWithSubRegs<18,"x18", [X18_H], ["s2"]>;
+  def X19_W : RISCVRegWithSubRegs<19,"x19", [X19_H], ["s3"]>;
+  def X20_W : RISCVRegWithSubRegs<20,"x20", [X20_H], ["s4"]>;
+  def X21_W : RISCVRegWithSubRegs<21,"x21", [X21_H], ["s5"]>;
+  def X22_W : RISCVRegWithSubRegs<22,"x22", [X22_H], ["s6"]>;
+  def X23_W : RISCVRegWithSubRegs<23,"x23", [X23_H], ["s7"]>;
+  def X24_W : RISCVRegWithSubRegs<24,"x24", [X24_H], ["s8"]>;
+  def X25_W : RISCVRegWithSubRegs<25,"x25", [X25_H], ["s9"]>;
+  def X26_W : RISCVRegWithSubRegs<26,"x26", [X26_H], ["s10"]>;
+  def X27_W : RISCVRegWithSubRegs<27,"x27", [X27_H], ["s11"]>;
+  def X28_W : RISCVRegWithSubRegs<28,"x28", [X28_H], ["t3"]>;
+  def X29_W : RISCVRegWithSubRegs<29,"x29", [X29_H], ["t4"]>;
+  def X30_W : RISCVRegWithSubRegs<30,"x30", [X30_H], ["t5"]>;
+  def X31_W : RISCVRegWithSubRegs<31,"x31", [X31_H], ["t6"]>;
+  }
+  }
+
+  let SubRegIndices = [sub_32] in {
+  let isConstant = true in
+  def X0  : RISCVRegWithSubRegs<0, "x0", [X0_W], ["zero"]>, DwarfRegNum<[0]>;
+  let CostPerUse = [0, 1] in {
+  def X1  : RISCVRegWithSubRegs<1, "x1", [X1_W], ["ra"]>, DwarfRegNum<[1]>;
+  def X2  : RISCVRegWithSubRegs<2, "x2", [X2_W], ["sp"]>, DwarfRegNum<[2]>;
+  def X3  : RISCVRegWithSubRegs<3, "x3", [X3_W], ["gp"]>, DwarfRegNum<[3]>;
+  def X4  : RISCVRegWithSubRegs<4, "x4", [X4_W], ["tp"]>, DwarfRegNum<[4]>;
+  def X5  : RISCVRegWithSubRegs<5, "x5", [X5_W], ["t0"]>, DwarfRegNum<[5]>;
+  def X6  : RISCVRegWithSubRegs<6, "x6", [X6_W], ["t1"]>, DwarfRegNum<[6]>;
+  def X7  : RISCVRegWithSubRegs<7, "x7", [X7_W], ["t2"]>, DwarfRegNum<[7]>;
+  }
+  def X8  : RISCVRegWithSubRegs<8, "x8", [X8_W], ["s0", "fp"]>, DwarfRegNum<[8]>;
+  def X9  : RISCVRegWithSubRegs<9, "x9", [X9_W], ["s1"]>, DwarfRegNum<[9]>;
+  def X10 : RISCVRegWithSubRegs<10,"x10", [X10_W], ["a0"]>, DwarfRegNum<[10]>;
+  def X11 : RISCVRegWithSubRegs<11,"x11", [X11_W], ["a1"]>, DwarfRegNum<[11]>;
+  def X12 : RISCVRegWithSubRegs<12,"x12", [X12_W], ["a2"]>, DwarfRegNum<[12]>;
+  def X13 : RISCVRegWithSubRegs<13,"x13", [X13_W], ["a3"]>, DwarfRegNum<[13]>;
+  def X14 : RISCVRegWithSubRegs<14,"x14", [X14_W], ["a4"]>, DwarfRegNum<[14]>;
+  def X15 : RISCVRegWithSubRegs<15,"x15", [X15_W], ["a5"]>, DwarfRegNum<[15]>;
+  let CostPerUse = [0, 1] in {
+  def X16 : RISCVRegWithSubRegs<16,"x16", [X16_W], ["a6"]>, DwarfRegNum<[16]>;
+  def X17 : RISCVRegWithSubRegs<17,"x17", [X17_W], ["a7"]>, DwarfRegNum<[17]>;
+  def X18 : RISCVRegWithSubRegs<18,"x18", [X18_W], ["s2"]>, DwarfRegNum<[18]>;
+  def X19 : RISCVRegWithSubRegs<19,"x19", [X19_W], ["s3"]>, DwarfRegNum<[19]>;
+  def X20 : RISCVRegWithSubRegs<20,"x20", [X20_W], ["s4"]>, DwarfRegNum<[20]>;
+  def X21 : RISCVRegWithSubRegs<21,"x21", [X21_W], ["s5"]>, DwarfRegNum<[21]>;
+  def X22 : RISCVRegWithSubRegs<22,"x22", [X22_W], ["s6"]>, DwarfRegNum<[22]>;
+  def X23 : RISCVRegWithSubRegs<23,"x23", [X23_W], ["s7"]>, DwarfRegNum<[23]>;
+  def X24 : RISCVRegWithSubRegs<24,"x24", [X24_W], ["s8"]>, DwarfRegNum<[24]>;
+  def X25 : RISCVRegWithSubRegs<25,"x25", [X25_W], ["s9"]>, DwarfRegNum<[25]>;
+  def X26 : RISCVRegWithSubRegs<26,"x26", [X26_W], ["s10"]>, DwarfRegNum<[26]>;
+  def X27 : RISCVRegWithSubRegs<27,"x27", [X27_W], ["s11"]>, DwarfRegNum<[27]>;
+  def X28 : RISCVRegWithSubRegs<28,"x28", [X28_W], ["t3"]>, DwarfRegNum<[28]>;
+  def X29 : RISCVRegWithSubRegs<29,"x29", [X29_W], ["t4"]>, DwarfRegNum<[29]>;
+  def X30 : RISCVRegWithSubRegs<30,"x30", [X30_W], ["t5"]>, DwarfRegNum<[30]>;
+  def X31 : RISCVRegWithSubRegs<31,"x31", [X31_W], ["t6"]>, DwarfRegNum<[31]>;
   }
   }
 }
@@ -617,9 +657,15 @@ def GPRF16 : RISCVRegisterClass<[f16], 16, (add (sequence "X%u_H", 10, 17),
 def GPRF16C : RISCVRegisterClass<[f16], 16, (add (sequence "X%u_H", 10, 15),
                                                  (sequence "X%u_H", 8, 9))>;
 
-let RegInfos = XLenRI in {
-def GPRF32  : RISCVRegisterClass<[f32], 32, (add GPR)>;
-} // RegInfos = XLenRI
+def GPRF32 : RISCVRegisterClass<[f32], 32, (add (sequence "X%u_W", 10, 17),
+                                                (sequence "X%u_W", 5, 7),
+                                                (sequence "X%u_W", 28, 31),
+                                                (sequence "X%u_W", 8, 9),
+                                                (sequence "X%u_W", 18, 27),
+                                                (sequence "X%u_W", 0, 4))>;
+def GPRF32C : RISCVRegisterClass<[f32], 32, (add (sequence "X%u_W", 10, 15),
+                                                 (sequence "X%u_W", 8, 9))>;
+def GPRF32NoX0 : RISCVRegisterClass<[f32], 32, (sub GPRF32, X0_W)>;
 
 // Dummy zero register for use in the register pair containing X0 (as X1 is
 // not read to or written when the X0 register pair is used).
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index b9d35a924669f..2dcac1320417c 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -19,6 +19,7 @@
 #include "TargetInfo/RISCVTargetInfo.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
 #include "llvm/CodeGen/GlobalISel/IRTranslator.h"
 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
 #include "llvm/CodeGen/GlobalISel/Legalizer.h"
@@ -374,6 +375,8 @@ class RISCVPassConfig : public TargetPassConfig {
   void addPreRegAlloc() override;
   void addPostRegAlloc() override;
   void addFastRegAlloc() override;
+
+  std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
 };
 } // namespace
 
@@ -381,6 +384,10 @@ TargetPassConfig *RISCVTargetMachine::createPassConfig(PassManagerBase &PM) {
   return new RISCVPassConfig(*this, PM);
 }
 
+std::unique_ptr<CSEConfigBase> RISCVPassConfig::getCSEConfig() const {
+  return getStandardCSEConfigForOpt(TM->getOptLevel());
+}
+
 FunctionPass *RISCVPassConfig::createRVVRegAllocPass(bool Optimized) {
   // Initialize the global default.
   llvm::call_once(InitializeDefaultRVVRegisterAllocatorFlag,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index c042782389f18..a61461681f79e 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1531,6 +1531,11 @@ RISCVTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
     Opcodes = {RISCV::VMV_S_X, RISCV::VREDAND_VS, RISCV::VMV_X_S};
     break;
   case ISD::FADD:
+    // We can't promote f16/bf16 fadd reductions.
+    if ((LT.second.getVectorElementType() == MVT::f16 &&
+         !ST->hasVInstructionsF16()) ||
+        LT.second.getVectorElementType() == MVT::bf16)
+      return InstructionCost::getInvalid();
     SplitOp = RISCV::VFADD_VV;
     Opcodes = {RISCV::VFMV_S_F, RISCV::VFREDUSUM_VS, RISCV::VFMV_F_S};
     break;
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index 026e5d653c38c..b883c50beadc0 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -419,8 +419,8 @@ bool RISCVVectorPeephole::convertSameMaskVMergeToVMv(MachineInstr &MI) {
   if (!NewOpc)
     return false;
   MachineInstr *True = MRI->getVRegDef(MI.getOperand(3).getReg());
-  if (!True || !RISCV::getMaskedPseudoInfo(True->getOpcode()) ||
-      !hasSameEEW(MI, *True))
+  if (!True || True->getParent() != MI.getParent() ||
+      !RISCV::getMaskedPseudoInfo(True->getOpcode()) || !hasSameEEW(MI, *True))
     return false;
 
   const MachineInstr *TrueV0Def = V0Defs.lookup(True);
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index 66cf163a1a0ac..ed95f40edd863 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -2678,8 +2678,19 @@ getImageType(const TargetExtType *ExtensionType,
          "SPIR-V image builtin type must have sampled type parameter!");
   const SPIRVType *SampledType =
       GR->getOrCreateSPIRVType(ExtensionType->getTypeParameter(0), MIRBuilder);
-  assert(ExtensionType->getNumIntParameters() == 7 &&
+  assert((ExtensionType->getNumIntParameters() == 7 ||
+          ExtensionType->getNumIntParameters() == 6) &&
          "Invalid number of parameters for SPIR-V image builtin!");
+
+  SPIRV::AccessQualifier::AccessQualifier accessQualifier =
+      SPIRV::AccessQualifier::None;
+  if (ExtensionType->getNumIntParameters() == 7) {
+    accessQualifier = Qualifier == SPIRV::AccessQualifier::WriteOnly
+                          ? SPIRV::AccessQualifier::WriteOnly
+                          : SPIRV::AccessQualifier::AccessQualifier(
+                                ExtensionType->getIntParameter(6));
+  }
+
   // Create or get an existing type from GlobalRegistry.
   return GR->getOrCreateOpTypeImage(
       MIRBuilder, SampledType,
@@ -2687,10 +2698,7 @@ getImageType(const TargetExtType *ExtensionType,
       ExtensionType->getIntParameter(1), ExtensionType->getIntParameter(2),
       ExtensionType->getIntParameter(3), ExtensionType->getIntParameter(4),
       SPIRV::ImageFormat::ImageFormat(ExtensionType->getIntParameter(5)),
-      Qualifier == SPIRV::AccessQualifier::WriteOnly
-          ? SPIRV::AccessQualifier::WriteOnly
-          : SPIRV::AccessQualifier::AccessQualifier(
-                ExtensionType->getIntParameter(6)));
+      accessQualifier);
 }
 
 static SPIRVType *getSampledImageType(const TargetExtType *OpaqueType,
diff --git a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp
index 832ca0ba5a82d..b82c2538a8136 100644
--- a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "SPIRVDuplicatesTracker.h"
+#include "SPIRVInstrInfo.h"
 
 #define DEBUG_TYPE "build-dep-graph"
 
@@ -19,13 +20,14 @@ using namespace llvm;
 
 template <typename T>
 void SPIRVGeneralDuplicatesTracker::prebuildReg2Entry(
-    SPIRVDuplicatesTracker<T> &DT, SPIRVReg2EntryTy &Reg2Entry) {
+    SPIRVDuplicatesTracker<T> &DT, SPIRVReg2EntryTy &Reg2Entry,
+    const SPIRVInstrInfo *TII) {
   for (auto &TPair : DT.getAllUses()) {
     for (auto &RegPair : TPair.second) {
       const MachineFunction *MF = RegPair.first;
       Register R = RegPair.second;
       MachineInstr *MI = MF->getRegInfo().getUniqueVRegDef(R);
-      if (!MI)
+      if (!MI || (TPair.second.getIsConst() && !TII->isConstantInstr(*MI)))
         continue;
       Reg2Entry[&MI->getOperand(0)] = &TPair.second;
     }
@@ -33,16 +35,16 @@ void SPIRVGeneralDuplicatesTracker::prebuildReg2Entry(
 }
 
 void SPIRVGeneralDuplicatesTracker::buildDepsGraph(
-    std::vector<SPIRV::DTSortableEntry *> &Graph,
+    std::vector<SPIRV::DTSortableEntry *> &Graph, const SPIRVInstrInfo *TII,
     MachineModuleInfo *MMI = nullptr) {
   SPIRVReg2EntryTy Reg2Entry;
-  prebuildReg2Entry(TT, Reg2Entry);
-  prebuildReg2Entry(CT, Reg2Entry);
-  prebuildReg2Entry(GT, Reg2Entry);
-  prebuildReg2Entry(FT, Reg2Entry);
-  prebuildReg2Entry(AT, Reg2Entry);
-  prebuildReg2Entry(MT, Reg2Entry);
-  prebuildReg2Entry(ST, Reg2Entry);
+  prebuildReg2Entry(TT, Reg2Entry, TII);
+  prebuildReg2Entry(CT, Reg2Entry, TII);
+  prebuildReg2Entry(GT, Reg2Entry, TII);
+  prebuildReg2Entry(FT, Reg2Entry, TII);
+  prebuildReg2Entry(AT, Reg2Entry, TII);
+  prebuildReg2Entry(MT, Reg2Entry, TII);
+  prebuildReg2Entry(ST, Reg2Entry, TII);
 
   for (auto &Op2E : Reg2Entry) {
     SPIRV::DTSortableEntry *E = Op2E.second;
@@ -65,20 +67,19 @@ void SPIRVGeneralDuplicatesTracker::buildDepsGraph(
         if (MI->getOpcode() == SPIRV::OpConstantFunctionPointerINTEL && i == 2)
           continue;
         MachineOperand *RegOp = &VRegDef->getOperand(0);
-        LLVM_DEBUG({
-          if (Reg2Entry.count(RegOp) == 0 &&
-              (MI->getOpcode() != SPIRV::OpVariable || i != 3)) {
-            dbgs() << "Unexpected pattern while building a dependency "
-                      "graph.\nInstruction: ";
-            MI->print(dbgs());
-            dbgs() << "Operand: ";
-            Op.print(dbgs());
-            dbgs() << "\nOperand definition: ";
-            VRegDef->print(dbgs());
-          }
-        });
-        assert((MI->getOpcode() == SPIRV::OpVariable && i == 3) ||
-               Reg2Entry.count(RegOp));
+        if (Reg2Entry.count(RegOp) == 0 &&
+            (MI->getOpcode() != SPIRV::OpVariable || i != 3)) {
+          std::string DiagMsg;
+          raw_string_ostream OS(DiagMsg);
+          OS << "Unexpected pattern while building a dependency "
+                "graph.\nInstruction: ";
+          MI->print(OS);
+          OS << "Operand: ";
+          Op.print(OS);
+          OS << "\nOperand definition: ";
+          VRegDef->print(OS);
+          report_fatal_error(DiagMsg.c_str());
+        }
         if (Reg2Entry.count(RegOp))
           E->addDep(Reg2Entry[RegOp]);
       }
diff --git a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h
index a37e65a47eda0..6847da0509797 100644
--- a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h
+++ b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h
@@ -26,6 +26,7 @@
 
 namespace llvm {
 namespace SPIRV {
+class SPIRVInstrInfo;
 // NOTE: using MapVector instead of DenseMap because it helps getting
 // everything ordered in a stable manner for a price of extra (NumKeys)*PtrSize
 // memory and expensive removals which do not happen anyway.
@@ -35,8 +36,9 @@ class DTSortableEntry : public MapVector<const MachineFunction *, Register> {
   struct FlagsTy {
     unsigned IsFunc : 1;
     unsigned IsGV : 1;
+    unsigned IsConst : 1;
     // NOTE: bit-field default init is a C++20 feature.
-    FlagsTy() : IsFunc(0), IsGV(0) {}
+    FlagsTy() : IsFunc(0), IsGV(0), IsConst(0) {}
   };
   FlagsTy Flags;
 
@@ -45,8 +47,10 @@ class DTSortableEntry : public MapVector<const MachineFunction *, Register> {
   // require hoisting of params as well.
   bool getIsFunc() const { return Flags.IsFunc; }
   bool getIsGV() const { return Flags.IsGV; }
+  bool getIsConst() const { return Flags.IsConst; }
   void setIsFunc(bool V) { Flags.IsFunc = V; }
   void setIsGV(bool V) { Flags.IsGV = V; }
+  void setIsConst(bool V) { Flags.IsConst = V; }
 
   const SmallVector<DTSortableEntry *, 2> &getDeps() const { return Deps; }
   void addDep(DTSortableEntry *E) { Deps.push_back(E); }
@@ -103,13 +107,15 @@ make_descr_image(const Type *SampledTy, unsigned Dim, unsigned Depth,
 inline SpecialTypeDescriptor
 make_descr_sampled_image(const Type *SampledTy, const MachineInstr *ImageTy) {
   assert(ImageTy->getOpcode() == SPIRV::OpTypeImage);
+  unsigned AC = AccessQualifier::AccessQualifier::None;
+  if (ImageTy->getNumOperands() > 8)
+    AC = ImageTy->getOperand(8).getImm();
   return std::make_tuple(
       SampledTy,
       ImageAttrs(
           ImageTy->getOperand(2).getImm(), ImageTy->getOperand(3).getImm(),
           ImageTy->getOperand(4).getImm(), ImageTy->getOperand(5).getImm(),
-          ImageTy->getOperand(6).getImm(), ImageTy->getOperand(7).getImm(),
-          ImageTy->getOperand(8).getImm())
+          ImageTy->getOperand(6).getImm(), ImageTy->getOperand(7).getImm(), AC)
           .Val,
       SpecialTypeKind::STK_SampledImage);
 }
@@ -160,6 +166,10 @@ template <typename KeyTy> class SPIRVDuplicatesTrackerBase {
                      typename std::remove_const<
                          typename std::remove_pointer<KeyTy>::type>::type>())
       Storage[V].setIsGV(true);
+    if (std::is_same<Constant,
+                     typename std::remove_const<
+                         typename std::remove_pointer<KeyTy>::type>::type>())
+      Storage[V].setIsConst(true);
   }
 
   Register find(KeyTy V, const MachineFunction *MF) const {
@@ -211,11 +221,12 @@ class SPIRVGeneralDuplicatesTracker {
 
   template <typename T>
   void prebuildReg2Entry(SPIRVDuplicatesTracker<T> &DT,
-                         SPIRVReg2EntryTy &Reg2Entry);
+                         SPIRVReg2EntryTy &Reg2Entry,
+                         const SPIRVInstrInfo *TII);
 
 public:
   void buildDepsGraph(std::vector<SPIRV::DTSortableEntry *> &Graph,
-                      MachineModuleInfo *MMI);
+                      const SPIRVInstrInfo *TII, MachineModuleInfo *MMI);
 
   void add(const Type *Ty, const MachineFunction *MF, Register R) {
     TT.add(unifyPtrType(Ty), MF, R);
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index ceca0a180c95b..f35c2435e60a4 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -1149,16 +1149,19 @@ SPIRVType *SPIRVGlobalRegistry::getOrCreateOpTypeImage(
     return Res;
   Register ResVReg = createTypeVReg(MIRBuilder);
   DT.add(TD, &MIRBuilder.getMF(), ResVReg);
-  return MIRBuilder.buildInstr(SPIRV::OpTypeImage)
-      .addDef(ResVReg)
-      .addUse(getSPIRVTypeID(SampledType))
-      .addImm(Dim)
-      .addImm(Depth)        // Depth (whether or not it is a Depth image).
-      .addImm(Arrayed)      // Arrayed.
-      .addImm(Multisampled) // Multisampled (0 = only single-sample).
-      .addImm(Sampled)      // Sampled (0 = usage known at runtime).
-      .addImm(ImageFormat)
-      .addImm(AccessQual);
+  auto MIB = MIRBuilder.buildInstr(SPIRV::OpTypeImage)
+                 .addDef(ResVReg)
+                 .addUse(getSPIRVTypeID(SampledType))
+                 .addImm(Dim)
+                 .addImm(Depth)   // Depth (whether or not it is a Depth image).
+                 .addImm(Arrayed) // Arrayed.
+                 .addImm(Multisampled) // Multisampled (0 = only single-sample).
+                 .addImm(Sampled)      // Sampled (0 = usage known at runtime).
+                 .addImm(ImageFormat);
+
+  if (AccessQual != SPIRV::AccessQualifier::None)
+    MIB.addImm(AccessQual);
+  return MIB;
 }
 
 SPIRVType *
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
index ace5cfe91ebe4..d301e119e16c8 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
@@ -152,8 +152,9 @@ class SPIRVGlobalRegistry {
   }
 
   void buildDepsGraph(std::vector<SPIRV::DTSortableEntry *> &Graph,
+                      const SPIRVInstrInfo *TII,
                       MachineModuleInfo *MMI = nullptr) {
-    DT.buildDepsGraph(Graph, MMI);
+    DT.buildDepsGraph(Graph, TII, MMI);
   }
 
   void setBound(unsigned V) { Bound = V; }
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 7a565249a342d..468e34a365826 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -145,24 +145,13 @@ class SPIRVInstructionSelector : public InstructionSelector {
 
   bool selectCmp(Register ResVReg, const SPIRVType *ResType,
                  unsigned comparisonOpcode, MachineInstr &I) const;
-
+  bool selectCross(Register ResVReg, const SPIRVType *ResType,
+                   MachineInstr &I) const;
   bool selectICmp(Register ResVReg, const SPIRVType *ResType,
                   MachineInstr &I) const;
   bool selectFCmp(Register ResVReg, const SPIRVType *ResType,
                   MachineInstr &I) const;
 
-  bool selectFmix(Register ResVReg, const SPIRVType *ResType,
-                  MachineInstr &I) const;
-
-  bool selectLength(Register ResVReg, const SPIRVType *ResType,
-                    MachineInstr &I) const;
-
-  bool selectFrac(Register ResVReg, const SPIRVType *ResType,
-                  MachineInstr &I) const;
-
-  bool selectRsqrt(Register ResVReg, const SPIRVType *ResType,
-                   MachineInstr &I) const;
-
   bool selectSign(Register ResVReg, const SPIRVType *ResType,
                   MachineInstr &I) const;
 
@@ -235,18 +224,12 @@ class SPIRVInstructionSelector : public InstructionSelector {
   bool selectLog10(Register ResVReg, const SPIRVType *ResType,
                    MachineInstr &I) const;
 
-  bool selectNormalize(Register ResVReg, const SPIRVType *ResType,
-                       MachineInstr &I) const;
-
   bool selectSaturate(Register ResVReg, const SPIRVType *ResType,
                       MachineInstr &I) const;
 
   bool selectSpvThreadId(Register ResVReg, const SPIRVType *ResType,
                          MachineInstr &I) const;
 
-  bool selectStep(Register ResVReg, const SPIRVType *ResType,
-                  MachineInstr &I) const;
-
   bool selectUnmergeValues(MachineInstr &I) const;
 
   // Utilities
@@ -802,8 +785,13 @@ bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
                      .addImm(static_cast<uint32_t>(Set))
                      .addImm(Opcode);
       const unsigned NumOps = I.getNumOperands();
-      for (unsigned i = 1; i < NumOps; ++i)
-        MIB.add(I.getOperand(i));
+      unsigned Index = 1;
+      if (Index < NumOps &&
+          I.getOperand(Index).getType() ==
+              MachineOperand::MachineOperandType::MO_IntrinsicID)
+        Index = 2;
+      for (; Index < NumOps; ++Index)
+        MIB.add(I.getOperand(Index));
       return MIB.constrainAllUses(TII, TRI, RBI);
     }
   }
@@ -1605,95 +1593,6 @@ bool SPIRVInstructionSelector::selectAny(Register ResVReg,
   return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny);
 }
 
-bool SPIRVInstructionSelector::selectFmix(Register ResVReg,
-                                          const SPIRVType *ResType,
-                                          MachineInstr &I) const {
-
-  assert(I.getNumOperands() == 5);
-  assert(I.getOperand(2).isReg());
-  assert(I.getOperand(3).isReg());
-  assert(I.getOperand(4).isReg());
-  MachineBasicBlock &BB = *I.getParent();
-
-  return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
-      .addDef(ResVReg)
-      .addUse(GR.getSPIRVTypeID(ResType))
-      .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
-      .addImm(GL::FMix)
-      .addUse(I.getOperand(2).getReg())
-      .addUse(I.getOperand(3).getReg())
-      .addUse(I.getOperand(4).getReg())
-      .constrainAllUses(TII, TRI, RBI);
-}
-
-bool SPIRVInstructionSelector::selectLength(Register ResVReg,
-                                            const SPIRVType *ResType,
-                                            MachineInstr &I) const {
-
-  assert(I.getNumOperands() == 3);
-  assert(I.getOperand(2).isReg());
-  MachineBasicBlock &BB = *I.getParent();
-
-  return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
-      .addDef(ResVReg)
-      .addUse(GR.getSPIRVTypeID(ResType))
-      .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
-      .addImm(GL::Length)
-      .addUse(I.getOperand(2).getReg())
-      .constrainAllUses(TII, TRI, RBI);
-}
-
-bool SPIRVInstructionSelector::selectFrac(Register ResVReg,
-                                           const SPIRVType *ResType,
-                                           MachineInstr &I) const {
-
-  assert(I.getNumOperands() == 3);
-  assert(I.getOperand(2).isReg());
-  MachineBasicBlock &BB = *I.getParent();
-
-  return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
-      .addDef(ResVReg)
-      .addUse(GR.getSPIRVTypeID(ResType))
-      .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
-      .addImm(GL::Fract)
-      .addUse(I.getOperand(2).getReg())
-      .constrainAllUses(TII, TRI, RBI);
-}
-
-bool SPIRVInstructionSelector::selectNormalize(Register ResVReg,
-                                               const SPIRVType *ResType,
-                                               MachineInstr &I) const {
-
-  assert(I.getNumOperands() == 3);
-  assert(I.getOperand(2).isReg());
-  MachineBasicBlock &BB = *I.getParent();
-
-  return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
-      .addDef(ResVReg)
-      .addUse(GR.getSPIRVTypeID(ResType))
-      .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
-      .addImm(GL::Normalize)
-      .addUse(I.getOperand(2).getReg())
-      .constrainAllUses(TII, TRI, RBI);
-}
-
-bool SPIRVInstructionSelector::selectRsqrt(Register ResVReg,
-                                           const SPIRVType *ResType,
-                                           MachineInstr &I) const {
-
-  assert(I.getNumOperands() == 3);
-  assert(I.getOperand(2).isReg());
-  MachineBasicBlock &BB = *I.getParent();
-
-  return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
-      .addDef(ResVReg)
-      .addUse(GR.getSPIRVTypeID(ResType))
-      .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
-      .addImm(GL::InverseSqrt)
-      .addUse(I.getOperand(2).getReg())
-      .constrainAllUses(TII, TRI, RBI);
-}
-
 // Select the OpDot instruction for the given float dot
 bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg,
                                               const SPIRVType *ResType,
@@ -1853,25 +1752,6 @@ bool SPIRVInstructionSelector::selectSign(Register ResVReg,
   return Result;
 }
 
-bool SPIRVInstructionSelector::selectStep(Register ResVReg,
-                                          const SPIRVType *ResType,
-                                          MachineInstr &I) const {
-
-  assert(I.getNumOperands() == 4);
-  assert(I.getOperand(2).isReg());
-  assert(I.getOperand(3).isReg());
-  MachineBasicBlock &BB = *I.getParent();
-
-  return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
-      .addDef(ResVReg)
-      .addUse(GR.getSPIRVTypeID(ResType))
-      .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
-      .addImm(GL::Step)
-      .addUse(I.getOperand(2).getReg())
-      .addUse(I.getOperand(3).getReg())
-      .constrainAllUses(TII, TRI, RBI);
-}
-
 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg,
                                                 const SPIRVType *ResType,
                                                 MachineInstr &I) const {
@@ -2621,16 +2501,18 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
     return selectAll(ResVReg, ResType, I);
   case Intrinsic::spv_any:
     return selectAny(ResVReg, ResType, I);
+  case Intrinsic::spv_cross:
+    return selectExtInst(ResVReg, ResType, I, CL::cross, GL::Cross);
   case Intrinsic::spv_lerp:
-    return selectFmix(ResVReg, ResType, I);
+    return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix);
   case Intrinsic::spv_length:
-    return selectLength(ResVReg, ResType, I);
+    return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length);
   case Intrinsic::spv_frac:
-    return selectFrac(ResVReg, ResType, I);
+    return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract);
   case Intrinsic::spv_normalize:
-    return selectNormalize(ResVReg, ResType, I);
+    return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize);
   case Intrinsic::spv_rsqrt:
-    return selectRsqrt(ResVReg, ResType, I);
+    return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt);
   case Intrinsic::spv_sign:
     return selectSign(ResVReg, ResType, I);
   case Intrinsic::spv_lifetime_start:
@@ -2654,7 +2536,9 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
         .addUse(GR.getOrCreateConstInt(3, I, IntTy, TII));
   }
   case Intrinsic::spv_step:
-    return selectStep(ResVReg, ResType, I);
+    return selectExtInst(ResVReg, ResType, I, CL::step, GL::Step);
+  case Intrinsic::spv_radians:
+    return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians);
   // Discard intrinsics which we do not expect to actually represent code after
   // lowering or intrinsics which are not implemented but should not crash when
   // found in a customer's LLVM IR input.
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index 8908d8965b67c..6af4dd3a34971 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -282,7 +282,7 @@ void SPIRVModuleAnalysis::collectGlobalEntities(
 void SPIRVModuleAnalysis::processDefInstrs(const Module &M) {
   std::vector<SPIRV::DTSortableEntry *> DepsGraph;
 
-  GR->buildDepsGraph(DepsGraph, SPVDumpDeps ? MMI : nullptr);
+  GR->buildDepsGraph(DepsGraph, TII, SPVDumpDeps ? MMI : nullptr);
 
   collectGlobalEntities(
       DepsGraph, SPIRV::MB_TypeConstVars,
@@ -691,7 +691,9 @@ void RequirementHandler::initAvailableCapabilitiesForVulkan(
 
   // Provided by all supported Vulkan versions.
   addAvailableCaps({Capability::Int16, Capability::Int64, Capability::Float16,
-                    Capability::Float64, Capability::GroupNonUniform});
+                    Capability::Float64, Capability::GroupNonUniform,
+                    Capability::Image1D, Capability::SampledBuffer,
+                    Capability::ImageBuffer});
 }
 
 } // namespace SPIRV
@@ -776,12 +778,13 @@ static void addOpTypeImageReqs(const MachineInstr &MI,
   }
 
   // Has optional access qualifier.
-  // TODO: check if it's OpenCL's kernel.
-  if (MI.getNumOperands() > 8 &&
-      MI.getOperand(8).getImm() == SPIRV::AccessQualifier::ReadWrite)
-    Reqs.addRequirements(SPIRV::Capability::ImageReadWrite);
-  else
-    Reqs.addRequirements(SPIRV::Capability::ImageBasic);
+  if (ST.isOpenCLEnv()) {
+    if (MI.getNumOperands() > 8 &&
+        MI.getOperand(8).getImm() == SPIRV::AccessQualifier::ReadWrite)
+      Reqs.addRequirements(SPIRV::Capability::ImageReadWrite);
+    else
+      Reqs.addRequirements(SPIRV::Capability::ImageBasic);
+  }
 }
 
 // Add requirements for handling atomic float instructions
diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
index 6bc27c7a0d193..8c1e7b922b61c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
+++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
@@ -1096,6 +1096,7 @@ multiclass AccessQualifierOperand<bits<32> value, list<Capability> reqCapabiliti
 defm ReadOnly : AccessQualifierOperand<0, [Kernel]>;
 defm WriteOnly : AccessQualifierOperand<1, [Kernel]>;
 defm ReadWrite : AccessQualifierOperand<2, [Kernel]>;
+defm None : AccessQualifierOperand<3, []>;
 
 //===----------------------------------------------------------------------===//
 // Multiclass used to define FunctionParameterAttribute enum values and at the
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
index 95db5500b0e1b..ee8686d1166a5 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
@@ -498,7 +498,9 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser {
 
   void addBlockTypeOperand(OperandVector &Operands, SMLoc NameLoc,
                            WebAssembly::BlockType BT) {
-    if (BT != WebAssembly::BlockType::Void) {
+    if (BT == WebAssembly::BlockType::Void) {
+      TC.setLastSig(wasm::WasmSignature{});
+    } else {
       wasm::WasmSignature Sig({static_cast<wasm::ValType>(BT)}, {});
       TC.setLastSig(Sig);
       NestingStack.back().Sig = Sig;
@@ -1002,7 +1004,8 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser {
       auto *Signature = Ctx.createWasmSignature();
       if (parseSignature(Signature))
         return ParseStatus::Failure;
-      TC.funcDecl(*Signature);
+      if (CurrentState == FunctionStart)
+        TC.funcDecl(*Signature);
       WasmSym->setSignature(Signature);
       WasmSym->setType(wasm::WASM_SYMBOL_TYPE_FUNCTION);
       TOut.emitFunctionType(WasmSym);
@@ -1154,8 +1157,8 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser {
           Inst.setOpcode(Opc64);
         }
       }
-      if (!SkipTypeCheck && TC.typeCheck(IDLoc, Inst, Operands))
-        return true;
+      if (!SkipTypeCheck)
+        TC.typeCheck(IDLoc, Inst, Operands);
       Out.emitInstruction(Inst, getSTI());
       if (CurrentState == EndFunction) {
         onEndOfFunction(IDLoc);
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp
index 6c71460201537..effc2e65223ca 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp
@@ -50,8 +50,7 @@ WebAssemblyAsmTypeCheck::WebAssemblyAsmTypeCheck(MCAsmParser &Parser,
 
 void WebAssemblyAsmTypeCheck::funcDecl(const wasm::WasmSignature &Sig) {
   LocalTypes.assign(Sig.Params.begin(), Sig.Params.end());
-  ReturnTypes.assign(Sig.Returns.begin(), Sig.Returns.end());
-  BrStack.emplace_back(Sig.Returns.begin(), Sig.Returns.end());
+  BlockInfoStack.push_back({Sig, 0, false});
 }
 
 void WebAssemblyAsmTypeCheck::localDecl(
@@ -64,14 +63,15 @@ void WebAssemblyAsmTypeCheck::dumpTypeStack(Twine Msg) {
 }
 
 bool WebAssemblyAsmTypeCheck::typeError(SMLoc ErrorLoc, const Twine &Msg) {
-  // If we're currently in unreachable code, we suppress errors completely.
-  if (Unreachable)
-    return false;
   dumpTypeStack("current stack: ");
   return Parser.Error(ErrorLoc, Msg);
 }
 
 bool WebAssemblyAsmTypeCheck::match(StackType TypeA, StackType TypeB) {
+  // These should have been filtered out in checkTypes()
+  assert(!std::get_if<Polymorphic>(&TypeA) &&
+         !std::get_if<Polymorphic>(&TypeB));
+
   if (TypeA == TypeB)
     return false;
   if (std::get_if<Any>(&TypeA) || std::get_if<Any>(&TypeB))
@@ -90,6 +90,10 @@ std::string WebAssemblyAsmTypeCheck::getTypesString(ArrayRef<StackType> Types,
                                                     size_t StartPos) {
   SmallVector<std::string, 4> TypeStrs;
   for (auto I = Types.size(); I > StartPos; I--) {
+    if (std::get_if<Polymorphic>(&Types[I - 1])) {
+      TypeStrs.push_back("...");
+      break;
+    }
     if (std::get_if<Any>(&Types[I - 1]))
       TypeStrs.push_back("any");
     else if (std::get_if<Ref>(&Types[I - 1]))
@@ -131,29 +135,48 @@ bool WebAssemblyAsmTypeCheck::checkTypes(SMLoc ErrorLoc,
                                          bool ExactMatch) {
   auto StackI = Stack.size();
   auto TypeI = Types.size();
+  assert(!BlockInfoStack.empty());
+  auto BlockStackStartPos = BlockInfoStack.back().StackStartPos;
   bool Error = false;
+  bool PolymorphicStack = false;
   // Compare elements one by one from the stack top
-  for (; StackI > 0 && TypeI > 0; StackI--, TypeI--) {
+  for (; StackI > BlockStackStartPos && TypeI > 0; StackI--, TypeI--) {
+    // If the stack is polymorphic, we assume all types in 'Types' have been
+    // compared and matched
+    if (std::get_if<Polymorphic>(&Stack[StackI - 1])) {
+      TypeI = 0;
+      break;
+    }
     if (match(Stack[StackI - 1], Types[TypeI - 1])) {
       Error = true;
       break;
     }
   }
+
+  // If the stack top is polymorphic, the stack is in the polymorphic state.
+  if (StackI > BlockStackStartPos &&
+      std::get_if<Polymorphic>(&Stack[StackI - 1]))
+    PolymorphicStack = true;
+
   // Even if no match failure has happened in the loop above, if not all
   // elements of Types has been matched, that means we don't have enough
   // elements on the stack.
   //
   // Also, if not all elements of the Stack has been matched and when
-  // 'ExactMatch' is true, that means we have superfluous elements remaining on
-  // the stack (e.g. at the end of a function).
-  if (TypeI > 0 || (ExactMatch && StackI > 0))
+  // 'ExactMatch' is true and the current stack is not polymorphic, that means
+  // we have superfluous elements remaining on the stack (e.g. at the end of a
+  // function).
+  if (TypeI > 0 ||
+      (ExactMatch && !PolymorphicStack && StackI > BlockStackStartPos))
     Error = true;
 
   if (!Error)
     return false;
 
-  auto StackStartPos =
-      ExactMatch ? 0 : std::max(0, (int)Stack.size() - (int)Types.size());
+  auto StackStartPos = ExactMatch
+                           ? BlockStackStartPos
+                           : std::max((int)BlockStackStartPos,
+                                      (int)Stack.size() - (int)Types.size());
   return typeError(ErrorLoc, "type mismatch, expected " +
                                  getTypesString(Types, 0) + " but got " +
                                  getTypesString(Stack, StackStartPos));
@@ -169,9 +192,13 @@ bool WebAssemblyAsmTypeCheck::popTypes(SMLoc ErrorLoc,
                                        ArrayRef<StackType> Types,
                                        bool ExactMatch) {
   bool Error = checkTypes(ErrorLoc, Types, ExactMatch);
-  auto NumPops = std::min(Stack.size(), Types.size());
-  for (size_t I = 0, E = NumPops; I != E; I++)
+  auto NumPops = std::min(Stack.size() - BlockInfoStack.back().StackStartPos,
+                          Types.size());
+  for (size_t I = 0, E = NumPops; I != E; I++) {
+    if (std::get_if<Polymorphic>(&Stack.back()))
+      break;
     Stack.pop_back();
+  }
   return Error;
 }
 
@@ -201,25 +228,6 @@ bool WebAssemblyAsmTypeCheck::getLocal(SMLoc ErrorLoc, const MCOperand &LocalOp,
   return false;
 }
 
-bool WebAssemblyAsmTypeCheck::checkBr(SMLoc ErrorLoc, size_t Level) {
-  if (Level >= BrStack.size())
-    return typeError(ErrorLoc,
-                     StringRef("br: invalid depth ") + std::to_string(Level));
-  const SmallVector<wasm::ValType, 4> &Expected =
-      BrStack[BrStack.size() - Level - 1];
-  return checkTypes(ErrorLoc, Expected);
-  return false;
-}
-
-bool WebAssemblyAsmTypeCheck::checkEnd(SMLoc ErrorLoc, bool PopVals) {
-  if (!PopVals)
-    BrStack.pop_back();
-
-  if (PopVals)
-    return popTypes(ErrorLoc, LastSig.Returns);
-  return checkTypes(ErrorLoc, LastSig.Returns);
-}
-
 bool WebAssemblyAsmTypeCheck::checkSig(SMLoc ErrorLoc,
                                        const wasm::WasmSignature &Sig) {
   bool Error = popTypes(ErrorLoc, Sig.Params);
@@ -309,9 +317,9 @@ bool WebAssemblyAsmTypeCheck::getSignature(SMLoc ErrorLoc,
 }
 
 bool WebAssemblyAsmTypeCheck::endOfFunction(SMLoc ErrorLoc, bool ExactMatch) {
-  bool Error = popTypes(ErrorLoc, ReturnTypes, ExactMatch);
-  Unreachable = true;
-  return Error;
+  assert(!BlockInfoStack.empty());
+  const auto &FuncInfo = BlockInfoStack[0];
+  return checkTypes(ErrorLoc, FuncInfo.Sig.Returns, ExactMatch);
 }
 
 bool WebAssemblyAsmTypeCheck::typeCheck(SMLoc ErrorLoc, const MCInst &Inst,
@@ -452,52 +460,91 @@ bool WebAssemblyAsmTypeCheck::typeCheck(SMLoc ErrorLoc, const MCInst &Inst,
     return popType(ErrorLoc, Any{});
   }
 
-  if (Name == "try" || Name == "block" || Name == "loop" || Name == "if") {
-    if (Name == "loop")
-      BrStack.emplace_back(LastSig.Params.begin(), LastSig.Params.end());
-    else
-      BrStack.emplace_back(LastSig.Returns.begin(), LastSig.Returns.end());
-    if (Name == "if" && popType(ErrorLoc, wasm::ValType::I32))
-      return true;
-    return false;
+  if (Name == "block" || Name == "loop" || Name == "if" || Name == "try") {
+    bool Error = Name == "if" && popType(ErrorLoc, wasm::ValType::I32);
+    // Pop block input parameters and check their types are correct
+    Error |= popTypes(ErrorLoc, LastSig.Params);
+    // Push a new block info
+    BlockInfoStack.push_back({LastSig, Stack.size(), Name == "loop"});
+    // Push back block input parameters
+    pushTypes(LastSig.Params);
+    return Error;
   }
 
   if (Name == "end_block" || Name == "end_loop" || Name == "end_if" ||
-      Name == "else" || Name == "end_try" || Name == "catch" ||
-      Name == "catch_all" || Name == "delegate") {
-    bool Error = checkEnd(ErrorLoc, Name == "else" || Name == "catch" ||
-                                        Name == "catch_all");
-    Unreachable = false;
-    if (Name == "catch") {
+      Name == "end_try" || Name == "delegate" || Name == "else" ||
+      Name == "catch" || Name == "catch_all") {
+    assert(!BlockInfoStack.empty());
+    // Check if the types on the stack match with the block return type
+    const auto &LastBlockInfo = BlockInfoStack.back();
+    bool Error = checkTypes(ErrorLoc, LastBlockInfo.Sig.Returns, true);
+    // Pop all types added to the stack for the current block level
+    Stack.truncate(LastBlockInfo.StackStartPos);
+    if (Name == "else") {
+      // 'else' expects the block input parameters to be on the stack, in the
+      // same way we entered 'if'
+      pushTypes(LastBlockInfo.Sig.Params);
+    } else if (Name == "catch") {
+      // 'catch' instruction pushes values whose types are specified in the
+      // tag's 'params' part
       const wasm::WasmSignature *Sig = nullptr;
       if (!getSignature(Operands[1]->getStartLoc(), Inst.getOperand(0),
                         wasm::WASM_SYMBOL_TYPE_TAG, Sig))
-        // catch instruction pushes values whose types are specified in the
-        // tag's "params" part
         pushTypes(Sig->Params);
       else
         Error = true;
+    } else if (Name == "catch_all") {
+      // 'catch_all' does not push anything onto the stack
+    } else {
+      // For normal end markers, push block return value types onto the stack
+      // and pop the block info
+      pushTypes(LastBlockInfo.Sig.Returns);
+      BlockInfoStack.pop_back();
     }
     return Error;
   }
 
-  if (Name == "br") {
+  if (Name == "br" || Name == "br_if") {
+    bool Error = false;
+    if (Name == "br_if")
+      Error |= popType(ErrorLoc, wasm::ValType::I32); // cond
     const MCOperand &Operand = Inst.getOperand(0);
-    if (!Operand.isImm())
-      return true;
-    return checkBr(ErrorLoc, static_cast<size_t>(Operand.getImm()));
+    if (Operand.isImm()) {
+      unsigned Level = Operand.getImm();
+      if (Level < BlockInfoStack.size()) {
+        const auto &DestBlockInfo =
+            BlockInfoStack[BlockInfoStack.size() - Level - 1];
+        if (DestBlockInfo.IsLoop)
+          Error |= checkTypes(ErrorLoc, DestBlockInfo.Sig.Params, false);
+        else
+          Error |= checkTypes(ErrorLoc, DestBlockInfo.Sig.Returns, false);
+      } else {
+        Error = typeError(ErrorLoc, StringRef("br: invalid depth ") +
+                                        std::to_string(Level));
+      }
+    } else {
+      Error =
+          typeError(Operands[1]->getStartLoc(), "depth should be an integer");
+    }
+    if (Name == "br")
+      pushType(Polymorphic{});
+    return Error;
   }
 
   if (Name == "return") {
-    return endOfFunction(ErrorLoc, false);
+    bool Error = endOfFunction(ErrorLoc, false);
+    pushType(Polymorphic{});
+    return Error;
   }
 
   if (Name == "call_indirect" || Name == "return_call_indirect") {
     // Function value.
     bool Error = popType(ErrorLoc, wasm::ValType::I32);
     Error |= checkSig(ErrorLoc, LastSig);
-    if (Name == "return_call_indirect" && endOfFunction(ErrorLoc, false))
-      return true;
+    if (Name == "return_call_indirect") {
+      Error |= endOfFunction(ErrorLoc, false);
+      pushType(Polymorphic{});
+    }
     return Error;
   }
 
@@ -509,13 +556,15 @@ bool WebAssemblyAsmTypeCheck::typeCheck(SMLoc ErrorLoc, const MCInst &Inst,
       Error |= checkSig(ErrorLoc, *Sig);
     else
       Error = true;
-    if (Name == "return_call" && endOfFunction(ErrorLoc, false))
-      return true;
+    if (Name == "return_call") {
+      Error |= endOfFunction(ErrorLoc, false);
+      pushType(Polymorphic{});
+    }
     return Error;
   }
 
   if (Name == "unreachable") {
-    Unreachable = true;
+    pushType(Polymorphic{});
     return false;
   }
 
@@ -526,11 +575,15 @@ bool WebAssemblyAsmTypeCheck::typeCheck(SMLoc ErrorLoc, const MCInst &Inst,
   }
 
   if (Name == "throw") {
+    bool Error = false;
     const wasm::WasmSignature *Sig = nullptr;
     if (!getSignature(Operands[1]->getStartLoc(), Inst.getOperand(0),
                       wasm::WASM_SYMBOL_TYPE_TAG, Sig))
-      return checkSig(ErrorLoc, *Sig);
-    return true;
+      Error |= checkSig(ErrorLoc, *Sig);
+    else
+      Error = true;
+    pushType(Polymorphic{});
+    return Error;
   }
 
   // The current instruction is a stack instruction which doesn't have
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.h b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.h
index df063d749e3b4..596fb27bce94e 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.h
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.h
@@ -31,13 +31,17 @@ class WebAssemblyAsmTypeCheck final {
 
   struct Ref : public std::monostate {};
   struct Any : public std::monostate {};
-  using StackType = std::variant<wasm::ValType, Ref, Any>;
+  struct Polymorphic : public std::monostate {};
+  using StackType = std::variant<wasm::ValType, Ref, Any, Polymorphic>;
   SmallVector<StackType, 16> Stack;
-  SmallVector<SmallVector<wasm::ValType, 4>, 8> BrStack;
+  struct BlockInfo {
+    wasm::WasmSignature Sig;
+    size_t StackStartPos;
+    bool IsLoop;
+  };
+  SmallVector<BlockInfo, 8> BlockInfoStack;
   SmallVector<wasm::ValType, 16> LocalTypes;
-  SmallVector<wasm::ValType, 4> ReturnTypes;
   wasm::WasmSignature LastSig;
-  bool Unreachable = false;
   bool Is64;
 
   // checkTypes checks 'Types' against the value stack. popTypes checks 'Types'
@@ -68,8 +72,6 @@ class WebAssemblyAsmTypeCheck final {
   void dumpTypeStack(Twine Msg);
   bool typeError(SMLoc ErrorLoc, const Twine &Msg);
   bool getLocal(SMLoc ErrorLoc, const MCOperand &LocalOp, wasm::ValType &Type);
-  bool checkEnd(SMLoc ErrorLoc, bool PopVals = false);
-  bool checkBr(SMLoc ErrorLoc, size_t Level);
   bool checkSig(SMLoc ErrorLoc, const wasm::WasmSignature &Sig);
   bool getSymRef(SMLoc ErrorLoc, const MCOperand &SymOp,
                  const MCSymbolRefExpr *&SymRef);
@@ -91,10 +93,8 @@ class WebAssemblyAsmTypeCheck final {
 
   void clear() {
     Stack.clear();
-    BrStack.clear();
+    BlockInfoStack.clear();
     LocalTypes.clear();
-    ReturnTypes.clear();
-    Unreachable = false;
   }
 };
 
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp
index 534717a4bea4e..587f923e789f0 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp
@@ -19,6 +19,7 @@
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/Support/raw_ostream.h"
+#include <string_view>
 
 using namespace llvm;
 
@@ -622,6 +623,270 @@ static bool printFMAComments(const MCInst *MI, raw_ostream &OS,
   return true;
 }
 
+// This table is indexed by the imm8 binary function specified in a
+// vpternlog{d,q} instruction. The symbols {a,b,c} correspond to the three
+// inputs to the binary function. This table was taken from
+// https://gist.github.com/dougallj/81a80cd381988466c4e1c4889ecac95b#file-2-x86-base-txt
+// with slight massaging.
+constexpr StringLiteral TernlogFunctions[] = {
+    "0",
+    "~(a | b | c)",
+    "c & ~(a | b)",
+    "~(a | b)",
+    "b & ~(a | c)",
+    "~(a | c)",
+    "~a & (b ^ c)",
+    "~(a | (b & c))",
+    "b & c & ~a",
+    "~(a | (b ^ c))",
+    "c & ~a",
+    "~a & (c | ~b)",
+    "b & ~a",
+    "~a & (b | ~c)",
+    "~a & (b | c)",
+    "~a",
+    "a & ~(b | c)",
+    "~(b | c)",
+    "~b & (a ^ c)",
+    "~((a & c) | b)",
+    "~c & (a ^ b)",
+    "~((a & b) | c)",
+    "a ^ ((a & b) | (b ^ c))",
+    "(a & (b ^ c)) ^ ~(b & c)",
+    "(a ^ b) & (a ^ c)",
+    "~((a & b) | (b ^ c))",
+    "a ^ ((a & b) | c)",
+    "(a & c) ^ (c | ~b)",
+    "a ^ ((a & c) | b)",
+    "(a & b) ^ (b | ~c)",
+    "a ^ (b | c)",
+    "~(a & (b | c))",
+    "a & c & ~b",
+    "~(b | (a ^ c))",
+    "c & ~b",
+    "~b & (c | ~a)",
+    "(a ^ b) & (b ^ c)",
+    "~((a & b) | (a ^ c))",
+    "b ^ ((a & b) | c)",
+    "(b & c) ^ (c | ~a)",
+    "c & (a ^ b)",
+    "(a | b) ^ ((a & b) | ~c)",
+    "c & ~(a & b)",
+    "(c & (a ^ b)) | ~(a | b)",
+    "(b | c) & (a ^ b)",
+    "a ^ (b | ~c)",
+    "(a & b) ^ (b | c)",
+    "(c & ~b) | ~a",
+    "a & ~b",
+    "~b & (a | ~c)",
+    "~b & (a | c)",
+    "~b",
+    "b ^ (a | (b & c))",
+    "(a & b) ^ (a | ~c)",
+    "b ^ (a | c)",
+    "~(b & (a | c))",
+    "(a | c) & (a ^ b)",
+    "b ^ (a | ~c)",
+    "(a & b) ^ (a | c)",
+    "(c & ~a) | ~b",
+    "a ^ b",
+    "~(a | c) | (a ^ b)",
+    "(c & ~a) | (a ^ b)",
+    "~(a & b)",
+    "a & b & ~c",
+    "~(c | (a ^ b))",
+    "(a ^ c) & (b ^ c)",
+    "~((a & c) | (a ^ b))",
+    "b & ~c",
+    "~c & (b | ~a)",
+    "c ^ ((a & c) | b)",
+    "(b & c) ^ (b | ~a)",
+    "b & (a ^ c)",
+    "(a | c) ^ ((a & c) | ~b)",
+    "(b | c) & (a ^ c)",
+    "a ^ (c | ~b)",
+    "b & ~(a & c)",
+    "(b & (a ^ c)) | ~(a | c)",
+    "(a & c) ^ (b | c)",
+    "(b & ~c) | ~a",
+    "a & ~c",
+    "~c & (a | ~b)",
+    "c ^ (a | (b & c))",
+    "(a & c) ^ (a | ~b)",
+    "~c & (a | b)",
+    "~c",
+    "c ^ (a | b)",
+    "~(c & (a | b))",
+    "(a | b) & (a ^ c)",
+    "c ^ (a | ~b)",
+    "a ^ c",
+    "~(a | b) | (a ^ c)",
+    "(a & c) ^ (a | b)",
+    "(b & ~a) | ~c",
+    "(b & ~a) | (a ^ c)",
+    "~(a & c)",
+    "a & (b ^ c)",
+    "~(b ^ c) ^ (a | (b & c))",
+    "(a | c) & (b ^ c)",
+    "b ^ (c | ~a)",
+    "(a | b) & (b ^ c)",
+    "c ^ (b | ~a)",
+    "b ^ c",
+    "~(a | b) | (b ^ c)",
+    "(a | b) & (c ^ (a & b))",
+    "b ^ c ^ ~a",
+    "c ^ (a & b)",
+    "~(a | b) | (c ^ (a & b))",
+    "b ^ (a & c)",
+    "~(a | c) | (b ^ (a & c))",
+    "(b & ~a) | (b ^ c)",
+    "~a | (b ^ c)",
+    "a & ~(b & c)",
+    "(a & (b ^ c)) | ~(b | c)",
+    "(b & c) ^ (a | c)",
+    "(a & ~c) | ~b",
+    "(b & c) ^ (a | b)",
+    "(a & ~b) | ~c",
+    "(a & ~b) | (b ^ c)",
+    "~(b & c)",
+    "a ^ (b & c)",
+    "~(b | c) | (a ^ (b & c))",
+    "(a & ~b) | (a ^ c)",
+    "~b | (a ^ c)",
+    "(a & ~c) | (a ^ b)",
+    "~c | (a ^ b)",
+    "(a ^ b) | (a ^ c)",
+    "~(a & b & c)",
+    "a & b & c",
+    "~((a ^ b) | (a ^ c))",
+    "c & ~(a ^ b)",
+    "~(a ^ b) & (c | ~a)",
+    "b & ~(a ^ c)",
+    "~(a ^ c) & (b | ~a)",
+    "(b | c) & (a ^ b ^ c)",
+    "(b & c) ^ ~a",
+    "b & c",
+    "~(b ^ c) & (b | ~a)",
+    "c & (b | ~a)",
+    "~((b & c) ^ (a | b))",
+    "b & (c | ~a)",
+    "~((b & c) ^ (a | c))",
+    "a ^ ((a ^ b) | (a ^ c))",
+    "(b & c) | ~a",
+    "a & ~(b ^ c)",
+    "~(b ^ c) & (a | ~b)",
+    "(a | c) & (a ^ b ^ c)",
+    "(a & c) ^ ~b",
+    "(a | b) & (a ^ b ^ c)",
+    "(a & b) ^ ~c",
+    "a ^ b ^ c",
+    "~(a | b) | (a ^ b ^ c)",
+    "~(b ^ c) & (a | b)",
+    "~(b ^ c)",
+    "c ^ (a & ~b)",
+    "~((a | b) & (b ^ c))",
+    "b ^ (a & ~c)",
+    "~((a | c) & (b ^ c))",
+    "(b & c) | (a ^ (b | c))",
+    "~(a & (b ^ c))",
+    "a & c",
+    "~(a ^ c) & (a | ~b)",
+    "c & (a | ~b)",
+    "~((a & c) ^ (a | b))",
+    "~(a ^ c) & (a | b)",
+    "~(a ^ c)",
+    "c ^ (b & ~a)",
+    "~((a | b) & (a ^ c))",
+    "c & (a | b)",
+    "~c ^ (a | b)",
+    "c",
+    "c | ~(a | b)",
+    "b ^ (a & (b ^ c))",
+    "(b & c) | ~(a ^ c)",
+    "(b & ~a) | c",
+    "c | ~a",
+    "a & (c | ~b)",
+    "~((a & c) ^ (b | c))",
+    "a ^ ((a ^ c) & (b ^ c))",
+    "(a & c) | ~b",
+    "a ^ (b & ~c)",
+    "~((b | c) & (a ^ c))",
+    "(a & c) | (a ^ b ^ c)",
+    "~(b & (a ^ c))",
+    "a ^ (b & (a ^ c))",
+    "(a & c) | ~(b ^ c)",
+    "(a & ~b) | c",
+    "c | ~b",
+    "(a & c) | (a ^ b)",
+    "~((a ^ c) & (b ^ c))",
+    "c | (a ^ b)",
+    "c | ~(a & b)",
+    "a & b",
+    "~(a ^ b) & (a | ~c)",
+    "~(a ^ b) & (a | c)",
+    "~(a ^ b)",
+    "b & (a | ~c)",
+    "~((a & b) ^ (a | c))",
+    "b ^ (c & ~a)",
+    "~((a | c) & (a ^ b))",
+    "b & (a | c)",
+    "~b ^ (a | c)",
+    "c ^ (a & (b ^ c))",
+    "(b & c) | ~(a ^ b)",
+    "b",
+    "b | ~(a | c)",
+    "(c & ~a) | b",
+    "b | ~a",
+    "a & (b | ~c)",
+    "~((a & b) ^ (b | c))",
+    "a ^ (c & ~b)",
+    "~((b | c) & (a ^ b))",
+    "a ^ ((a ^ b) & (b ^ c))",
+    "(a & b) | ~c",
+    "(a & b) | (a ^ b ^ c)",
+    "~(c & (a ^ b))",
+    "a ^ (c & (a ^ b))",
+    "(a & b) | ~(b ^ c)",
+    "(a & b) | (a ^ c)",
+    "~((a ^ b) & (b ^ c))",
+    "(a & ~c) | b",
+    "b | ~c",
+    "b | (a ^ c)",
+    "b | ~(a & c)",
+    "a & (b | c)",
+    "~a ^ (b | c)",
+    "c ^ (b & (a ^ c))",
+    "(a & c) | ~(a ^ b)",
+    "b ^ (c & (a ^ b))",
+    "(a & b) | ~(a ^ c)",
+    "(a & b) | (b ^ c)",
+    "~((a ^ b) & (a ^ c))",
+    "(a | b) & ((a & b) | c)",
+    "(a & b) | (b ^ c ^ ~a)",
+    "(a & b) | c",
+    "c | ~(a ^ b)",
+    "(a & c) | b",
+    "b | ~(a ^ c)",
+    "b | c",
+    "~a | b | c",
+    "a",
+    "a | ~(b | c)",
+    "a | (c & ~b)",
+    "a | ~b",
+    "a | (b & ~c)",
+    "a | ~c",
+    "a | (b ^ c)",
+    "a | ~(b & c)",
+    "a | (b & c)",
+    "a | ~(b ^ c)",
+    "a | c",
+    "~b | a | c",
+    "a | b",
+    "~c | a | b",
+    "a | b | c",
+    "-1",
+};
+
 static bool printPTERNLOGComments(const MCInst *MI, raw_ostream &OS,
                                   const MCInstrInfo &MCII) {
   unsigned NumOperands = MI->getNumOperands();
@@ -650,58 +915,35 @@ static bool printPTERNLOGComments(const MCInst *MI, raw_ostream &OS,
   default:
     return false;
   }
-  const char *DestName = getRegName(MI->getOperand(0).getReg());
-  const char *Src1Name = getRegName(MI->getOperand(1).getReg());
-  const char *Src2Name = getRegName(MI->getOperand(Src2Idx).getReg());
-  const char *Src3Name =
+  StringRef DestName = getRegName(MI->getOperand(0).getReg());
+  StringRef Src1Name = getRegName(MI->getOperand(1).getReg());
+  StringRef Src2Name = getRegName(MI->getOperand(Src2Idx).getReg());
+  StringRef Src3Name =
       Src3Idx != -1 ? getRegName(MI->getOperand(Src3Idx).getReg()) : "mem";
   uint8_t TruthTable = MI->getOperand(NumOperands - 1).getImm();
 
+  StringRef SrcNames[] = {Src1Name, Src2Name, Src3Name};
+
   OS << DestName;
   printMasking(OS, MI, MCII);
   OS << " = ";
 
-  constexpr unsigned kNumVariables = 3;
-  constexpr unsigned kNumTruthTableEntries = 1 << kNumVariables;
-  int NumMinterms = llvm::popcount(TruthTable);
-  if (NumMinterms == 0) {
-    OS << '0';
-  } else if (NumMinterms == kNumTruthTableEntries) {
-    OS << "-1";
-  } else {
-    while (TruthTable != 0) {
-      // Index of the lowest bit set.
-      unsigned I = llvm::countr_zero(TruthTable);
-      // Clear the lowest bit set.
-      TruthTable &= TruthTable - 1;
-      // Our index tells us which sources are and are not complemented. Note
-      // that the indexing goes left-to-right.
-      bool Src1 = I & 0b100;
-      bool Src2 = I & 0b010;
-      bool Src3 = I & 0b001;
-
-      // Group in parenthesis to make the output more obvious but only if there
-      // are multiple terms.
-      if (NumMinterms > 1)
-        OS << '(';
-
-      if (!Src1)
-        OS << '~';
-      OS << Src1Name << " & ";
-      if (!Src2)
-        OS << '~';
-      OS << Src2Name << " & ";
-      if (!Src3)
-        OS << '~';
-      OS << Src3Name;
-
-      if (NumMinterms > 1)
-        OS << ')';
-
-      // Output an OR if there is another term in the table.
-      if (TruthTable != 0)
-        OS << " | ";
+  static_assert(std::size(TernlogFunctions) == 256);
+  std::string_view BooleanFunction = TernlogFunctions[TruthTable];
+
+  while (!BooleanFunction.empty()) {
+    // Print the expression up to the next symbol.
+    size_t SymbolOffset = BooleanFunction.find_first_of("abc");
+    OS << BooleanFunction.substr(0, SymbolOffset);
+    if (SymbolOffset == std::string_view::npos) {
+      // No more symbols, that means we just printed everything.
+      break;
     }
+    // Let's replace {a,b,c} with Src{1,2,3}Name.
+    char Symbol = BooleanFunction[SymbolOffset];
+    OS << SrcNames[Symbol - 'a'];
+    // Consume the part of the expression we handled.
+    BooleanFunction.remove_prefix(SymbolOffset + 1);
   }
   OS << '\n';
   return true;
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
index 3ce044387ada2..47eb617c06ac5 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
@@ -43,8 +43,6 @@ X86MCAsmInfoDarwin::X86MCAsmInfoDarwin(const Triple &T) {
 
   AssemblerDialect = AsmWriterFlavor;
 
-  TextAlignFillValue = 0x90;
-
   if (!is64Bit)
     Data64bitsDirective = nullptr;       // we can't emit a 64-bit unit
 
@@ -93,8 +91,6 @@ X86ELFMCAsmInfo::X86ELFMCAsmInfo(const Triple &T) {
 
   AssemblerDialect = AsmWriterFlavor;
 
-  TextAlignFillValue = 0x90;
-
   // Debug Information
   SupportsDebugInformation = true;
 
@@ -132,8 +128,6 @@ X86MCAsmInfoMicrosoft::X86MCAsmInfoMicrosoft(const Triple &Triple) {
 
   AssemblerDialect = AsmWriterFlavor;
 
-  TextAlignFillValue = 0x90;
-
   AllowAtInName = true;
 }
 
@@ -167,7 +161,5 @@ X86MCAsmInfoGNUCOFF::X86MCAsmInfoGNUCOFF(const Triple &Triple) {
 
   AssemblerDialect = AsmWriterFlavor;
 
-  TextAlignFillValue = 0x90;
-
   AllowAtInName = true;
 }
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 73f7f52846f62..4389b65210436 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -48499,7 +48499,8 @@ static SDValue combineMul(SDNode *N, SelectionDAG &DAG,
     if (VT.isVector())
       if (auto *RawC = getTargetConstantFromNode(N->getOperand(1)))
         if (auto *SplatC = RawC->getSplatValue())
-          C = &(SplatC->getUniqueInteger());
+          if (auto *SplatCI = dyn_cast<ConstantInt>(SplatC))
+            C = &(SplatCI->getValue());
 
     if (!C || C->getBitWidth() != VT.getScalarSizeInBits())
       return SDValue();
diff --git a/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp b/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp
index 670930e99334f..5450222a7b2e1 100644
--- a/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp
+++ b/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp
@@ -13,6 +13,7 @@
 
 #include "XtensaISelLowering.h"
 #include "XtensaConstantPoolValue.h"
+#include "XtensaInstrInfo.h"
 #include "XtensaSubtarget.h"
 #include "XtensaTargetMachine.h"
 #include "llvm/CodeGen/CallingConvLower.h"
@@ -1104,10 +1105,26 @@ XtensaTargetLowering::emitSelectCC(MachineInstr &MI,
 MachineBasicBlock *XtensaTargetLowering::EmitInstrWithCustomInserter(
     MachineInstr &MI, MachineBasicBlock *MBB) const {
   DebugLoc DL = MI.getDebugLoc();
+  const XtensaInstrInfo &TII = *Subtarget.getInstrInfo();
 
   switch (MI.getOpcode()) {
   case Xtensa::SELECT:
     return emitSelectCC(MI, MBB);
+  case Xtensa::S8I:
+  case Xtensa::S16I:
+  case Xtensa::S32I:
+  case Xtensa::L8UI:
+  case Xtensa::L16SI:
+  case Xtensa::L16UI:
+  case Xtensa::L32I: {
+    // Insert memory wait instruction "memw" before volatile load/store as it is
+    // implemented in gcc. If memoperands is empty then assume that it aslo
+    // maybe volatile load/store and insert "memw".
+    if (MI.memoperands_empty() || (*MI.memoperands_begin())->isVolatile()) {
+      BuildMI(*MBB, MI, DL, TII.get(Xtensa::MEMW));
+    }
+    return MBB;
+  }
   default:
     llvm_unreachable("Unexpected instr type to insert");
   }
diff --git a/llvm/lib/Target/Xtensa/XtensaInstrInfo.td b/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
index 0d01864b54bc3..9773480624e92 100644
--- a/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
+++ b/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
@@ -195,7 +195,7 @@ def SSAI : RRR_Inst<0x00, 0x00, 0x04, (outs), (ins uimm5:$imm),
 //===----------------------------------------------------------------------===//
 
 // Load instructions
-let mayLoad = 1 in {
+let mayLoad = 1, usesCustomInserter = 1 in {
 
   class Load_RRI8<bits<4> oper, string instrAsm, SDPatternOperator opNode,
         ComplexPattern addrOp, Operand memOp>
@@ -216,7 +216,7 @@ def L16UI : Load_RRI8<0x01, "l16ui", zextloadi16, addr_ish2, mem16>;
 def L32I  : Load_RRI8<0x02, "l32i", load, addr_ish4, mem32>;
 
 // Store instructions
-let mayStore = 1 in {
+let mayStore = 1, usesCustomInserter = 1 in {
   class Store_II8<bits<4> oper, string instrAsm, SDPatternOperator opNode,
         ComplexPattern addrOp, Operand memOp>
 	  : RRI8_Inst<0x02, (outs), (ins AR:$t, memOp:$addr),
diff --git a/llvm/lib/TargetParser/ARMTargetParser.cpp b/llvm/lib/TargetParser/ARMTargetParser.cpp
index 9d9917d86a368..f073bf66f20ba 100644
--- a/llvm/lib/TargetParser/ARMTargetParser.cpp
+++ b/llvm/lib/TargetParser/ARMTargetParser.cpp
@@ -87,6 +87,7 @@ unsigned ARM::parseArchVersion(StringRef Arch) {
   case ArchKind::ARMV9_3A:
   case ArchKind::ARMV9_4A:
   case ArchKind::ARMV9_5A:
+  case ArchKind::ARMV9_6A:
     return 9;
   case ArchKind::INVALID:
     return 0;
@@ -125,6 +126,7 @@ static ARM::ProfileKind getProfileKind(ARM::ArchKind AK) {
   case ARM::ArchKind::ARMV9_3A:
   case ARM::ArchKind::ARMV9_4A:
   case ARM::ArchKind::ARMV9_5A:
+  case ARM::ArchKind::ARMV9_6A:
     return ARM::ProfileKind::A;
   case ARM::ArchKind::ARMV4:
   case ARM::ArchKind::ARMV4T:
diff --git a/llvm/lib/TargetParser/ARMTargetParserCommon.cpp b/llvm/lib/TargetParser/ARMTargetParserCommon.cpp
index d6ce6581bb1a9..e2ed8df78f95e 100644
--- a/llvm/lib/TargetParser/ARMTargetParserCommon.cpp
+++ b/llvm/lib/TargetParser/ARMTargetParserCommon.cpp
@@ -45,6 +45,7 @@ StringRef ARM::getArchSynonym(StringRef Arch) {
       .Case("v9.3a", "v9.3-a")
       .Case("v9.4a", "v9.4-a")
       .Case("v9.5a", "v9.5-a")
+      .Case("v9.6a", "v9.6-a")
       .Case("v8m.base", "v8-m.base")
       .Case("v8m.main", "v8-m.main")
       .Case("v8.1m.main", "v8.1-m.main")
diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp
index 616e4eda1dd29..14567af8590e6 100644
--- a/llvm/lib/TargetParser/Host.cpp
+++ b/llvm/lib/TargetParser/Host.cpp
@@ -1898,7 +1898,8 @@ const StringMap<bool> sys::getHostCPUFeatures() {
     }
 
 #if defined(__aarch64__)
-  // Keep track of which crypto features we have seen
+  // All of these are "crypto" features, but we must sift out actual features
+  // as the former meaning of "crypto" as a single feature is no more.
   enum { CAP_AES = 0x1, CAP_PMULL = 0x2, CAP_SHA1 = 0x4, CAP_SHA2 = 0x8 };
   uint32_t crypto = 0;
 #endif
@@ -1941,9 +1942,13 @@ const StringMap<bool> sys::getHostCPUFeatures() {
   }
 
 #if defined(__aarch64__)
-  // If we have all crypto bits we can add the feature
-  if (crypto == (CAP_AES | CAP_PMULL | CAP_SHA1 | CAP_SHA2))
-    Features["crypto"] = true;
+  // LLVM has decided some AArch64 CPUs have all the instructions they _may_
+  // have, as opposed to all the instructions they _must_ have, so allow runtime
+  // information to correct us on that.
+  uint32_t Aes = CAP_AES | CAP_PMULL;
+  uint32_t Sha2 = CAP_SHA1 | CAP_SHA2;
+  Features["aes"] = (crypto & Aes) == Aes;
+  Features["sha2"] = (crypto & Sha2) == Sha2;
 #endif
 
   return Features;
@@ -1952,12 +1957,17 @@ const StringMap<bool> sys::getHostCPUFeatures() {
 const StringMap<bool> sys::getHostCPUFeatures() {
   StringMap<bool> Features;
 
-  if (IsProcessorFeaturePresent(PF_ARM_NEON_INSTRUCTIONS_AVAILABLE))
-    Features["neon"] = true;
-  if (IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE))
-    Features["crc"] = true;
-  if (IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE))
-    Features["crypto"] = true;
+  // If we're asking the OS at runtime, believe what the OS says
+  Features["neon"] =
+      IsProcessorFeaturePresent(PF_ARM_NEON_INSTRUCTIONS_AVAILABLE);
+  Features["crc"] =
+      IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE);
+
+  // Avoid inferring "crypto" means more than the traditional AES + SHA2
+  bool TradCrypto =
+      IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE);
+  Features["aes"] = TradCrypto;
+  Features["sha2"] = TradCrypto;
 
   return Features;
 }
diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp
index 7fa3d8edec84d..caa5a97747ee5 100644
--- a/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -1049,7 +1049,7 @@ constexpr static RISCVExtBit RISCVBitPositions[] = {
     {"zvksed", 0, 57},    {"zvksh", 0, 58},
     {"zvkt", 0, 59},      {"zve32x", 0, 60},
     {"zve32f", 0, 61},    {"zve64x", 0, 62},
-    {"zve64x", 0, 63},    {"zve64d", 1, 0},
+    {"zve64f", 0, 63},    {"zve64d", 1, 0},
     {"zimop", 1, 1},      {"zca", 1, 2},
     {"zcb", 1, 3},        {"zcd", 1, 4},
     {"zcf", 1, 5},        {"zcmop", 1, 6},
diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp
index 26ab1be4eb5a5..229cf782e52ed 100644
--- a/llvm/lib/TargetParser/Triple.cpp
+++ b/llvm/lib/TargetParser/Triple.cpp
@@ -867,6 +867,8 @@ static Triple::SubArchType parseSubArch(StringRef SubArchName) {
     return Triple::ARMSubArch_v9_4a;
   case ARM::ArchKind::ARMV9_5A:
     return Triple::ARMSubArch_v9_5a;
+  case ARM::ArchKind::ARMV9_6A:
+    return Triple::ARMSubArch_v9_6a;
   case ARM::ArchKind::ARMV8R:
     return Triple::ARMSubArch_v8r;
   case ARM::ArchKind::ARMV8MBaseline:
diff --git a/llvm/lib/Transforms/Coroutines/ABI.h b/llvm/lib/Transforms/Coroutines/ABI.h
new file mode 100644
index 0000000000000..c94bf7d356b65
--- /dev/null
+++ b/llvm/lib/Transforms/Coroutines/ABI.h
@@ -0,0 +1,102 @@
+//===- ABI.h - Coroutine lowering class definitions (ABIs) ----*- C++ -*---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// This file defines coroutine lowering classes. The interface for coroutine
+// lowering is defined by BaseABI. Each lowering method (ABI) implements the
+// interface. Note that the enum class ABI, such as ABI::Switch, determines
+// which ABI class, such as SwitchABI, is used to lower the coroutine. Both the
+// ABI enum and ABI class are used by the Coroutine passes when lowering.
+//===----------------------------------------------------------------------===//
+
+#ifndef LIB_TRANSFORMS_COROUTINES_ABI_H
+#define LIB_TRANSFORMS_COROUTINES_ABI_H
+
+#include "CoroShape.h"
+#include "SuspendCrossingInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+
+namespace llvm {
+
+class Function;
+
+namespace coro {
+
+// This interface/API is to provide an object oriented way to implement ABI
+// functionality. This is intended to replace use of the ABI enum to perform
+// ABI operations. The ABIs (e.g. Switch, Async, Retcon{Once}) are the common
+// ABIs.
+
+class LLVM_LIBRARY_VISIBILITY BaseABI {
+public:
+  BaseABI(Function &F, coro::Shape &S,
+          std::function<bool(Instruction &)> IsMaterializable)
+      : F(F), Shape(S), IsMaterializable(IsMaterializable) {}
+  virtual ~BaseABI() = default;
+
+  // Initialize the coroutine ABI
+  virtual void init() = 0;
+
+  // Allocate the coroutine frame and do spill/reload as needed.
+  virtual void buildCoroutineFrame();
+
+  // Perform the function splitting according to the ABI.
+  virtual void splitCoroutine(Function &F, coro::Shape &Shape,
+                              SmallVectorImpl<Function *> &Clones,
+                              TargetTransformInfo &TTI) = 0;
+
+  Function &F;
+  coro::Shape &Shape;
+
+  // Callback used by coro::BaseABI::buildCoroutineFrame for rematerialization.
+  // It is provided to coro::doMaterializations(..).
+  std::function<bool(Instruction &I)> IsMaterializable;
+};
+
+class LLVM_LIBRARY_VISIBILITY SwitchABI : public BaseABI {
+public:
+  SwitchABI(Function &F, coro::Shape &S,
+            std::function<bool(Instruction &)> IsMaterializable)
+      : BaseABI(F, S, IsMaterializable) {}
+
+  void init() override;
+
+  void splitCoroutine(Function &F, coro::Shape &Shape,
+                      SmallVectorImpl<Function *> &Clones,
+                      TargetTransformInfo &TTI) override;
+};
+
+class LLVM_LIBRARY_VISIBILITY AsyncABI : public BaseABI {
+public:
+  AsyncABI(Function &F, coro::Shape &S,
+           std::function<bool(Instruction &)> IsMaterializable)
+      : BaseABI(F, S, IsMaterializable) {}
+
+  void init() override;
+
+  void splitCoroutine(Function &F, coro::Shape &Shape,
+                      SmallVectorImpl<Function *> &Clones,
+                      TargetTransformInfo &TTI) override;
+};
+
+class LLVM_LIBRARY_VISIBILITY AnyRetconABI : public BaseABI {
+public:
+  AnyRetconABI(Function &F, coro::Shape &S,
+               std::function<bool(Instruction &)> IsMaterializable)
+      : BaseABI(F, S, IsMaterializable) {}
+
+  void init() override;
+
+  void splitCoroutine(Function &F, coro::Shape &Shape,
+                      SmallVectorImpl<Function *> &Clones,
+                      TargetTransformInfo &TTI) override;
+};
+
+} // end namespace coro
+
+} // end namespace llvm
+
+#endif // LLVM_TRANSFORMS_COROUTINES_ABI_H
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index c08f56b024dfc..021b1f7a4156b 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -15,6 +15,7 @@
 // the value into the coroutine frame.
 //===----------------------------------------------------------------------===//
 
+#include "ABI.h"
 #include "CoroInternal.h"
 #include "MaterializationUtils.h"
 #include "SpillUtils.h"
@@ -2055,11 +2056,9 @@ void coro::normalizeCoroutine(Function &F, coro::Shape &Shape,
   rewritePHIs(F);
 }
 
-void coro::buildCoroutineFrame(
-    Function &F, Shape &Shape,
-    const std::function<bool(Instruction &)> &MaterializableCallback) {
+void coro::BaseABI::buildCoroutineFrame() {
   SuspendCrossingInfo Checker(F, Shape.CoroSuspends, Shape.CoroEnds);
-  doRematerializations(F, Checker, MaterializableCallback);
+  doRematerializations(F, Checker, IsMaterializable);
 
   const DominatorTree DT(F);
   if (Shape.ABI != coro::ABI::Async && Shape.ABI != coro::ABI::Retcon &&
diff --git a/llvm/lib/Transforms/Coroutines/CoroInternal.h b/llvm/lib/Transforms/Coroutines/CoroInternal.h
index fcbd31878bdea..88d0f83c98c9e 100644
--- a/llvm/lib/Transforms/Coroutines/CoroInternal.h
+++ b/llvm/lib/Transforms/Coroutines/CoroInternal.h
@@ -62,9 +62,6 @@ struct LowererBase {
 bool defaultMaterializable(Instruction &V);
 void normalizeCoroutine(Function &F, coro::Shape &Shape,
                         TargetTransformInfo &TTI);
-void buildCoroutineFrame(
-    Function &F, Shape &Shape,
-    const std::function<bool(Instruction &)> &MaterializableCallback);
 CallInst *createMustTailCall(DebugLoc Loc, Function *MustTailCallFn,
                              TargetTransformInfo &TTI,
                              ArrayRef<Value *> Arguments, IRBuilder<> &);
diff --git a/llvm/lib/Transforms/Coroutines/CoroShape.h b/llvm/lib/Transforms/Coroutines/CoroShape.h
index dcfe94ca076bd..f4fb4baa6df31 100644
--- a/llvm/lib/Transforms/Coroutines/CoroShape.h
+++ b/llvm/lib/Transforms/Coroutines/CoroShape.h
@@ -275,7 +275,6 @@ struct LLVM_LIBRARY_VISIBILITY Shape {
       invalidateCoroutine(F, CoroFrames);
       return;
     }
-    initABI();
     cleanCoroutine(CoroFrames, UnusedCoroSaves);
   }
 };
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index 382bdfff1926f..4fbda077129fa 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -19,8 +19,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Coroutines/CoroSplit.h"
+#include "ABI.h"
 #include "CoroInstr.h"
 #include "CoroInternal.h"
+#include "MaterializationUtils.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/PriorityWorklist.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -1779,9 +1781,9 @@ CallInst *coro::createMustTailCall(DebugLoc Loc, Function *MustTailCallFn,
   return TailCall;
 }
 
-static void splitAsyncCoroutine(Function &F, coro::Shape &Shape,
-                                SmallVectorImpl<Function *> &Clones,
-                                TargetTransformInfo &TTI) {
+void coro::AsyncABI::splitCoroutine(Function &F, coro::Shape &Shape,
+                                    SmallVectorImpl<Function *> &Clones,
+                                    TargetTransformInfo &TTI) {
   assert(Shape.ABI == coro::ABI::Async);
   assert(Clones.empty());
   // Reset various things that the optimizer might have decided it
@@ -1874,9 +1876,9 @@ static void splitAsyncCoroutine(Function &F, coro::Shape &Shape,
   }
 }
 
-static void splitRetconCoroutine(Function &F, coro::Shape &Shape,
-                                 SmallVectorImpl<Function *> &Clones,
-                                 TargetTransformInfo &TTI) {
+void coro::AnyRetconABI::splitCoroutine(Function &F, coro::Shape &Shape,
+                                        SmallVectorImpl<Function *> &Clones,
+                                        TargetTransformInfo &TTI) {
   assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce);
   assert(Clones.empty());
 
@@ -2044,26 +2046,27 @@ static bool hasSafeElideCaller(Function &F) {
   return false;
 }
 
-static coro::Shape
-splitCoroutine(Function &F, SmallVectorImpl<Function *> &Clones,
-               TargetTransformInfo &TTI, bool OptimizeFrame,
-               std::function<bool(Instruction &)> MaterializableCallback) {
-  PrettyStackTraceFunction prettyStackTrace(F);
+void coro::SwitchABI::splitCoroutine(Function &F, coro::Shape &Shape,
+                                     SmallVectorImpl<Function *> &Clones,
+                                     TargetTransformInfo &TTI) {
+  SwitchCoroutineSplitter::split(F, Shape, Clones, TTI);
+}
 
-  // The suspend-crossing algorithm in buildCoroutineFrame get tripped
-  // up by uses in unreachable blocks, so remove them as a first pass.
-  removeUnreachableBlocks(F);
+static void doSplitCoroutine(Function &F, SmallVectorImpl<Function *> &Clones,
+                             coro::BaseABI &ABI, TargetTransformInfo &TTI) {
+  PrettyStackTraceFunction prettyStackTrace(F);
 
-  coro::Shape Shape(F, OptimizeFrame);
-  if (!Shape.CoroBegin)
-    return Shape;
+  auto &Shape = ABI.Shape;
+  assert(Shape.CoroBegin);
 
   lowerAwaitSuspends(F, Shape);
 
   simplifySuspendPoints(Shape);
+
   normalizeCoroutine(F, Shape, TTI);
-  buildCoroutineFrame(F, Shape, MaterializableCallback);
+  ABI.buildCoroutineFrame();
   replaceFrameSizeAndAlignment(Shape);
+
   bool isNoSuspendCoroutine = Shape.CoroSuspends.empty();
 
   bool shouldCreateNoAllocVariant = !isNoSuspendCoroutine &&
@@ -2075,18 +2078,7 @@ splitCoroutine(Function &F, SmallVectorImpl<Function *> &Clones,
   if (isNoSuspendCoroutine) {
     handleNoSuspendCoroutine(Shape);
   } else {
-    switch (Shape.ABI) {
-    case coro::ABI::Switch:
-      SwitchCoroutineSplitter::split(F, Shape, Clones, TTI);
-      break;
-    case coro::ABI::Async:
-      splitAsyncCoroutine(F, Shape, Clones, TTI);
-      break;
-    case coro::ABI::Retcon:
-    case coro::ABI::RetconOnce:
-      splitRetconCoroutine(F, Shape, Clones, TTI);
-      break;
-    }
+    ABI.splitCoroutine(F, Shape, Clones, TTI);
   }
 
   // Replace all the swifterror operations in the original function.
@@ -2107,8 +2099,6 @@ splitCoroutine(Function &F, SmallVectorImpl<Function *> &Clones,
 
   if (shouldCreateNoAllocVariant)
     SwitchCoroutineSplitter::createNoAllocVariant(F, Shape, Clones);
-
-  return Shape;
 }
 
 static LazyCallGraph::SCC &updateCallGraphAfterCoroutineSplit(
@@ -2207,8 +2197,44 @@ static void addPrepareFunction(const Module &M,
     Fns.push_back(PrepareFn);
 }
 
+static std::unique_ptr<coro::BaseABI>
+CreateNewABI(Function &F, coro::Shape &S,
+             std::function<bool(Instruction &)> IsMatCallback) {
+  switch (S.ABI) {
+  case coro::ABI::Switch:
+    return std::unique_ptr<coro::BaseABI>(
+        new coro::SwitchABI(F, S, IsMatCallback));
+  case coro::ABI::Async:
+    return std::unique_ptr<coro::BaseABI>(
+        new coro::AsyncABI(F, S, IsMatCallback));
+  case coro::ABI::Retcon:
+    return std::unique_ptr<coro::BaseABI>(
+        new coro::AnyRetconABI(F, S, IsMatCallback));
+  case coro::ABI::RetconOnce:
+    return std::unique_ptr<coro::BaseABI>(
+        new coro::AnyRetconABI(F, S, IsMatCallback));
+  }
+  llvm_unreachable("Unknown ABI");
+}
+
 CoroSplitPass::CoroSplitPass(bool OptimizeFrame)
-    : MaterializableCallback(coro::defaultMaterializable),
+    : CreateAndInitABI([](Function &F, coro::Shape &S) {
+        std::unique_ptr<coro::BaseABI> ABI =
+            CreateNewABI(F, S, coro::isTriviallyMaterializable);
+        ABI->init();
+        return ABI;
+      }),
+      OptimizeFrame(OptimizeFrame) {}
+
+// For back compatibility, constructor takes a materializable callback and
+// creates a generator for an ABI with a modified materializable callback.
+CoroSplitPass::CoroSplitPass(std::function<bool(Instruction &)> IsMatCallback,
+                             bool OptimizeFrame)
+    : CreateAndInitABI([=](Function &F, coro::Shape &S) {
+        std::unique_ptr<coro::BaseABI> ABI = CreateNewABI(F, S, IsMatCallback);
+        ABI->init();
+        return ABI;
+      }),
       OptimizeFrame(OptimizeFrame) {}
 
 PreservedAnalyses CoroSplitPass::run(LazyCallGraph::SCC &C,
@@ -2241,12 +2267,23 @@ PreservedAnalyses CoroSplitPass::run(LazyCallGraph::SCC &C,
     Function &F = N->getFunction();
     LLVM_DEBUG(dbgs() << "CoroSplit: Processing coroutine '" << F.getName()
                       << "\n");
+
+    // The suspend-crossing algorithm in buildCoroutineFrame gets tripped up
+    // by unreachable blocks, so remove them as a first pass. Remove the
+    // unreachable blocks before collecting intrinsics into Shape.
+    removeUnreachableBlocks(F);
+
+    coro::Shape Shape(F, OptimizeFrame);
+    if (!Shape.CoroBegin)
+      continue;
+
     F.setSplittedCoroutine();
 
+    std::unique_ptr<coro::BaseABI> ABI = CreateAndInitABI(F, Shape);
+
     SmallVector<Function *, 4> Clones;
-    coro::Shape Shape =
-        splitCoroutine(F, Clones, FAM.getResult<TargetIRAnalysis>(F),
-                       OptimizeFrame, MaterializableCallback);
+    auto &TTI = FAM.getResult<TargetIRAnalysis>(F);
+    doSplitCoroutine(F, Clones, *ABI, TTI);
     CurrentSCC = &updateCallGraphAfterCoroutineSplit(
         *N, Shape, Clones, *CurrentSCC, CG, AM, UR, FAM);
 
diff --git a/llvm/lib/Transforms/Coroutines/Coroutines.cpp b/llvm/lib/Transforms/Coroutines/Coroutines.cpp
index 453736912a8c5..184efbfe903d2 100644
--- a/llvm/lib/Transforms/Coroutines/Coroutines.cpp
+++ b/llvm/lib/Transforms/Coroutines/Coroutines.cpp
@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "ABI.h"
 #include "CoroInstr.h"
 #include "CoroInternal.h"
 #include "CoroShape.h"
@@ -373,11 +374,10 @@ void coro::Shape::invalidateCoroutine(
   }
 }
 
-// Perform semantic checking and initialization of the ABI
-void coro::Shape::initABI() {
-  switch (ABI) {
-  case coro::ABI::Switch: {
-    for (auto *AnySuspend : CoroSuspends) {
+void coro::SwitchABI::init() {
+  assert(Shape.ABI == coro::ABI::Switch);
+  {
+    for (auto *AnySuspend : Shape.CoroSuspends) {
       auto Suspend = dyn_cast<CoroSuspendInst>(AnySuspend);
       if (!Suspend) {
 #ifndef NDEBUG
@@ -387,21 +387,22 @@ void coro::Shape::initABI() {
       }
 
       if (!Suspend->getCoroSave())
-        createCoroSave(CoroBegin, Suspend);
+        createCoroSave(Shape.CoroBegin, Suspend);
     }
-    break;
   }
-  case coro::ABI::Async: {
-    break;
-  };
-  case coro::ABI::Retcon:
-  case coro::ABI::RetconOnce: {
+}
+
+void coro::AsyncABI::init() { assert(Shape.ABI == coro::ABI::Async); }
+
+void coro::AnyRetconABI::init() {
+  assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce);
+  {
     // Determine the result value types, and make sure they match up with
     // the values passed to the suspends.
-    auto ResultTys = getRetconResultTypes();
-    auto ResumeTys = getRetconResumeTypes();
+    auto ResultTys = Shape.getRetconResultTypes();
+    auto ResumeTys = Shape.getRetconResumeTypes();
 
-    for (auto *AnySuspend : CoroSuspends) {
+    for (auto *AnySuspend : Shape.CoroSuspends) {
       auto Suspend = dyn_cast<CoroSuspendRetconInst>(AnySuspend);
       if (!Suspend) {
 #ifndef NDEBUG
@@ -428,7 +429,7 @@ void coro::Shape::initABI() {
 
 #ifndef NDEBUG
           Suspend->dump();
-          RetconLowering.ResumePrototype->getFunctionType()->dump();
+          Shape.RetconLowering.ResumePrototype->getFunctionType()->dump();
 #endif
           report_fatal_error("argument to coro.suspend.retcon does not "
                              "match corresponding prototype function result");
@@ -437,7 +438,7 @@ void coro::Shape::initABI() {
       if (SI != SE || RI != RE) {
 #ifndef NDEBUG
         Suspend->dump();
-        RetconLowering.ResumePrototype->getFunctionType()->dump();
+        Shape.RetconLowering.ResumePrototype->getFunctionType()->dump();
 #endif
         report_fatal_error("wrong number of arguments to coro.suspend.retcon");
       }
@@ -456,7 +457,7 @@ void coro::Shape::initABI() {
       if (SuspendResultTys.size() != ResumeTys.size()) {
 #ifndef NDEBUG
         Suspend->dump();
-        RetconLowering.ResumePrototype->getFunctionType()->dump();
+        Shape.RetconLowering.ResumePrototype->getFunctionType()->dump();
 #endif
         report_fatal_error("wrong number of results from coro.suspend.retcon");
       }
@@ -464,15 +465,13 @@ void coro::Shape::initABI() {
         if (SuspendResultTys[I] != ResumeTys[I]) {
 #ifndef NDEBUG
           Suspend->dump();
-          RetconLowering.ResumePrototype->getFunctionType()->dump();
+          Shape.RetconLowering.ResumePrototype->getFunctionType()->dump();
 #endif
           report_fatal_error("result from coro.suspend.retcon does not "
                              "match corresponding prototype function param");
         }
       }
     }
-    break;
-  }
   }
 }
 
diff --git a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
index 90e8c39e5a90d..afc13232ff195 100644
--- a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
+++ b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
@@ -491,10 +491,7 @@ static bool isArgUnmodifiedByAllCalls(Argument *Arg,
                                       FunctionAnalysisManager &FAM) {
   for (User *U : Arg->getParent()->users()) {
 
-    // Bail if we find an unexpected (non CallInst) use of the function.
-    auto *Call = dyn_cast<CallInst>(U);
-    if (!Call)
-      return false;
+    auto *Call = cast<CallBase>(U);
 
     MemoryLocation Loc =
         MemoryLocation::getForArgument(Call, Arg->getArgNo(), nullptr);
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index 238bdf9c344b0..45e2a0d0b9336 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -827,7 +827,7 @@ struct AA::PointerInfo::State : public AbstractState {
     AccessList = R.AccessList;
     OffsetBins = R.OffsetBins;
     RemoteIMap = R.RemoteIMap;
-    ReachesReturn = R.ReachesReturn;
+    ReturnedOffsets = R.ReturnedOffsets;
     return *this;
   }
 
@@ -838,7 +838,7 @@ struct AA::PointerInfo::State : public AbstractState {
     std::swap(AccessList, R.AccessList);
     std::swap(OffsetBins, R.OffsetBins);
     std::swap(RemoteIMap, R.RemoteIMap);
-    std::swap(ReachesReturn, R.ReachesReturn);
+    std::swap(ReturnedOffsets, R.ReturnedOffsets);
     return *this;
   }
 
@@ -883,13 +883,13 @@ struct AA::PointerInfo::State : public AbstractState {
   /// Flag to determine if the underlying pointer is reaching a return statement
   /// in the associated function or not. Returns in other functions cause
   /// invalidation.
-  bool ReachesReturn = false;
+  AAPointerInfo::OffsetInfo ReturnedOffsets;
 
   /// See AAPointerInfo::forallInterferingAccesses.
   bool forallInterferingAccesses(
       AA::RangeTy Range,
       function_ref<bool(const AAPointerInfo::Access &, bool)> CB) const {
-    if (!isValidState() || ReachesReturn)
+    if (!isValidState() || !ReturnedOffsets.isUnassigned())
       return false;
 
     for (const auto &It : OffsetBins) {
@@ -911,7 +911,7 @@ struct AA::PointerInfo::State : public AbstractState {
       Instruction &I,
       function_ref<bool(const AAPointerInfo::Access &, bool)> CB,
       AA::RangeTy &Range) const {
-    if (!isValidState() || ReachesReturn)
+    if (!isValidState() || !ReturnedOffsets.isUnassigned())
       return false;
 
     auto LocalList = RemoteIMap.find(&I);
@@ -1010,54 +1010,9 @@ ChangeStatus AA::PointerInfo::State::addAccess(
 
 namespace {
 
-/// A helper containing a list of offsets computed for a Use. Ideally this
-/// list should be strictly ascending, but we ensure that only when we
-/// actually translate the list of offsets to a RangeList.
-struct OffsetInfo {
-  using VecTy = SmallVector<int64_t>;
-  using const_iterator = VecTy::const_iterator;
-  VecTy Offsets;
-
-  const_iterator begin() const { return Offsets.begin(); }
-  const_iterator end() const { return Offsets.end(); }
-
-  bool operator==(const OffsetInfo &RHS) const {
-    return Offsets == RHS.Offsets;
-  }
-
-  bool operator!=(const OffsetInfo &RHS) const { return !(*this == RHS); }
-
-  void insert(int64_t Offset) { Offsets.push_back(Offset); }
-  bool isUnassigned() const { return Offsets.size() == 0; }
-
-  bool isUnknown() const {
-    if (isUnassigned())
-      return false;
-    if (Offsets.size() == 1)
-      return Offsets.front() == AA::RangeTy::Unknown;
-    return false;
-  }
-
-  void setUnknown() {
-    Offsets.clear();
-    Offsets.push_back(AA::RangeTy::Unknown);
-  }
-
-  void addToAll(int64_t Inc) {
-    for (auto &Offset : Offsets) {
-      Offset += Inc;
-    }
-  }
-
-  /// Copy offsets from \p R into the current list.
-  ///
-  /// Ideally all lists should be strictly ascending, but we defer that to the
-  /// actual use of the list. So we just blindly append here.
-  void merge(const OffsetInfo &R) { Offsets.append(R.Offsets); }
-};
-
 #ifndef NDEBUG
-static raw_ostream &operator<<(raw_ostream &OS, const OffsetInfo &OI) {
+static raw_ostream &operator<<(raw_ostream &OS,
+                               const AAPointerInfo::OffsetInfo &OI) {
   ListSeparator LS;
   OS << "[";
   for (auto Offset : OI) {
@@ -1079,7 +1034,13 @@ struct AAPointerInfoImpl
            (isValidState() ? (std::string("#") +
                               std::to_string(OffsetBins.size()) + " bins")
                            : "<invalid>") +
-           (ReachesReturn ? " (returned)" : "");
+           (reachesReturn()
+                ? (" (returned:" +
+                   join(map_range(ReturnedOffsets,
+                                  [](int64_t O) { return std::to_string(O); }),
+                        ", ") +
+                   ")")
+                : "");
   }
 
   /// See AbstractAttribute::manifest(...).
@@ -1092,13 +1053,34 @@ struct AAPointerInfoImpl
   virtual int64_t numOffsetBins() const override {
     return State::numOffsetBins();
   }
-  virtual bool reachesReturn() const override { return ReachesReturn; }
-  ChangeStatus setReachesReturn(bool Val) {
-    if (ReachesReturn == Val)
-      return ChangeStatus::UNCHANGED;
+  virtual bool reachesReturn() const override {
+    return !ReturnedOffsets.isUnassigned();
+  }
+  virtual void addReturnedOffsetsTo(OffsetInfo &OI) const override {
+    if (ReturnedOffsets.isUnknown()) {
+      OI.setUnknown();
+      return;
+    }
 
-    ReachesReturn = Val;
-    return ChangeStatus::CHANGED;
+    OffsetInfo MergedOI;
+    for (auto Offset : ReturnedOffsets) {
+      OffsetInfo TmpOI = OI;
+      TmpOI.addToAll(Offset);
+      MergedOI.merge(TmpOI);
+    }
+    OI = std::move(MergedOI);
+  }
+
+  ChangeStatus setReachesReturn(const OffsetInfo &ReachedReturnedOffsets) {
+    if (ReturnedOffsets.isUnknown())
+      return ChangeStatus::UNCHANGED;
+    if (ReachedReturnedOffsets.isUnknown()) {
+      ReturnedOffsets.setUnknown();
+      return ChangeStatus::CHANGED;
+    }
+    if (ReturnedOffsets.merge(ReachedReturnedOffsets))
+      return ChangeStatus::CHANGED;
+    return ChangeStatus::UNCHANGED;
   }
 
   bool forallInterferingAccesses(
@@ -1390,7 +1372,7 @@ struct AAPointerInfoImpl
     ChangeStatus Changed = ChangeStatus::UNCHANGED;
     const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
     bool IsByval = OtherAAImpl.getAssociatedArgument()->hasByValAttr();
-    Changed |= setReachesReturn(OtherAAImpl.ReachesReturn);
+    Changed |= setReachesReturn(OtherAAImpl.ReturnedOffsets);
 
     // Combine the accesses bin by bin.
     const auto &State = OtherAAImpl.getState();
@@ -1485,7 +1467,7 @@ struct AAPointerInfoFloating : public AAPointerInfoImpl {
   /// Deal with an access and signal if it was handled successfully.
   bool handleAccess(Attributor &A, Instruction &I,
                     std::optional<Value *> Content, AccessKind Kind,
-                    SmallVectorImpl<int64_t> &Offsets, ChangeStatus &Changed,
+                    OffsetInfo::VecTy &Offsets, ChangeStatus &Changed,
                     Type &Ty) {
     using namespace AA::PointerInfo;
     auto Size = AA::RangeTy::Unknown;
@@ -1495,16 +1477,16 @@ struct AAPointerInfoFloating : public AAPointerInfoImpl {
       Size = AccessSize.getFixedValue();
 
     // Make a strictly ascending list of offsets as required by addAccess()
-    llvm::sort(Offsets);
-    auto *Last = llvm::unique(Offsets);
-    Offsets.erase(Last, Offsets.end());
+    SmallVector<int64_t> OffsetsSorted(Offsets.begin(), Offsets.end());
+    llvm::sort(OffsetsSorted);
 
     VectorType *VT = dyn_cast<VectorType>(&Ty);
     if (!VT || VT->getElementCount().isScalable() ||
         !Content.value_or(nullptr) || !isa<Constant>(*Content) ||
         (*Content)->getType() != VT ||
         DL.getTypeStoreSize(VT->getElementType()).isScalable()) {
-      Changed = Changed | addAccess(A, {Offsets, Size}, I, Content, Kind, &Ty);
+      Changed =
+          Changed | addAccess(A, {OffsetsSorted, Size}, I, Content, Kind, &Ty);
     } else {
       // Handle vector stores with constant content element-wise.
       // TODO: We could look for the elements or create instructions
@@ -1689,8 +1671,12 @@ ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {
     // then check the call site return. Returns from other functions can't be
     // tracked and are cause for invalidation.
     if (auto *RI = dyn_cast<ReturnInst>(Usr)) {
-      Changed |= setReachesReturn(RI->getFunction() == getAssociatedFunction());
-      return ReachesReturn;
+      if (RI->getFunction() == getAssociatedFunction()) {
+        auto &PtrOI = OffsetInfoMap[CurPtr];
+        Changed |= setReachesReturn(PtrOI);
+        return true;
+      }
+      return false;
     }
 
     // For PHIs we need to take care of the recurrence explicitly as the value
@@ -1946,9 +1932,10 @@ ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {
             *this, IRPosition::callsite_returned(*CB), DepClassTy::REQUIRED);
         if (!CSRetPI)
           return false;
-        Changed = translateAndAddState(A, *CSRetPI, OffsetInfoMap[CurPtr], *CB,
-                                       IsRetMustAcc) |
-                  Changed;
+        OffsetInfo OI = OffsetInfoMap[CurPtr];
+        CSArgPI->addReturnedOffsetsTo(OI);
+        Changed =
+            translateAndAddState(A, *CSRetPI, OI, *CB, IsRetMustAcc) | Changed;
         return isValidState();
       }
       LLVM_DEBUG(dbgs() << "[AAPointerInfo] Call user not handled " << *CB
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index e8c0b00661654..688601a8ffa54 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -181,14 +181,15 @@ static unsigned conjugateICmpMask(unsigned Mask) {
 // Adapts the external decomposeBitTestICmp for local use.
 static bool decomposeBitTestICmp(Value *LHS, Value *RHS, CmpInst::Predicate &Pred,
                                  Value *&X, Value *&Y, Value *&Z) {
-  auto Res = llvm::decomposeBitTestICmp(LHS, RHS, Pred);
+  auto Res = llvm::decomposeBitTestICmp(
+      LHS, RHS, Pred, /*LookThroughTrunc=*/true, /*AllowNonZeroC=*/true);
   if (!Res)
     return false;
 
   Pred = Res->Pred;
   X = Res->X;
   Y = ConstantInt::get(X->getType(), Res->Mask);
-  Z = ConstantInt::get(X->getType(), 0);
+  Z = ConstantInt::get(X->getType(), Res->C);
   return true;
 }
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index d4d45384ec90e..12eec99e7d22b 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -423,7 +423,7 @@ Instruction *InstCombinerImpl::foldCmpLoadFromIndexedGlobal(
 /// Returns true if we can rewrite Start as a GEP with pointer Base
 /// and some integer offset. The nodes that need to be re-written
 /// for this transformation will be added to Explored.
-static bool canRewriteGEPAsOffset(Value *Start, Value *Base,
+static bool canRewriteGEPAsOffset(Value *Start, Value *Base, GEPNoWrapFlags &NW,
                                   const DataLayout &DL,
                                   SetVector<Value *> &Explored) {
   SmallVector<Value *, 16> WorkList(1, Start);
@@ -462,6 +462,7 @@ static bool canRewriteGEPAsOffset(Value *Start, Value *Base,
         if (!GEP->isInBounds() || count_if(GEP->indices(), IsNonConst) > 1)
           return false;
 
+        NW = NW.intersectForOffsetAdd(GEP->getNoWrapFlags());
         if (!Explored.contains(GEP->getOperand(0)))
           WorkList.push_back(GEP->getOperand(0));
       }
@@ -536,7 +537,7 @@ static void setInsertionPoint(IRBuilder<> &Builder, Value *V,
 
 /// Returns a re-written value of Start as an indexed GEP using Base as a
 /// pointer.
-static Value *rewriteGEPAsOffset(Value *Start, Value *Base,
+static Value *rewriteGEPAsOffset(Value *Start, Value *Base, GEPNoWrapFlags NW,
                                  const DataLayout &DL,
                                  SetVector<Value *> &Explored,
                                  InstCombiner &IC) {
@@ -578,8 +579,10 @@ static Value *rewriteGEPAsOffset(Value *Start, Value *Base,
       if (isa<ConstantInt>(Op) && cast<ConstantInt>(Op)->isZero())
         NewInsts[GEP] = OffsetV;
       else
-        NewInsts[GEP] = Builder.CreateNSWAdd(
-            Op, OffsetV, GEP->getOperand(0)->getName() + ".add");
+        NewInsts[GEP] = Builder.CreateAdd(
+            Op, OffsetV, GEP->getOperand(0)->getName() + ".add",
+            /*NUW=*/NW.hasNoUnsignedWrap(),
+            /*NSW=*/NW.hasNoUnsignedSignedWrap());
       continue;
     }
     if (isa<PHINode>(Val))
@@ -613,8 +616,8 @@ static Value *rewriteGEPAsOffset(Value *Start, Value *Base,
 
     setInsertionPoint(Builder, Val, false);
     // Create GEP for external users.
-    Value *NewVal = Builder.CreateInBoundsGEP(
-        Builder.getInt8Ty(), Base, NewInsts[Val], Val->getName() + ".ptr");
+    Value *NewVal = Builder.CreateGEP(Builder.getInt8Ty(), Base, NewInsts[Val],
+                                      Val->getName() + ".ptr", NW);
     IC.replaceInstUsesWith(*cast<Instruction>(Val), NewVal);
     // Add old instruction to worklist for DCE. We don't directly remove it
     // here because the original compare is one of the users.
@@ -649,8 +652,8 @@ static Instruction *transformToIndexedCompare(GEPOperator *GEPLHS, Value *RHS,
 
   // The set of nodes that will take part in this transformation.
   SetVector<Value *> Nodes;
-
-  if (!canRewriteGEPAsOffset(RHS, PtrBase, DL, Nodes))
+  GEPNoWrapFlags NW = GEPLHS->getNoWrapFlags();
+  if (!canRewriteGEPAsOffset(RHS, PtrBase, NW, DL, Nodes))
     return nullptr;
 
   // We know we can re-write this as
@@ -659,7 +662,7 @@ static Instruction *transformToIndexedCompare(GEPOperator *GEPLHS, Value *RHS,
   // can't have overflow on either side. We can therefore re-write
   // this as:
   //   OFFSET1 cmp OFFSET2
-  Value *NewRHS = rewriteGEPAsOffset(RHS, PtrBase, DL, Nodes, IC);
+  Value *NewRHS = rewriteGEPAsOffset(RHS, PtrBase, NW, DL, Nodes, IC);
 
   // RewriteGEPAsOffset has replaced RHS and all of its uses with a re-written
   // GEP having PtrBase as the pointer base, and has returned in NewRHS the
@@ -5012,6 +5015,18 @@ Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I,
     }
   }
 
+  // (icmp eq/ne (X, -P2), INT_MIN)
+  //	-> (icmp slt/sge X, INT_MIN + P2)
+  if (ICmpInst::isEquality(Pred) && BO0 &&
+      match(I.getOperand(1), m_SignMask()) &&
+      match(BO0, m_And(m_Value(), m_NegatedPower2OrZero()))) {
+    // Will Constant fold.
+    Value *NewC = Builder.CreateSub(I.getOperand(1), BO0->getOperand(1));
+    return new ICmpInst(Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_SLT
+                                                  : ICmpInst::ICMP_SGE,
+                        BO0->getOperand(0), NewC);
+  }
+
   {
     // Similar to above: an unsigned overflow comparison may use offset + mask:
     // ((Op1 + C) & C) u<  Op1 --> Op1 != 0
@@ -5917,31 +5932,15 @@ Instruction *InstCombinerImpl::foldICmpWithTrunc(ICmpInst &ICmp) {
     return nullptr;
 
   // This matches patterns corresponding to tests of the signbit as well as:
-  // (trunc X) u< C --> (X & -C) == 0 (are all masked-high-bits clear?)
-  // (trunc X) u> C --> (X & ~C) != 0 (are any masked-high-bits set?)
-  if (auto Res = decomposeBitTestICmp(Op0, Op1, Pred, /*WithTrunc=*/true)) {
+  // (trunc X) pred C2 --> (X & Mask) == C
+  if (auto Res = decomposeBitTestICmp(Op0, Op1, Pred, /*WithTrunc=*/true,
+                                      /*AllowNonZeroC=*/true)) {
     Value *And = Builder.CreateAnd(Res->X, Res->Mask);
-    Constant *Zero = ConstantInt::getNullValue(Res->X->getType());
-    return new ICmpInst(Res->Pred, And, Zero);
+    Constant *C = ConstantInt::get(Res->X->getType(), Res->C);
+    return new ICmpInst(Res->Pred, And, C);
   }
 
   unsigned SrcBits = X->getType()->getScalarSizeInBits();
-  if (Pred == ICmpInst::ICMP_ULT && C->isNegatedPowerOf2()) {
-    // If C is a negative power-of-2 (high-bit mask):
-    // (trunc X) u< C --> (X & C) != C (are any masked-high-bits clear?)
-    Constant *MaskC = ConstantInt::get(X->getType(), C->zext(SrcBits));
-    Value *And = Builder.CreateAnd(X, MaskC);
-    return new ICmpInst(ICmpInst::ICMP_NE, And, MaskC);
-  }
-
-  if (Pred == ICmpInst::ICMP_UGT && (~*C).isPowerOf2()) {
-    // If C is not-of-power-of-2 (one clear bit):
-    // (trunc X) u> C --> (X & (C+1)) == C+1 (are all masked-high-bits set?)
-    Constant *MaskC = ConstantInt::get(X->getType(), (*C + 1).zext(SrcBits));
-    Value *And = Builder.CreateAnd(X, MaskC);
-    return new ICmpInst(ICmpInst::ICMP_EQ, And, MaskC);
-  }
-
   if (auto *II = dyn_cast<IntrinsicInst>(X)) {
     if (II->getIntrinsicID() == Intrinsic::cttz ||
         II->getIntrinsicID() == Intrinsic::ctlz) {
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index da6f991ad4cd1..7c6f42de77fc7 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -739,6 +739,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
   Instruction *foldSelectOfBools(SelectInst &SI);
   Instruction *foldSelectToCmp(SelectInst &SI);
   Instruction *foldSelectExtConst(SelectInst &Sel);
+  Instruction *foldSelectEqualityTest(SelectInst &SI);
   Instruction *foldSelectOpOp(SelectInst &SI, Instruction *TI, Instruction *FI);
   Instruction *foldSelectIntoOp(SelectInst &SI, Value *, Value *);
   Instruction *foldSPFofSPF(Instruction *Inner, SelectPatternFlavor SPF1,
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index 3dbe95897d635..3f780285efe42 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -1410,6 +1410,44 @@ Instruction *InstCombinerImpl::foldSelectValueEquivalence(SelectInst &Sel,
   return nullptr;
 }
 
+/// Fold the following code sequence:
+/// \code
+///   %XeqZ = icmp eq i64 %X, %Z
+///   %YeqZ = icmp eq i64 %Y, %Z
+///   %XeqY = icmp eq i64 %X, %Y
+///   %not.YeqZ = xor i1 %YeqZ, true
+///   %and = select i1 %not.YeqZ, i1 %XeqY, i1 false
+///   %equal = select i1 %XeqZ, i1 %YeqZ, i1 %and
+/// \code
+///
+/// into:
+///   %equal = icmp eq i64 %X, %Y
+Instruction *InstCombinerImpl::foldSelectEqualityTest(SelectInst &Sel) {
+  Value *X, *Y, *Z;
+  Value *XeqY, *XeqZ = Sel.getCondition(), *YeqZ = Sel.getTrueValue();
+
+  if (!match(XeqZ, m_SpecificICmp(ICmpInst::ICMP_EQ, m_Value(X), m_Value(Z))))
+    return nullptr;
+
+  if (!match(YeqZ,
+             m_c_SpecificICmp(ICmpInst::ICMP_EQ, m_Value(Y), m_Specific(Z))))
+    std::swap(X, Z);
+
+  if (!match(YeqZ,
+             m_c_SpecificICmp(ICmpInst::ICMP_EQ, m_Value(Y), m_Specific(Z))))
+    return nullptr;
+
+  if (!match(Sel.getFalseValue(),
+             m_c_LogicalAnd(m_Not(m_Specific(YeqZ)), m_Value(XeqY))))
+    return nullptr;
+
+  if (!match(XeqY,
+             m_c_SpecificICmp(ICmpInst::ICMP_EQ, m_Specific(X), m_Specific(Y))))
+    return nullptr;
+
+  return replaceInstUsesWith(Sel, XeqY);
+}
+
 // See if this is a pattern like:
 //   %old_cmp1 = icmp slt i32 %x, C2
 //   %old_replacement = select i1 %old_cmp1, i32 %target_low, i32 %target_high
@@ -4112,6 +4150,9 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) {
   if (Instruction *I = foldSelectToCmp(SI))
     return I;
 
+  if (Instruction *I = foldSelectEqualityTest(SI))
+    return I;
+
   // Fold:
   // (select A && B, T, F) -> (select A, (select B, T, F), F)
   // (select A || B, T, F) -> (select A, T, (select B, T, F))
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index 0ad178594be03..3d4461dc1a87f 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -592,6 +592,15 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
     if (DemandedFromOps.isOne() && DemandedFromOps.isSubsetOf(LHSKnown.Zero))
       return I->getOperand(1);
 
+    // Canonicalize sub mask, X -> ~X
+    const APInt *LHSC;
+    if (match(I->getOperand(0), m_LowBitMask(LHSC)) &&
+        DemandedFromOps.isSubsetOf(*LHSC)) {
+      IRBuilderBase::InsertPointGuard Guard(Builder);
+      Builder.SetInsertPoint(I);
+      return Builder.CreateNot(I->getOperand(1));
+    }
+
     // Otherwise just compute the known bits of the result.
     bool NSW = cast<OverflowingBinaryOperator>(I)->hasNoSignedWrap();
     bool NUW = cast<OverflowingBinaryOperator>(I)->hasNoUnsignedWrap();
@@ -1910,17 +1919,23 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V,
 /// For floating-point classes that resolve to a single bit pattern, return that
 /// value.
 static Constant *getFPClassConstant(Type *Ty, FPClassTest Mask) {
+  if (Mask == fcNone)
+    return PoisonValue::get(Ty);
+
+  if (Mask == fcPosZero)
+    return Constant::getNullValue(Ty);
+
+  // TODO: Support aggregate types that are allowed by FPMathOperator.
+  if (Ty->isAggregateType())
+    return nullptr;
+
   switch (Mask) {
-  case fcPosZero:
-    return ConstantFP::getZero(Ty);
   case fcNegZero:
     return ConstantFP::getZero(Ty, true);
   case fcPosInf:
     return ConstantFP::getInfinity(Ty);
   case fcNegInf:
     return ConstantFP::getInfinity(Ty, true);
-  case fcNone:
-    return PoisonValue::get(Ty);
   default:
     return nullptr;
   }
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 916a14e13ff2a..1ea3efb27937d 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -5170,8 +5170,13 @@ bool InstCombinerImpl::run() {
         LLVM_DEBUG(dbgs() << "IC: Old = " << *I << '\n'
                           << "    New = " << *Result << '\n');
 
-        Result->copyMetadata(*I,
-                             {LLVMContext::MD_dbg, LLVMContext::MD_annotation});
+        // We copy the old instruction's DebugLoc to the new instruction, unless
+        // InstCombine already assigned a DebugLoc to it, in which case we
+        // should trust the more specifically selected DebugLoc.
+        if (!Result->getDebugLoc())
+          Result->setDebugLoc(I->getDebugLoc());
+        // We also copy annotation metadata to the new instruction.
+        Result->copyMetadata(*I, LLVMContext::MD_annotation);
         // Everything uses the new instruction now.
         I->replaceAllUsesWith(Result);
 
diff --git a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
index 76d6033708614..86637109d9408 100644
--- a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
+++ b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
@@ -132,6 +132,15 @@ static cl::opt<int> ICPMaxNumVTableLastCandidate(
     "icp-max-num-vtable-last-candidate", cl::init(1), cl::Hidden,
     cl::desc("The maximum number of vtable for the last candidate."));
 
+static cl::list<std::string> ICPIgnoredBaseTypes(
+    "icp-ignored-base-types", cl::Hidden,
+    cl::desc(
+        "A list of mangled vtable type info names. Classes specified by the "
+        "type info names and their derived ones will not be vtable-ICP'ed. "
+        "Useful when the profiled types and actual types in the optimized "
+        "binary could be different due to profiling limitations. Type info "
+        "names are those string literals used in LLVM type metadata"));
+
 namespace {
 
 // The key is a vtable global variable, and the value is a map.
@@ -316,6 +325,8 @@ class IndirectCallPromoter {
 
   OptimizationRemarkEmitter &ORE;
 
+  const DenseSet<StringRef> &IgnoredBaseTypes;
+
   // A struct that records the direct target and it's call count.
   struct PromotionCandidate {
     Function *const TargetFunction;
@@ -366,6 +377,10 @@ class IndirectCallPromoter {
   bool isProfitableToCompareVTables(const CallBase &CB,
                                     ArrayRef<PromotionCandidate> Candidates);
 
+  // Return true if the vtable corresponding to VTableGUID should be skipped
+  // for vtable-based comparison.
+  bool shouldSkipVTable(uint64_t VTableGUID);
+
   // Given an indirect callsite and the list of function candidates, compute
   // the following vtable information in output parameters and return vtable
   // pointer if type profiles exist.
@@ -391,10 +406,12 @@ class IndirectCallPromoter {
       Function &Func, Module &M, InstrProfSymtab *Symtab, bool SamplePGO,
       const VirtualCallSiteTypeInfoMap &VirtualCSInfo,
       VTableAddressPointOffsetValMap &VTableAddressPointOffsetVal,
+      const DenseSet<StringRef> &IgnoredBaseTypes,
       OptimizationRemarkEmitter &ORE)
       : F(Func), M(M), Symtab(Symtab), SamplePGO(SamplePGO),
         VirtualCSInfo(VirtualCSInfo),
-        VTableAddressPointOffsetVal(VTableAddressPointOffsetVal), ORE(ORE) {}
+        VTableAddressPointOffsetVal(VTableAddressPointOffsetVal), ORE(ORE),
+        IgnoredBaseTypes(IgnoredBaseTypes) {}
   IndirectCallPromoter(const IndirectCallPromoter &) = delete;
   IndirectCallPromoter &operator=(const IndirectCallPromoter &) = delete;
 
@@ -851,9 +868,14 @@ bool IndirectCallPromoter::isProfitableToCompareVTables(
     LLVM_DEBUG(dbgs() << "\n");
 
     uint64_t CandidateVTableCount = 0;
-    for (auto &[GUID, Count] : VTableGUIDAndCounts)
+
+    for (auto &[GUID, Count] : VTableGUIDAndCounts) {
       CandidateVTableCount += Count;
 
+      if (shouldSkipVTable(GUID))
+        return false;
+    }
+
     if (CandidateVTableCount < Candidate.Count * ICPVTablePercentageThreshold) {
       LLVM_DEBUG(
           dbgs() << "    function count " << Candidate.Count
@@ -883,6 +905,27 @@ bool IndirectCallPromoter::isProfitableToCompareVTables(
   return true;
 }
 
+bool IndirectCallPromoter::shouldSkipVTable(uint64_t VTableGUID) {
+  if (IgnoredBaseTypes.empty())
+    return false;
+
+  auto *VTableVar = Symtab->getGlobalVariable(VTableGUID);
+
+  assert(VTableVar && "VTableVar must exist for GUID in VTableGUIDAndCounts");
+
+  SmallVector<MDNode *, 2> Types;
+  VTableVar->getMetadata(LLVMContext::MD_type, Types);
+
+  for (auto *Type : Types)
+    if (auto *TypeId = dyn_cast<MDString>(Type->getOperand(1).get()))
+      if (IgnoredBaseTypes.contains(TypeId->getString())) {
+        LLVM_DEBUG(dbgs() << "    vtable profiles should be ignored. Bail "
+                             "out of vtable comparison.");
+        return true;
+      }
+  return false;
+}
+
 // For virtual calls in the module, collect per-callsite information which will
 // be used to associate an ICP candidate with a vtable and a specific function
 // in the vtable. With type intrinsics (llvm.type.test), we can find virtual
@@ -956,9 +999,15 @@ static bool promoteIndirectCalls(Module &M, ProfileSummaryInfo *PSI, bool InLTO,
   bool Changed = false;
   VirtualCallSiteTypeInfoMap VirtualCSInfo;
 
-  if (EnableVTableProfileUse)
+  DenseSet<StringRef> IgnoredBaseTypes;
+
+  if (EnableVTableProfileUse) {
     computeVirtualCallSiteTypeInfoMap(M, MAM, VirtualCSInfo);
 
+    for (StringRef Str : ICPIgnoredBaseTypes)
+      IgnoredBaseTypes.insert(Str);
+  }
+
   // VTableAddressPointOffsetVal stores the vtable address points. The vtable
   // address point of a given <vtable, address point offset> is static (doesn't
   // change after being computed once).
@@ -977,7 +1026,8 @@ static bool promoteIndirectCalls(Module &M, ProfileSummaryInfo *PSI, bool InLTO,
     auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
 
     IndirectCallPromoter CallPromoter(F, M, &Symtab, SamplePGO, VirtualCSInfo,
-                                      VTableAddressPointOffsetVal, ORE);
+                                      VTableAddressPointOffsetVal,
+                                      IgnoredBaseTypes, ORE);
     bool FuncChanged = CallPromoter.processFunction(PSI);
     if (ICPDUMPAFTER && FuncChanged) {
       LLVM_DEBUG(dbgs() << "\n== IR Dump After =="; F.print(dbgs()));
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index fe882164656df..1c95a4606ecc5 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -267,10 +267,6 @@ class InstrLowerer final {
   GlobalVariable *NamesVar = nullptr;
   size_t NamesSize = 0;
 
-  /// The instance of [[alwaysinline]] rmw_or(ptr, i8).
-  /// This is name-insensitive.
-  Function *RMWOrFunc = nullptr;
-
   // vector of counter load/store pairs to be register promoted.
   std::vector<LoadStorePair> PromotionCandidates;
 
@@ -337,14 +333,6 @@ class InstrLowerer final {
                                        StringRef Name,
                                        GlobalValue::LinkageTypes Linkage);
 
-  /// Create [[alwaysinline]] rmw_or(ptr, i8).
-  /// This doesn't update `RMWOrFunc`.
-  Function *createRMWOrFunc();
-
-  /// Get the call to `rmw_or`.
-  /// Create the instance if it is unknown.
-  CallInst *getRMWOrCall(Value *Addr, Value *Val);
-
   /// Compute the address of the test vector bitmap that this profiling
   /// instruction acts on.
   Value *getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I);
@@ -1137,68 +1125,6 @@ Value *InstrLowerer::getCounterAddress(InstrProfCntrInstBase *I) {
   return Builder.CreateIntToPtr(Add, Addr->getType());
 }
 
-/// Create `void [[alwaysinline]] rmw_or(uint8_t *ArgAddr, uint8_t ArgVal)`
-/// "Basic" sequence is `*ArgAddr |= ArgVal`
-Function *InstrLowerer::createRMWOrFunc() {
-  auto &Ctx = M.getContext();
-  auto *Int8Ty = Type::getInt8Ty(Ctx);
-  Function *Fn = Function::Create(
-      FunctionType::get(Type::getVoidTy(Ctx),
-                        {PointerType::getUnqual(Ctx), Int8Ty}, false),
-      Function::LinkageTypes::PrivateLinkage, "rmw_or", M);
-  Fn->addFnAttr(Attribute::AlwaysInline);
-  auto *ArgAddr = Fn->getArg(0);
-  auto *ArgVal = Fn->getArg(1);
-  IRBuilder<> Builder(BasicBlock::Create(Ctx, "", Fn));
-
-  // Load profile bitmap byte.
-  //  %mcdc.bits = load i8, ptr %4, align 1
-  auto *Bitmap = Builder.CreateLoad(Int8Ty, ArgAddr, "mcdc.bits");
-
-  if (Options.Atomic || AtomicCounterUpdateAll) {
-    // If ((Bitmap & Val) != Val), then execute atomic (Bitmap |= Val).
-    // Note, just-loaded Bitmap might not be up-to-date. Use it just for
-    // early testing.
-    auto *Masked = Builder.CreateAnd(Bitmap, ArgVal);
-    auto *ShouldStore = Builder.CreateICmpNE(Masked, ArgVal);
-    auto *ThenTerm = BasicBlock::Create(Ctx, "", Fn);
-    auto *ElseTerm = BasicBlock::Create(Ctx, "", Fn);
-    // Assume updating will be rare.
-    auto *Unlikely = MDBuilder(Ctx).createUnlikelyBranchWeights();
-    Builder.CreateCondBr(ShouldStore, ThenTerm, ElseTerm, Unlikely);
-
-    IRBuilder<> ThenBuilder(ThenTerm);
-    ThenBuilder.CreateAtomicRMW(AtomicRMWInst::Or, ArgAddr, ArgVal,
-                                MaybeAlign(), AtomicOrdering::Monotonic);
-    ThenBuilder.CreateRetVoid();
-
-    IRBuilder<> ElseBuilder(ElseTerm);
-    ElseBuilder.CreateRetVoid();
-
-    return Fn;
-  }
-
-  // Perform logical OR of profile bitmap byte and shifted bit offset.
-  //  %8 = or i8 %mcdc.bits, %7
-  auto *Result = Builder.CreateOr(Bitmap, ArgVal);
-
-  // Store the updated profile bitmap byte.
-  //  store i8 %8, ptr %3, align 1
-  Builder.CreateStore(Result, ArgAddr);
-
-  // Terminator
-  Builder.CreateRetVoid();
-
-  return Fn;
-}
-
-CallInst *InstrLowerer::getRMWOrCall(Value *Addr, Value *Val) {
-  if (!RMWOrFunc)
-    RMWOrFunc = createRMWOrFunc();
-
-  return CallInst::Create(RMWOrFunc, {Addr, Val});
-}
-
 Value *InstrLowerer::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) {
   auto *Bitmaps = getOrCreateRegionBitmaps(I);
   if (!isRuntimeCounterRelocationEnabled())
@@ -1291,9 +1217,10 @@ void InstrLowerer::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
 
 void InstrLowerer::lowerMCDCTestVectorBitmapUpdate(
     InstrProfMCDCTVBitmapUpdate *Update) {
+  auto &Ctx = M.getContext();
   IRBuilder<> Builder(Update);
-  auto *Int8Ty = Type::getInt8Ty(M.getContext());
-  auto *Int32Ty = Type::getInt32Ty(M.getContext());
+  auto *Int8Ty = Type::getInt8Ty(Ctx);
+  auto *Int32Ty = Type::getInt32Ty(Ctx);
   auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr();
   auto *BitmapAddr = getBitmapAddress(Update);
 
@@ -1321,7 +1248,36 @@ void InstrLowerer::lowerMCDCTestVectorBitmapUpdate(
   //  %7 = shl i8 1, %6
   auto *ShiftedVal = Builder.CreateShl(Builder.getInt8(0x1), BitToSet);
 
-  Builder.Insert(getRMWOrCall(BitmapByteAddr, ShiftedVal));
+  // Load profile bitmap byte.
+  //  %mcdc.bits = load i8, ptr %4, align 1
+  auto *Bitmap = Builder.CreateLoad(Int8Ty, BitmapByteAddr, "mcdc.bits");
+
+  if (Options.Atomic || AtomicCounterUpdateAll) {
+    // If ((Bitmap & Val) != Val), then execute atomic (Bitmap |= Val).
+    // Note, just-loaded Bitmap might not be up-to-date. Use it just for
+    // early testing.
+    auto *Masked = Builder.CreateAnd(Bitmap, ShiftedVal);
+    auto *ShouldStore = Builder.CreateICmpNE(Masked, ShiftedVal);
+
+    // Assume updating will be rare.
+    auto *Unlikely = MDBuilder(Ctx).createUnlikelyBranchWeights();
+    Instruction *ThenBranch =
+        SplitBlockAndInsertIfThen(ShouldStore, Update, false, Unlikely);
+
+    // Execute if (unlikely(ShouldStore)).
+    Builder.SetInsertPoint(ThenBranch);
+    Builder.CreateAtomicRMW(AtomicRMWInst::Or, BitmapByteAddr, ShiftedVal,
+                            MaybeAlign(), AtomicOrdering::Monotonic);
+  } else {
+    // Perform logical OR of profile bitmap byte and shifted bit offset.
+    //  %8 = or i8 %mcdc.bits, %7
+    auto *Result = Builder.CreateOr(Bitmap, ShiftedVal);
+
+    // Store the updated profile bitmap byte.
+    //  store i8 %8, ptr %3, align 1
+    Builder.CreateStore(Result, BitmapByteAddr);
+  }
+
   Update->eraseFromParent();
 }
 
diff --git a/llvm/lib/Transforms/Instrumentation/RealtimeSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/RealtimeSanitizer.cpp
index 7854cf4f2c625..c4cb72ab2e4da 100644
--- a/llvm/lib/Transforms/Instrumentation/RealtimeSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/RealtimeSanitizer.cpp
@@ -15,49 +15,77 @@
 
 #include "llvm/IR/Analysis.h"
 #include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstIterator.h"
 #include "llvm/IR/Module.h"
 
+#include "llvm/Demangle/Demangle.h"
 #include "llvm/Transforms/Instrumentation/RealtimeSanitizer.h"
 
 using namespace llvm;
 
+static SmallVector<Type *> getArgTypes(ArrayRef<Value *> FunctionArgs) {
+  SmallVector<Type *> Types;
+  for (Value *Arg : FunctionArgs)
+    Types.push_back(Arg->getType());
+  return Types;
+}
+
 static void insertCallBeforeInstruction(Function &Fn, Instruction &Instruction,
-                                        const char *FunctionName) {
+                                        const char *FunctionName,
+                                        ArrayRef<Value *> FunctionArgs) {
   LLVMContext &Context = Fn.getContext();
-  FunctionType *FuncType = FunctionType::get(Type::getVoidTy(Context), false);
+  FunctionType *FuncType = FunctionType::get(Type::getVoidTy(Context),
+                                             getArgTypes(FunctionArgs), false);
   FunctionCallee Func =
       Fn.getParent()->getOrInsertFunction(FunctionName, FuncType);
   IRBuilder<> Builder{&Instruction};
-  Builder.CreateCall(Func, {});
+  Builder.CreateCall(Func, FunctionArgs);
 }
 
 static void insertCallAtFunctionEntryPoint(Function &Fn,
-                                           const char *InsertFnName) {
-
-  insertCallBeforeInstruction(Fn, Fn.front().front(), InsertFnName);
+                                           const char *InsertFnName,
+                                           ArrayRef<Value *> FunctionArgs) {
+  insertCallBeforeInstruction(Fn, Fn.front().front(), InsertFnName,
+                              FunctionArgs);
 }
 
 static void insertCallAtAllFunctionExitPoints(Function &Fn,
-                                              const char *InsertFnName) {
-  for (auto &BB : Fn)
-    for (auto &I : BB)
-      if (isa<ReturnInst>(&I))
-        insertCallBeforeInstruction(Fn, I, InsertFnName);
+                                              const char *InsertFnName,
+                                              ArrayRef<Value *> FunctionArgs) {
+  for (auto &I : instructions(Fn))
+    if (isa<ReturnInst>(&I))
+      insertCallBeforeInstruction(Fn, I, InsertFnName, FunctionArgs);
+}
+
+static PreservedAnalyses rtsanPreservedCFGAnalyses() {
+  PreservedAnalyses PA;
+  PA.preserveSet<CFGAnalyses>();
+  return PA;
+}
+
+static PreservedAnalyses runSanitizeRealtime(Function &Fn) {
+  insertCallAtFunctionEntryPoint(Fn, "__rtsan_realtime_enter", {});
+  insertCallAtAllFunctionExitPoints(Fn, "__rtsan_realtime_exit", {});
+  return rtsanPreservedCFGAnalyses();
+}
+
+static PreservedAnalyses runSanitizeRealtimeUnsafe(Function &Fn) {
+  IRBuilder<> Builder(&Fn.front().front());
+  Value *Name = Builder.CreateGlobalString(demangle(Fn.getName()));
+  insertCallAtFunctionEntryPoint(Fn, "__rtsan_notify_blocking_call", {Name});
+  return rtsanPreservedCFGAnalyses();
 }
 
 RealtimeSanitizerPass::RealtimeSanitizerPass(
     const RealtimeSanitizerOptions &Options) {}
 
-PreservedAnalyses RealtimeSanitizerPass::run(Function &F,
+PreservedAnalyses RealtimeSanitizerPass::run(Function &Fn,
                                              AnalysisManager<Function> &AM) {
-  if (F.hasFnAttribute(Attribute::SanitizeRealtime)) {
-    insertCallAtFunctionEntryPoint(F, "__rtsan_realtime_enter");
-    insertCallAtAllFunctionExitPoints(F, "__rtsan_realtime_exit");
-
-    PreservedAnalyses PA;
-    PA.preserveSet<CFGAnalyses>();
-    return PA;
-  }
+  if (Fn.hasFnAttribute(Attribute::SanitizeRealtime))
+    return runSanitizeRealtime(Fn);
+
+  if (Fn.hasFnAttribute(Attribute::SanitizeRealtimeUnsafe))
+    return runSanitizeRealtimeUnsafe(Fn);
 
   return PreservedAnalyses::all();
 }
diff --git a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
index cf11f5bc885a7..a1dbb4e1d5e75 100644
--- a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
+++ b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
@@ -362,7 +362,7 @@ static bool isEqualImpl(SimpleValue LHS, SimpleValue RHS) {
 
   if (LHSI->getOpcode() != RHSI->getOpcode())
     return false;
-  if (LHSI->isIdenticalToWhenDefined(RHSI)) {
+  if (LHSI->isIdenticalToWhenDefined(RHSI, /*IntersectAttrs=*/true)) {
     // Convergent calls implicitly depend on the set of threads that is
     // currently executing, so conservatively return false if they are in
     // different basic blocks.
@@ -551,7 +551,7 @@ bool DenseMapInfo<CallValue>::isEqual(CallValue LHS, CallValue RHS) {
   if (LHSI->isConvergent() && LHSI->getParent() != RHSI->getParent())
     return false;
 
-  return LHSI->isIdenticalTo(RHSI);
+  return LHSI->isIdenticalToWhenDefined(RHSI, /*IntersectAttrs=*/true);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1307,6 +1307,23 @@ static void combineIRFlags(Instruction &From, Value *To) {
         (I->hasPoisonGeneratingFlags() && !programUndefinedIfPoison(I)))
       I->andIRFlags(&From);
   }
+  if (isa<CallBase>(&From) && isa<CallBase>(To)) {
+    // NB: Intersection of attrs between InVal.first and Inst is overly
+    // conservative. Since we only CSE readonly functions that have the same
+    // memory state, we can preserve (or possibly in some cases combine)
+    // more attributes. Likewise this implies when checking equality of
+    // callsite for CSEing, we can probably ignore more attributes.
+    // Generally poison generating attributes need to be handled with more
+    // care as they can create *new* UB if preserved/combined and violated.
+    // Attributes that imply immediate UB on the other hand would have been
+    // violated either way.
+    bool Success =
+        cast<CallBase>(To)->tryIntersectAttributes(cast<CallBase>(&From));
+    assert(Success && "Failed to intersect attributes in callsites that "
+                      "passed identical check");
+    // For NDEBUG Compile.
+    (void)Success;
+  }
 }
 
 bool EarlyCSE::overridingStores(const ParseMemoryInst &Earlier,
@@ -1632,6 +1649,7 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
           LLVM_DEBUG(dbgs() << "Skipping due to debug counter\n");
           continue;
         }
+        combineIRFlags(Inst, InVal.first);
         if (!Inst.use_empty())
           Inst.replaceAllUsesWith(InVal.first);
         salvageKnowledge(&Inst, &AC);
diff --git a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
index 7e76e4a0f387e..2668305e9c844 100644
--- a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -1920,7 +1920,7 @@ bool IndVarSimplify::run(Loop *L) {
 
   // Create a rewriter object which we'll use to transform the code with.
   SCEVExpander Rewriter(*SE, DL, "indvars");
-#ifndef NDEBUG
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
   Rewriter.setDebugType(DEBUG_TYPE);
 #endif
 
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index d73b550741fb3..50f7637b07f6f 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -2820,18 +2820,7 @@ static bool hoistBOAssociation(Instruction &I, Loop &L,
   if (!BO || !BO->isAssociative())
     return false;
 
-  // TODO: Only hoist ADDs, MULs, FADDs, and FMULs for now.
   Instruction::BinaryOps Opcode = BO->getOpcode();
-  switch (Opcode) {
-  case Instruction::Add:
-  case Instruction::Mul:
-  case Instruction::FAdd:
-  case Instruction::FMul:
-    break;
-  default:
-    return false;
-  }
-
   bool LVInRHS = L.isLoopInvariant(BO->getOperand(0));
   auto *BO0 = dyn_cast<BinaryOperator>(BO->getOperand(LVInRHS));
   if (!BO0 || BO0->getOpcode() != Opcode || !BO0->isAssociative() ||
diff --git a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
index a4f2dbf9a5828..5ef25c21162fe 100644
--- a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
@@ -293,7 +293,7 @@ PreservedAnalyses FunctionToLoopPassAdaptor::run(Function &F,
     Updater.CurrentL = L;
     Updater.SkipCurrentLoop = false;
 
-#ifndef NDEBUG
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
     // Save a parent loop pointer for asserts.
     Updater.ParentL = L->getParentLoop();
 #endif
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index f966ccaa83842..575395eda1c5b 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -6188,7 +6188,7 @@ LSRInstance::LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE,
 
   // Configure SCEVExpander already now, so the correct mode is used for
   // isSafeToExpand() checks.
-#ifndef NDEBUG
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
   Rewriter.setDebugType(DEBUG_TYPE);
 #endif
   Rewriter.disableCanonicalMode();
@@ -6870,6 +6870,8 @@ static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE,
             SE.computeConstantDifference(DVIRec.SCEVs[i], SCEVInductionVar)) {
       if (Offset->getSignificantBits() <= 64)
         SalvageExpr->createOffsetExpr(Offset->getSExtValue(), LSRInductionVar);
+      else
+        return false;
     } else if (!SalvageExpr->createIterCountExpr(DVIRec.SCEVs[i], IterCountExpr,
                                                  SE))
       return false;
@@ -7084,7 +7086,7 @@ static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
     SmallVector<WeakTrackingVH, 16> DeadInsts;
     const DataLayout &DL = L->getHeader()->getDataLayout();
     SCEVExpander Rewriter(SE, DL, "lsr", false);
-#ifndef NDEBUG
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
     Rewriter.setDebugType(DEBUG_TYPE);
 #endif
     unsigned numFolded = Rewriter.replaceCongruentIVs(L, &DT, DeadInsts, &TTI);
diff --git a/llvm/lib/Transforms/Scalar/Scalarizer.cpp b/llvm/lib/Transforms/Scalar/Scalarizer.cpp
index da9b804e94a74..ee86e2e6c9751 100644
--- a/llvm/lib/Transforms/Scalar/Scalarizer.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalarizer.cpp
@@ -39,7 +39,6 @@
 #include "llvm/IR/Value.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Support/Casting.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include <cassert>
 #include <cstdint>
@@ -51,28 +50,6 @@ using namespace llvm;
 
 #define DEBUG_TYPE "scalarizer"
 
-static cl::opt<bool> ClScalarizeVariableInsertExtract(
-    "scalarize-variable-insert-extract", cl::init(true), cl::Hidden,
-    cl::desc("Allow the scalarizer pass to scalarize "
-             "insertelement/extractelement with variable index"));
-
-// This is disabled by default because having separate loads and stores
-// makes it more likely that the -combiner-alias-analysis limits will be
-// reached.
-static cl::opt<bool> ClScalarizeLoadStore(
-    "scalarize-load-store", cl::init(false), cl::Hidden,
-    cl::desc("Allow the scalarizer pass to scalarize loads and store"));
-
-// Split vectors larger than this size into fragments, where each fragment is
-// either a vector no larger than this size or a scalar.
-//
-// Instructions with operands or results of different sizes that would be split
-// into a different number of fragments are currently left as-is.
-static cl::opt<unsigned> ClScalarizeMinBits(
-    "scalarize-min-bits", cl::init(0), cl::Hidden,
-    cl::desc("Instruct the scalarizer pass to attempt to keep values of a "
-             "minimum number of bits"));
-
 namespace {
 
 BasicBlock::iterator skipPastPhiNodesAndDbg(BasicBlock::iterator Itr) {
@@ -273,24 +250,14 @@ static Value *concatenate(IRBuilder<> &Builder, ArrayRef<Value *> Fragments,
   return Res;
 }
 
-template <typename T>
-T getWithDefaultOverride(const cl::opt<T> &ClOption,
-                         const std::optional<T> &DefaultOverride) {
-  return ClOption.getNumOccurrences() ? ClOption
-                                      : DefaultOverride.value_or(ClOption);
-}
-
 class ScalarizerVisitor : public InstVisitor<ScalarizerVisitor, bool> {
 public:
   ScalarizerVisitor(DominatorTree *DT, const TargetTransformInfo *TTI,
                     ScalarizerPassOptions Options)
-      : DT(DT), TTI(TTI), ScalarizeVariableInsertExtract(getWithDefaultOverride(
-                              ClScalarizeVariableInsertExtract,
-                              Options.ScalarizeVariableInsertExtract)),
-        ScalarizeLoadStore(getWithDefaultOverride(ClScalarizeLoadStore,
-                                                  Options.ScalarizeLoadStore)),
-        ScalarizeMinBits(getWithDefaultOverride(ClScalarizeMinBits,
-                                                Options.ScalarizeMinBits)) {}
+      : DT(DT), TTI(TTI),
+        ScalarizeVariableInsertExtract(Options.ScalarizeVariableInsertExtract),
+        ScalarizeLoadStore(Options.ScalarizeLoadStore),
+        ScalarizeMinBits(Options.ScalarizeMinBits) {}
 
   bool visit(Function &F);
 
diff --git a/llvm/lib/Transforms/Utils/CloneFunction.cpp b/llvm/lib/Transforms/Utils/CloneFunction.cpp
index dc9ca1423f3e7..fc03643e3542c 100644
--- a/llvm/lib/Transforms/Utils/CloneFunction.cpp
+++ b/llvm/lib/Transforms/Utils/CloneFunction.cpp
@@ -70,6 +70,7 @@ BasicBlock *llvm::CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap,
     if (isa<CallInst>(I) && !I.isDebugOrPseudoInst()) {
       hasCalls = true;
       hasMemProfMetadata |= I.hasMetadata(LLVMContext::MD_memprof);
+      hasMemProfMetadata |= I.hasMetadata(LLVMContext::MD_callsite);
     }
     if (const AllocaInst *AI = dyn_cast<AllocaInst>(&I)) {
       if (!AI->isStaticAlloca()) {
@@ -556,6 +557,7 @@ void PruningFunctionCloner::CloneBlock(
     if (isa<CallInst>(II) && !II->isDebugOrPseudoInst()) {
       hasCalls = true;
       hasMemProfMetadata |= II->hasMetadata(LLVMContext::MD_memprof);
+      hasMemProfMetadata |= II->hasMetadata(LLVMContext::MD_callsite);
     }
 
     CloneDbgRecordsToHere(NewInst, II);
diff --git a/llvm/lib/Transforms/Utils/IntegerDivision.cpp b/llvm/lib/Transforms/Utils/IntegerDivision.cpp
index cea095408b0c3..11956816a6ec3 100644
--- a/llvm/lib/Transforms/Utils/IntegerDivision.cpp
+++ b/llvm/lib/Transforms/Utils/IntegerDivision.cpp
@@ -70,8 +70,8 @@ static Value *generateSignedRemainderCode(Value *Dividend, Value *Divisor,
 /// code generated, e.g. at the urem instruction. This will generate a udiv in
 /// the process, and Builder's insert point will be pointing at the udiv (if
 /// present, i.e. not folded), ready to be expanded if the user wishes
-static Value *generatedUnsignedRemainderCode(Value *Dividend, Value *Divisor,
-                                             IRBuilder<> &Builder) {
+static Value *generateUnsignedRemainderCode(Value *Dividend, Value *Divisor,
+                                            IRBuilder<> &Builder) {
   // Remainder = Dividend - Quotient*Divisor
 
   // Following instructions are generated for both i32 and i64
@@ -381,9 +381,8 @@ bool llvm::expandRemainder(BinaryOperator *Rem) {
     Rem = BO;
   }
 
-  Value *Remainder = generatedUnsignedRemainderCode(Rem->getOperand(0),
-                                                    Rem->getOperand(1),
-                                                    Builder);
+  Value *Remainder = generateUnsignedRemainderCode(Rem->getOperand(0),
+                                                   Rem->getOperand(1), Builder);
 
   Rem->replaceAllUsesWith(Remainder);
   Rem->dropAllReferences();
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index 1088547e1f3ef..1ff3cd78aa987 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -28,7 +28,7 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
 
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
 #define SCEV_DEBUG_WITH_TYPE(TYPE, X) DEBUG_WITH_TYPE(TYPE, X)
 #else
 #define SCEV_DEBUG_WITH_TYPE(TYPE, X)
@@ -352,16 +352,19 @@ Value *SCEVExpander::InsertBinop(Instruction::BinaryOps Opcode,
 /// loop-invariant portions of expressions, after considering what
 /// can be folded using target addressing modes.
 ///
-Value *SCEVExpander::expandAddToGEP(const SCEV *Offset, Value *V) {
+Value *SCEVExpander::expandAddToGEP(const SCEV *Offset, Value *V,
+                                    SCEV::NoWrapFlags Flags) {
   assert(!isa<Instruction>(V) ||
          SE.DT.dominates(cast<Instruction>(V), &*Builder.GetInsertPoint()));
 
   Value *Idx = expand(Offset);
+  GEPNoWrapFlags NW = (Flags & SCEV::FlagNUW) ? GEPNoWrapFlags::noUnsignedWrap()
+                                              : GEPNoWrapFlags::none();
 
   // Fold a GEP with constant operands.
   if (Constant *CLHS = dyn_cast<Constant>(V))
     if (Constant *CRHS = dyn_cast<Constant>(Idx))
-      return Builder.CreatePtrAdd(CLHS, CRHS);
+      return Builder.CreatePtrAdd(CLHS, CRHS, "", NW);
 
   // Do a quick scan to see if we have this GEP nearby.  If so, reuse it.
   unsigned ScanLimit = 6;
@@ -380,7 +383,7 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *Offset, Value *V) {
             GEP->getSourceElementType() == Builder.getInt8Ty() &&
             GEP->getOperand(1) == Idx) {
           rememberFlags(GEP);
-          GEP->setNoWrapFlags(GEPNoWrapFlags::none());
+          GEP->setNoWrapFlags(GEP->getNoWrapFlags() & NW);
           return &*IP;
         }
       }
@@ -402,7 +405,7 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *Offset, Value *V) {
   }
 
   // Emit a GEP.
-  return Builder.CreatePtrAdd(V, Idx, "scevgep");
+  return Builder.CreatePtrAdd(V, Idx, "scevgep", NW);
 }
 
 /// PickMostRelevantLoop - Given two loops pick the one that's most relevant for
@@ -549,7 +552,7 @@ Value *SCEVExpander::visitAddExpr(const SCEVAddExpr *S) {
             X = SE.getSCEV(U->getValue());
         NewOps.push_back(X);
       }
-      Sum = expandAddToGEP(SE.getAddExpr(NewOps), Sum);
+      Sum = expandAddToGEP(SE.getAddExpr(NewOps), Sum, S->getNoWrapFlags());
     } else if (Op->isNonConstantNegative()) {
       // Instead of doing a negate and add, just do a subtract.
       Value *W = expand(SE.getNegativeSCEV(Op));
@@ -1251,7 +1254,8 @@ Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) {
   if (!S->getStart()->isZero()) {
     if (isa<PointerType>(S->getType())) {
       Value *StartV = expand(SE.getPointerBase(S));
-      return expandAddToGEP(SE.removePointerBase(S), StartV);
+      return expandAddToGEP(SE.removePointerBase(S), StartV,
+                            S->getNoWrapFlags(SCEV::FlagNUW));
     }
 
     SmallVector<const SCEV *, 4> NewOps(S->operands());
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index ffd4037f23ffa..18d26aaf46066 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -1594,7 +1594,7 @@ static void hoistLockstepIdenticalDbgVariableRecords(
 
 static bool areIdenticalUpToCommutativity(const Instruction *I1,
                                           const Instruction *I2) {
-  if (I1->isIdenticalToWhenDefined(I2))
+  if (I1->isIdenticalToWhenDefined(I2, /*IntersectAttrs=*/true))
     return true;
 
   if (auto *Cmp1 = dyn_cast<CmpInst>(I1))
@@ -1910,6 +1910,14 @@ bool SimplifyCFGOpt::hoistCommonCodeFromSuccessors(Instruction *TI,
           if (!I2->use_empty())
             I2->replaceAllUsesWith(I1);
           I1->andIRFlags(I2);
+          if (auto *CB = dyn_cast<CallBase>(I1)) {
+            bool Success = CB->tryIntersectAttributes(cast<CallBase>(I2));
+            assert(Success && "We should not be trying to hoist callbases "
+                              "with non-intersectable attributes");
+            // For NDEBUG Compile.
+            (void)Success;
+          }
+
           combineMetadataForCSE(I1, I2, true);
           // I1 and I2 are being combined into a single instruction.  Its debug
           // location is the merged locations of the original instructions.
@@ -2130,7 +2138,7 @@ static bool canSinkInstructions(
   const Instruction *I0 = Insts.front();
   const auto I0MMRA = MMRAMetadata(*I0);
   for (auto *I : Insts) {
-    if (!I->isSameOperationAs(I0))
+    if (!I->isSameOperationAs(I0, Instruction::CompareUsingIntersectedAttrs))
       return false;
 
     // swifterror pointers can only be used by a load or store; sinking a load
@@ -2287,6 +2295,13 @@ static void sinkLastInstruction(ArrayRef<BasicBlock*> Blocks) {
       I0->applyMergedLocation(I0->getDebugLoc(), I->getDebugLoc());
       combineMetadataForCSE(I0, I, true);
       I0->andIRFlags(I);
+      if (auto *CB = dyn_cast<CallBase>(I0)) {
+        bool Success = CB->tryIntersectAttributes(cast<CallBase>(I));
+        assert(Success && "We should not be trying to sink callbases "
+                          "with non-intersectable attributes");
+        // For NDEBUG Compile.
+        (void)Success;
+      }
     }
 
   for (User *U : make_early_inc_range(I0->users())) {
diff --git a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
index ff13c0653e9b9..7fca1a6aa5260 100644
--- a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
@@ -1024,7 +1024,7 @@ bool simplifyLoopIVs(Loop *L, ScalarEvolution *SE, DominatorTree *DT,
                      LoopInfo *LI, const TargetTransformInfo *TTI,
                      SmallVectorImpl<WeakTrackingVH> &Dead) {
   SCEVExpander Rewriter(*SE, SE->getDataLayout(), "indvars");
-#ifndef NDEBUG
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
   Rewriter.setDebugType(DEBUG_TYPE);
 #endif
   bool Changed = false;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f5ef50934f59f..6e082b1c134de 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4054,7 +4054,13 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
     unsigned MaxVFtimesIC =
         UserIC ? *MaxPowerOf2RuntimeVF * UserIC : *MaxPowerOf2RuntimeVF;
     ScalarEvolution *SE = PSE.getSE();
-    const SCEV *BackedgeTakenCount = PSE.getBackedgeTakenCount();
+    // Currently only loops with countable exits are vectorized, but calling
+    // getSymbolicMaxBackedgeTakenCount allows enablement work for loops with
+    // uncountable exits whilst also ensuring the symbolic maximum and known
+    // back-edge taken count remain identical for loops with countable exits.
+    const SCEV *BackedgeTakenCount = PSE.getSymbolicMaxBackedgeTakenCount();
+    assert(BackedgeTakenCount == PSE.getBackedgeTakenCount() &&
+           "Invalid loop count");
     const SCEV *ExitCount = SE->getAddExpr(
         BackedgeTakenCount, SE->getOne(BackedgeTakenCount->getType()));
     const SCEV *Rem = SE->getURemExpr(
@@ -6545,14 +6551,17 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
       Op2 = cast<SCEVConstant>(PSE.getSCEV(Op2))->getValue();
     }
     auto Op2Info = TTI.getOperandInfo(Op2);
-    auto IsInvariant = [this](Value *Op) {
+    std::function<bool(Value *)> IsInvariant =
+        [this, &IsInvariant](Value *Op) -> bool {
       if (!Legal->isInvariant(Op))
         return false;
-      // Consider Op2 invariant, if it is not a predicated instruction in the
-      // loop. In that case, it is not trivially hoistable.
+      // Consider Op2invariant, if it or its operands aren't predicated
+      // instruction in the loop. In that case, it is not trivially hoistable.
       return !isa<Instruction>(Op) ||
              !TheLoop->contains(cast<Instruction>(Op)) ||
-             !isPredicatedInst(cast<Instruction>(Op));
+             (!isPredicatedInst(cast<Instruction>(Op)) &&
+              all_of(cast<Instruction>(Op)->operands(),
+                     [&IsInvariant](Value *Op) { return IsInvariant(Op); }));
     };
     if (Op2Info.Kind == TargetTransformInfo::OK_AnyValue && IsInvariant(Op2))
       Op2Info.Kind = TargetTransformInfo::OK_UniformValue;
@@ -8414,21 +8423,28 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
   case Instruction::Sub:
   case Instruction::Xor:
   case Instruction::Freeze:
-    if (I->getOpcode() == Instruction::Mul) {
-      // Simplify operands of multiplications using SCEV. This is needed at the
-      // moment to match the behavior of the legacy cost-model.
-      // TODO: Generalize to any opcode and move to VPlan transformation.
-      SmallVector<VPValue *> NewOps(Operands);
+    SmallVector<VPValue *> NewOps(Operands);
+    if (Instruction::isBinaryOp(I->getOpcode())) {
+      // The legacy cost model uses SCEV to check if some of the operands are
+      // constants. To match the legacy cost model's behavior, use SCEV to try
+      // to replace operands with constants.
       ScalarEvolution &SE = *PSE.getSE();
-      for (unsigned I = 0; I < Operands.size(); ++I) {
-        Value *V = NewOps[I]->getUnderlyingValue();
-        if (!isa<Constant>(V) && SE.isSCEVable(V->getType()))
-          if (auto *C = dyn_cast<SCEVConstant>(PSE.getSE()->getSCEV(V)))
-            NewOps[I] = Plan.getOrAddLiveIn(C->getValue());
-      }
-      return new VPWidenRecipe(*I, make_range(NewOps.begin(), NewOps.end()));
+      auto GetConstantViaSCEV = [this, &SE](VPValue *Op) {
+        Value *V = Op->getUnderlyingValue();
+        if (isa<Constant>(V) || !SE.isSCEVable(V->getType()))
+          return Op;
+        auto *C = dyn_cast<SCEVConstant>(SE.getSCEV(V));
+        if (!C)
+          return Op;
+        return Plan.getOrAddLiveIn(C->getValue());
+      };
+      // For Mul, the legacy cost model checks both operands.
+      if (I->getOpcode() == Instruction::Mul)
+        NewOps[0] = GetConstantViaSCEV(NewOps[0]);
+      // For other binops, the legacy cost model only checks the second operand.
+      NewOps[1] = GetConstantViaSCEV(NewOps[1]);
     }
-    return new VPWidenRecipe(*I, make_range(Operands.begin(), Operands.end()));
+    return new VPWidenRecipe(*I, make_range(NewOps.begin(), NewOps.end()));
   };
 }
 
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 54a1b99606bcb..dc9ad5335f8a5 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -79,6 +79,7 @@
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/DOTGraphTraits.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugCounter.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/GraphWriter.h"
 #include "llvm/Support/InstructionCost.h"
@@ -109,6 +110,9 @@ using namespace slpvectorizer;
 
 STATISTIC(NumVectorInstructions, "Number of vector instructions generated");
 
+DEBUG_COUNTER(VectorizedGraphs, "slp-vectorized",
+              "Controls which SLP graphs should be vectorized.");
+
 static cl::opt<bool>
     RunSLPVectorization("vectorize-slp", cl::init(true), cl::Hidden,
                         cl::desc("Run the SLP vectorization passes"));
@@ -827,8 +831,7 @@ static bool isValidForAlternation(unsigned Opcode) {
 }
 
 static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
-                                       const TargetLibraryInfo &TLI,
-                                       unsigned BaseIndex = 0);
+                                       const TargetLibraryInfo &TLI);
 
 /// Checks if the provided operands of 2 cmp instructions are compatible, i.e.
 /// compatible instructions or constants, or just some other regular values.
@@ -869,8 +872,8 @@ static bool isCmpSameOrSwapped(const CmpInst *BaseCI, const CmpInst *CI,
 /// InstructionsState, the Opcode that we suppose the whole list
 /// could be vectorized even if its structure is diverse.
 static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
-                                       const TargetLibraryInfo &TLI,
-                                       unsigned BaseIndex) {
+                                       const TargetLibraryInfo &TLI) {
+  constexpr unsigned BaseIndex = 0;
   // Make sure these are all Instructions.
   if (llvm::any_of(VL, [](Value *V) { return !isa<Instruction>(V); }))
     return InstructionsState(VL[BaseIndex], nullptr, nullptr);
@@ -2309,10 +2312,10 @@ class BoUpSLP {
       assert((empty() || VL.size() == getNumLanes()) &&
              "Expected same number of lanes");
       assert(isa<Instruction>(VL[0]) && "Expected instruction");
-      unsigned NumOperands = cast<Instruction>(VL[0])->getNumOperands();
       constexpr unsigned IntrinsicNumOperands = 2;
-      if (isa<IntrinsicInst>(VL[0]))
-        NumOperands = IntrinsicNumOperands;
+      unsigned NumOperands = isa<IntrinsicInst>(VL[0])
+                                 ? IntrinsicNumOperands
+                                 : cast<Instruction>(VL[0])->getNumOperands();
       OpsVec.resize(NumOperands);
       unsigned NumLanes = VL.size();
       for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
@@ -4575,6 +4578,8 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE) {
               continue;
             if (!TE.ReuseShuffleIndices.empty())
               K = TE.ReuseShuffleIndices[K];
+            if (K == PoisonMaskElem)
+              continue;
             if (!TE.ReorderIndices.empty())
               K = std::distance(TE.ReorderIndices.begin(),
                                 find(TE.ReorderIndices, K));
@@ -5443,6 +5448,33 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) {
     if (!TE.ReorderIndices.empty())
       return TE.ReorderIndices;
 
+    SmallVector<Instruction *> UserBVHead(TE.Scalars.size());
+    for (auto [I, V] : zip(UserBVHead, TE.Scalars)) {
+      if (!V->hasNUsesOrMore(1))
+        continue;
+      auto *II = dyn_cast<InsertElementInst>(*V->user_begin());
+      if (!II)
+        continue;
+      Instruction *BVHead = nullptr;
+      BasicBlock *BB = II->getParent();
+      while (II && II->hasOneUse() && II->getParent() == BB) {
+        BVHead = II;
+        II = dyn_cast<InsertElementInst>(II->getOperand(0));
+      }
+      I = BVHead;
+    }
+
+    auto CompareByBasicBlocks = [&](BasicBlock *BB1, BasicBlock *BB2) {
+      assert(BB1 != BB2 && "Expected different basic blocks.");
+      auto *NodeA = DT->getNode(BB1);
+      auto *NodeB = DT->getNode(BB2);
+      assert(NodeA && "Should only process reachable instructions");
+      assert(NodeB && "Should only process reachable instructions");
+      assert((NodeA == NodeB) ==
+                 (NodeA->getDFSNumIn() == NodeB->getDFSNumIn()) &&
+             "Different nodes should have different DFS numbers");
+      return NodeA->getDFSNumIn() < NodeB->getDFSNumIn();
+    };
     auto PHICompare = [&](unsigned I1, unsigned I2) {
       Value *V1 = TE.Scalars[I1];
       Value *V2 = TE.Scalars[I2];
@@ -5454,21 +5486,56 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) {
         return false;
       auto *FirstUserOfPhi1 = cast<Instruction>(*V1->user_begin());
       auto *FirstUserOfPhi2 = cast<Instruction>(*V2->user_begin());
-      if (auto *IE1 = dyn_cast<InsertElementInst>(FirstUserOfPhi1))
-        if (auto *IE2 = dyn_cast<InsertElementInst>(FirstUserOfPhi2)) {
-          if (!areTwoInsertFromSameBuildVector(
-                  IE1, IE2,
-                  [](InsertElementInst *II) { return II->getOperand(0); }))
-            return I1 < I2;
+      if (FirstUserOfPhi1->getParent() != FirstUserOfPhi2->getParent())
+        return CompareByBasicBlocks(FirstUserOfPhi1->getParent(),
+                                    FirstUserOfPhi2->getParent());
+      auto *IE1 = dyn_cast<InsertElementInst>(FirstUserOfPhi1);
+      auto *IE2 = dyn_cast<InsertElementInst>(FirstUserOfPhi2);
+      auto *EE1 = dyn_cast<ExtractElementInst>(FirstUserOfPhi1);
+      auto *EE2 = dyn_cast<ExtractElementInst>(FirstUserOfPhi2);
+      if (IE1 && !IE2)
+        return true;
+      if (!IE1 && IE2)
+        return false;
+      if (IE1 && IE2) {
+        if (UserBVHead[I1] && !UserBVHead[I2])
+          return true;
+        if (!UserBVHead[I1])
+          return false;
+        if (UserBVHead[I1] == UserBVHead[I2])
           return getElementIndex(IE1) < getElementIndex(IE2);
-        }
-      if (auto *EE1 = dyn_cast<ExtractElementInst>(FirstUserOfPhi1))
-        if (auto *EE2 = dyn_cast<ExtractElementInst>(FirstUserOfPhi2)) {
-          if (EE1->getOperand(0) != EE2->getOperand(0))
-            return I1 < I2;
+        if (UserBVHead[I1]->getParent() != UserBVHead[I2]->getParent())
+          return CompareByBasicBlocks(UserBVHead[I1]->getParent(),
+                                      UserBVHead[I2]->getParent());
+        return UserBVHead[I1]->comesBefore(UserBVHead[I2]);
+      }
+      if (EE1 && !EE2)
+        return true;
+      if (!EE1 && EE2)
+        return false;
+      if (EE1 && EE2) {
+        auto *Inst1 = dyn_cast<Instruction>(EE1->getOperand(0));
+        auto *Inst2 = dyn_cast<Instruction>(EE2->getOperand(0));
+        auto *P1 = dyn_cast<Argument>(EE1->getOperand(0));
+        auto *P2 = dyn_cast<Argument>(EE2->getOperand(0));
+        if (!Inst2 && !P2)
+          return Inst1 || P1;
+        if (EE1->getOperand(0) == EE2->getOperand(0))
           return getElementIndex(EE1) < getElementIndex(EE2);
+        if (!Inst1 && Inst2)
+          return false;
+        if (Inst1 && Inst2) {
+          if (Inst1->getParent() != Inst2->getParent())
+            return CompareByBasicBlocks(Inst1->getParent(), Inst2->getParent());
+          return Inst1->comesBefore(Inst2);
         }
-      return I1 < I2;
+        if (!P1 && P2)
+          return false;
+        assert(P1 && P2 &&
+               "Expected either instructions or arguments vector operands.");
+        return P1->getArgNo() < P2->getArgNo();
+      }
+      return false;
     };
     SmallDenseMap<unsigned, unsigned, 16> PhiToId;
     SmallVector<unsigned> Phis(TE.Scalars.size());
@@ -6516,7 +6583,8 @@ static void gatherPossiblyVectorizableLoads(
           if (Idx < Start)
             continue;
           ToAdd.clear();
-          if (LI->getParent() != Data.front().first->getParent())
+          if (LI->getParent() != Data.front().first->getParent() ||
+              LI->getType() != Data.front().first->getType())
             continue;
           std::optional<int> Dist =
               getPointersDiff(LI->getType(), LI->getPointerOperand(),
@@ -9181,12 +9249,13 @@ void BoUpSLP::transformNodes() {
         for (unsigned Cnt : Slices) {
           ArrayRef<Value *> Slice = VL.slice(Cnt, VF);
           // If any instruction is vectorized already - do not try again.
-          if (const TreeEntry *SE = getTreeEntry(Slice.front());
+          if (TreeEntry *SE = getTreeEntry(Slice.front());
               SE || getTreeEntry(Slice.back())) {
             if (!SE)
               continue;
             if (VF != SE->getVectorFactor() || !SE->isSame(Slice))
               continue;
+            SE->UserTreeIndices.emplace_back(&E, UINT_MAX);
             AddCombinedNode(SE->Idx, Cnt);
             continue;
           }
@@ -11175,6 +11244,9 @@ bool BoUpSLP::isLoadCombineCandidate(ArrayRef<Value *> Stores) const {
 }
 
 bool BoUpSLP::isTreeTinyAndNotFullyVectorizable(bool ForReduction) const {
+  if (!DebugCounter::shouldExecute(VectorizedGraphs))
+    return true;
+
   // No need to vectorize inserts of gathered values.
   if (VectorizableTree.size() == 2 &&
       isa<InsertElementInst>(VectorizableTree[0]->Scalars[0]) &&
@@ -11571,6 +11643,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
   DenseSet<std::pair<const TreeEntry *, Type *>> VectorCasts;
   std::optional<DenseMap<Value *, unsigned>> ValueToExtUses;
   DenseMap<const TreeEntry *, DenseSet<Value *>> ExtractsCount;
+  SmallPtrSet<Value *, 4> ScalarOpsFromCasts;
   for (ExternalUser &EU : ExternalUses) {
     // Uses by ephemeral values are free (because the ephemeral value will be
     // removed prior to code generation, and so the extraction will be
@@ -11706,7 +11779,8 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
       // Can use original instruction, if no operands vectorized or they are
       // marked as externally used already.
       auto *Inst = cast<Instruction>(EU.Scalar);
-      bool CanBeUsedAsScalar = all_of(Inst->operands(), [&](Value *V) {
+      InstructionCost ScalarCost = TTI->getInstructionCost(Inst, CostKind);
+      auto OperandIsScalar = [&](Value *V) {
         if (!getTreeEntry(V)) {
           // Some extractelements might be not vectorized, but
           // transformed into shuffle and removed from the function,
@@ -11716,9 +11790,23 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
           return true;
         }
         return ValueToExtUses->contains(V);
-      });
+      };
+      bool CanBeUsedAsScalar = all_of(Inst->operands(), OperandIsScalar);
+      bool CanBeUsedAsScalarCast = false;
+      if (auto *CI = dyn_cast<CastInst>(Inst); CI && !CanBeUsedAsScalar) {
+        if (auto *Op = dyn_cast<Instruction>(CI->getOperand(0));
+            Op && all_of(Op->operands(), OperandIsScalar)) {
+          InstructionCost OpCost =
+              (getTreeEntry(Op) && !ValueToExtUses->contains(Op))
+                  ? TTI->getInstructionCost(Op, CostKind)
+                  : 0;
+          if (ScalarCost + OpCost <= ExtraCost) {
+            CanBeUsedAsScalar = CanBeUsedAsScalarCast = true;
+            ScalarCost += OpCost;
+          }
+        }
+      }
       if (CanBeUsedAsScalar) {
-        InstructionCost ScalarCost = TTI->getInstructionCost(Inst, CostKind);
         bool KeepScalar = ScalarCost <= ExtraCost;
         // Try to keep original scalar if the user is the phi node from the same
         // block as the root phis, currently vectorized. It allows to keep
@@ -11774,12 +11862,34 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
           ExtraCost = ScalarCost;
           if (!IsPhiInLoop(EU))
             ExtractsCount[Entry].insert(Inst);
+          if (CanBeUsedAsScalarCast) {
+            ScalarOpsFromCasts.insert(Inst->getOperand(0));
+            // Update the users of the operands of the cast operand to avoid
+            // compiler crash.
+            if (auto *IOp = dyn_cast<Instruction>(Inst->getOperand(0))) {
+              for_each(IOp->operands(), [&](Value *V) {
+                auto It = ValueToExtUses->find(V);
+                if (It != ValueToExtUses->end()) {
+                  // Replace all uses to avoid compiler crash.
+                  ExternalUses[It->second].User = nullptr;
+                }
+              });
+            }
+          }
         }
       }
     }
 
     ExtractCost += ExtraCost;
   }
+  // Insert externals for extract of operands of casts to be emitted as scalars
+  // instead of extractelement.
+  for (Value *V : ScalarOpsFromCasts) {
+    ExternalUsesAsOriginalScalar.insert(V);
+    if (const TreeEntry *E = getTreeEntry(V)) {
+      ExternalUses.emplace_back(V, nullptr, E->findLaneForValue(V));
+    }
+  }
   // Add reduced value cost, if resized.
   if (!VectorizedVals.empty()) {
     const TreeEntry &Root = *VectorizableTree.front();
@@ -13095,7 +13205,8 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
       UniqueBases.insert(VecBase);
       // If the only one use is vectorized - can delete the extractelement
       // itself.
-      if (!EI->hasOneUse() || (NumParts != 1 && count(E->Scalars, EI) > 1) ||
+      if (!EI->hasOneUse() || R.ExternalUsesAsOriginalScalar.contains(EI) ||
+          (NumParts != 1 && count(E->Scalars, EI) > 1) ||
           any_of(EI->users(), [&](User *U) {
             const TreeEntry *UTE = R.getTreeEntry(U);
             return !UTE || R.MultiNodeScalars.contains(U) ||
@@ -13400,7 +13511,12 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
         if (CommonMask[Idx] != PoisonMaskElem)
           CommonMask[Idx] = Idx;
       for (auto [E, Idx] : SubVectors) {
-        Value *V = castToScalarTyElem(E->VectorizedValue);
+        Value *V = E->VectorizedValue;
+        if (V->getType()->isIntOrIntVectorTy())
+          V = castToScalarTyElem(V, any_of(E->Scalars, [&](Value *V) {
+                                   return !isKnownNonNegative(
+                                       V, SimplifyQuery(*R.DL));
+                                 }));
         Vec = Builder.CreateInsertVector(Vec->getType(), Vec, V,
                                          Builder.getInt64(Idx));
         if (!CommonMask.empty()) {
@@ -18809,6 +18925,7 @@ class HorizontalReduction {
             ++SameValuesCounter.try_emplace(OrigV).first->second;
           }
         }
+        V.transformNodes();
         SmallPtrSet<Value *, 4> VLScalars(VL.begin(), VL.end());
         // Gather externally used values.
         SmallPtrSet<Value *, 4> Visited;
@@ -18838,7 +18955,6 @@ class HorizontalReduction {
         for (Value *RdxVal : VL)
           if (RequiredExtract.contains(RdxVal))
             LocalExternallyUsedValues[RdxVal];
-        V.transformNodes();
         V.buildExternalUses(LocalExternallyUsedValues);
 
         V.computeMinimumValueSizes();
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp
index 67b56451c7b59..10da07c24940d 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp
@@ -31,6 +31,25 @@ void DGNode::dump() const {
 }
 #endif // NDEBUG
 
+Interval<MemDGNode>
+MemDGNodeIntervalBuilder::make(const Interval<Instruction> &Instrs,
+                               DependencyGraph &DAG) {
+  // If top or bottom instructions are not mem-dep candidate nodes we need to
+  // walk down/up the chain and find the mem-dep ones.
+  Instruction *MemTopI = Instrs.top();
+  Instruction *MemBotI = Instrs.bottom();
+  while (!DGNode::isMemDepNodeCandidate(MemTopI) && MemTopI != MemBotI)
+    MemTopI = MemTopI->getNextNode();
+  while (!DGNode::isMemDepNodeCandidate(MemBotI) && MemBotI != MemTopI)
+    MemBotI = MemBotI->getPrevNode();
+  // If we couldn't find a mem node in range TopN - BotN then it's empty.
+  if (!DGNode::isMemDepNodeCandidate(MemTopI))
+    return {};
+  // Now that we have the mem-dep nodes, create and return the range.
+  return Interval<MemDGNode>(cast<MemDGNode>(DAG.getNode(MemTopI)),
+                             cast<MemDGNode>(DAG.getNode(MemBotI)));
+}
+
 Interval<Instruction> DependencyGraph::extend(ArrayRef<Instruction *> Instrs) {
   if (Instrs.empty())
     return {};
@@ -39,10 +58,18 @@ Interval<Instruction> DependencyGraph::extend(ArrayRef<Instruction *> Instrs) {
   auto *TopI = Interval.top();
   auto *BotI = Interval.bottom();
   DGNode *LastN = getOrCreateNode(TopI);
+  MemDGNode *LastMemN = dyn_cast<MemDGNode>(LastN);
   for (Instruction *I = TopI->getNextNode(), *E = BotI->getNextNode(); I != E;
        I = I->getNextNode()) {
     auto *N = getOrCreateNode(I);
-    N->addMemPred(LastN);
+    N->addMemPred(LastMemN);
+    // Build the Mem node chain.
+    if (auto *MemN = dyn_cast<MemDGNode>(N)) {
+      MemN->setPrevNode(LastMemN);
+      if (LastMemN != nullptr)
+        LastMemN->setNextNode(MemN);
+      LastMemN = MemN;
+    }
     LastN = N;
   }
   return Interval;
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 4247d20cb0e53..5e3a638809494 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -881,11 +881,18 @@ VPlanPtr VPlan::createInitialVPlan(Type *InductionTy,
   auto Plan = std::make_unique<VPlan>(Entry, VecPreheader);
 
   // Create SCEV and VPValue for the trip count.
-  const SCEV *BackedgeTakenCount = PSE.getBackedgeTakenCount();
-  assert(!isa<SCEVCouldNotCompute>(BackedgeTakenCount) && "Invalid loop count");
+
+  // Currently only loops with countable exits are vectorized, but calling
+  // getSymbolicMaxBackedgeTakenCount allows enablement work for loops with
+  // uncountable exits whilst also ensuring the symbolic maximum and known
+  // back-edge taken count remain identical for loops with countable exits.
+  const SCEV *BackedgeTakenCountSCEV = PSE.getSymbolicMaxBackedgeTakenCount();
+  assert((!isa<SCEVCouldNotCompute>(BackedgeTakenCountSCEV) &&
+          BackedgeTakenCountSCEV == PSE.getBackedgeTakenCount()) &&
+         "Invalid loop count");
   ScalarEvolution &SE = *PSE.getSE();
-  const SCEV *TripCount =
-      SE.getTripCountFromExitCount(BackedgeTakenCount, InductionTy, TheLoop);
+  const SCEV *TripCount = SE.getTripCountFromExitCount(BackedgeTakenCountSCEV,
+                                                       InductionTy, TheLoop);
   Plan->TripCount =
       vputils::getOrCreateVPValueForSCEVExpr(*Plan, TripCount, SE);
 
diff --git a/llvm/test/Analysis/CostModel/PowerPC/vsr_load_32_64.ll b/llvm/test/Analysis/CostModel/PowerPC/vsr_load_32_64.ll
index 54cafa0ae59f3..17bcdd4d3f44c 100644
--- a/llvm/test/Analysis/CostModel/PowerPC/vsr_load_32_64.ll
+++ b/llvm/test/Analysis/CostModel/PowerPC/vsr_load_32_64.ll
@@ -1,15 +1,16 @@
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 -mattr=+vsx | FileCheck %s
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 -mattr=+vsx | FileCheck -DCOST32=1 %s
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -mattr=+vsx | FileCheck -DCOST32=2 %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
 define i32 @loads(i32 %arg) {
-  ; CHECK: cost of 1 {{.*}} load
+  ; CHECK: cost of [[COST32]] {{.*}} load
   load <4 x i8>, ptr undef, align 1
 
   ; CHECK: cost of 1 {{.*}} load
   load <8 x i8>, ptr undef, align 1
 
-  ; CHECK: cost of 1 {{.*}} load
+  ; CHECK: cost of [[COST32]] {{.*}} load
   load <2 x i16>, ptr undef, align 2
 
   ; CHECK: cost of 1 {{.*}} load
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-fadd.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-fadd.ll
index 71f614d7003fa..afb2b64464521 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-fadd.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-fadd.ll
@@ -1,18 +1,30 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfh -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s --check-prefix=FP-REDUCE
+; RUN: opt < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfh -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s --check-prefixes=FP-REDUCE,FP-REDUCE-ZVFH
+; RUN: opt < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfhmin -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s --check-prefixes=FP-REDUCE,FP-REDUCE-ZVFHMIN
 ; RUN: opt < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfh -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output | FileCheck %s  --check-prefix=SIZE
 
 define void @reduce_fadd_half() {
-; FP-REDUCE-LABEL: 'reduce_fadd_half'
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call fast half @llvm.vector.reduce.fadd.v1f16(half 0xH0000, <1 x half> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call fast half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call fast half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call fast half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call fast half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32 = call fast half @llvm.vector.reduce.fadd.v32f16(half 0xH0000, <32 x half> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call fast half @llvm.vector.reduce.fadd.v64f16(half 0xH0000, <64 x half> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128 = call fast half @llvm.vector.reduce.fadd.v128f16(half 0xH0000, <128 x half> undef)
-; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; FP-REDUCE-ZVFH-LABEL: 'reduce_fadd_half'
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call fast half @llvm.vector.reduce.fadd.v1f16(half 0xH0000, <1 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call fast half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call fast half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call fast half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call fast half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32 = call fast half @llvm.vector.reduce.fadd.v32f16(half 0xH0000, <32 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call fast half @llvm.vector.reduce.fadd.v64f16(half 0xH0000, <64 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128 = call fast half @llvm.vector.reduce.fadd.v128f16(half 0xH0000, <128 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; FP-REDUCE-ZVFHMIN-LABEL: 'reduce_fadd_half'
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %V1 = call fast half @llvm.vector.reduce.fadd.v1f16(half 0xH0000, <1 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %V2 = call fast half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %V4 = call fast half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %V8 = call fast half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %V16 = call fast half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %v32 = call fast half @llvm.vector.reduce.fadd.v32f16(half 0xH0000, <32 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %V64 = call fast half @llvm.vector.reduce.fadd.v64f16(half 0xH0000, <64 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Invalid cost for instruction: %V128 = call fast half @llvm.vector.reduce.fadd.v128f16(half 0xH0000, <128 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SIZE-LABEL: 'reduce_fadd_half'
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call fast half @llvm.vector.reduce.fadd.v1f16(half 0xH0000, <1 x half> undef)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-fmul.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-fmul.ll
new file mode 100644
index 0000000000000..da1336aa724c9
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-fmul.ll
@@ -0,0 +1,219 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfh -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s --check-prefixes=FP-REDUCE,FP-REDUCE-ZVFH
+; RUN: opt < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfhmin -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s --check-prefixes=FP-REDUCE,FP-REDUCE-ZVFHMIN
+; RUN: opt < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfh -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output | FileCheck %s  --check-prefix=SIZE
+
+define void @reduce_fmul_half() {
+; FP-REDUCE-ZVFH-LABEL: 'reduce_fmul_half'
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call fast half @llvm.vector.reduce.fmul.v1f16(half 0xH0000, <1 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call fast half @llvm.vector.reduce.fmul.v2f16(half 0xH0000, <2 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4 = call fast half @llvm.vector.reduce.fmul.v4f16(half 0xH0000, <4 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8 = call fast half @llvm.vector.reduce.fmul.v8f16(half 0xH0000, <8 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16 = call fast half @llvm.vector.reduce.fmul.v16f16(half 0xH0000, <16 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 151 for instruction: %v32 = call fast half @llvm.vector.reduce.fmul.v32f16(half 0xH0000, <32 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 541 for instruction: %V64 = call fast half @llvm.vector.reduce.fmul.v64f16(half 0xH0000, <64 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 573 for instruction: %V128 = call fast half @llvm.vector.reduce.fmul.v128f16(half 0xH0000, <128 x half> undef)
+; FP-REDUCE-ZVFH-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; FP-REDUCE-ZVFHMIN-LABEL: 'reduce_fmul_half'
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call fast half @llvm.vector.reduce.fmul.v1f16(half 0xH0000, <1 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call fast half @llvm.vector.reduce.fmul.v2f16(half 0xH0000, <2 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4 = call fast half @llvm.vector.reduce.fmul.v4f16(half 0xH0000, <4 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8 = call fast half @llvm.vector.reduce.fmul.v8f16(half 0xH0000, <8 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 73 for instruction: %V16 = call fast half @llvm.vector.reduce.fmul.v16f16(half 0xH0000, <16 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 211 for instruction: %v32 = call fast half @llvm.vector.reduce.fmul.v32f16(half 0xH0000, <32 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 541 for instruction: %V64 = call fast half @llvm.vector.reduce.fmul.v64f16(half 0xH0000, <64 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 573 for instruction: %V128 = call fast half @llvm.vector.reduce.fmul.v128f16(half 0xH0000, <128 x half> undef)
+; FP-REDUCE-ZVFHMIN-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SIZE-LABEL: 'reduce_fmul_half'
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call fast half @llvm.vector.reduce.fmul.v1f16(half 0xH0000, <1 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call fast half @llvm.vector.reduce.fmul.v2f16(half 0xH0000, <2 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call fast half @llvm.vector.reduce.fmul.v4f16(half 0xH0000, <4 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call fast half @llvm.vector.reduce.fmul.v8f16(half 0xH0000, <8 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16 = call fast half @llvm.vector.reduce.fmul.v16f16(half 0xH0000, <16 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %v32 = call fast half @llvm.vector.reduce.fmul.v32f16(half 0xH0000, <32 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64 = call fast half @llvm.vector.reduce.fmul.v64f16(half 0xH0000, <64 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V128 = call fast half @llvm.vector.reduce.fmul.v128f16(half 0xH0000, <128 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V1 = call fast half @llvm.vector.reduce.fmul.v1f16(half 0.0, <1 x half> undef)
+  %V2 = call fast half @llvm.vector.reduce.fmul.v2f16(half 0.0, <2 x half> undef)
+  %V4 = call fast half @llvm.vector.reduce.fmul.v4f16(half 0.0, <4 x half> undef)
+  %V8 = call fast half @llvm.vector.reduce.fmul.v8f16(half 0.0, <8 x half> undef)
+  %V16 = call fast half @llvm.vector.reduce.fmul.v16f16(half 0.0, <16 x half> undef)
+  %v32 = call fast half @llvm.vector.reduce.fmul.v32f16(half 0.0, <32 x half> undef)
+  %V64 = call fast half @llvm.vector.reduce.fmul.v64f16(half 0.0, <64 x half> undef)
+  %V128 = call fast half @llvm.vector.reduce.fmul.v128f16(half 0.0, <128 x half> undef)
+  ret void
+}
+
+define void @reduce_fmul_float() {
+; FP-REDUCE-LABEL: 'reduce_fmul_float'
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call fast float @llvm.vector.reduce.fmul.v1f32(float 0.000000e+00, <1 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call fast float @llvm.vector.reduce.fmul.v2f32(float 0.000000e+00, <2 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4 = call fast float @llvm.vector.reduce.fmul.v4f32(float 0.000000e+00, <4 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8 = call fast float @llvm.vector.reduce.fmul.v8f32(float 0.000000e+00, <8 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 121 for instruction: %V16 = call fast float @llvm.vector.reduce.fmul.v16f32(float 0.000000e+00, <16 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 451 for instruction: %v32 = call fast float @llvm.vector.reduce.fmul.v32f32(float 0.000000e+00, <32 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 483 for instruction: %V64 = call fast float @llvm.vector.reduce.fmul.v64f32(float 0.000000e+00, <64 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 547 for instruction: %V128 = call fast float @llvm.vector.reduce.fmul.v128f32(float 0.000000e+00, <128 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SIZE-LABEL: 'reduce_fmul_float'
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call fast float @llvm.vector.reduce.fmul.v1f32(float 0.000000e+00, <1 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call fast float @llvm.vector.reduce.fmul.v2f32(float 0.000000e+00, <2 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call fast float @llvm.vector.reduce.fmul.v4f32(float 0.000000e+00, <4 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call fast float @llvm.vector.reduce.fmul.v8f32(float 0.000000e+00, <8 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16 = call fast float @llvm.vector.reduce.fmul.v16f32(float 0.000000e+00, <16 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %v32 = call fast float @llvm.vector.reduce.fmul.v32f32(float 0.000000e+00, <32 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V64 = call fast float @llvm.vector.reduce.fmul.v64f32(float 0.000000e+00, <64 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V128 = call fast float @llvm.vector.reduce.fmul.v128f32(float 0.000000e+00, <128 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V1 = call fast float @llvm.vector.reduce.fmul.v1f32(float 0.0, <1 x float> undef)
+  %V2 = call fast float @llvm.vector.reduce.fmul.v2f32(float 0.0, <2 x float> undef)
+  %V4 = call fast float @llvm.vector.reduce.fmul.v4f32(float 0.0, <4 x float> undef)
+  %V8 = call fast float @llvm.vector.reduce.fmul.v8f32(float 0.0, <8 x float> undef)
+  %V16 = call fast float @llvm.vector.reduce.fmul.v16f32(float 0.0, <16 x float> undef)
+  %v32 = call fast float @llvm.vector.reduce.fmul.v32f32(float 0.0, <32 x float> undef)
+  %V64 = call fast float @llvm.vector.reduce.fmul.v64f32(float 0.0, <64 x float> undef)
+  %V128 = call fast float @llvm.vector.reduce.fmul.v128f32(float 0.0, <128 x float> undef)
+  ret void
+}
+
+define void @reduce_fmul_double() {
+; FP-REDUCE-LABEL: 'reduce_fmul_double'
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call fast double @llvm.vector.reduce.fmul.v1f64(double 0.000000e+00, <1 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call fast double @llvm.vector.reduce.fmul.v2f64(double 0.000000e+00, <2 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4 = call fast double @llvm.vector.reduce.fmul.v4f64(double 0.000000e+00, <4 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 91 for instruction: %V8 = call fast double @llvm.vector.reduce.fmul.v8f64(double 0.000000e+00, <8 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 361 for instruction: %V16 = call fast double @llvm.vector.reduce.fmul.v16f64(double 0.000000e+00, <16 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 393 for instruction: %v32 = call fast double @llvm.vector.reduce.fmul.v32f64(double 0.000000e+00, <32 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 457 for instruction: %V64 = call fast double @llvm.vector.reduce.fmul.v64f64(double 0.000000e+00, <64 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 585 for instruction: %V128 = call fast double @llvm.vector.reduce.fmul.v128f64(double 0.000000e+00, <128 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SIZE-LABEL: 'reduce_fmul_double'
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1 = call fast double @llvm.vector.reduce.fmul.v1f64(double 0.000000e+00, <1 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call fast double @llvm.vector.reduce.fmul.v2f64(double 0.000000e+00, <2 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call fast double @llvm.vector.reduce.fmul.v4f64(double 0.000000e+00, <4 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call fast double @llvm.vector.reduce.fmul.v8f64(double 0.000000e+00, <8 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16 = call fast double @llvm.vector.reduce.fmul.v16f64(double 0.000000e+00, <16 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v32 = call fast double @llvm.vector.reduce.fmul.v32f64(double 0.000000e+00, <32 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V64 = call fast double @llvm.vector.reduce.fmul.v64f64(double 0.000000e+00, <64 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V128 = call fast double @llvm.vector.reduce.fmul.v128f64(double 0.000000e+00, <128 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V1 = call fast double @llvm.vector.reduce.fmul.v1f64(double 0.0, <1 x double> undef)
+  %V2 = call fast double @llvm.vector.reduce.fmul.v2f64(double 0.0, <2 x double> undef)
+  %V4 = call fast double @llvm.vector.reduce.fmul.v4f64(double 0.0, <4 x double> undef)
+  %V8 = call fast double @llvm.vector.reduce.fmul.v8f64(double 0.0, <8 x double> undef)
+  %V16 = call fast double @llvm.vector.reduce.fmul.v16f64(double 0.0, <16 x double> undef)
+  %v32 = call fast double @llvm.vector.reduce.fmul.v32f64(double 0.0, <32 x double> undef)
+  %V64 = call fast double @llvm.vector.reduce.fmul.v64f64(double 0.0, <64 x double> undef)
+  %V128 = call fast double @llvm.vector.reduce.fmul.v128f64(double 0.0, <128 x double> undef)
+  ret void
+}
+
+define void @reduce_ordered_fmul_half() {
+; FP-REDUCE-LABEL: 'reduce_ordered_fmul_half'
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call half @llvm.vector.reduce.fmul.v1f16(half 0xH0000, <1 x half> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call half @llvm.vector.reduce.fmul.v2f16(half 0xH0000, <2 x half> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4 = call half @llvm.vector.reduce.fmul.v4f16(half 0xH0000, <4 x half> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8 = call half @llvm.vector.reduce.fmul.v8f16(half 0xH0000, <8 x half> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V16 = call half @llvm.vector.reduce.fmul.v16f16(half 0xH0000, <16 x half> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v32 = call half @llvm.vector.reduce.fmul.v32f16(half 0xH0000, <32 x half> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %V64 = call half @llvm.vector.reduce.fmul.v64f16(half 0xH0000, <64 x half> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 510 for instruction: %V128 = call half @llvm.vector.reduce.fmul.v128f16(half 0xH0000, <128 x half> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SIZE-LABEL: 'reduce_ordered_fmul_half'
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call half @llvm.vector.reduce.fmul.v1f16(half 0xH0000, <1 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2 = call half @llvm.vector.reduce.fmul.v2f16(half 0xH0000, <2 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call half @llvm.vector.reduce.fmul.v4f16(half 0xH0000, <4 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8 = call half @llvm.vector.reduce.fmul.v8f16(half 0xH0000, <8 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16 = call half @llvm.vector.reduce.fmul.v16f16(half 0xH0000, <16 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 95 for instruction: %v32 = call half @llvm.vector.reduce.fmul.v32f16(half 0xH0000, <32 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 191 for instruction: %V64 = call half @llvm.vector.reduce.fmul.v64f16(half 0xH0000, <64 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 382 for instruction: %V128 = call half @llvm.vector.reduce.fmul.v128f16(half 0xH0000, <128 x half> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V1 = call half @llvm.vector.reduce.fmul.v1f16(half 0.0, <1 x half> undef)
+  %V2 = call half @llvm.vector.reduce.fmul.v2f16(half 0.0, <2 x half> undef)
+  %V4 = call half @llvm.vector.reduce.fmul.v4f16(half 0.0, <4 x half> undef)
+  %V8 = call half @llvm.vector.reduce.fmul.v8f16(half 0.0, <8 x half> undef)
+  %V16 = call half @llvm.vector.reduce.fmul.v16f16(half 0.0, <16 x half> undef)
+  %v32 = call half @llvm.vector.reduce.fmul.v32f16(half 0.0, <32 x half> undef)
+  %V64 = call half @llvm.vector.reduce.fmul.v64f16(half 0.0, <64 x half> undef)
+  %V128 = call half @llvm.vector.reduce.fmul.v128f16(half 0.0, <128 x half> undef)
+  ret void
+}
+
+define void @reduce_ordered_fmul_float() {
+; FP-REDUCE-LABEL: 'reduce_ordered_fmul_float'
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call float @llvm.vector.reduce.fmul.v1f32(float 0.000000e+00, <1 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call float @llvm.vector.reduce.fmul.v2f32(float 0.000000e+00, <2 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4 = call float @llvm.vector.reduce.fmul.v4f32(float 0.000000e+00, <4 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8 = call float @llvm.vector.reduce.fmul.v8f32(float 0.000000e+00, <8 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V16 = call float @llvm.vector.reduce.fmul.v16f32(float 0.000000e+00, <16 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v32 = call float @llvm.vector.reduce.fmul.v32f32(float 0.000000e+00, <32 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 254 for instruction: %V64 = call float @llvm.vector.reduce.fmul.v64f32(float 0.000000e+00, <64 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 508 for instruction: %V128 = call float @llvm.vector.reduce.fmul.v128f32(float 0.000000e+00, <128 x float> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SIZE-LABEL: 'reduce_ordered_fmul_float'
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call float @llvm.vector.reduce.fmul.v1f32(float 0.000000e+00, <1 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2 = call float @llvm.vector.reduce.fmul.v2f32(float 0.000000e+00, <2 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call float @llvm.vector.reduce.fmul.v4f32(float 0.000000e+00, <4 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8 = call float @llvm.vector.reduce.fmul.v8f32(float 0.000000e+00, <8 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16 = call float @llvm.vector.reduce.fmul.v16f32(float 0.000000e+00, <16 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 95 for instruction: %v32 = call float @llvm.vector.reduce.fmul.v32f32(float 0.000000e+00, <32 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 190 for instruction: %V64 = call float @llvm.vector.reduce.fmul.v64f32(float 0.000000e+00, <64 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 380 for instruction: %V128 = call float @llvm.vector.reduce.fmul.v128f32(float 0.000000e+00, <128 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V1 = call float @llvm.vector.reduce.fmul.v1f32(float 0.0, <1 x float> undef)
+  %V2 = call float @llvm.vector.reduce.fmul.v2f32(float 0.0, <2 x float> undef)
+  %V4 = call float @llvm.vector.reduce.fmul.v4f32(float 0.0, <4 x float> undef)
+  %V8 = call float @llvm.vector.reduce.fmul.v8f32(float 0.0, <8 x float> undef)
+  %V16 = call float @llvm.vector.reduce.fmul.v16f32(float 0.0, <16 x float> undef)
+  %v32 = call float @llvm.vector.reduce.fmul.v32f32(float 0.0, <32 x float> undef)
+  %V64 = call float @llvm.vector.reduce.fmul.v64f32(float 0.0, <64 x float> undef)
+  %V128 = call float @llvm.vector.reduce.fmul.v128f32(float 0.0, <128 x float> undef)
+  ret void
+}
+
+define void @reduce_ordered_fmul_double() {
+; FP-REDUCE-LABEL: 'reduce_ordered_fmul_double'
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call double @llvm.vector.reduce.fmul.v1f64(double 0.000000e+00, <1 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call double @llvm.vector.reduce.fmul.v2f64(double 0.000000e+00, <2 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4 = call double @llvm.vector.reduce.fmul.v4f64(double 0.000000e+00, <4 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8 = call double @llvm.vector.reduce.fmul.v8f64(double 0.000000e+00, <8 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V16 = call double @llvm.vector.reduce.fmul.v16f64(double 0.000000e+00, <16 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %v32 = call double @llvm.vector.reduce.fmul.v32f64(double 0.000000e+00, <32 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 252 for instruction: %V64 = call double @llvm.vector.reduce.fmul.v64f64(double 0.000000e+00, <64 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 504 for instruction: %V128 = call double @llvm.vector.reduce.fmul.v128f64(double 0.000000e+00, <128 x double> undef)
+; FP-REDUCE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SIZE-LABEL: 'reduce_ordered_fmul_double'
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1 = call double @llvm.vector.reduce.fmul.v1f64(double 0.000000e+00, <1 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2 = call double @llvm.vector.reduce.fmul.v2f64(double 0.000000e+00, <2 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call double @llvm.vector.reduce.fmul.v4f64(double 0.000000e+00, <4 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8 = call double @llvm.vector.reduce.fmul.v8f64(double 0.000000e+00, <8 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16 = call double @llvm.vector.reduce.fmul.v16f64(double 0.000000e+00, <16 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 94 for instruction: %v32 = call double @llvm.vector.reduce.fmul.v32f64(double 0.000000e+00, <32 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 188 for instruction: %V64 = call double @llvm.vector.reduce.fmul.v64f64(double 0.000000e+00, <64 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 376 for instruction: %V128 = call double @llvm.vector.reduce.fmul.v128f64(double 0.000000e+00, <128 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %V1 = call double @llvm.vector.reduce.fmul.v1f64(double 0.0, <1 x double> undef)
+  %V2 = call double @llvm.vector.reduce.fmul.v2f64(double 0.0, <2 x double> undef)
+  %V4 = call double @llvm.vector.reduce.fmul.v4f64(double 0.0, <4 x double> undef)
+  %V8 = call double @llvm.vector.reduce.fmul.v8f64(double 0.0, <8 x double> undef)
+  %V16 = call double @llvm.vector.reduce.fmul.v16f64(double 0.0, <16 x double> undef)
+  %v32 = call double @llvm.vector.reduce.fmul.v32f64(double 0.0, <32 x double> undef)
+  %V64 = call double @llvm.vector.reduce.fmul.v64f64(double 0.0, <64 x double> undef)
+  %V128 = call double @llvm.vector.reduce.fmul.v128f64(double 0.0, <128 x double> undef)
+  ret void
+}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/no-dep-via-loop-guards.ll b/llvm/test/Analysis/LoopAccessAnalysis/no-dep-via-loop-guards.ll
new file mode 100644
index 0000000000000..2cb98b1a81993
--- /dev/null
+++ b/llvm/test/Analysis/LoopAccessAnalysis/no-dep-via-loop-guards.ll
@@ -0,0 +1,351 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes='print<access-info>' -disable-output < %s  2>&1 | FileCheck %s
+
+; Loop guard for %off guarantees the accesses in the loop do not overlap.
+define void @access_after_via_loop_guard(ptr %a, i64 %off) {
+; CHECK-LABEL: 'access_after_via_loop_guard'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+  %c = icmp eq i64 %off, 100
+  br i1 %c, label %ph, label %exit
+
+ph:
+  %gep.after = getelementptr inbounds nuw i32, ptr %a, i64 %off
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %ph ], [ %iv.next, %loop ]
+  %l = load i32 , ptr %gep.after, align 4
+  %add = add i32 %l, %l
+  %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %add, ptr %gep, align 4
+  %iv.next = add nsw nuw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; Loop guard for %off guarantees the accesses in the loop do not overlap.
+; TODO: currently missed by LAA
+define void @access_after_via_loop_guard_sge(ptr %a, i64 %off) {
+; CHECK-LABEL: 'access_after_via_loop_guard_sge'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %gep.after, align 4 ->
+; CHECK-NEXT:            store i32 %add, ptr %gep, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+  %c = icmp sge i64 %off, 100
+  br i1 %c, label %ph, label %exit
+
+ph:
+  %gep.after = getelementptr inbounds nuw i32, ptr %a, i64 %off
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %ph ], [ %iv.next, %loop ]
+  %l = load i32 , ptr %gep.after, align 4
+  %add = add i32 %l, %l
+  %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %add, ptr %gep, align 4
+  %iv.next = add nsw nuw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @access_after_via_loop_guard_99(ptr %a, i64 %off) {
+; CHECK-LABEL: 'access_after_via_loop_guard_99'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %gep.after, align 4 ->
+; CHECK-NEXT:            store i32 %add, ptr %gep, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+  %c = icmp eq i64 %off, 99
+  br i1 %c, label %ph, label %exit
+
+ph:
+  %gep.after = getelementptr inbounds nuw i32, ptr %a, i64 %off
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %ph ], [ %iv.next, %loop ]
+  %l = load i32 , ptr %gep.after, align 4
+  %add = add i32 %l, %l
+  %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %add, ptr %gep, align 4
+  %iv.next = add nsw nuw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; Loop guard for %off guarantees the accesses in the loop do not overlap.
+; TODO: currently missed by LAA
+define void @access_after_via_loop_guard_sge_99(ptr %a, i64 %off) {
+; CHECK-LABEL: 'access_after_via_loop_guard_sge_99'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %gep.after, align 4 ->
+; CHECK-NEXT:            store i32 %add, ptr %gep, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+  %c = icmp sge i64 %off, 99
+  br i1 %c, label %ph, label %exit
+
+ph:
+  %gep.after = getelementptr inbounds nuw i32, ptr %a, i64 %off
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %ph ], [ %iv.next, %loop ]
+  %l = load i32 , ptr %gep.after, align 4
+  %add = add i32 %l, %l
+  %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %add, ptr %gep, align 4
+  %iv.next = add nsw nuw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @access_after_via_loop_guard_uge(ptr %a, i64 %off) {
+; CHECK-LABEL: 'access_after_via_loop_guard_uge'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %gep.after, align 4 ->
+; CHECK-NEXT:            store i32 %add, ptr %gep, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+  %c = icmp uge i64 %off, 100
+  br i1 %c, label %ph, label %exit
+
+ph:
+  %gep.after = getelementptr inbounds nuw i32, ptr %a, i64 %off
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %ph ], [ %iv.next, %loop ]
+  %l = load i32 , ptr %gep.after, align 4
+  %add = add i32 %l, %l
+  %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %add, ptr %gep, align 4
+  %iv.next = add nsw nuw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; Loop guard for %off guarantees the accesses in the loop do not overlap.
+define void @access_after_via_loop_guard_eq_loop_cond(ptr %a, i64 %off) {
+; CHECK-LABEL: 'access_after_via_loop_guard_eq_loop_cond'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+  %c = icmp eq i64 %off, 100
+  br i1 %c, label %ph, label %exit
+
+ph:
+  %gep.after = getelementptr inbounds nuw i32, ptr %a, i64 100
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %ph ], [ %iv.next, %loop ]
+  %l = load i32 , ptr %gep.after, align 4
+  %add = add i32 %l, %l
+  %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %add, ptr %gep, align 4
+  %iv.next = add nsw nuw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %off
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @access_after_via_loop_guard_eq_loop_cond_100(ptr %a, i64 %off) {
+; CHECK-LABEL: 'access_after_via_loop_guard_eq_loop_cond_100'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %gep.after, align 4 ->
+; CHECK-NEXT:            store i32 %add, ptr %gep, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+  %c = icmp eq i64 %off, 101
+  br i1 %c, label %ph, label %exit
+
+ph:
+  %gep.after = getelementptr inbounds nuw i32, ptr %a, i64 100
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %ph ], [ %iv.next, %loop ]
+  %l = load i32 , ptr %gep.after, align 4
+  %add = add i32 %l, %l
+  %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %add, ptr %gep, align 4
+  %iv.next = add nsw nuw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %off
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; Loop guard for %off guarantees the accesses in the loop do not overlap.
+; TODO: currently missed by LAA
+define void @access_after_via_loop_guard_sge_loop_cond(ptr %a, i64 %off) {
+; CHECK-LABEL: 'access_after_via_loop_guard_sge_loop_cond'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %gep.after, align 4 ->
+; CHECK-NEXT:            store i32 %add, ptr %gep, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+  %c = icmp sge i64 %off, 100
+  br i1 %c, label %ph, label %exit
+
+ph:
+  %gep.after = getelementptr inbounds nuw i32, ptr %a, i64 100
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %ph ], [ %iv.next, %loop ]
+  %l = load i32 , ptr %gep.after, align 4
+  %add = add i32 %l, %l
+  %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %add, ptr %gep, align 4
+  %iv.next = add nsw nuw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %off
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @access_after_via_loop_guard_sge_loop_cond_101(ptr %a, i64 %off) {
+; CHECK-LABEL: 'access_after_via_loop_guard_sge_loop_cond_101'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %gep.after, align 4 ->
+; CHECK-NEXT:            store i32 %add, ptr %gep, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+  %c = icmp sge i64 %off, 101
+  br i1 %c, label %ph, label %exit
+
+ph:
+  %gep.after = getelementptr inbounds nuw i32, ptr %a, i64 100
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %ph ], [ %iv.next, %loop ]
+  %l = load i32 , ptr %gep.after, align 4
+  %add = add i32 %l, %l
+  %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+  store i32 %add, ptr %gep, align 4
+  %iv.next = add nsw nuw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %off
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll
index 124d895e5aef5..30c3bbfb12e5d 100644
--- a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll
+++ b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll
@@ -1625,6 +1625,176 @@ exit:
   ret void
 }
 
+define void @ptr_induction_early_exit_eq_1_with_align_on_load(ptr %a, ptr %b, ptr %c) {
+; CHECK-LABEL: 'ptr_induction_early_exit_eq_1_with_align_on_load'
+; CHECK-NEXT:  Classifying expressions for: @ptr_induction_early_exit_eq_1_with_align_on_load
+; CHECK-NEXT:    %a_ = load ptr, ptr %a, align 8, !align !0
+; CHECK-NEXT:    --> %a_ U: [0,-7) S: [-9223372036854775808,9223372036854775801)
+; CHECK-NEXT:    %b_ = load ptr, ptr %b, align 8, !align !0
+; CHECK-NEXT:    --> %b_ U: [0,-7) S: [-9223372036854775808,9223372036854775801)
+; CHECK-NEXT:    %ptr.iv = phi ptr [ %ptr.iv.next, %loop.inc ], [ %a_, %entry ]
+; CHECK-NEXT:    --> {%a_,+,8}<nuw><%loop> U: [0,-7) S: [-9223372036854775808,9223372036854775801) Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %ld1 = load ptr, ptr %ptr.iv, align 8
+; CHECK-NEXT:    --> %ld1 U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:    %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 8
+; CHECK-NEXT:    --> {(8 + %a_),+,8}<nw><%loop> U: [0,-7) S: [-9223372036854775808,9223372036854775801) Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @ptr_induction_early_exit_eq_1_with_align_on_load
+; CHECK-NEXT:  Loop %loop: <multiple exits> Unpredictable backedge-taken count.
+; CHECK-NEXT:    exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    exit count for loop.inc: ((-8 + (-1 * (ptrtoint ptr %a_ to i64)) + (ptrtoint ptr %b_ to i64)) /u 8)
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i64 2305843009213693951
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is ((-8 + (-1 * (ptrtoint ptr %a_ to i64)) + (ptrtoint ptr %b_ to i64)) /u 8)
+; CHECK-NEXT:    symbolic max exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    symbolic max exit count for loop.inc: ((-8 + (-1 * (ptrtoint ptr %a_ to i64)) + (ptrtoint ptr %b_ to i64)) /u 8)
+;
+entry:
+  %a_ = load ptr, ptr %a, !align !{i64 8}
+  %b_ = load ptr, ptr %b, !align !{i64 8}
+  %cmp = icmp eq ptr %a_, %b_
+  br i1 %cmp, label %exit, label %loop
+
+loop:
+  %ptr.iv = phi ptr [ %ptr.iv.next, %loop.inc ], [ %a_, %entry ]
+  %ld1 = load ptr, ptr %ptr.iv, align 8
+  %earlyexitcond = icmp eq ptr %ld1, %c
+  br i1 %earlyexitcond, label %exit, label %loop.inc
+
+loop.inc:
+  %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 8
+  %exitcond = icmp eq ptr %ptr.iv.next, %b_
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @ptr_induction_early_exit_eq_1_with_align_on_arguments(ptr align 8 %a, ptr align 8 %b, ptr %c) {
+; CHECK-LABEL: 'ptr_induction_early_exit_eq_1_with_align_on_arguments'
+; CHECK-NEXT:  Classifying expressions for: @ptr_induction_early_exit_eq_1_with_align_on_arguments
+; CHECK-NEXT:    %ptr.iv = phi ptr [ %ptr.iv.next, %loop.inc ], [ %a, %entry ]
+; CHECK-NEXT:    --> {%a,+,8}<nuw><%loop> U: [0,-7) S: [-9223372036854775808,9223372036854775801) Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %ld1 = load ptr, ptr %ptr.iv, align 8
+; CHECK-NEXT:    --> %ld1 U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:    %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 8
+; CHECK-NEXT:    --> {(8 + %a),+,8}<nw><%loop> U: [0,-7) S: [-9223372036854775808,9223372036854775801) Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @ptr_induction_early_exit_eq_1_with_align_on_arguments
+; CHECK-NEXT:  Loop %loop: <multiple exits> Unpredictable backedge-taken count.
+; CHECK-NEXT:    exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    exit count for loop.inc: ((-8 + (-1 * (ptrtoint ptr %a to i64)) + (ptrtoint ptr %b to i64)) /u 8)
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i64 2305843009213693951
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is ((-8 + (-1 * (ptrtoint ptr %a to i64)) + (ptrtoint ptr %b to i64)) /u 8)
+; CHECK-NEXT:    symbolic max exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    symbolic max exit count for loop.inc: ((-8 + (-1 * (ptrtoint ptr %a to i64)) + (ptrtoint ptr %b to i64)) /u 8)
+;
+entry:
+  %cmp = icmp eq ptr %a, %b
+  br i1 %cmp, label %exit, label %loop
+
+loop:
+  %ptr.iv = phi ptr [ %ptr.iv.next, %loop.inc ], [ %a, %entry ]
+  %ld1 = load ptr, ptr %ptr.iv, align 8
+  %earlyexitcond = icmp eq ptr %ld1, %c
+  br i1 %earlyexitcond, label %exit, label %loop.inc
+
+loop.inc:
+  %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 8
+  %exitcond = icmp eq ptr %ptr.iv.next, %b
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @ptr_induction_early_exit_eq_1_align_assumption_1(ptr %a, ptr %b, ptr %c) {
+; CHECK-LABEL: 'ptr_induction_early_exit_eq_1_align_assumption_1'
+; CHECK-NEXT:  Classifying expressions for: @ptr_induction_early_exit_eq_1_align_assumption_1
+; CHECK-NEXT:    %a_ = load ptr, ptr %a, align 8
+; CHECK-NEXT:    --> %a_ U: [0,-7) S: [-9223372036854775808,9223372036854775801)
+; CHECK-NEXT:    %b_ = load ptr, ptr %b, align 8
+; CHECK-NEXT:    --> %b_ U: [0,-7) S: [-9223372036854775808,9223372036854775801)
+; CHECK-NEXT:    %ptr.iv = phi ptr [ %ptr.iv.next, %loop.inc ], [ %a_, %entry ]
+; CHECK-NEXT:    --> {%a_,+,8}<nuw><%loop> U: [0,-7) S: [-9223372036854775808,9223372036854775801) Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %ld1 = load ptr, ptr %ptr.iv, align 8
+; CHECK-NEXT:    --> %ld1 U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:    %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 8
+; CHECK-NEXT:    --> {(8 + %a_),+,8}<nw><%loop> U: [0,-7) S: [-9223372036854775808,9223372036854775801) Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @ptr_induction_early_exit_eq_1_align_assumption_1
+; CHECK-NEXT:  Loop %loop: <multiple exits> Unpredictable backedge-taken count.
+; CHECK-NEXT:    exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    exit count for loop.inc: ((-8 + (-1 * (ptrtoint ptr %a_ to i64)) + (ptrtoint ptr %b_ to i64)) /u 8)
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i64 2305843009213693951
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is ((-8 + (-1 * (ptrtoint ptr %a_ to i64)) + (ptrtoint ptr %b_ to i64)) /u 8)
+; CHECK-NEXT:    symbolic max exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    symbolic max exit count for loop.inc: ((-8 + (-1 * (ptrtoint ptr %a_ to i64)) + (ptrtoint ptr %b_ to i64)) /u 8)
+;
+entry:
+  %a_ = load ptr, ptr %a
+  call void @llvm.assume(i1 true) [ "align"(ptr %a_, i64 8) ]
+  %b_ = load ptr, ptr %b
+  call void @llvm.assume(i1 true) [ "align"(ptr %b_, i64 8) ]
+  %cmp = icmp eq ptr %a_, %b_
+  br i1 %cmp, label %exit, label %loop
+
+loop:
+  %ptr.iv = phi ptr [ %ptr.iv.next, %loop.inc ], [ %a_, %entry ]
+  %ld1 = load ptr, ptr %ptr.iv, align 8
+  %earlyexitcond = icmp eq ptr %ld1, %c
+  br i1 %earlyexitcond, label %exit, label %loop.inc
+
+loop.inc:
+  %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 8
+  %exitcond = icmp eq ptr %ptr.iv.next, %b_
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @ptr_induction_early_exit_eq_1_align_assumption_2(ptr %a, ptr %b, ptr %c) {
+; CHECK-LABEL: 'ptr_induction_early_exit_eq_1_align_assumption_2'
+; CHECK-NEXT:  Classifying expressions for: @ptr_induction_early_exit_eq_1_align_assumption_2
+; CHECK-NEXT:    %a_ = load ptr, ptr %a, align 8
+; CHECK-NEXT:    --> %a_ U: [0,-7) S: [-9223372036854775808,9223372036854775801)
+; CHECK-NEXT:    %b_ = load ptr, ptr %b, align 8
+; CHECK-NEXT:    --> %b_ U: [0,-7) S: [-9223372036854775808,9223372036854775801)
+; CHECK-NEXT:    %ptr.iv = phi ptr [ %ptr.iv.next, %loop.inc ], [ %a_, %entry ]
+; CHECK-NEXT:    --> {%a_,+,8}<nuw><%loop> U: [0,-7) S: [-9223372036854775808,9223372036854775801) Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %ld1 = load ptr, ptr %ptr.iv, align 8
+; CHECK-NEXT:    --> %ld1 U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:    %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 8
+; CHECK-NEXT:    --> {(8 + %a_),+,8}<nw><%loop> U: [0,-7) S: [-9223372036854775808,9223372036854775801) Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @ptr_induction_early_exit_eq_1_align_assumption_2
+; CHECK-NEXT:  Loop %loop: <multiple exits> Unpredictable backedge-taken count.
+; CHECK-NEXT:    exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    exit count for loop.inc: ((-8 + (-1 * (ptrtoint ptr %a_ to i64)) + (ptrtoint ptr %b_ to i64)) /u 8)
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i64 2305843009213693951
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is ((-8 + (-1 * (ptrtoint ptr %a_ to i64)) + (ptrtoint ptr %b_ to i64)) /u 8)
+; CHECK-NEXT:    symbolic max exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    symbolic max exit count for loop.inc: ((-8 + (-1 * (ptrtoint ptr %a_ to i64)) + (ptrtoint ptr %b_ to i64)) /u 8)
+;
+entry:
+  %a_ = load ptr, ptr %a
+  %b_ = load ptr, ptr %b
+  call void @llvm.assume(i1 true) [ "align"(ptr %a_, i64 8) ]
+  call void @llvm.assume(i1 true) [ "align"(ptr %b_, i64 8) ]
+  %cmp = icmp eq ptr %a_, %b_
+  br i1 %cmp, label %exit, label %loop
+
+loop:
+  %ptr.iv = phi ptr [ %ptr.iv.next, %loop.inc ], [ %a_, %entry ]
+  %ld1 = load ptr, ptr %ptr.iv, align 8
+  %earlyexitcond = icmp eq ptr %ld1, %c
+  br i1 %earlyexitcond, label %exit, label %loop.inc
+
+loop.inc:
+  %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 8
+  %exitcond = icmp eq ptr %ptr.iv.next, %b_
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
 define void @ptr_induction_early_exit_eq_2(ptr %a, i64 %n, ptr %c) {
 ; CHECK-LABEL: 'ptr_induction_early_exit_eq_2'
 ; CHECK-NEXT:  Classifying expressions for: @ptr_induction_early_exit_eq_2
diff --git a/llvm/test/Analysis/ScalarEvolution/scev-expander-reuse-gep.ll b/llvm/test/Analysis/ScalarEvolution/scev-expander-reuse-gep.ll
index 32d6a958198e6..579f5149340b5 100644
--- a/llvm/test/Analysis/ScalarEvolution/scev-expander-reuse-gep.ll
+++ b/llvm/test/Analysis/ScalarEvolution/scev-expander-reuse-gep.ll
@@ -3,7 +3,7 @@
 ; RUN: opt -mtriple=i386-apple-macosx10.12.0 < %s -loop-reduce -S | FileCheck %s
 
 ; CHECK: %ptr4.ptr1 = select i1 %cmp.i, ptr %ptr4, ptr %ptr1
-; CHECK-NEXT: %scevgep = getelementptr i8, ptr %ptr4.ptr1, i32 1
+; CHECK-NEXT: %scevgep = getelementptr nuw i8, ptr %ptr4.ptr1, i32 1
 ; CHECK-NEXT: br label %while.cond.i
 
 target datalayout = "e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128"
diff --git a/llvm/test/Analysis/ScalarEvolution/umin-seq-operand-may-trigger-ub.ll b/llvm/test/Analysis/ScalarEvolution/umin-seq-operand-may-trigger-ub.ll
new file mode 100644
index 0000000000000..8085592111ff5
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/umin-seq-operand-may-trigger-ub.ll
@@ -0,0 +1,111 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes='print<scalar-evolution>' -scalar-evolution-classify-expressions=0 -disable-output %s 2>&1 | FileCheck %s
+
+; The UDiv in the latch may never be executed. The backedge-taken-count
+; expressions must account for the fact that evaluating the UDiv
+; unconditionally may trigger UB.
+; FIXME: umin_seq should be used instead of umin  for BTCs.
+define i64 @multi_exit_exit_count_with_udiv_by_value_in_latch(ptr %dst, i64 %N) {
+; CHECK-LABEL: 'multi_exit_exit_count_with_udiv_by_value_in_latch'
+; CHECK-NEXT:  Determining loop execution counts for: @multi_exit_exit_count_with_udiv_by_value_in_latch
+; CHECK-NEXT:  Loop %loop.header: <multiple exits> backedge-taken count is ((42 /u %N) umin (0 smax %N))
+; CHECK-NEXT:    exit count for loop.header: (0 smax %N)
+; CHECK-NEXT:    exit count for loop.latch: (42 /u %N)
+; CHECK-NEXT:  Loop %loop.header: constant max backedge-taken count is i64 42
+; CHECK-NEXT:  Loop %loop.header: symbolic max backedge-taken count is ((42 /u %N) umin (0 smax %N))
+; CHECK-NEXT:    symbolic max exit count for loop.header: (0 smax %N)
+; CHECK-NEXT:    symbolic max exit count for loop.latch: (42 /u %N)
+; CHECK-NEXT:  Loop %loop.header: Trip multiple is 1
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+  store i32 1, ptr %gep
+  %c.0 = icmp slt i64 %iv, %N
+  br i1 %c.0, label %loop.latch, label %exit
+
+loop.latch:
+  %iv.next = add i64 %iv, 1
+  %d = udiv i64 42, %N
+  %c.1 = icmp slt i64 %iv, %d
+  br i1 %c.1, label %loop.header, label %exit
+
+exit:
+  %p = phi i64 [ 1, %loop.header ], [ 0, %loop.latch]
+  ret i64 %p
+}
+
+; The UDiv in the latch may never be executed. The backedge-taken-count
+; expressions must account for the fact that evaluating the UDiv
+; unconditionally may trigger UB.
+; FIXME: umin_seq should be used instead of umin  for BTCs.
+define i64 @multi_exit_exit_count_with_udiv_by_value_in_latch_different_bounds(ptr %dst, i64 %N, i64 %M) {
+; CHECK-LABEL: 'multi_exit_exit_count_with_udiv_by_value_in_latch_different_bounds'
+; CHECK-NEXT:  Determining loop execution counts for: @multi_exit_exit_count_with_udiv_by_value_in_latch_different_bounds
+; CHECK-NEXT:  Loop %loop.header: <multiple exits> backedge-taken count is ((0 smax %N) umin_seq (42 /u %M))
+; CHECK-NEXT:    exit count for loop.header: (0 smax %N)
+; CHECK-NEXT:    exit count for loop.latch: (42 /u %M)
+; CHECK-NEXT:  Loop %loop.header: constant max backedge-taken count is i64 42
+; CHECK-NEXT:  Loop %loop.header: symbolic max backedge-taken count is ((0 smax %N) umin_seq (42 /u %M))
+; CHECK-NEXT:    symbolic max exit count for loop.header: (0 smax %N)
+; CHECK-NEXT:    symbolic max exit count for loop.latch: (42 /u %M)
+; CHECK-NEXT:  Loop %loop.header: Trip multiple is 1
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+  store i32 1, ptr %gep
+  %c.0 = icmp slt i64 %iv, %N
+  br i1 %c.0, label %loop.latch, label %exit
+
+loop.latch:
+  %iv.next = add i64 %iv, 1
+  %d = udiv i64 42, %M
+  %c.1 = icmp slt i64 %iv, %d
+  br i1 %c.1, label %loop.header, label %exit
+
+exit:
+  %p = phi i64 [ 1, %loop.header ], [ 0, %loop.latch]
+  ret i64 %p
+}
+
+; The UDiv in the latch cannot trigger UB, evaluating it unconditionally in the
+; trip count expression is fine.
+define i64 @multi_exit_exit_count_with_udiv_by_constant_in_latch(ptr %dst, i64 %N) {
+; CHECK-LABEL: 'multi_exit_exit_count_with_udiv_by_constant_in_latch'
+; CHECK-NEXT:  Determining loop execution counts for: @multi_exit_exit_count_with_udiv_by_constant_in_latch
+; CHECK-NEXT:  Loop %loop.header: <multiple exits> backedge-taken count is ((%N /u 42) umin (0 smax %N))
+; CHECK-NEXT:    exit count for loop.header: (0 smax %N)
+; CHECK-NEXT:    exit count for loop.latch: (%N /u 42)
+; CHECK-NEXT:  Loop %loop.header: constant max backedge-taken count is i64 439208192231179800
+; CHECK-NEXT:  Loop %loop.header: symbolic max backedge-taken count is ((%N /u 42) umin (0 smax %N))
+; CHECK-NEXT:    symbolic max exit count for loop.header: (0 smax %N)
+; CHECK-NEXT:    symbolic max exit count for loop.latch: (%N /u 42)
+; CHECK-NEXT:  Loop %loop.header: Trip multiple is 1
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+  store i32 1, ptr %gep
+  %c.0 = icmp slt i64 %iv, %N
+  br i1 %c.0, label %loop.latch, label %exit
+
+loop.latch:
+  %iv.next = add i64 %iv, 1
+  %d = udiv i64 %N, 42
+  %c.1 = icmp slt i64 %iv, %d
+  br i1 %c.1, label %loop.header, label %exit
+
+exit:
+  %p = phi i64 [ 1, %loop.header ], [ 0, %loop.latch]
+  ret i64 %p
+}
diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll
index ea29ff634a43b..a1b2370a87b82 100644
--- a/llvm/test/Bitcode/compatibility.ll
+++ b/llvm/test/Bitcode/compatibility.ll
@@ -1122,6 +1122,26 @@ define void @fastMathFlagsForArrayCalls([2 x float] %f, [2 x double] %d1, [2 x <
   ret void
 }
 
+declare { float, float } @fmf_struct_f32()
+declare { double, double, double } @fmf_struct_f64()
+declare { <4 x double> } @fmf_struct_v4f64()
+
+; CHECK-LABEL: fastMathFlagsForStructCalls(
+define void @fastMathFlagsForStructCalls() {
+  %call.fast = call fast { float, float } @fmf_struct_f32()
+  ; CHECK: %call.fast = call fast { float, float } @fmf_struct_f32()
+
+  ; Throw in some other attributes to make sure those stay in the right places.
+
+  %call.nsz.arcp = notail call nsz arcp { double, double, double } @fmf_struct_f64()
+  ; CHECK: %call.nsz.arcp = notail call nsz arcp { double, double, double } @fmf_struct_f64()
+
+  %call.nnan.ninf = tail call nnan ninf fastcc { <4 x double> } @fmf_struct_v4f64()
+  ; CHECK: %call.nnan.ninf = tail call nnan ninf fastcc { <4 x double> } @fmf_struct_v4f64()
+
+  ret void
+}
+
 ;; Type System
 %opaquety = type opaque
 define void @typesystem() {
@@ -2077,9 +2097,14 @@ declare nofpclass(sub zero) float @nofpclass_sub_zero(float nofpclass(sub zero))
 ; CHECK: declare nofpclass(inf sub) float @nofpclass_sub_inf(float nofpclass(inf sub))
 declare nofpclass(sub inf) float @nofpclass_sub_inf(float nofpclass(sub inf))
 
+; CHECK: declare nofpclass(nan) { float, float } @nofpclass_struct({ double } nofpclass(nan))
+declare nofpclass(nan) { float, float } @nofpclass_struct({ double } nofpclass(nan))
+
 declare float @unknown_fpclass_func(float)
 
-define float @nofpclass_callsites(float %arg) {
+declare { <4 x double>, <4 x double>, <4 x double> } @unknown_fpclass_struct_func({ float })
+
+define float @nofpclass_callsites(float %arg, { float } %arg1) {
   ; CHECK: %call0 = call nofpclass(nan) float @unknown_fpclass_func(float nofpclass(ninf) %arg)
   %call0 = call nofpclass(nan) float @unknown_fpclass_func(float nofpclass(ninf) %arg)
 
@@ -2088,6 +2113,10 @@ define float @nofpclass_callsites(float %arg) {
 
   ; CHECK: %call2 = call nofpclass(zero) float @unknown_fpclass_func(float nofpclass(norm) %arg)
   %call2 = call nofpclass(zero) float @unknown_fpclass_func(float nofpclass(norm) %arg)
+
+  ; CHECK: %call3 = call nofpclass(pinf) { <4 x double>, <4 x double>, <4 x double> } @unknown_fpclass_struct_func({ float } nofpclass(all) %arg1)
+  %call3 = call nofpclass(pinf) { <4 x double>, <4 x double>, <4 x double> } @unknown_fpclass_struct_func({ float } nofpclass(all) %arg1)
+
   %add0 = fadd float %call0, %call1
   %add1 = fadd float %add0, %call2
   ret float %add1
diff --git a/llvm/test/CodeGen/AArch64/DAGCombine_vscale.ll b/llvm/test/CodeGen/AArch64/DAGCombine_vscale.ll
index 71f4da2b465c1..487b726253cc7 100644
--- a/llvm/test/CodeGen/AArch64/DAGCombine_vscale.ll
+++ b/llvm/test/CodeGen/AArch64/DAGCombine_vscale.ll
@@ -10,10 +10,8 @@ define <vscale x 4 x i32> @sext_inreg(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    sxth z0.s, p0/m, z0.s
 ; CHECK-NEXT:    ret
-  %in = insertelement <vscale x 4 x i32> undef, i32 16, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %in, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %sext = shl <vscale x 4 x i32> %a, %splat
-  %conv = ashr <vscale x 4 x i32> %sext, %splat
+  %sext = shl <vscale x 4 x i32> %a, splat(i32 16)
+  %conv = ashr <vscale x 4 x i32> %sext, splat(i32 16)
   ret <vscale x 4 x i32> %conv
 }
 
@@ -23,12 +21,8 @@ define <vscale x 4 x i32> @ashr_shl(<vscale x 4 x i32> %a)  {
 ; CHECK-NEXT:    lsl z0.s, z0.s, #8
 ; CHECK-NEXT:    asr z0.s, z0.s, #16
 ; CHECK-NEXT:    ret
-  %in1 = insertelement <vscale x 4 x i32> undef, i32 8, i32 0
-  %splat1 = shufflevector <vscale x 4 x i32> %in1, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %in2 = insertelement <vscale x 4 x i32> undef, i32 16, i32 0
-  %splat2 = shufflevector <vscale x 4 x i32> %in2, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %shl = shl <vscale x 4 x i32> %a, %splat1
-  %r = ashr <vscale x 4 x i32> %shl, %splat2
+  %shl = shl <vscale x 4 x i32> %a, splat(i32 8)
+  %r = ashr <vscale x 4 x i32> %shl, splat(i32 16)
   ret <vscale x 4 x i32> %r
 }
 
@@ -38,12 +32,8 @@ define <vscale x 4 x i32> @ashr_shl_illegal_trunc_vec_ty(<vscale x 4 x i32> %a)
 ; CHECK-NEXT:    lsl z0.s, z0.s, #8
 ; CHECK-NEXT:    asr z0.s, z0.s, #11
 ; CHECK-NEXT:    ret
-  %in1 = insertelement <vscale x 4 x i32> undef, i32 8, i32 0
-  %splat1 = shufflevector <vscale x 4 x i32> %in1, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %in2 = insertelement <vscale x 4 x i32> undef, i32 11, i32 0
-  %splat2 = shufflevector <vscale x 4 x i32> %in2, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %shl = shl <vscale x 4 x i32> %a, %splat1
-  %r = ashr <vscale x 4 x i32> %shl, %splat2
+  %shl = shl <vscale x 4 x i32> %a, splat(i32 8)
+  %r = ashr <vscale x 4 x i32> %shl, splat(i32 11)
   ret <vscale x 4 x i32> %r
 }
 
@@ -55,12 +45,8 @@ define <vscale x 4 x i32> @ashr_add_shl_nxv4i8(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    add z0.s, z0.s, z1.s
 ; CHECK-NEXT:    asr z0.s, z0.s, #24
 ; CHECK-NEXT:    ret
-  %in1 = insertelement <vscale x 4 x i32> undef, i32 24, i32 0
-  %splat1 = shufflevector <vscale x 4 x i32> %in1, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %in2 = insertelement <vscale x 4 x i32> undef, i32 16777216, i32 0
-  %splat2 = shufflevector <vscale x 4 x i32> %in2, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %conv = shl <vscale x 4 x i32> %a, %splat1
-  %sext = add <vscale x 4 x i32> %conv, %splat2
-  %conv1 = ashr <vscale x 4 x i32> %sext, %splat1
+  %conv = shl <vscale x 4 x i32> %a, splat(i32 24)
+  %sext = add <vscale x 4 x i32> %conv, splat(i32 16777216)
+  %conv1 = ashr <vscale x 4 x i32> %sext, splat(i32 24)
   ret <vscale x 4 x i32> %conv1
 }
diff --git a/llvm/test/CodeGen/AArch64/fexplog.ll b/llvm/test/CodeGen/AArch64/fexplog.ll
index 30ce389f23128..f13e2fcd1c448 100644
--- a/llvm/test/CodeGen/AArch64/fexplog.ll
+++ b/llvm/test/CodeGen/AArch64/fexplog.ll
@@ -36,16 +36,26 @@ entry:
   ret half %c
 }
 
-define <1 x double> @exp_v1f64(<1 x double> %x) {
+define fp128 @exp_fp128(fp128 %a) {
+; CHECK-LABEL: exp_fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    b expl
+entry:
+  %c = call fp128 @llvm.exp.fp128(fp128 %a)
+  ret fp128 %c
+}
+
+define <1 x double> @exp_v1f64(<1 x double> %a) {
 ; CHECK-LABEL: exp_v1f64:
-; CHECK:       // %bb.0:
+; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl exp
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
-  %c = call <1 x double> @llvm.exp.v1f64(<1 x double> %x)
+entry:
+  %c = call <1 x double> @llvm.exp.v1f64(<1 x double> %a)
   ret <1 x double> %c
 }
 
@@ -1273,6 +1283,28 @@ entry:
   ret <16 x half> %c
 }
 
+define <2 x fp128> @exp_v2fp128(<2 x fp128> %a) {
+; CHECK-LABEL: exp_v2fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    bl expl
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    bl expl
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+entry:
+  %c = call <2 x fp128> @llvm.exp.v2fp128(<2 x fp128> %a)
+  ret <2 x fp128> %c
+}
+
 define double @exp2_f64(double %a) {
 ; CHECK-LABEL: exp2_f64:
 ; CHECK:       // %bb.0: // %entry
@@ -1307,16 +1339,26 @@ entry:
   ret half %c
 }
 
-define <1 x double> @exp2_v1f64(<1 x double> %x) {
+define fp128 @exp2_fp128(fp128 %a) {
+; CHECK-LABEL: exp2_fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    b exp2l
+entry:
+  %c = call fp128 @llvm.exp2.fp128(fp128 %a)
+  ret fp128 %c
+}
+
+define <1 x double> @exp2_v1f64(<1 x double> %a) {
 ; CHECK-LABEL: exp2_v1f64:
-; CHECK:       // %bb.0:
+; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl exp2
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
-  %c = call <1 x double> @llvm.exp2.v1f64(<1 x double> %x)
+entry:
+  %c = call <1 x double> @llvm.exp2.v1f64(<1 x double> %a)
   ret <1 x double> %c
 }
 
@@ -2544,6 +2586,28 @@ entry:
   ret <16 x half> %c
 }
 
+define <2 x fp128> @exp2_v2fp128(<2 x fp128> %a) {
+; CHECK-LABEL: exp2_v2fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    bl exp2l
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    bl exp2l
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+entry:
+  %c = call <2 x fp128> @llvm.exp2.v2fp128(<2 x fp128> %a)
+  ret <2 x fp128> %c
+}
+
 define double @log_f64(double %a) {
 ; CHECK-LABEL: log_f64:
 ; CHECK:       // %bb.0: // %entry
@@ -2578,16 +2642,26 @@ entry:
   ret half %c
 }
 
-define <1 x double> @log_v1f64(<1 x double> %x) {
+define fp128 @log_fp128(fp128 %a) {
+; CHECK-LABEL: log_fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    b logl
+entry:
+  %c = call fp128 @llvm.log.fp128(fp128 %a)
+  ret fp128 %c
+}
+
+define <1 x double> @log_v1f64(<1 x double> %a) {
 ; CHECK-LABEL: log_v1f64:
-; CHECK:       // %bb.0:
+; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl log
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
-  %c = call <1 x double> @llvm.log.v1f64(<1 x double> %x)
+entry:
+  %c = call <1 x double> @llvm.log.v1f64(<1 x double> %a)
   ret <1 x double> %c
 }
 
@@ -3815,6 +3889,28 @@ entry:
   ret <16 x half> %c
 }
 
+define <2 x fp128> @log_v2fp128(<2 x fp128> %a) {
+; CHECK-LABEL: log_v2fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    bl logl
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    bl logl
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+entry:
+  %c = call <2 x fp128> @llvm.log.v2fp128(<2 x fp128> %a)
+  ret <2 x fp128> %c
+}
+
 define double @log2_f64(double %a) {
 ; CHECK-LABEL: log2_f64:
 ; CHECK:       // %bb.0: // %entry
@@ -3849,16 +3945,26 @@ entry:
   ret half %c
 }
 
-define <1 x double> @log2_v1f64(<1 x double> %x) {
+define fp128 @log2_fp128(fp128 %a) {
+; CHECK-LABEL: log2_fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    b log2l
+entry:
+  %c = call fp128 @llvm.log2.fp128(fp128 %a)
+  ret fp128 %c
+}
+
+define <1 x double> @log2_v1f64(<1 x double> %a) {
 ; CHECK-LABEL: log2_v1f64:
-; CHECK:       // %bb.0:
+; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl log2
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
-  %c = call <1 x double> @llvm.log2.v1f64(<1 x double> %x)
+entry:
+  %c = call <1 x double> @llvm.log2.v1f64(<1 x double> %a)
   ret <1 x double> %c
 }
 
@@ -5086,6 +5192,28 @@ entry:
   ret <16 x half> %c
 }
 
+define <2 x fp128> @log2_v2fp128(<2 x fp128> %a) {
+; CHECK-LABEL: log2_v2fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    bl log2l
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    bl log2l
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+entry:
+  %c = call <2 x fp128> @llvm.log2.v2fp128(<2 x fp128> %a)
+  ret <2 x fp128> %c
+}
+
 define double @log10_f64(double %a) {
 ; CHECK-LABEL: log10_f64:
 ; CHECK:       // %bb.0: // %entry
@@ -5120,16 +5248,26 @@ entry:
   ret half %c
 }
 
-define <1 x double> @log10_v1f64(<1 x double> %x) {
+define fp128 @log10_fp128(fp128 %a) {
+; CHECK-LABEL: log10_fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    b log10l
+entry:
+  %c = call fp128 @llvm.log10.fp128(fp128 %a)
+  ret fp128 %c
+}
+
+define <1 x double> @log10_v1f64(<1 x double> %a) {
 ; CHECK-LABEL: log10_v1f64:
-; CHECK:       // %bb.0:
+; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl log10
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
-  %c = call <1 x double> @llvm.log10.v1f64(<1 x double> %x)
+entry:
+  %c = call <1 x double> @llvm.log10.v1f64(<1 x double> %a)
   ret <1 x double> %c
 }
 
@@ -6357,6 +6495,33 @@ entry:
   ret <16 x half> %c
 }
 
+define <2 x fp128> @log10_v2fp128(<2 x fp128> %a) {
+; CHECK-LABEL: log10_v2fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    bl log10l
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    bl log10l
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+entry:
+  %c = call <2 x fp128> @llvm.log10.v2fp128(<2 x fp128> %a)
+  ret <2 x fp128> %c
+}
+
+declare <1 x double> @llvm.exp.v1f64(<1 x double>)
+declare <1 x double> @llvm.exp2.v1f64(<1 x double>)
+declare <1 x double> @llvm.log.v1f64(<1 x double>)
+declare <1 x double> @llvm.log10.v1f64(<1 x double>)
+declare <1 x double> @llvm.log2.v1f64(<1 x double>)
 declare <16 x half> @llvm.exp.v16f16(<16 x half>)
 declare <16 x half> @llvm.exp2.v16f16(<16 x half>)
 declare <16 x half> @llvm.log.v16f16(<16 x half>)
@@ -6372,6 +6537,11 @@ declare <2 x float> @llvm.exp2.v2f32(<2 x float>)
 declare <2 x float> @llvm.log.v2f32(<2 x float>)
 declare <2 x float> @llvm.log10.v2f32(<2 x float>)
 declare <2 x float> @llvm.log2.v2f32(<2 x float>)
+declare <2 x fp128> @llvm.exp.v2fp128(<2 x fp128>)
+declare <2 x fp128> @llvm.exp2.v2fp128(<2 x fp128>)
+declare <2 x fp128> @llvm.log.v2fp128(<2 x fp128>)
+declare <2 x fp128> @llvm.log10.v2fp128(<2 x fp128>)
+declare <2 x fp128> @llvm.log2.v2fp128(<2 x fp128>)
 declare <3 x double> @llvm.exp.v3f64(<3 x double>)
 declare <3 x double> @llvm.exp2.v3f64(<3 x double>)
 declare <3 x double> @llvm.log.v3f64(<3 x double>)
@@ -6422,6 +6592,11 @@ declare float @llvm.exp2.f32(float)
 declare float @llvm.log.f32(float)
 declare float @llvm.log10.f32(float)
 declare float @llvm.log2.f32(float)
+declare fp128 @llvm.exp.fp128(fp128)
+declare fp128 @llvm.exp2.fp128(fp128)
+declare fp128 @llvm.log.fp128(fp128)
+declare fp128 @llvm.log10.fp128(fp128)
+declare fp128 @llvm.log2.fp128(fp128)
 declare half @llvm.exp.f16(half)
 declare half @llvm.exp2.f16(half)
 declare half @llvm.log.f16(half)
diff --git a/llvm/test/CodeGen/AArch64/fpow.ll b/llvm/test/CodeGen/AArch64/fpow.ll
index 6e8cd0c8c00b4..dc93d5be9b3f3 100644
--- a/llvm/test/CodeGen/AArch64/fpow.ll
+++ b/llvm/test/CodeGen/AArch64/fpow.ll
@@ -37,14 +37,23 @@ entry:
   ret half %c
 }
 
+define fp128 @pow_fp128(fp128 %a, fp128 %b) {
+; CHECK-LABEL: pow_fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    b powl
+entry:
+  %c = call fp128 @llvm.pow.fp128(fp128 %a, fp128 %b)
+  ret fp128 %c
+}
+
 define <1 x double> @pow_v1f64(<1 x double> %x) {
 ; CHECK-LABEL: pow_v1f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset w30, -16
-; CHECK-NEXT:    adrp x8, .LCPI3_0
-; CHECK-NEXT:    ldr d1, [x8, :lo12:.LCPI3_0]
+; CHECK-NEXT:    adrp x8, .LCPI4_0
+; CHECK-NEXT:    ldr d1, [x8, :lo12:.LCPI4_0]
 ; CHECK-NEXT:    bl pow
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
@@ -1591,9 +1600,51 @@ entry:
   ret <16 x half> %c
 }
 
+define <2 x fp128> @pow_v2fp128(<2 x fp128> %a, <2 x fp128> %b) {
+; CHECK-SD-LABEL: pow_v2fp128:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sub sp, sp, #64
+; CHECK-SD-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-SD-NEXT:    stp q1, q3, [sp, #16] // 32-byte Folded Spill
+; CHECK-SD-NEXT:    mov v1.16b, v2.16b
+; CHECK-SD-NEXT:    bl powl
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    ldp q0, q1, [sp, #16] // 32-byte Folded Reload
+; CHECK-SD-NEXT:    bl powl
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; CHECK-SD-NEXT:    add sp, sp, #64
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: pow_v2fp128:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sub sp, sp, #64
+; CHECK-GI-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-GI-NEXT:    stp q3, q1, [sp, #16] // 32-byte Folded Spill
+; CHECK-GI-NEXT:    mov v1.16b, v2.16b
+; CHECK-GI-NEXT:    bl powl
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    ldp q1, q0, [sp, #16] // 32-byte Folded Reload
+; CHECK-GI-NEXT:    bl powl
+; CHECK-GI-NEXT:    mov v1.16b, v0.16b
+; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; CHECK-GI-NEXT:    add sp, sp, #64
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = call <2 x fp128> @llvm.pow.v2fp128(<2 x fp128> %a, <2 x fp128> %b)
+  ret <2 x fp128> %c
+}
+
 declare <16 x half> @llvm.pow.v16f16(<16 x half>, <16 x half>)
 declare <2 x double> @llvm.pow.v2f64(<2 x double>, <2 x double>)
 declare <2 x float> @llvm.pow.v2f32(<2 x float>, <2 x float>)
+declare <2 x fp128> @llvm.pow.v2fp128(<2 x fp128>, <2 x fp128>)
 declare <3 x double> @llvm.pow.v3f64(<3 x double>, <3 x double>)
 declare <3 x float> @llvm.pow.v3f32(<3 x float>, <3 x float>)
 declare <4 x double> @llvm.pow.v4f64(<4 x double>, <4 x double>)
@@ -1604,4 +1655,5 @@ declare <8 x float> @llvm.pow.v8f32(<8 x float>, <8 x float>)
 declare <8 x half> @llvm.pow.v8f16(<8 x half>, <8 x half>)
 declare double @llvm.pow.f64(double, double)
 declare float @llvm.pow.f32(float, float)
+declare fp128 @llvm.pow.fp128(fp128, fp128)
 declare half @llvm.pow.f16(half, half)
diff --git a/llvm/test/CodeGen/AArch64/fpowi.ll b/llvm/test/CodeGen/AArch64/fpowi.ll
index 62fc1c0854ca8..8948556d1b380 100644
--- a/llvm/test/CodeGen/AArch64/fpowi.ll
+++ b/llvm/test/CodeGen/AArch64/fpowi.ll
@@ -36,6 +36,29 @@ entry:
   ret half %c
 }
 
+define fp128 @powi_fp128(fp128 %a, i32 %b) {
+; CHECK-LABEL: powi_fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    b __powitf2
+entry:
+  %c = call fp128 @llvm.powi.fp128.i32(fp128 %a, i32 %b)
+  ret fp128 %c
+}
+
+define <1 x double> @powi_v1f64(<1 x double> %a, i32 %b) {
+; CHECK-LABEL: powi_v1f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    bl __powidf2
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %c = call <1 x double> @llvm.powi.v1f64.i32(<1 x double> %a, i32 %b)
+  ret <1 x double> %c
+}
+
 define <2 x double> @powi_v2f64(<2 x double> %a, i32 %b) {
 ; CHECK-SD-LABEL: powi_v2f64:
 ; CHECK-SD:       // %bb.0: // %entry
@@ -1406,9 +1429,56 @@ entry:
   ret <16 x half> %c
 }
 
+define <2 x fp128> @powi_v2fp128(<2 x fp128> %a, i32 %b) {
+; CHECK-SD-LABEL: powi_v2fp128:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sub sp, sp, #48
+; CHECK-SD-NEXT:    stp x30, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-SD-NEXT:    mov w19, w0
+; CHECK-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    bl __powitf2
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    mov w0, w19
+; CHECK-SD-NEXT:    bl __powitf2
+; CHECK-SD-NEXT:    ldp x30, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    add sp, sp, #48
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: powi_v2fp128:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sub sp, sp, #48
+; CHECK-GI-NEXT:    stp x30, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-GI-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    mov w19, w0
+; CHECK-GI-NEXT:    bl __powitf2
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov w0, w19
+; CHECK-GI-NEXT:    bl __powitf2
+; CHECK-GI-NEXT:    ldp x30, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v1.16b, v0.16b
+; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    add sp, sp, #48
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = call <2 x fp128> @llvm.powi.v2fp128.i32(<2 x fp128> %a, i32 %b)
+  ret <2 x fp128> %c
+}
+
+declare <1 x double> @llvm.powi.v1f64.i32(<1 x double>, i32)
 declare <16 x half> @llvm.powi.v16f16.i32(<16 x half>, i32)
 declare <2 x double> @llvm.powi.v2f64.i32(<2 x double>, i32)
 declare <2 x float> @llvm.powi.v2f32.i32(<2 x float>, i32)
+declare <2 x fp128> @llvm.powi.v2fp128.i32(<2 x fp128>, i32)
 declare <3 x double> @llvm.powi.v3f64.i32(<3 x double>, i32)
 declare <3 x float> @llvm.powi.v3f32.i32(<3 x float>, i32)
 declare <4 x double> @llvm.powi.v4f64.i32(<4 x double>, i32)
@@ -1419,4 +1489,5 @@ declare <8 x float> @llvm.powi.v8f32.i32(<8 x float>, i32)
 declare <8 x half> @llvm.powi.v8f16.i32(<8 x half>, i32)
 declare double @llvm.powi.f64.i32(double, i32)
 declare float @llvm.powi.f32.i32(float, i32)
+declare fp128 @llvm.powi.fp128.i32(fp128, i32)
 declare half @llvm.powi.f16.i32(half, i32)
diff --git a/llvm/test/CodeGen/AArch64/frem.ll b/llvm/test/CodeGen/AArch64/frem.ll
index fe5146d79895c..feb13da64cbf8 100644
--- a/llvm/test/CodeGen/AArch64/frem.ll
+++ b/llvm/test/CodeGen/AArch64/frem.ll
@@ -39,6 +39,29 @@ entry:
   ret half %c
 }
 
+define fp128 @frem_fp128(fp128 %a, fp128 %b) {
+; CHECK-LABEL: frem_fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    b fmodl
+entry:
+  %c = frem fp128 %a, %b
+  ret fp128 %c
+}
+
+define <1 x double> @frem_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: frem_v1f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    bl fmod
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %c = frem <1 x double> %a, %b
+  ret <1 x double> %c
+}
+
 define <2 x double> @frem_v2f64(<2 x double> %a, <2 x double> %b) {
 ; CHECK-SD-LABEL: frem_v2f64:
 ; CHECK-SD:       // %bb.0: // %entry
@@ -1577,3 +1600,44 @@ entry:
   %c = frem <16 x half> %a, %b
   ret <16 x half> %c
 }
+
+define <2 x fp128> @frem_v2fp128(<2 x fp128> %a, <2 x fp128> %b) {
+; CHECK-SD-LABEL: frem_v2fp128:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sub sp, sp, #64
+; CHECK-SD-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-SD-NEXT:    stp q1, q3, [sp, #16] // 32-byte Folded Spill
+; CHECK-SD-NEXT:    mov v1.16b, v2.16b
+; CHECK-SD-NEXT:    bl fmodl
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    ldp q0, q1, [sp, #16] // 32-byte Folded Reload
+; CHECK-SD-NEXT:    bl fmodl
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; CHECK-SD-NEXT:    add sp, sp, #64
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: frem_v2fp128:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sub sp, sp, #64
+; CHECK-GI-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-GI-NEXT:    stp q3, q1, [sp, #16] // 32-byte Folded Spill
+; CHECK-GI-NEXT:    mov v1.16b, v2.16b
+; CHECK-GI-NEXT:    bl fmodl
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    ldp q1, q0, [sp, #16] // 32-byte Folded Reload
+; CHECK-GI-NEXT:    bl fmodl
+; CHECK-GI-NEXT:    mov v1.16b, v0.16b
+; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; CHECK-GI-NEXT:    add sp, sp, #64
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = frem <2 x fp128> %a, %b
+  ret <2 x fp128> %c
+}
diff --git a/llvm/test/CodeGen/AArch64/fsincos.ll b/llvm/test/CodeGen/AArch64/fsincos.ll
index 557add3a4eaeb..2afc56a7139fb 100644
--- a/llvm/test/CodeGen/AArch64/fsincos.ll
+++ b/llvm/test/CodeGen/AArch64/fsincos.ll
@@ -36,6 +36,15 @@ entry:
   ret half %c
 }
 
+define fp128 @sin_fp128(fp128 %a) {
+; CHECK-LABEL: sin_fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    b sinl
+entry:
+  %c = call fp128 @llvm.sin.fp128(fp128 %a)
+  ret fp128 %c
+}
+
 define <1 x double> @sin_v1f64(<1 x double> %x) {
 ; CHECK-LABEL: sin_v1f64:
 ; CHECK:       // %bb.0:
@@ -1273,6 +1282,28 @@ entry:
   ret <16 x half> %c
 }
 
+define <2 x fp128> @sin_v2fp128(<2 x fp128> %a) {
+; CHECK-LABEL: sin_v2fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    bl sinl
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    bl sinl
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+entry:
+  %c = call <2 x fp128> @llvm.sin.v2fp128(<2 x fp128> %a)
+  ret <2 x fp128> %c
+}
+
 define double @cos_f64(double %a) {
 ; CHECK-LABEL: cos_f64:
 ; CHECK:       // %bb.0: // %entry
@@ -1307,6 +1338,15 @@ entry:
   ret half %c
 }
 
+define fp128 @cos_fp128(fp128 %a) {
+; CHECK-LABEL: cos_fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    b cosl
+entry:
+  %c = call fp128 @llvm.cos.fp128(fp128 %a)
+  ret fp128 %c
+}
+
 define <1 x double> @cos_v1f64(<1 x double> %x) {
 ; CHECK-LABEL: cos_v1f64:
 ; CHECK:       // %bb.0:
@@ -2544,6 +2584,28 @@ entry:
   ret <16 x half> %c
 }
 
+define <2 x fp128> @cos_v2fp128(<2 x fp128> %a) {
+; CHECK-LABEL: cos_v2fp128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    bl cosl
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    bl cosl
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+entry:
+  %c = call <2 x fp128> @llvm.cos.v2fp128(<2 x fp128> %a)
+  ret <2 x fp128> %c
+}
+
 ; This is testing that we do not produce incorrect tailcall lowerings
 define i64 @donttailcall(double noundef %x, double noundef %y) {
 ; CHECK-LABEL: donttailcall:
@@ -2568,6 +2630,8 @@ declare <2 x double> @llvm.cos.v2f64(<2 x double>)
 declare <2 x double> @llvm.sin.v2f64(<2 x double>)
 declare <2 x float> @llvm.cos.v2f32(<2 x float>)
 declare <2 x float> @llvm.sin.v2f32(<2 x float>)
+declare <2 x fp128> @llvm.cos.v2fp128(<2 x fp128>)
+declare <2 x fp128> @llvm.sin.v2fp128(<2 x fp128>)
 declare <3 x double> @llvm.cos.v3f64(<3 x double>)
 declare <3 x double> @llvm.sin.v3f64(<3 x double>)
 declare <3 x float> @llvm.cos.v3f32(<3 x float>)
@@ -2588,5 +2652,7 @@ declare double @llvm.cos.f64(double)
 declare double @llvm.sin.f64(double)
 declare float @llvm.cos.f32(float)
 declare float @llvm.sin.f32(float)
+declare fp128 @llvm.cos.fp128(fp128)
+declare fp128 @llvm.sin.fp128(fp128)
 declare half @llvm.cos.f16(half)
 declare half @llvm.sin.f16(half)
diff --git a/llvm/test/CodeGen/AArch64/icmp-ult-eq-fold.ll b/llvm/test/CodeGen/AArch64/icmp-ult-eq-fold.ll
new file mode 100644
index 0000000000000..33c5ba7987974
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/icmp-ult-eq-fold.ll
@@ -0,0 +1,216 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=aarch64 | FileCheck %s
+
+
+define i1 @lt8_u8(i8 %0) {
+; CHECK-LABEL: lt8_u8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0xf8
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %2 = icmp ult i8 %0, 8
+  ret i1 %2
+}
+
+define i1 @lt32_u8(i8 %0) {
+; CHECK-LABEL: lt32_u8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0xe0
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %2 = icmp ult i8 %0, 32
+  ret i1 %2
+}
+
+define i1 @lt64_u8(i8 %0) {
+; CHECK-LABEL: lt64_u8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0xc0
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %2 = icmp ult i8 %0, 64
+  ret i1 %2
+}
+
+define i1 @lt8_u32(i32 %0) {
+; CHECK-LABEL: lt8_u32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmp w0, #8
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %2 = icmp ult i32 %0, 8
+  ret i1 %2
+}
+
+define i1 @lt32_u32(i32 %0) {
+; CHECK-LABEL: lt32_u32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmp w0, #32
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %2 = icmp ult i32 %0, 32
+  ret i1 %2
+}
+
+define i1 @lt64_u32(i32 %0) {
+; CHECK-LABEL: lt64_u32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmp w0, #64
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %2 = icmp ult i32 %0, 64
+  ret i1 %2
+}
+
+define i1 @lt8_u64(i64 %0) {
+; CHECK-LABEL: lt8_u64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmp x0, #8
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %2 = icmp ult i64 %0, 8
+  ret i1 %2
+}
+
+define i1 @lt32_u64(i64 %0) {
+; CHECK-LABEL: lt32_u64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmp x0, #32
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %2 = icmp ult i64 %0, 32
+  ret i1 %2
+}
+
+define i1 @lt64_u64(i64 %0) {
+; CHECK-LABEL: lt64_u64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmp x0, #64
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %2 = icmp ult i64 %0, 64
+  ret i1 %2
+}
+
+define i1 @lt8_u16_and_5(i8 %0) {
+; CHECK-LABEL: lt8_u16_and_5:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    cmp w8, #0
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %2 = and i8 %0, 5
+  %3 = icmp ult i8 %2, 16
+  ret i1 %3
+}
+
+define i1 @lt8_u16_and_19(i8 %0) {
+; CHECK-LABEL: lt8_u16_and_19:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x10
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %2 = and i8 %0, 19
+  %3 = icmp ult i8 %2, 16
+  ret i1 %3
+}
+
+define i1 @lt32_u16_and_7(i32 %0) {
+; CHECK-LABEL: lt32_u16_and_7:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    cmp w8, #0
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %2 = and i32 %0, 7
+  %3 = icmp ult i32 %2, 16
+  ret i1 %3
+}
+
+define i1 @lt32_u16_and_21(i32 %0) {
+; CHECK-LABEL: lt32_u16_and_21:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst w0, #0x10
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %2 = and i32 %0, 21
+  %3 = icmp ult i32 %2, 16
+  ret i1 %3
+}
+
+define i1 @lt64_u16_and_9(i64 %0) {
+; CHECK-LABEL: lt64_u16_and_9:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    cmp x8, #0
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %2 = and i64 %0, 9
+  %3 = icmp ult i64 %2, 16
+  ret i1 %3
+}
+
+define i1 @lt64_u16_and_23(i64 %0) {
+; CHECK-LABEL: lt64_u16_and_23:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tst x0, #0x10
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+  %2 = and i64 %0, 23
+  %3 = icmp ult i64 %2, 16
+  ret i1 %3
+}
+
+; negative test
+define i1 @lt3_u8(i8 %0) {
+; CHECK-LABEL: lt3_u8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w8, w0, #0xff
+; CHECK-NEXT:    cmp w8, #3
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %2 = icmp ult i8 %0, 3
+  ret i1 %2
+}
+
+; negative test
+define i1 @lt3_u32(i32 %0) {
+; CHECK-LABEL: lt3_u32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmp w0, #3
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %2 = icmp ult i32 %0, 3
+  ret i1 %2
+}
+
+; negative test
+define i1 @lt3_u64(i64 %0) {
+; CHECK-LABEL: lt3_u64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmp x0, #3
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+  %2 = icmp ult i64 %0, 3
+  ret i1 %2
+}
+
+; negative test
+define i32 @lt32_u16_multiple_use(i32 %0) {
+; CHECK-LABEL: lt32_u16_multiple_use:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #21 // =0x15
+; CHECK-NEXT:    mov w9, #10 // =0xa
+; CHECK-NEXT:    and w8, w0, w8
+; CHECK-NEXT:    cmp w8, #16
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    cset w10, lo
+; CHECK-NEXT:    mul w0, w8, w10
+; CHECK-NEXT:    ret
+  %2 = and i32 %0, 21
+  %3 = icmp ult i32 %2, 16
+  %4 = add i32 %2, 10
+  %5 = zext i1 %3 to i32
+  %6 = mul i32 %4, %5
+  ret i32 %6
+}
diff --git a/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll b/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll
index 59a460923e8b7..47fae5a01c931 100644
--- a/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll
+++ b/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll
@@ -1165,9 +1165,7 @@ define <vscale x 2 x i64> @fshl_rot_const_i64(<vscale x 2 x i64> %a){
 ; CHECK-NEXT:    lsl z0.d, z0.d, #3
 ; CHECK-NEXT:    orr z0.d, z0.d, z1.d
 ; CHECK-NEXT:    ret
-  %insert = insertelement <vscale x 2 x i64> poison, i64 3, i32 0
-  %shuf = shufflevector <vscale x 2 x i64> %insert, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-  %fshl = call <vscale x 2 x i64> @llvm.fshl.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %a, <vscale x 2 x i64> %shuf)
+  %fshl = call <vscale x 2 x i64> @llvm.fshl.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 3))
   ret <vscale x 2 x i64> %fshl
 }
 
diff --git a/llvm/test/CodeGen/AArch64/signed-truncation-check.ll b/llvm/test/CodeGen/AArch64/signed-truncation-check.ll
index bb4df6d8935b1..7c80f9320faec 100644
--- a/llvm/test/CodeGen/AArch64/signed-truncation-check.ll
+++ b/llvm/test/CodeGen/AArch64/signed-truncation-check.ll
@@ -287,9 +287,8 @@ define i1 @add_ultcmp_bad_i16_i8_add(i16 %x, i16 %y) nounwind {
 ; CHECK-LABEL: add_ultcmp_bad_i16_i8_add:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add w8, w0, w1
-; CHECK-NEXT:    and w8, w8, #0xffff
-; CHECK-NEXT:    cmp w8, #256
-; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    tst w8, #0xff00
+; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %tmp0 = add i16 %x, %y
   %tmp1 = icmp ult i16 %tmp0, 256 ; 1U << 8
@@ -328,9 +327,8 @@ define i1 @add_ultcmp_bad_i16_i8_c0notpoweroftwo(i16 %x) nounwind {
 ; CHECK-LABEL: add_ultcmp_bad_i16_i8_c0notpoweroftwo:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add w8, w0, #192
-; CHECK-NEXT:    and w8, w8, #0xffff
-; CHECK-NEXT:    cmp w8, #256
-; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    tst w8, #0xff00
+; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %tmp0 = add i16 %x, 192 ; (1U << (8-1)) + (1U << (8-1-1))
   %tmp1 = icmp ult i16 %tmp0, 256 ; 1U << 8
@@ -356,9 +354,8 @@ define i1 @add_ultcmp_bad_i16_i8_magic(i16 %x) nounwind {
 ; CHECK-LABEL: add_ultcmp_bad_i16_i8_magic:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add w8, w0, #64
-; CHECK-NEXT:    and w8, w8, #0xffff
-; CHECK-NEXT:    cmp w8, #256
-; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    tst w8, #0xff00
+; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %tmp0 = add i16 %x, 64 ; 1U << (8-1-1)
   %tmp1 = icmp ult i16 %tmp0, 256 ; 1U << 8
@@ -370,9 +367,8 @@ define i1 @add_ultcmp_bad_i16_i4(i16 %x) nounwind {
 ; CHECK-LABEL: add_ultcmp_bad_i16_i4:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add w8, w0, #8
-; CHECK-NEXT:    and w8, w8, #0xffff
-; CHECK-NEXT:    cmp w8, #16
-; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    tst w8, #0xfff0
+; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %tmp0 = add i16 %x, 8 ; 1U << (4-1)
   %tmp1 = icmp ult i16 %tmp0, 16 ; 1U << 4
@@ -384,9 +380,8 @@ define i1 @add_ultcmp_bad_i24_i8(i24 %x) nounwind {
 ; CHECK-LABEL: add_ultcmp_bad_i24_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add w8, w0, #128
-; CHECK-NEXT:    and w8, w8, #0xffffff
-; CHECK-NEXT:    cmp w8, #256
-; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    tst w8, #0xffff00
+; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
   %tmp0 = add i24 %x, 128 ; 1U << (8-1)
   %tmp1 = icmp ult i24 %tmp0, 256 ; 1U << 8
diff --git a/llvm/test/CodeGen/AArch64/sme-disable-rematerialize-with-streaming-mode-changes.ll b/llvm/test/CodeGen/AArch64/sme-disable-rematerialize-with-streaming-mode-changes.ll
index b3aeb1fcc42da..5b2aeb04b71da 100644
--- a/llvm/test/CodeGen/AArch64/sme-disable-rematerialize-with-streaming-mode-changes.ll
+++ b/llvm/test/CodeGen/AArch64/sme-disable-rematerialize-with-streaming-mode-changes.ll
@@ -54,9 +54,7 @@ entry:
 for.body:                                         ; preds = %entry, %for.body
   %index.03 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
   call void asm sideeffect "", "~{x19},~{x20},~{x21},~{x22},~{x23},~{x24},~{x25},~{x26},~{x27}"() nounwind
-  %ptrue.ins = insertelement <vscale x 16 x i1> poison, i1 1, i32 0
-  %ptrue = shufflevector <vscale x 16 x i1> %ptrue.ins, <vscale x 16 x i1> poison, <vscale x 16 x i32> zeroinitializer
-  call void @bar(<vscale x 16 x i1> %ptrue)
+  call void @bar(<vscale x 16 x i1> splat(i1 true))
   %inc = add nuw nsw i32 %index.03, 1
   %exitcond.not = icmp eq i32 %inc, %N
   br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-dbg-assign-tag-offset.ll b/llvm/test/CodeGen/AArch64/stack-tagging-dbg-assign-tag-offset.ll
index a48a885a6c68e..1e78e98bb78b2 100644
--- a/llvm/test/CodeGen/AArch64/stack-tagging-dbg-assign-tag-offset.ll
+++ b/llvm/test/CodeGen/AArch64/stack-tagging-dbg-assign-tag-offset.ll
@@ -3,7 +3,7 @@
 
 source_filename = "test.ll"
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
-target triple = "aarch64--linux-android10000"
+target triple = "aarch64--linux-android35"
 
 declare void @g(ptr, ptr, ptr, ptr, ptr, ptr)
 
diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-dbg-declare-tag-offset.ll b/llvm/test/CodeGen/AArch64/stack-tagging-dbg-declare-tag-offset.ll
index e0efd37aad2c0..a2e6096d8cb92 100644
--- a/llvm/test/CodeGen/AArch64/stack-tagging-dbg-declare-tag-offset.ll
+++ b/llvm/test/CodeGen/AArch64/stack-tagging-dbg-declare-tag-offset.ll
@@ -6,7 +6,7 @@
 ; RUN: opt --try-experimental-debuginfo-iterators -aarch64-stack-tagging -stack-tagging-record-stack-history=instr -S -o - %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
-target triple = "aarch64--linux-android10000"
+target triple = "aarch64--linux-android35"
 
 declare void @g(ptr, ptr, ptr, ptr, ptr, ptr)
 
diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-dbg-value-tag-offset-nopad.ll b/llvm/test/CodeGen/AArch64/stack-tagging-dbg-value-tag-offset-nopad.ll
index 093ddd3be3a56..dc7c4039a44d2 100644
--- a/llvm/test/CodeGen/AArch64/stack-tagging-dbg-value-tag-offset-nopad.ll
+++ b/llvm/test/CodeGen/AArch64/stack-tagging-dbg-value-tag-offset-nopad.ll
@@ -1,7 +1,7 @@
 ; RUN: opt -aarch64-stack-tagging -stack-tagging-record-stack-history=instr  -S -o - %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
-target triple = "aarch64-unknown-linux-android10000"
+target triple = "aarch64-unknown-linux-android35"
 
 define dso_local void @f() sanitize_memtag !dbg !14 {
   %a1 = alloca i128, align 4
diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-prologue.ll b/llvm/test/CodeGen/AArch64/stack-tagging-prologue.ll
index 40bc8bdd70703..3eed6d45c7710 100644
--- a/llvm/test/CodeGen/AArch64/stack-tagging-prologue.ll
+++ b/llvm/test/CodeGen/AArch64/stack-tagging-prologue.ll
@@ -4,7 +4,7 @@
 
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
-target triple = "aarch64--linux-android10000"
+target triple = "aarch64--linux-android35"
 
 declare void @use32(ptr)
 
diff --git a/llvm/test/CodeGen/AArch64/sve-adr.ll b/llvm/test/CodeGen/AArch64/sve-adr.ll
index 8f8477d8f3f15..388c07cf0cb6d 100644
--- a/llvm/test/CodeGen/AArch64/sve-adr.ll
+++ b/llvm/test/CodeGen/AArch64/sve-adr.ll
@@ -14,10 +14,8 @@ define <vscale x 4 x i32>  @adr_32bit_lsl1(<vscale x 4 x i32>  %base, <vscale x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adr z0.s, [z0.s, z1.s, lsl #1]
 ; CHECK-NEXT:    ret
-  %splat_insert = insertelement <vscale x 4 x i32> poison, i32 1, i32 0
-  %one = shufflevector <vscale x 4 x i32> %splat_insert, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-  %shiftedOffset = shl <vscale x 4 x i32> %idx, %one
-  %address = add <vscale x 4 x i32> %base, %shiftedOffset
+  %offset = shl <vscale x 4 x i32> %idx, splat (i32 1)
+  %address = add <vscale x 4 x i32> %base, %offset
   ret <vscale x 4 x i32>  %address
 }
 
@@ -26,10 +24,8 @@ define <vscale x 4 x i32>  @adr_32bit_lsl2(<vscale x 4 x i32>  %base, <vscale x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adr z0.s, [z0.s, z1.s, lsl #2]
 ; CHECK-NEXT:    ret
-  %splat_insert = insertelement <vscale x 4 x i32> poison, i32 2, i32 0
-  %two = shufflevector <vscale x 4 x i32> %splat_insert, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-  %shiftedOffset = shl <vscale x 4 x i32> %idx, %two
-  %address = add <vscale x 4 x i32> %base, %shiftedOffset
+  %offset = shl <vscale x 4 x i32> %idx, splat (i32 2)
+  %address = add <vscale x 4 x i32> %base, %offset
   ret <vscale x 4 x i32>  %address
 }
 
@@ -38,10 +34,8 @@ define <vscale x 4 x i32>  @adr_32bit_lsl3(<vscale x 4 x i32>  %base, <vscale x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adr z0.s, [z0.s, z1.s, lsl #3]
 ; CHECK-NEXT:    ret
-  %splat_insert = insertelement <vscale x 4 x i32> poison, i32 3, i32 0
-  %three = shufflevector <vscale x 4 x i32> %splat_insert, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-  %shiftedOffset = shl <vscale x 4 x i32> %idx, %three
-  %address = add <vscale x 4 x i32> %base, %shiftedOffset
+  %offset = shl <vscale x 4 x i32> %idx, splat (i32 3)
+  %address = add <vscale x 4 x i32> %base, %offset
   ret <vscale x 4 x i32>  %address
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve-cmp-folds.ll b/llvm/test/CodeGen/AArch64/sve-cmp-folds.ll
index 1b78c253c58dd..981cc88298a3e 100644
--- a/llvm/test/CodeGen/AArch64/sve-cmp-folds.ll
+++ b/llvm/test/CodeGen/AArch64/sve-cmp-folds.ll
@@ -8,9 +8,7 @@ define <vscale x 8 x i1> @not_icmp_sle_nxv8i16(<vscale x 8 x i16> %a, <vscale x
 ; CHECK-NEXT:    cmpgt p0.h, p0/z, z0.h, z1.h
 ; CHECK-NEXT:    ret
   %icmp = icmp sle <vscale x 8 x i16> %a, %b
-  %tmp = insertelement <vscale x 8 x i1> undef, i1 true, i32 0
-  %ones = shufflevector <vscale x 8 x i1> %tmp, <vscale x 8 x i1> undef, <vscale x 8 x i32> zeroinitializer
-  %not = xor <vscale x 8 x i1> %ones, %icmp
+  %not = xor <vscale x 8 x i1> splat(i1 true), %icmp
   ret <vscale x 8 x i1> %not
 }
 
@@ -21,9 +19,7 @@ define <vscale x 4 x i1> @not_icmp_sgt_nxv4i32(<vscale x 4 x i32> %a, <vscale x
 ; CHECK-NEXT:    cmpge p0.s, p0/z, z1.s, z0.s
 ; CHECK-NEXT:    ret
   %icmp = icmp sgt <vscale x 4 x i32> %a, %b
-  %tmp = insertelement <vscale x 4 x i1> undef, i1 true, i32 0
-  %ones = shufflevector <vscale x 4 x i1> %tmp, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer
-  %not = xor <vscale x 4 x i1> %icmp, %ones
+  %not = xor <vscale x 4 x i1> %icmp, splat(i1 true)
   ret <vscale x 4 x i1> %not
 }
 
@@ -34,9 +30,7 @@ define <vscale x 2 x i1> @not_fcmp_une_nxv2f64(<vscale x 2 x double> %a, <vscale
 ; CHECK-NEXT:    fcmeq p0.d, p0/z, z0.d, z1.d
 ; CHECK-NEXT:    ret
   %icmp = fcmp une <vscale x 2 x double> %a, %b
-  %tmp = insertelement <vscale x 2 x i1> undef, i1 true, i32 0
-  %ones = shufflevector <vscale x 2 x i1> %tmp, <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer
-  %not = xor <vscale x 2 x i1> %icmp, %ones
+  %not = xor <vscale x 2 x i1> %icmp, splat(i1 true)
   ret <vscale x 2 x i1> %not
 }
 
@@ -47,9 +41,7 @@ define <vscale x 4 x i1> @not_fcmp_uge_nxv4f32(<vscale x 4 x float> %a, <vscale
 ; CHECK-NEXT:    fcmgt p0.s, p0/z, z1.s, z0.s
 ; CHECK-NEXT:    ret
   %icmp = fcmp uge <vscale x 4 x float> %a, %b
-  %tmp = insertelement <vscale x 4 x i1> undef, i1 true, i32 0
-  %ones = shufflevector <vscale x 4 x i1> %tmp, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer
-  %not = xor <vscale x 4 x i1> %icmp, %ones
+  %not = xor <vscale x 4 x i1> %icmp, splat(i1 true)
   ret <vscale x 4 x i1> %not
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve-dead-masked-store.ll b/llvm/test/CodeGen/AArch64/sve-dead-masked-store.ll
index c5db3dfdf5e54..2ff32bde25d61 100644
--- a/llvm/test/CodeGen/AArch64/sve-dead-masked-store.ll
+++ b/llvm/test/CodeGen/AArch64/sve-dead-masked-store.ll
@@ -17,10 +17,8 @@ define void @dead_masked_store_alltrue_same(<vscale x 4 x i32> %val, ptr %a, <vs
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
 ; CHECK-NEXT:    ret
-  %alltrue.ins = insertelement <vscale x 4 x i1> poison, i1 true, i32 0
-  %alltrue = shufflevector  <vscale x 4 x i1> %alltrue.ins,  <vscale x 4 x i1> poison,  <vscale x 4 x i32> zeroinitializer
   call void @llvm.masked.store.nxv4i32(<vscale x 4 x i32> %val, ptr %a, i32 4, <vscale x 4 x i1> %mask)
-  call void @llvm.masked.store.nxv4i32(<vscale x 4 x i32> %val, ptr %a, i32 4, <vscale x 4 x i1> %alltrue)
+  call void @llvm.masked.store.nxv4i32(<vscale x 4 x i32> %val, ptr %a, i32 4, <vscale x 4 x i1> splat(i1 true))
   ret void
 }
 
@@ -30,10 +28,8 @@ define void @dead_masked_store_alltrue_bigger(<vscale x 4 x i16> %val, <vscale x
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    st1w { z1.s }, p0, [x0]
 ; CHECK-NEXT:    ret
-  %alltrue.ins = insertelement <vscale x 4 x i1> poison, i1 true, i32 0
-  %alltrue = shufflevector  <vscale x 4 x i1> %alltrue.ins,  <vscale x 4 x i1> poison,  <vscale x 4 x i32> zeroinitializer
   call void @llvm.masked.store.nxv4i16(<vscale x 4 x i16> %val, ptr %a, i32 4, <vscale x 4 x i1> %mask)
-  call void @llvm.masked.store.nxv4i32(<vscale x 4 x i32> %val1, ptr %a, i32 4, <vscale x 4 x i1> %alltrue)
+  call void @llvm.masked.store.nxv4i32(<vscale x 4 x i32> %val1, ptr %a, i32 4, <vscale x 4 x i1> splat(i1 true))
   ret void
 }
 
@@ -44,10 +40,8 @@ define void @dead_masked_store_alltrue_smaller(<vscale x 4 x i32> %val, <vscale
 ; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
 ; CHECK-NEXT:    st1h { z1.s }, p1, [x0]
 ; CHECK-NEXT:    ret
-  %alltrue.ins = insertelement <vscale x 4 x i1> poison, i1 true, i32 0
-  %alltrue = shufflevector  <vscale x 4 x i1> %alltrue.ins,  <vscale x 4 x i1> poison,  <vscale x 4 x i32> zeroinitializer
   call void @llvm.masked.store.nxv4i32(<vscale x 4 x i32> %val, ptr %a, i32 4, <vscale x 4 x i1> %mask)
-  call void @llvm.masked.store.nxv4i16(<vscale x 4 x i16> %val1, ptr %a, i32 4, <vscale x 4 x i1> %alltrue)
+  call void @llvm.masked.store.nxv4i16(<vscale x 4 x i16> %val1, ptr %a, i32 4, <vscale x 4 x i1> splat(i1 true))
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll b/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll
index b05b46a75b698..678afc4dea309 100644
--- a/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll
@@ -392,9 +392,7 @@ define <2 x float> @extract_v2f32_nxv4f32_splat_const() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmov v0.2s, #1.00000000
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 4 x float> poison, float 1.0, i32 0
-  %splat = shufflevector <vscale x 4 x float> %ins, <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
-  %ext = call <2 x float> @llvm.vector.extract.v2f32.nxv4f32(<vscale x 4 x float> %splat, i64 0)
+  %ext = call <2 x float> @llvm.vector.extract.v2f32.nxv4f32(<vscale x 4 x float> splat(float 1.0), i64 0)
   ret <2 x float> %ext
 }
 
@@ -403,9 +401,7 @@ define <4 x i32> @extract_v4i32_nxv8i32_splat_const() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v0.4s, #1
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 8 x i32> poison, i32 1, i32 0
-  %splat = shufflevector <vscale x 8 x i32> %ins, <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
-  %ext = call <4 x i32> @llvm.vector.extract.v4i32.nxv8i32(<vscale x 8 x i32> %splat, i64 0)
+  %ext = call <4 x i32> @llvm.vector.extract.v4i32.nxv8i32(<vscale x 8 x i32> splat(i32 1), i64 0)
   ret <4 x i32> %ext
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve-extract-scalable-vector.ll b/llvm/test/CodeGen/AArch64/sve-extract-scalable-vector.ll
index 3c0bd501f45d8..cbede1bf8bb74 100644
--- a/llvm/test/CodeGen/AArch64/sve-extract-scalable-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-extract-scalable-vector.ll
@@ -1039,9 +1039,7 @@ define <vscale x 2 x float> @extract_nxv2f32_nxv4f32_splat_const() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmov z0.s, #1.00000000
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 4 x float> poison, float 1.0, i32 0
-  %splat = shufflevector <vscale x 4 x float> %ins, <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
-  %ext = call <vscale x 2 x float> @llvm.vector.extract.nxv2f32.nxv4f32(<vscale x 4 x float> %splat, i64 0)
+  %ext = call <vscale x 2 x float> @llvm.vector.extract.nxv2f32.nxv4f32(<vscale x 4 x float> splat(float 1.0), i64 0)
   ret <vscale x 2 x float> %ext
 }
 
@@ -1050,9 +1048,7 @@ define <vscale x 4 x i32> @extract_nxv4i32_nxv8i32_splat_const() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z0.s, #1 // =0x1
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 8 x i32> poison, i32 1, i32 0
-  %splat = shufflevector <vscale x 8 x i32> %ins, <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
-  %ext = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> %splat, i64 0)
+  %ext = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> splat(i32 1), i64 0)
   ret <vscale x 4 x i32> %ext
 }
 
@@ -1061,9 +1057,7 @@ define <vscale x 2 x i1> @extract_nxv2i1_nxv16i1_all_ones() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 16 x i1> poison, i1 1, i32 0
-  %splat = shufflevector <vscale x 16 x i1> %ins, <vscale x 16 x i1> poison, <vscale x 16 x i32> zeroinitializer
-  %ext = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv16i1(<vscale x 16 x i1> %splat, i64 0)
+  %ext = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv16i1(<vscale x 16 x i1> splat(i1 true), i64 0)
   ret <vscale x 2 x i1> %ext
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve-fadda-select.ll b/llvm/test/CodeGen/AArch64/sve-fadda-select.ll
index b4d6aa5e09cd9..e7681cae08372 100644
--- a/llvm/test/CodeGen/AArch64/sve-fadda-select.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fadda-select.ll
@@ -10,9 +10,7 @@ define float @pred_fadda_nxv2f32(float %x, <vscale x 2 x float> %y, <vscale x 2
 ; CHECK-NEXT:    fadda s0, p0, s0, z1.s
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $z0
 ; CHECK-NEXT:    ret
-  %i = insertelement <vscale x 2 x float> poison, float -0.000000e+00, i32 0
-  %minus0 = shufflevector <vscale x 2 x float> %i, <vscale x 2 x float> poison, <vscale x 2 x i32> zeroinitializer
-  %sel = select <vscale x 2 x i1> %mask, <vscale x 2 x float> %y, <vscale x 2 x float> %minus0
+  %sel = select <vscale x 2 x i1> %mask, <vscale x 2 x float> %y, <vscale x 2 x float> splat(float -0.000000e+00)
   %fadda = call float @llvm.vector.reduce.fadd.nxv2f32(float %x, <vscale x 2 x float> %sel)
   ret float %fadda
 }
@@ -24,9 +22,7 @@ define float @pred_fadda_nxv4f32(float %x, <vscale x 4 x float> %y, <vscale x 4
 ; CHECK-NEXT:    fadda s0, p0, s0, z1.s
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $z0
 ; CHECK-NEXT:    ret
-  %i = insertelement <vscale x 4 x float> poison, float -0.000000e+00, i32 0
-  %minus0 = shufflevector <vscale x 4 x float> %i, <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
-  %sel = select <vscale x 4 x i1> %mask, <vscale x 4 x float> %y, <vscale x 4 x float> %minus0
+  %sel = select <vscale x 4 x i1> %mask, <vscale x 4 x float> %y, <vscale x 4 x float> splat(float -0.000000e+00)
   %fadda = call float @llvm.vector.reduce.fadd.nxv4f32(float %x, <vscale x 4 x float> %sel)
   ret float %fadda
 }
@@ -38,9 +34,7 @@ define double @pred_fadda_nxv2f64(double %x, <vscale x 2 x double> %y, <vscale x
 ; CHECK-NEXT:    fadda d0, p0, d0, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-  %i = insertelement <vscale x 2 x double> poison, double -0.000000e+00, i32 0
-  %minus0 = shufflevector <vscale x 2 x double> %i, <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
-  %sel = select <vscale x 2 x i1> %mask, <vscale x 2 x double> %y, <vscale x 2 x double> %minus0
+  %sel = select <vscale x 2 x i1> %mask, <vscale x 2 x double> %y, <vscale x 2 x double> splat(double -0.000000e+00)
   %fadda = call double @llvm.vector.reduce.fadd.nxv2f64(double %x, <vscale x 2 x double> %sel)
   ret double %fadda
 }
@@ -52,9 +46,7 @@ define half @pred_fadda_nxv2f16(half %x, <vscale x 2 x half> %y, <vscale x 2 x i
 ; CHECK-NEXT:    fadda h0, p0, h0, z1.h
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
-  %i = insertelement <vscale x 2 x half> poison, half -0.000000e+00, i32 0
-  %minus0 = shufflevector <vscale x 2 x half> %i, <vscale x 2 x half> poison, <vscale x 2 x i32> zeroinitializer
-  %sel = select <vscale x 2 x i1> %mask, <vscale x 2 x half> %y, <vscale x 2 x half> %minus0
+  %sel = select <vscale x 2 x i1> %mask, <vscale x 2 x half> %y, <vscale x 2 x half> splat(half -0.000000e+00)
   %fadda = call half @llvm.vector.reduce.fadd.nxv2f16(half %x, <vscale x 2 x half> %sel)
   ret half %fadda
 }
@@ -66,9 +58,7 @@ define half @pred_fadda_nxv4f16(half %x, <vscale x 4 x half> %y, <vscale x 4 x i
 ; CHECK-NEXT:    fadda h0, p0, h0, z1.h
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
-  %i = insertelement <vscale x 4 x half> poison, half -0.000000e+00, i32 0
-  %minus0 = shufflevector <vscale x 4 x half> %i, <vscale x 4 x half> poison, <vscale x 4 x i32> zeroinitializer
-  %sel = select <vscale x 4 x i1> %mask, <vscale x 4 x half> %y, <vscale x 4 x half> %minus0
+  %sel = select <vscale x 4 x i1> %mask, <vscale x 4 x half> %y, <vscale x 4 x half> splat(half -0.000000e+00)
   %fadda = call half @llvm.vector.reduce.fadd.nxv4f16(half %x, <vscale x 4 x half> %sel)
   ret half %fadda
 }
@@ -80,9 +70,7 @@ define half @pred_fadda_nxv8f16(half %x, <vscale x 8 x half> %y, <vscale x 8 x i
 ; CHECK-NEXT:    fadda h0, p0, h0, z1.h
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
-  %i = insertelement <vscale x 8 x half> poison, half -0.000000e+00, i32 0
-  %minus0 = shufflevector <vscale x 8 x half> %i, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
-  %sel = select <vscale x 8 x i1> %mask, <vscale x 8 x half> %y, <vscale x 8 x half> %minus0
+  %sel = select <vscale x 8 x i1> %mask, <vscale x 8 x half> %y, <vscale x 8 x half> splat(half -0.000000e+00)
   %fadda = call half @llvm.vector.reduce.fadd.nxv8f16(half %x, <vscale x 8 x half> %sel)
   ret half %fadda
 }
diff --git a/llvm/test/CodeGen/AArch64/sve-fold-loadext-and-splat-vector.ll b/llvm/test/CodeGen/AArch64/sve-fold-loadext-and-splat-vector.ll
index e532743a8658b..42ec96ad60498 100644
--- a/llvm/test/CodeGen/AArch64/sve-fold-loadext-and-splat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fold-loadext-and-splat-vector.ll
@@ -13,9 +13,7 @@ define <vscale x 2 x i64> @fold_loadext_and(ptr %ptr, i32 %needle, <vscale x 2 x
 ; CHECK-NEXT:    ret
   %load = load <vscale x 2 x i32>, ptr %ptr, align 4
   %ext = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
-  %splatinsert = insertelement <vscale x 2 x i64> poison, i64 4294967295, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %splatinsert, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-  %and = and <vscale x 2 x i64> %ext, %splat
+  %and = and <vscale x 2 x i64> %ext, splat(i64 4294967295)
   ret <vscale x 2 x i64> %and
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve-fp-immediates-merging.ll b/llvm/test/CodeGen/AArch64/sve-fp-immediates-merging.ll
index ef62ffd58bd5d..e1d883b0e7899 100644
--- a/llvm/test/CodeGen/AArch64/sve-fp-immediates-merging.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fp-immediates-merging.ll
@@ -13,9 +13,7 @@ define <vscale x 8 x half> @fadd_h_immhalf(<vscale x 8 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    fadd z0.h, p0/m, z0.h, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
-  %out = fadd <vscale x 8 x half> %a, %splat
+  %out = fadd <vscale x 8 x half> %a, splat(half 0.500000e+00)
   ret <vscale x 8 x half> %out
 }
 
@@ -25,9 +23,7 @@ define <vscale x 8 x half> @fadd_h_immone(<vscale x 8 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    fadd z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
-  %out = fadd <vscale x 8 x half> %a, %splat
+  %out = fadd <vscale x 8 x half> %a, splat(half 1.000000e+00)
   ret <vscale x 8 x half> %out
 }
 
@@ -37,9 +33,7 @@ define <vscale x 4 x half> @fadd_4h_immhalf(<vscale x 4 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fadd z0.h, p0/m, z0.h, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x half> undef, half 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x half> %elt, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
-  %out = fadd <vscale x 4 x half> %a, %splat
+  %out = fadd <vscale x 4 x half> %a, splat(half 0.500000e+00)
   ret <vscale x 4 x half> %out
 }
 
@@ -49,9 +43,7 @@ define <vscale x 4 x half> @fadd_4h_immone(<vscale x 4 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fadd z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x half> %elt, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
-  %out = fadd <vscale x 4 x half> %a, %splat
+  %out = fadd <vscale x 4 x half> %a, splat(half 1.000000e+00)
   ret <vscale x 4 x half> %out
 }
 
@@ -61,9 +53,7 @@ define <vscale x 2 x half> @fadd_2h_immhalf(<vscale x 2 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fadd z0.h, p0/m, z0.h, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x half> undef, half 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x half> %elt, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fadd <vscale x 2 x half> %a, %splat
+  %out = fadd <vscale x 2 x half> %a, splat(half 0.500000e+00)
   ret <vscale x 2 x half> %out
 }
 
@@ -73,9 +63,7 @@ define <vscale x 2 x half> @fadd_2h_immone(<vscale x 2 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fadd z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x half> %elt, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fadd <vscale x 2 x half> %a, %splat
+  %out = fadd <vscale x 2 x half> %a, splat(half 1.000000e+00)
   ret <vscale x 2 x half> %out
 }
 
@@ -85,9 +73,7 @@ define <vscale x 4 x float> @fadd_s_immhalf(<vscale x 4 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fadd z0.s, p0/m, z0.s, #0.5
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x float> undef, float 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
-  %out = fadd <vscale x 4 x float> %a, %splat
+  %out = fadd <vscale x 4 x float> %a, splat(float 0.500000e+00)
   ret <vscale x 4 x float> %out
 }
 
@@ -97,9 +83,7 @@ define <vscale x 4 x float> @fadd_s_immone(<vscale x 4 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fadd z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
-  %out = fadd <vscale x 4 x float> %a, %splat
+  %out = fadd <vscale x 4 x float> %a, splat(float 1.000000e+00)
   ret <vscale x 4 x float> %out
 }
 
@@ -109,9 +93,7 @@ define <vscale x 2 x float> @fadd_2s_immhalf(<vscale x 2 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fadd z0.s, p0/m, z0.s, #0.5
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x float> undef, float 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x float> %elt, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fadd <vscale x 2 x float> %a, %splat
+  %out = fadd <vscale x 2 x float> %a, splat(float 0.500000e+00)
   ret <vscale x 2 x float> %out
 }
 
@@ -121,9 +103,7 @@ define <vscale x 2 x float> @fadd_2s_immone(<vscale x 2 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fadd z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x float> %elt, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fadd <vscale x 2 x float> %a, %splat
+  %out = fadd <vscale x 2 x float> %a, splat(float 1.000000e+00)
   ret <vscale x 2 x float> %out
 }
 
@@ -134,9 +114,7 @@ define <vscale x 2 x double> @fadd_d_immhalf(<vscale x 2 x double> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fadd z0.d, p0/m, z0.d, #0.5
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x double> undef, double 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fadd <vscale x 2 x double> %a, %splat
+  %out = fadd <vscale x 2 x double> %a, splat(double 0.500000e+00)
   ret <vscale x 2 x double> %out
 }
 
@@ -146,9 +124,7 @@ define <vscale x 2 x double> @fadd_d_immone(<vscale x 2 x double> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fadd z0.d, p0/m, z0.d, #1.0
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x double> undef, double 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fadd <vscale x 2 x double> %a, %splat
+  %out = fadd <vscale x 2 x double> %a, splat(double 1.000000e+00)
   ret <vscale x 2 x double> %out
 }
 
@@ -162,9 +138,7 @@ define <vscale x 8 x half> @fmax_h_immzero(<vscale x 8 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    fmax z0.h, p0/m, z0.h, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
-  %out = call <vscale x 8 x half> @llvm.maximum.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %splat)
+  %out = call <vscale x 8 x half> @llvm.maximum.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> zeroinitializer)
   ret <vscale x 8 x half> %out
 }
 
@@ -174,9 +148,7 @@ define <vscale x 8 x half> @fmax_h_immone(<vscale x 8 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    fmax z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
-  %out = call <vscale x 8 x half> @llvm.maximum.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %splat)
+  %out = call <vscale x 8 x half> @llvm.maximum.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> splat(half 1.000000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -186,9 +158,7 @@ define <vscale x 4 x half> @fmax_4h_immzero(<vscale x 4 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fmax z0.h, p0/m, z0.h, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x half> undef, half 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x half> %elt, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
-  %out = call <vscale x 4 x half> @llvm.maximum.nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %splat)
+  %out = call <vscale x 4 x half> @llvm.maximum.nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> zeroinitializer)
   ret <vscale x 4 x half> %out
 }
 
@@ -198,9 +168,7 @@ define <vscale x 4 x half> @fmax_4h_immone(<vscale x 4 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fmax z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x half> %elt, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
-  %out = call <vscale x 4 x half> @llvm.maximum.nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %splat)
+  %out = call <vscale x 4 x half> @llvm.maximum.nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> splat(half 1.000000e+00))
   ret <vscale x 4 x half> %out
 }
 
@@ -210,9 +178,7 @@ define <vscale x 2 x half> @fmax_2h_immzero(<vscale x 2 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmax z0.h, p0/m, z0.h, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x half> undef, half 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x half> %elt, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x half> @llvm.maximum.nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> %splat)
+  %out = call <vscale x 2 x half> @llvm.maximum.nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> zeroinitializer)
   ret <vscale x 2 x half> %out
 }
 
@@ -222,9 +188,7 @@ define <vscale x 2 x half> @fmax_2h_immone(<vscale x 2 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmax z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x half> %elt, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x half> @llvm.maximum.nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> %splat)
+  %out = call <vscale x 2 x half> @llvm.maximum.nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> splat(half 1.000000e+00))
   ret <vscale x 2 x half> %out
 }
 
@@ -234,9 +198,7 @@ define <vscale x 4 x float> @fmax_s_immzero(<vscale x 4 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fmax z0.s, p0/m, z0.s, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
-  %out = call <vscale x 4 x float> @llvm.maximum.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %splat)
+  %out = call <vscale x 4 x float> @llvm.maximum.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> zeroinitializer)
   ret <vscale x 4 x float> %out
 }
 
@@ -246,9 +208,7 @@ define <vscale x 4 x float> @fmax_s_immone(<vscale x 4 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fmax z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
-  %out = call <vscale x 4 x float> @llvm.maximum.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %splat)
+  %out = call <vscale x 4 x float> @llvm.maximum.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> splat(float 1.000000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -258,9 +218,7 @@ define <vscale x 2 x float> @fmax_2s_immzero(<vscale x 2 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmax z0.s, p0/m, z0.s, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x float> undef, float 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x float> %elt, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x float> @llvm.maximum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %splat)
+  %out = call <vscale x 2 x float> @llvm.maximum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> zeroinitializer)
   ret <vscale x 2 x float> %out
 }
 
@@ -270,9 +228,7 @@ define <vscale x 2 x float> @fmax_2s_immone(<vscale x 2 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmax z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x float> %elt, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x float> @llvm.maximum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %splat)
+  %out = call <vscale x 2 x float> @llvm.maximum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> splat(float 1.000000e+00))
   ret <vscale x 2 x float> %out
 }
 
@@ -282,9 +238,7 @@ define <vscale x 2 x double> @fmax_d_immzero(<vscale x 2 x double> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmax z0.d, p0/m, z0.d, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x double> @llvm.maximum.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %splat)
+  %out = call <vscale x 2 x double> @llvm.maximum.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer)
   ret <vscale x 2 x double> %out
 }
 
@@ -294,9 +248,7 @@ define <vscale x 2 x double> @fmax_d_immone(<vscale x 2 x double> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmax z0.d, p0/m, z0.d, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x double> @llvm.maximum.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %splat)
+  %out = call <vscale x 2 x double> @llvm.maximum.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> splat(double 1.000000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -310,9 +262,7 @@ define <vscale x 8 x half> @fmaxnm_h_immzero(<vscale x 8 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    fmaxnm z0.h, p0/m, z0.h, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
-  %out = call <vscale x 8 x half> @llvm.maxnum.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %splat)
+  %out = call <vscale x 8 x half> @llvm.maxnum.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> zeroinitializer)
   ret <vscale x 8 x half> %out
 }
 
@@ -322,9 +272,7 @@ define <vscale x 8 x half> @fmaxnm_h_immone(<vscale x 8 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    fmaxnm z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
-  %out = call <vscale x 8 x half> @llvm.maxnum.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %splat)
+  %out = call <vscale x 8 x half> @llvm.maxnum.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> splat(half 1.000000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -334,9 +282,7 @@ define <vscale x 4 x half> @fmaxnm_4h_immzero(<vscale x 4 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fmaxnm z0.h, p0/m, z0.h, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x half> undef, half 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x half> %elt, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
-  %out = call <vscale x 4 x half> @llvm.maxnum.nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %splat)
+  %out = call <vscale x 4 x half> @llvm.maxnum.nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> zeroinitializer)
   ret <vscale x 4 x half> %out
 }
 
@@ -346,9 +292,7 @@ define <vscale x 4 x half> @fmaxnm_4h_immone(<vscale x 4 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fmaxnm z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x half> %elt, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
-  %out = call <vscale x 4 x half> @llvm.maxnum.nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %splat)
+  %out = call <vscale x 4 x half> @llvm.maxnum.nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> splat(half 1.000000e+00))
   ret <vscale x 4 x half> %out
 }
 
@@ -358,9 +302,7 @@ define <vscale x 2 x half> @fmaxnm_2h_immzero(<vscale x 2 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmaxnm z0.h, p0/m, z0.h, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x half> undef, half 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x half> %elt, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x half> @llvm.maxnum.nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> %splat)
+  %out = call <vscale x 2 x half> @llvm.maxnum.nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> zeroinitializer)
   ret <vscale x 2 x half> %out
 }
 
@@ -370,9 +312,7 @@ define <vscale x 2 x half> @fmaxnm_2h_immone(<vscale x 2 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmaxnm z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x half> %elt, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x half> @llvm.maxnum.nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> %splat)
+  %out = call <vscale x 2 x half> @llvm.maxnum.nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> splat(half 1.000000e+00))
   ret <vscale x 2 x half> %out
 }
 
@@ -382,9 +322,7 @@ define <vscale x 4 x float> @fmaxnm_s_immzero(<vscale x 4 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fmaxnm z0.s, p0/m, z0.s, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
-  %out = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %splat)
+  %out = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> zeroinitializer)
   ret <vscale x 4 x float> %out
 }
 
@@ -394,9 +332,7 @@ define <vscale x 4 x float> @fmaxnm_s_immone(<vscale x 4 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fmaxnm z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
-  %out = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %splat)
+  %out = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> splat(float 1.000000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -406,9 +342,7 @@ define <vscale x 2 x float> @fmaxnm_2s_immzero(<vscale x 2 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmaxnm z0.s, p0/m, z0.s, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x float> undef, float 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x float> %elt, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x float> @llvm.maxnum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %splat)
+  %out = call <vscale x 2 x float> @llvm.maxnum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> zeroinitializer)
   ret <vscale x 2 x float> %out
 }
 
@@ -418,9 +352,7 @@ define <vscale x 2 x float> @fmaxnm_2s_immone(<vscale x 2 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmaxnm z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x float> %elt, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x float> @llvm.maxnum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %splat)
+  %out = call <vscale x 2 x float> @llvm.maxnum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> splat(float 1.000000e+00))
   ret <vscale x 2 x float> %out
 }
 
@@ -430,9 +362,7 @@ define <vscale x 2 x double> @fmaxnm_d_immzero(<vscale x 2 x double> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmaxnm z0.d, p0/m, z0.d, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x double> @llvm.maxnum.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %splat)
+  %out = call <vscale x 2 x double> @llvm.maxnum.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer)
   ret <vscale x 2 x double> %out
 }
 
@@ -442,9 +372,7 @@ define <vscale x 2 x double> @fmaxnm_d_immone(<vscale x 2 x double> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmaxnm z0.d, p0/m, z0.d, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x double> @llvm.maxnum.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %splat)
+  %out = call <vscale x 2 x double> @llvm.maxnum.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> splat(double 1.000000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -458,9 +386,7 @@ define <vscale x 8 x half> @fmin_h_immzero(<vscale x 8 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    fmin z0.h, p0/m, z0.h, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
-  %out = call <vscale x 8 x half> @llvm.minimum.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %splat)
+  %out = call <vscale x 8 x half> @llvm.minimum.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> zeroinitializer)
   ret <vscale x 8 x half> %out
 }
 
@@ -470,9 +396,7 @@ define <vscale x 8 x half> @fmin_h_immone(<vscale x 8 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    fmin z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
-  %out = call <vscale x 8 x half> @llvm.minimum.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %splat)
+  %out = call <vscale x 8 x half> @llvm.minimum.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> splat(half 1.000000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -482,9 +406,7 @@ define <vscale x 4 x half> @fmin_4h_immzero(<vscale x 4 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fmin z0.h, p0/m, z0.h, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x half> undef, half 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x half> %elt, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
-  %out = call <vscale x 4 x half> @llvm.minimum.nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %splat)
+  %out = call <vscale x 4 x half> @llvm.minimum.nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> zeroinitializer)
   ret <vscale x 4 x half> %out
 }
 
@@ -494,9 +416,7 @@ define <vscale x 4 x half> @fmin_4h_immone(<vscale x 4 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fmin z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x half> %elt, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
-  %out = call <vscale x 4 x half> @llvm.minimum.nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %splat)
+  %out = call <vscale x 4 x half> @llvm.minimum.nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> splat(half 1.000000e+00))
   ret <vscale x 4 x half> %out
 }
 
@@ -506,9 +426,7 @@ define <vscale x 2 x half> @fmin_2h_immzero(<vscale x 2 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmin z0.h, p0/m, z0.h, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x half> undef, half 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x half> %elt, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x half> @llvm.minimum.nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> %splat)
+  %out = call <vscale x 2 x half> @llvm.minimum.nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> zeroinitializer)
   ret <vscale x 2 x half> %out
 }
 
@@ -518,9 +436,7 @@ define <vscale x 2 x half> @fmin_2h_immone(<vscale x 2 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmin z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x half> %elt, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x half> @llvm.minimum.nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> %splat)
+  %out = call <vscale x 2 x half> @llvm.minimum.nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> splat(half 1.000000e+00))
   ret <vscale x 2 x half> %out
 }
 
@@ -530,9 +446,7 @@ define <vscale x 4 x float> @fmin_s_immzero(<vscale x 4 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fmin z0.s, p0/m, z0.s, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
-  %out = call <vscale x 4 x float> @llvm.minimum.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %splat)
+  %out = call <vscale x 4 x float> @llvm.minimum.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> zeroinitializer)
   ret <vscale x 4 x float> %out
 }
 
@@ -542,9 +456,7 @@ define <vscale x 4 x float> @fmin_s_immone(<vscale x 4 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fmin z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
-  %out = call <vscale x 4 x float> @llvm.minimum.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %splat)
+  %out = call <vscale x 4 x float> @llvm.minimum.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> splat(float 1.000000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -554,9 +466,7 @@ define <vscale x 2 x float> @fmin_2s_immzero(<vscale x 2 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmin z0.s, p0/m, z0.s, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x float> undef, float 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x float> %elt, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x float> @llvm.minimum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %splat)
+  %out = call <vscale x 2 x float> @llvm.minimum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> zeroinitializer)
   ret <vscale x 2 x float> %out
 }
 
@@ -566,9 +476,7 @@ define <vscale x 2 x float> @fmin_2s_immone(<vscale x 2 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmin z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x float> %elt, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x float> @llvm.minimum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %splat)
+  %out = call <vscale x 2 x float> @llvm.minimum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> splat(float 1.000000e+00))
   ret <vscale x 2 x float> %out
 }
 
@@ -578,9 +486,7 @@ define <vscale x 2 x double> @fmin_d_immzero(<vscale x 2 x double> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmin z0.d, p0/m, z0.d, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x double> @llvm.minimum.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %splat)
+  %out = call <vscale x 2 x double> @llvm.minimum.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer)
   ret <vscale x 2 x double> %out
 }
 
@@ -590,9 +496,7 @@ define <vscale x 2 x double> @fmin_d_immone(<vscale x 2 x double> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmin z0.d, p0/m, z0.d, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x double> @llvm.minimum.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %splat)
+  %out = call <vscale x 2 x double> @llvm.minimum.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> splat(double 1.000000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -606,9 +510,7 @@ define <vscale x 8 x half> @fminnm_h_immzero(<vscale x 8 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    fminnm z0.h, p0/m, z0.h, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
-  %out = call <vscale x 8 x half> @llvm.minnum.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %splat)
+  %out = call <vscale x 8 x half> @llvm.minnum.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> zeroinitializer)
   ret <vscale x 8 x half> %out
 }
 
@@ -618,9 +520,7 @@ define <vscale x 8 x half> @fminnm_h_immone(<vscale x 8 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    fminnm z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
-  %out = call <vscale x 8 x half> @llvm.minnum.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %splat)
+  %out = call <vscale x 8 x half> @llvm.minnum.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> splat(half 1.000000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -630,9 +530,7 @@ define <vscale x 4 x half> @fminnm_4h_immzero(<vscale x 4 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fminnm z0.h, p0/m, z0.h, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x half> undef, half 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x half> %elt, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
-  %out = call <vscale x 4 x half> @llvm.minnum.nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %splat)
+  %out = call <vscale x 4 x half> @llvm.minnum.nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> zeroinitializer)
   ret <vscale x 4 x half> %out
 }
 
@@ -642,9 +540,7 @@ define <vscale x 4 x half> @fminnm_4h_immone(<vscale x 4 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fminnm z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x half> %elt, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
-  %out = call <vscale x 4 x half> @llvm.minnum.nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %splat)
+  %out = call <vscale x 4 x half> @llvm.minnum.nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> splat(half 1.000000e+00))
   ret <vscale x 4 x half> %out
 }
 
@@ -654,9 +550,7 @@ define <vscale x 2 x half> @fminnm_2h_immzero(<vscale x 2 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fminnm z0.h, p0/m, z0.h, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x half> undef, half 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x half> %elt, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x half> @llvm.minnum.nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> %splat)
+  %out = call <vscale x 2 x half> @llvm.minnum.nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> zeroinitializer)
   ret <vscale x 2 x half> %out
 }
 
@@ -666,9 +560,7 @@ define <vscale x 2 x half> @fminnm_2h_immone(<vscale x 2 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fminnm z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x half> %elt, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x half> @llvm.minnum.nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> %splat)
+  %out = call <vscale x 2 x half> @llvm.minnum.nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> splat(half 1.000000e+00))
   ret <vscale x 2 x half> %out
 }
 
@@ -678,9 +570,7 @@ define <vscale x 4 x float> @fminnm_s_immzero(<vscale x 4 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fminnm z0.s, p0/m, z0.s, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
-  %out = call <vscale x 4 x float> @llvm.minnum.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %splat)
+  %out = call <vscale x 4 x float> @llvm.minnum.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> zeroinitializer)
   ret <vscale x 4 x float> %out
 }
 
@@ -690,9 +580,7 @@ define <vscale x 4 x float> @fminnm_s_immone(<vscale x 4 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fminnm z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
-  %out = call <vscale x 4 x float> @llvm.minnum.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %splat)
+  %out = call <vscale x 4 x float> @llvm.minnum.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> splat(float 1.000000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -702,9 +590,7 @@ define <vscale x 2 x float> @fminnm_2s_immzero(<vscale x 2 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fminnm z0.s, p0/m, z0.s, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x float> undef, float 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x float> %elt, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x float> @llvm.minnum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %splat)
+  %out = call <vscale x 2 x float> @llvm.minnum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> zeroinitializer)
   ret <vscale x 2 x float> %out
 }
 
@@ -714,9 +600,7 @@ define <vscale x 2 x float> @fminnm_2s_immone(<vscale x 2 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fminnm z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x float> %elt, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x float> @llvm.minnum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %splat)
+  %out = call <vscale x 2 x float> @llvm.minnum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> splat(float 1.000000e+00))
   ret <vscale x 2 x float> %out
 }
 
@@ -726,9 +610,7 @@ define <vscale x 2 x double> @fminnm_d_immzero(<vscale x 2 x double> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fminnm z0.d, p0/m, z0.d, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x double> @llvm.minnum.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %splat)
+  %out = call <vscale x 2 x double> @llvm.minnum.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer)
   ret <vscale x 2 x double> %out
 }
 
@@ -738,9 +620,7 @@ define <vscale x 2 x double> @fminnm_d_immone(<vscale x 2 x double> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fminnm z0.d, p0/m, z0.d, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 2 x double> @llvm.minnum.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %splat)
+  %out = call <vscale x 2 x double> @llvm.minnum.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> splat(double 1.000000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -754,9 +634,7 @@ define <vscale x 8 x half> @fmul_h_immhalf(<vscale x 8 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    fmul z0.h, p0/m, z0.h, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
-  %out = fmul <vscale x 8 x half> %a, %splat
+  %out = fmul <vscale x 8 x half> %a, splat(half 0.500000e+00)
   ret <vscale x 8 x half> %out
 }
 
@@ -765,9 +643,7 @@ define <vscale x 8 x half> @fmul_h_immtwo(<vscale x 8 x half> %a) #0 {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fadd z0.h, z0.h, z0.h
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 2.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
-  %out = fmul <vscale x 8 x half> %a, %splat
+  %out = fmul <vscale x 8 x half> %a, splat(half 2.000000e+00)
   ret <vscale x 8 x half> %out
 }
 
@@ -777,9 +653,7 @@ define <vscale x 4 x half> @fmul_4h_immhalf(<vscale x 4 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fmul z0.h, p0/m, z0.h, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x half> undef, half 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x half> %elt, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
-  %out = fmul <vscale x 4 x half> %a, %splat
+  %out = fmul <vscale x 4 x half> %a, splat(half 0.500000e+00)
   ret <vscale x 4 x half> %out
 }
 
@@ -789,9 +663,7 @@ define <vscale x 4 x half> @fmul_4h_immtwo(<vscale x 4 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fadd z0.h, p0/m, z0.h, z0.h
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x half> undef, half 2.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x half> %elt, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
-  %out = fmul <vscale x 4 x half> %a, %splat
+  %out = fmul <vscale x 4 x half> %a, splat(half 2.000000e+00)
   ret <vscale x 4 x half> %out
 }
 
@@ -801,9 +673,7 @@ define <vscale x 2 x half> @fmul_2h_immhalf(<vscale x 2 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmul z0.h, p0/m, z0.h, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x half> undef, half 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x half> %elt, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fmul <vscale x 2 x half> %a, %splat
+  %out = fmul <vscale x 2 x half> %a, splat(half 0.500000e+00)
   ret <vscale x 2 x half> %out
 }
 
@@ -813,9 +683,7 @@ define <vscale x 2 x half> @fmul_2h_immtwo(<vscale x 2 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fadd z0.h, p0/m, z0.h, z0.h
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x half> undef, half 2.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x half> %elt, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fmul <vscale x 2 x half> %a, %splat
+  %out = fmul <vscale x 2 x half> %a, splat(half 2.000000e+00)
   ret <vscale x 2 x half> %out
 }
 
@@ -825,9 +693,7 @@ define <vscale x 4 x float> @fmul_s_immhalf(<vscale x 4 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fmul z0.s, p0/m, z0.s, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
-  %out = fmul <vscale x 4 x float> %a, %splat
+  %out = fmul <vscale x 4 x float> %a, splat(float 0.500000e+00)
   ret <vscale x 4 x float> %out
 }
 
@@ -836,9 +702,7 @@ define <vscale x 4 x float> @fmul_s_immtwo(<vscale x 4 x float> %a) #0 {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fadd z0.s, z0.s, z0.s
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 2.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
-  %out = fmul <vscale x 4 x float> %a, %splat
+  %out = fmul <vscale x 4 x float> %a, splat(float 2.000000e+00)
   ret <vscale x 4 x float> %out
 }
 
@@ -848,9 +712,7 @@ define <vscale x 2 x float> @fmul_2s_immhalf(<vscale x 2 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmul z0.s, p0/m, z0.s, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x float> undef, float 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x float> %elt, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fmul <vscale x 2 x float> %a, %splat
+  %out = fmul <vscale x 2 x float> %a, splat(float 0.500000e+00)
   ret <vscale x 2 x float> %out
 }
 
@@ -860,9 +722,7 @@ define <vscale x 2 x float> @fmul_2s_immtwo(<vscale x 2 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fadd z0.s, p0/m, z0.s, z0.s
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x float> undef, float 2.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x float> %elt, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fmul <vscale x 2 x float> %a, %splat
+  %out = fmul <vscale x 2 x float> %a, splat(float 2.000000e+00)
   ret <vscale x 2 x float> %out
 }
 
@@ -872,9 +732,7 @@ define <vscale x 2 x double> @fmul_d_immhalf(<vscale x 2 x double> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fmul z0.d, p0/m, z0.d, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fmul <vscale x 2 x double> %a, %splat
+  %out = fmul <vscale x 2 x double> %a, splat(double 0.500000e+00)
   ret <vscale x 2 x double> %out
 }
 
@@ -883,9 +741,7 @@ define <vscale x 2 x double> @fmul_d_immtwo(<vscale x 2 x double> %a) #0 {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fadd z0.d, z0.d, z0.d
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 2.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fmul <vscale x 2 x double> %a, %splat
+  %out = fmul <vscale x 2 x double> %a, splat(double 2.000000e+00)
   ret <vscale x 2 x double> %out
 }
 
@@ -899,9 +755,7 @@ define <vscale x 8 x half> @fsub_h_immhalf(<vscale x 8 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    fsub z0.h, p0/m, z0.h, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
-  %out = fsub <vscale x 8 x half> %a, %splat
+  %out = fsub <vscale x 8 x half> %a, splat(half 0.500000e+00)
   ret <vscale x 8 x half> %out
 }
 
@@ -911,9 +765,7 @@ define <vscale x 8 x half> @fsub_h_immone(<vscale x 8 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    fsub z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
-  %out = fsub <vscale x 8 x half> %a, %splat
+  %out = fsub <vscale x 8 x half> %a, splat(half 1.000000e+00)
   ret <vscale x 8 x half> %out
 }
 
@@ -923,9 +775,7 @@ define <vscale x 4 x half> @fsub_4h_immhalf(<vscale x 4 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fsub z0.h, p0/m, z0.h, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x half> undef, half 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x half> %elt, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
-  %out = fsub <vscale x 4 x half> %a, %splat
+  %out = fsub <vscale x 4 x half> %a, splat(half 0.500000e+00)
   ret <vscale x 4 x half> %out
 }
 
@@ -935,9 +785,7 @@ define <vscale x 4 x half> @fsub_4h_immone(<vscale x 4 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fsub z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x half> %elt, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
-  %out = fsub <vscale x 4 x half> %a, %splat
+  %out = fsub <vscale x 4 x half> %a, splat(half 1.000000e+00)
   ret <vscale x 4 x half> %out
 }
 
@@ -947,9 +795,7 @@ define <vscale x 2 x half> @fsub_2h_immhalf(<vscale x 2 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fsub z0.h, p0/m, z0.h, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x half> undef, half 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x half> %elt, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fsub <vscale x 2 x half> %a, %splat
+  %out = fsub <vscale x 2 x half> %a, splat(half 0.500000e+00)
   ret <vscale x 2 x half> %out
 }
 
@@ -959,9 +805,7 @@ define <vscale x 2 x half> @fsub_2h_immone(<vscale x 2 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fsub z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x half> %elt, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fsub <vscale x 2 x half> %a, %splat
+  %out = fsub <vscale x 2 x half> %a, splat(half 1.000000e+00)
   ret <vscale x 2 x half> %out
 }
 
@@ -971,9 +815,7 @@ define <vscale x 4 x float> @fsub_s_immhalf(<vscale x 4 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fsub z0.s, p0/m, z0.s, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
-  %out = fsub <vscale x 4 x float> %a, %splat
+  %out = fsub <vscale x 4 x float> %a, splat(float 0.500000e+00)
   ret <vscale x 4 x float> %out
 }
 
@@ -983,9 +825,7 @@ define <vscale x 4 x float> @fsub_s_immone(<vscale x 4 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fsub z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
-  %out = fsub <vscale x 4 x float> %a, %splat
+  %out = fsub <vscale x 4 x float> %a, splat(float 1.000000e+00)
   ret <vscale x 4 x float> %out
 }
 
@@ -995,9 +835,7 @@ define <vscale x 2 x float> @fsub_2s_immhalf(<vscale x 2 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fsub z0.s, p0/m, z0.s, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x float> undef, float 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x float> %elt, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fsub <vscale x 2 x float> %a, %splat
+  %out = fsub <vscale x 2 x float> %a, splat(float 0.500000e+00)
   ret <vscale x 2 x float> %out
 }
 
@@ -1007,9 +845,7 @@ define <vscale x 2 x float> @fsub_2s_immone(<vscale x 2 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fsub z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x float> %elt, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fsub <vscale x 2 x float> %a, %splat
+  %out = fsub <vscale x 2 x float> %a, splat(float 1.000000e+00)
   ret <vscale x 2 x float> %out
 }
 
@@ -1019,9 +855,7 @@ define <vscale x 2 x double> @fsub_d_immhalf(<vscale x 2 x double> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fsub z0.d, p0/m, z0.d, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fsub <vscale x 2 x double> %a, %splat
+  %out = fsub <vscale x 2 x double> %a, splat(double 0.500000e+00)
   ret <vscale x 2 x double> %out
 }
 
@@ -1031,9 +865,7 @@ define <vscale x 2 x double> @fsub_d_immone(<vscale x 2 x double> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fsub z0.d, p0/m, z0.d, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fsub <vscale x 2 x double> %a, %splat
+  %out = fsub <vscale x 2 x double> %a, splat(double 1.000000e+00)
   ret <vscale x 2 x double> %out
 }
 
@@ -1047,9 +879,7 @@ define <vscale x 8 x half> @fsubr_h_immhalf(<vscale x 8 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    fsubr z0.h, p0/m, z0.h, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
-  %out = fsub <vscale x 8 x half> %splat, %a
+  %out = fsub <vscale x 8 x half> splat(half 0.500000e+00), %a
   ret <vscale x 8 x half> %out
 }
 
@@ -1059,9 +889,7 @@ define <vscale x 8 x half> @fsubr_h_immone(<vscale x 8 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    fsubr z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
-  %out = fsub <vscale x 8 x half> %splat, %a
+  %out = fsub <vscale x 8 x half> splat(half 1.000000e+00), %a
   ret <vscale x 8 x half> %out
 }
 
@@ -1071,9 +899,7 @@ define <vscale x 4 x half> @fsubr_4h_immhalf(<vscale x 4 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fsubr z0.h, p0/m, z0.h, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x half> undef, half 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x half> %elt, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
-  %out = fsub <vscale x 4 x half> %splat, %a
+  %out = fsub <vscale x 4 x half> splat(half 0.500000e+00), %a
   ret <vscale x 4 x half> %out
 }
 
@@ -1083,9 +909,7 @@ define <vscale x 4 x half> @fsubr_4h_immone(<vscale x 4 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fsubr z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x half> %elt, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
-  %out = fsub <vscale x 4 x half> %splat, %a
+  %out = fsub <vscale x 4 x half> splat(half 1.000000e+00), %a
   ret <vscale x 4 x half> %out
 }
 
@@ -1095,9 +919,7 @@ define <vscale x 2 x half> @fsubr_2h_immhalf(<vscale x 2 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fsubr z0.h, p0/m, z0.h, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x half> undef, half 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x half> %elt, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fsub <vscale x 2 x half> %splat, %a
+  %out = fsub <vscale x 2 x half> splat(half 0.500000e+00), %a
   ret <vscale x 2 x half> %out
 }
 
@@ -1107,9 +929,7 @@ define <vscale x 2 x half> @fsubr_2h_immone(<vscale x 2 x half> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fsubr z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x half> %elt, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fsub <vscale x 2 x half> %splat, %a
+  %out = fsub <vscale x 2 x half> splat(half 1.000000e+00), %a
   ret <vscale x 2 x half> %out
 }
 
@@ -1119,9 +939,7 @@ define <vscale x 4 x float> @fsubr_s_immhalf(<vscale x 4 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fsubr z0.s, p0/m, z0.s, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
-  %out = fsub <vscale x 4 x float> %splat, %a
+  %out = fsub <vscale x 4 x float> splat(float 0.500000e+00), %a
   ret <vscale x 4 x float> %out
 }
 
@@ -1131,9 +949,7 @@ define <vscale x 4 x float> @fsubr_s_immone(<vscale x 4 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fsubr z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
-  %out = fsub <vscale x 4 x float> %splat, %a
+  %out = fsub <vscale x 4 x float> splat(float 1.000000e+00), %a
   ret <vscale x 4 x float> %out
 }
 
@@ -1143,9 +959,7 @@ define <vscale x 2 x float> @fsubr_2s_immhalf(<vscale x 2 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fsubr z0.s, p0/m, z0.s, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x float> undef, float 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x float> %elt, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fsub <vscale x 2 x float> %splat, %a
+  %out = fsub <vscale x 2 x float> splat(float 0.500000e+00), %a
   ret <vscale x 2 x float> %out
 }
 
@@ -1155,9 +969,7 @@ define <vscale x 2 x float> @fsubr_2s_immone(<vscale x 2 x float> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fsubr z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x float> %elt, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fsub <vscale x 2 x float> %splat, %a
+  %out = fsub <vscale x 2 x float> splat(float 1.000000e+00), %a
   ret <vscale x 2 x float> %out
 }
 
@@ -1167,9 +979,7 @@ define <vscale x 2 x double> @fsubr_d_immhalf(<vscale x 2 x double> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fsubr z0.d, p0/m, z0.d, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fsub <vscale x 2 x double> %splat, %a
+  %out = fsub <vscale x 2 x double> splat(double 0.500000e+00), %a
   ret <vscale x 2 x double> %out
 }
 
@@ -1179,14 +989,10 @@ define <vscale x 2 x double> @fsubr_d_immone(<vscale x 2 x double> %a) #0 {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    fsubr z0.d, p0/m, z0.d, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
-  %out = fsub <vscale x 2 x double> %splat, %a
+  %out = fsub <vscale x 2 x double> splat(double 1.000000e+00), %a
   ret <vscale x 2 x double> %out
 }
 
-;; Arithmetic intrinsic declarations
-
 declare <vscale x 8 x half> @llvm.maximum.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
 declare <vscale x 4 x half> @llvm.maximum.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>)
 declare <vscale x 2 x half> @llvm.maximum.nxv2f16(<vscale x 2 x half>, <vscale x 2 x half>)
diff --git a/llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll b/llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll
index 3030c38e13bf4..047716292ac3b 100644
--- a/llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll
+++ b/llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll
@@ -13,10 +13,8 @@ define void @scatter_i8_index_offset_maximum(ptr %base, i64 %offset, <vscale x 4
 ; CHECK-NEXT:    ret
   %t0 = insertelement <vscale x 4 x i64> undef, i64 %offset, i32 0
   %t1 = shufflevector <vscale x 4 x i64> %t0, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
-  %t2 = insertelement <vscale x 4 x i64> undef, i64 33554431, i32 0
-  %t3 = shufflevector <vscale x 4 x i64> %t2, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
   %step = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-  %t4 = mul <vscale x 4 x i64> %t3, %step
+  %t4 = mul <vscale x 4 x i64> splat(i64 33554431), %step
   %t5 = add <vscale x 4 x i64> %t1, %t4
   %t6 = getelementptr i8, ptr %base, <vscale x 4 x i64> %t5
   call void @llvm.masked.scatter.nxv4i8(<vscale x 4 x i8> %data, <vscale x 4 x ptr> %t6, i32 2, <vscale x 4 x i1> %pg)
@@ -34,10 +32,8 @@ define void @scatter_i16_index_offset_minimum(ptr %base, i64 %offset, <vscale x
 ; CHECK-NEXT:    ret
   %t0 = insertelement <vscale x 4 x i64> undef, i64 %offset, i32 0
   %t1 = shufflevector <vscale x 4 x i64> %t0, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
-  %t2 = insertelement <vscale x 4 x i64> undef, i64 -33554432, i32 0
-  %t3 = shufflevector <vscale x 4 x i64> %t2, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
   %step = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-  %t4 = mul <vscale x 4 x i64> %t3, %step
+  %t4 = mul <vscale x 4 x i64> splat(i64 -33554432), %step
   %t5 = add <vscale x 4 x i64> %t1, %t4
   %t6 = getelementptr i16, ptr %base, <vscale x 4 x i64> %t5
   call void @llvm.masked.scatter.nxv4i16(<vscale x 4 x i16> %data, <vscale x 4 x ptr> %t6, i32 2, <vscale x 4 x i1> %pg)
@@ -55,9 +51,7 @@ define <vscale x 4 x i8> @gather_i8_index_offset_8(ptr %base, i64 %offset, <vsca
   %splat.insert0 = insertelement <vscale x 4 x i64> undef, i64 %offset, i32 0
   %splat0 = shufflevector <vscale x 4 x i64> %splat.insert0, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
   %step = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-  %splat.insert1 = insertelement <vscale x 4 x i64> undef, i64 1, i32 0
-  %splat1 = shufflevector <vscale x 4 x i64> %splat.insert1, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
-  %t1 = mul <vscale x 4 x i64> %splat1, %step
+  %t1 = mul <vscale x 4 x i64> splat(i64 1), %step
   %t2 = add <vscale x 4 x i64> %splat0, %t1
   %t3 = getelementptr i8, ptr %base, <vscale x 4 x i64> %t2
   %load = call <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x ptr> %t3, i32 4, <vscale x 4 x i1> %pg, <vscale x 4 x i8> undef)
@@ -117,10 +111,8 @@ define void @scatter_i8_index_offset_maximum_plus_one(ptr %base, i64 %offset, <v
 ; CHECK-NEXT:    ret
   %t0 = insertelement <vscale x 4 x i64> undef, i64 %offset, i32 0
   %t1 = shufflevector <vscale x 4 x i64> %t0, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
-  %t2 = insertelement <vscale x 4 x i64> undef, i64 33554432, i32 0
-  %t3 = shufflevector <vscale x 4 x i64> %t2, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
   %step = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-  %t4 = mul <vscale x 4 x i64> %t3, %step
+  %t4 = mul <vscale x 4 x i64> splat(i64 33554432), %step
   %t5 = add <vscale x 4 x i64> %t1, %t4
   %t6 = getelementptr i8, ptr %base, <vscale x 4 x i64> %t5
   call void @llvm.masked.scatter.nxv4i8(<vscale x 4 x i8> %data, <vscale x 4 x ptr> %t6, i32 2, <vscale x 4 x i1> %pg)
@@ -148,10 +140,8 @@ define void @scatter_i8_index_offset_minimum_minus_one(ptr %base, i64 %offset, <
 ; CHECK-NEXT:    ret
   %t0 = insertelement <vscale x 4 x i64> undef, i64 %offset, i32 0
   %t1 = shufflevector <vscale x 4 x i64> %t0, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
-  %t2 = insertelement <vscale x 4 x i64> undef, i64 -33554433, i32 0
-  %t3 = shufflevector <vscale x 4 x i64> %t2, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
   %step = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-  %t4 = mul <vscale x 4 x i64> %t3, %step
+  %t4 = mul <vscale x 4 x i64> splat(i64 -33554433), %step
   %t5 = add <vscale x 4 x i64> %t1, %t4
   %t6 = getelementptr i8, ptr %base, <vscale x 4 x i64> %t5
   call void @llvm.masked.scatter.nxv4i8(<vscale x 4 x i8> %data, <vscale x 4 x ptr> %t6, i32 2, <vscale x 4 x i1> %pg)
@@ -178,10 +168,8 @@ define void @scatter_i8_index_stride_too_big(ptr %base, i64 %offset, <vscale x 4
 ; CHECK-NEXT:    ret
   %t0 = insertelement <vscale x 4 x i64> undef, i64 %offset, i32 0
   %t1 = shufflevector <vscale x 4 x i64> %t0, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
-  %t2 = insertelement <vscale x 4 x i64> undef, i64 4611686018427387904, i32 0
-  %t3 = shufflevector <vscale x 4 x i64> %t2, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
   %step = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-  %t4 = mul <vscale x 4 x i64> %t3, %step
+  %t4 = mul <vscale x 4 x i64> splat(i64 4611686018427387904), %step
   %t5 = add <vscale x 4 x i64> %t1, %t4
   %t6 = getelementptr i8, ptr %base, <vscale x 4 x i64> %t5
   call void @llvm.masked.scatter.nxv4i8(<vscale x 4 x i8> %data, <vscale x 4 x ptr> %t6, i32 2, <vscale x 4 x i1> %pg)
@@ -310,9 +298,7 @@ define void @scatter_f16_index_add_add_mul(ptr %base, i64 %offset, i64 %offset2,
   %step = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
   %add1 = add <vscale x 4 x i64> %splat.offset, %step
   %add2 = add <vscale x 4 x i64> %add1, %splat.offset2
-  %splat.const8.ins = insertelement <vscale x 4 x i64> undef, i64 8, i32 0
-  %splat.const8 = shufflevector <vscale x 4 x i64> %splat.const8.ins, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
-  %mul = mul <vscale x 4 x i64> %add2, %splat.const8
+  %mul = mul <vscale x 4 x i64> %add2, splat(i64 8)
   %gep = getelementptr [8 x half], ptr %base, <vscale x 4 x i64> %mul
   %gep.bc = bitcast <vscale x 4 x ptr> %gep to <vscale x 4 x ptr>
   call void @llvm.masked.scatter.nxv4f16(<vscale x 4 x half> %data, <vscale x 4 x ptr> %gep.bc, i32 2, <vscale x 4 x i1> %pg)
@@ -350,9 +336,7 @@ define <vscale x 2 x i64> @masked_gather_nxv2i64_null_with__vec_plus_imm_offsets
 ; CHECK-NEXT:    mov w8, #8 // =0x8
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x8, z0.d, lsl #3]
 ; CHECK-NEXT:    ret
-  %scalar_offset.ins = insertelement <vscale x 2 x i64> undef, i64 1, i64 0
-  %scalar_offset.splat = shufflevector <vscale x 2 x i64> %scalar_offset.ins, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %offsets = add <vscale x 2 x i64> %vector_offsets, %scalar_offset.splat
+  %offsets = add <vscale x 2 x i64> %vector_offsets, splat(i64 1)
   %ptrs = getelementptr i64, ptr null, <vscale x 2 x i64> %offsets
   %data = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x ptr> %ptrs, i32 8, <vscale x 2 x i1> %pg, <vscale x 2 x i64> undef)
   ret <vscale x 2 x i64> %data
@@ -428,9 +412,7 @@ define void @masked_scatter_nxv2i64_null_with__vec_plus_imm_offsets(<vscale x 2
 ; CHECK-NEXT:    mov w8, #8 // =0x8
 ; CHECK-NEXT:    st1d { z1.d }, p0, [x8, z0.d, lsl #3]
 ; CHECK-NEXT:    ret
-  %scalar_offset.ins = insertelement <vscale x 2 x i64> undef, i64 1, i64 0
-  %scalar_offset.splat = shufflevector <vscale x 2 x i64> %scalar_offset.ins, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %offsets = add <vscale x 2 x i64> %vector_offsets, %scalar_offset.splat
+  %offsets = add <vscale x 2 x i64> %vector_offsets, splat(i64 1)
   %ptrs = getelementptr i64, ptr null, <vscale x 2 x i64> %offsets
   call void @llvm.masked.scatter.nxv2i64(<vscale x 2 x i64> %data, <vscale x 2 x ptr> %ptrs, i32 8, <vscale x 2 x i1> %pg)
   ret void
diff --git a/llvm/test/CodeGen/AArch64/sve-insert-vector.ll b/llvm/test/CodeGen/AArch64/sve-insert-vector.ll
index 5efe9e2819d5e..5d1d7cf65c09e 100644
--- a/llvm/test/CodeGen/AArch64/sve-insert-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-insert-vector.ll
@@ -267,9 +267,7 @@ define <vscale x 4 x i32> @insert_nxv1i32_nxv4i32_undef() nounwind {
 ; CHECK-NEXT:    mov z0.s, #1 // =0x1
 ; CHECK-NEXT:    ret
 entry:
-  %0 = insertelement <vscale x 1 x i32> undef, i32 1, i32 0
-  %subvec = shufflevector <vscale x 1 x i32> %0, <vscale x 1 x i32> undef, <vscale x 1 x i32> zeroinitializer
-  %retval = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.nxv1i32(<vscale x 4 x i32> undef, <vscale x 1 x i32> %subvec, i64 0)
+  %retval = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.nxv1i32(<vscale x 4 x i32> undef, <vscale x 1 x i32> splat(i32 1), i64 0)
   ret <vscale x 4 x i32> %retval
 }
 
@@ -279,9 +277,7 @@ define <vscale x 6 x i16> @insert_nxv1i16_nxv6i16_undef() nounwind {
 ; CHECK-NEXT:    mov z0.h, #1 // =0x1
 ; CHECK-NEXT:    ret
 entry:
-  %0 = insertelement <vscale x 1 x i16> undef, i16 1, i32 0
-  %subvec = shufflevector <vscale x 1 x i16> %0, <vscale x 1 x i16> undef, <vscale x 1 x i32> zeroinitializer
-  %retval = call <vscale x 6 x i16> @llvm.vector.insert.nxv6i16.nxv1i16(<vscale x 6 x i16> undef, <vscale x 1 x i16> %subvec, i64 0)
+  %retval = call <vscale x 6 x i16> @llvm.vector.insert.nxv6i16.nxv1i16(<vscale x 6 x i16> undef, <vscale x 1 x i16> splat(i16 1), i64 0)
   ret <vscale x 6 x i16> %retval
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll b/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll
index 52bd79e7a7e60..25b27c47f3d7a 100644
--- a/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll
+++ b/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll
@@ -9,10 +9,8 @@ define <vscale x 16 x i8> @smax_i8_pos(<vscale x 16 x i8> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smax z0.b, z0.b, #27
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 27, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %cmp = icmp sgt <vscale x 16 x i8> %a, %splat
-  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  %cmp = icmp sgt <vscale x 16 x i8> %a, splat(i8 27)
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> splat(i8 27)
   ret <vscale x 16 x i8> %res
 }
 
@@ -21,10 +19,8 @@ define <vscale x 16 x i8> @smax_i8_neg(<vscale x 16 x i8> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smax z0.b, z0.b, #-58
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 -58, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %cmp = icmp sgt <vscale x 16 x i8> %a, %splat
-  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  %cmp = icmp sgt <vscale x 16 x i8> %a, splat(i8 -58)
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> splat(i8 -58)
   ret <vscale x 16 x i8> %res
 }
 
@@ -33,10 +29,8 @@ define <vscale x 8 x i16> @smax_i16_pos(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smax z0.h, z0.h, #27
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 27, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %cmp = icmp sgt <vscale x 8 x i16> %a, %splat
-  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  %cmp = icmp sgt <vscale x 8 x i16> %a, splat(i16 27)
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 27)
   ret <vscale x 8 x i16> %res
 }
 
@@ -45,10 +39,8 @@ define <vscale x 8 x i16> @smax_i16_neg(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smax z0.h, z0.h, #-58
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 -58, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %cmp = icmp sgt <vscale x 8 x i16> %a, %splat
-  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  %cmp = icmp sgt <vscale x 8 x i16> %a, splat(i16 -58)
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 -58)
   ret <vscale x 8 x i16> %res
 }
 
@@ -59,10 +51,8 @@ define <vscale x 8 x i16> @smax_i16_out_of_range(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    smax z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 257, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %cmp = icmp sgt <vscale x 8 x i16> %a, %splat
-  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  %cmp = icmp sgt <vscale x 8 x i16> %a, splat(i16 257)
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 257)
   ret <vscale x 8 x i16> %res
 }
 
@@ -71,10 +61,8 @@ define <vscale x 4 x i32> @smax_i32_pos(<vscale x 4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smax z0.s, z0.s, #27
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 27, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %cmp = icmp sgt <vscale x 4 x i32> %a, %splat
-  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  %cmp = icmp sgt <vscale x 4 x i32> %a, splat(i32 27)
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 27)
   ret <vscale x 4 x i32> %res
 }
 
@@ -83,10 +71,8 @@ define <vscale x 4 x i32> @smax_i32_neg(<vscale x 4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smax z0.s, z0.s, #-58
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 -58, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %cmp = icmp sgt <vscale x 4 x i32> %a, %splat
-  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  %cmp = icmp sgt <vscale x 4 x i32> %a, splat(i32 -58)
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 -58)
   ret <vscale x 4 x i32> %res
 }
 
@@ -97,10 +83,8 @@ define <vscale x 4 x i32> @smax_i32_out_of_range(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    smax z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 -129, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %cmp = icmp sgt <vscale x 4 x i32> %a, %splat
-  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  %cmp = icmp sgt <vscale x 4 x i32> %a, splat(i32 -129)
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 -129)
   ret <vscale x 4 x i32> %res
 }
 
@@ -109,10 +93,8 @@ define <vscale x 2 x i64> @smax_i64_pos(<vscale x 2 x i64> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smax z0.d, z0.d, #27
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 27, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %cmp = icmp sgt <vscale x 2 x i64> %a, %splat
-  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  %cmp = icmp sgt <vscale x 2 x i64> %a, splat(i64 27)
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 27)
   ret <vscale x 2 x i64> %res
 }
 
@@ -121,10 +103,8 @@ define <vscale x 2 x i64> @smax_i64_neg(<vscale x 2 x i64> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smax z0.d, z0.d, #-58
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 -58, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %cmp = icmp sgt <vscale x 2 x i64> %a, %splat
-  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  %cmp = icmp sgt <vscale x 2 x i64> %a, splat(i64 -58)
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 -58)
   ret <vscale x 2 x i64> %res
 }
 
@@ -135,10 +115,8 @@ define <vscale x 2 x i64> @smax_i64_out_of_range(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    smax z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 65535, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %cmp = icmp sgt <vscale x 2 x i64> %a, %splat
-  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  %cmp = icmp sgt <vscale x 2 x i64> %a, splat(i64 65535)
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 65535)
   ret <vscale x 2 x i64> %res
 }
 
@@ -150,10 +128,8 @@ define <vscale x 16 x i8> @smin_i8_pos(<vscale x 16 x i8> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smin z0.b, z0.b, #27
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 27, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %cmp = icmp slt <vscale x 16 x i8> %a, %splat
-  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  %cmp = icmp slt <vscale x 16 x i8> %a, splat(i8 27)
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> splat(i8 27)
   ret <vscale x 16 x i8> %res
 }
 
@@ -162,10 +138,8 @@ define <vscale x 16 x i8> @smin_i8_neg(<vscale x 16 x i8> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smin z0.b, z0.b, #-58
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 -58, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %cmp = icmp slt <vscale x 16 x i8> %a, %splat
-  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  %cmp = icmp slt <vscale x 16 x i8> %a, splat(i8 -58)
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> splat(i8 -58)
   ret <vscale x 16 x i8> %res
 }
 
@@ -174,10 +148,8 @@ define <vscale x 8 x i16> @smin_i16_pos(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smin z0.h, z0.h, #27
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 27, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %cmp = icmp slt <vscale x 8 x i16> %a, %splat
-  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  %cmp = icmp slt <vscale x 8 x i16> %a, splat(i16 27)
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 27)
   ret <vscale x 8 x i16> %res
 }
 
@@ -186,10 +158,8 @@ define <vscale x 8 x i16> @smin_i16_neg(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smin z0.h, z0.h, #-58
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 -58, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %cmp = icmp slt <vscale x 8 x i16> %a, %splat
-  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  %cmp = icmp slt <vscale x 8 x i16> %a, splat(i16 -58)
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 -58)
   ret <vscale x 8 x i16> %res
 }
 
@@ -200,10 +170,8 @@ define <vscale x 8 x i16> @smin_i16_out_of_range(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    smin z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 257, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %cmp = icmp slt <vscale x 8 x i16> %a, %splat
-  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  %cmp = icmp slt <vscale x 8 x i16> %a, splat(i16 257)
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 257)
   ret <vscale x 8 x i16> %res
 }
 
@@ -212,10 +180,8 @@ define <vscale x 4 x i32> @smin_i32_pos(<vscale x 4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smin z0.s, z0.s, #27
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 27, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %cmp = icmp slt <vscale x 4 x i32> %a, %splat
-  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  %cmp = icmp slt <vscale x 4 x i32> %a, splat(i32 27)
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 27)
   ret <vscale x 4 x i32> %res
 }
 
@@ -224,10 +190,8 @@ define <vscale x 4 x i32> @smin_i32_neg(<vscale x 4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smin z0.s, z0.s, #-58
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 -58, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %cmp = icmp slt <vscale x 4 x i32> %a, %splat
-  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  %cmp = icmp slt <vscale x 4 x i32> %a, splat(i32 -58)
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 -58)
   ret <vscale x 4 x i32> %res
 }
 
@@ -238,10 +202,8 @@ define <vscale x 4 x i32> @smin_i32_out_of_range(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    smin z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 -129, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %cmp = icmp slt <vscale x 4 x i32> %a, %splat
-  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  %cmp = icmp slt <vscale x 4 x i32> %a, splat(i32 -129)
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 -129)
   ret <vscale x 4 x i32> %res
 }
 
@@ -250,10 +212,8 @@ define <vscale x 2 x i64> @smin_i64_pos(<vscale x 2 x i64> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smin z0.d, z0.d, #27
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 27, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %cmp = icmp slt <vscale x 2 x i64> %a, %splat
-  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  %cmp = icmp slt <vscale x 2 x i64> %a, splat(i64 27)
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 27)
   ret <vscale x 2 x i64> %res
 }
 
@@ -262,10 +222,8 @@ define <vscale x 2 x i64> @smin_i64_neg(<vscale x 2 x i64> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smin z0.d, z0.d, #-58
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 -58, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %cmp = icmp slt <vscale x 2 x i64> %a, %splat
-  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  %cmp = icmp slt <vscale x 2 x i64> %a, splat(i64 -58)
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 -58)
   ret <vscale x 2 x i64> %res
 }
 
@@ -276,10 +234,8 @@ define <vscale x 2 x i64> @smin_i64_out_of_range(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    smin z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 65535, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %cmp = icmp slt <vscale x 2 x i64> %a, %splat
-  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  %cmp = icmp slt <vscale x 2 x i64> %a, splat(i64 65535)
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 65535)
   ret <vscale x 2 x i64> %res
 }
 
@@ -291,10 +247,8 @@ define <vscale x 16 x i8> @umax_i8_pos(<vscale x 16 x i8> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umax z0.b, z0.b, #27
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 27, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %cmp = icmp ugt <vscale x 16 x i8> %a, %splat
-  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  %cmp = icmp ugt <vscale x 16 x i8> %a, splat(i8 27)
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> splat(i8 27)
   ret <vscale x 16 x i8> %res
 }
 
@@ -303,10 +257,8 @@ define <vscale x 16 x i8> @umax_i8_large(<vscale x 16 x i8> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umax z0.b, z0.b, #129
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 129, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %cmp = icmp ugt <vscale x 16 x i8> %a, %splat
-  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  %cmp = icmp ugt <vscale x 16 x i8> %a, splat(i8 129)
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> splat(i8 129)
   ret <vscale x 16 x i8> %res
 }
 
@@ -315,10 +267,8 @@ define <vscale x 8 x i16> @umax_i16_pos(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umax z0.h, z0.h, #27
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 27, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %cmp = icmp ugt <vscale x 8 x i16> %a, %splat
-  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  %cmp = icmp ugt <vscale x 8 x i16> %a, splat(i16 27)
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 27)
   ret <vscale x 8 x i16> %res
 }
 
@@ -329,10 +279,8 @@ define <vscale x 8 x i16> @umax_i16_out_of_range(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    umax z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 257, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %cmp = icmp ugt <vscale x 8 x i16> %a, %splat
-  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  %cmp = icmp ugt <vscale x 8 x i16> %a, splat(i16 257)
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 257)
   ret <vscale x 8 x i16> %res
 }
 
@@ -341,10 +289,8 @@ define <vscale x 4 x i32> @umax_i32_pos(<vscale x 4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umax z0.s, z0.s, #27
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 27, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %cmp = icmp ugt <vscale x 4 x i32> %a, %splat
-  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  %cmp = icmp ugt <vscale x 4 x i32> %a, splat(i32 27)
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 27)
   ret <vscale x 4 x i32> %res
 }
 
@@ -356,10 +302,8 @@ define <vscale x 4 x i32> @umax_i32_out_of_range(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    mov z1.s, w8
 ; CHECK-NEXT:    umax z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 257, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %cmp = icmp ugt <vscale x 4 x i32> %a, %splat
-  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  %cmp = icmp ugt <vscale x 4 x i32> %a, splat(i32 257)
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 257)
   ret <vscale x 4 x i32> %res
 }
 
@@ -368,10 +312,8 @@ define <vscale x 2 x i64> @umax_i64_pos(<vscale x 2 x i64> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umax z0.d, z0.d, #27
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 27, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %cmp = icmp ugt <vscale x 2 x i64> %a, %splat
-  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  %cmp = icmp ugt <vscale x 2 x i64> %a, splat(i64 27)
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 27)
   ret <vscale x 2 x i64> %res
 }
 
@@ -382,10 +324,8 @@ define <vscale x 2 x i64> @umax_i64_out_of_range(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    umax z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 65535, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %cmp = icmp ugt <vscale x 2 x i64> %a, %splat
-  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  %cmp = icmp ugt <vscale x 2 x i64> %a, splat(i64 65535)
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 65535)
   ret <vscale x 2 x i64> %res
 }
 
@@ -397,10 +337,8 @@ define <vscale x 16 x i8> @umin_i8_pos(<vscale x 16 x i8> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umin z0.b, z0.b, #27
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 27, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %cmp = icmp ult <vscale x 16 x i8> %a, %splat
-  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  %cmp = icmp ult <vscale x 16 x i8> %a, splat(i8 27)
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> splat(i8 27)
   ret <vscale x 16 x i8> %res
 }
 
@@ -409,10 +347,8 @@ define <vscale x 16 x i8> @umin_i8_large(<vscale x 16 x i8> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umin z0.b, z0.b, #129
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 129, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %cmp = icmp ult <vscale x 16 x i8> %a, %splat
-  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> %splat
+  %cmp = icmp ult <vscale x 16 x i8> %a, splat(i8 129)
+  %res = select <vscale x 16 x i1> %cmp, <vscale x 16 x i8> %a, <vscale x 16 x i8> splat(i8 129)
   ret <vscale x 16 x i8> %res
 }
 
@@ -421,10 +357,8 @@ define <vscale x 8 x i16> @umin_i16_pos(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umin z0.h, z0.h, #27
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 27, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %cmp = icmp ult <vscale x 8 x i16> %a, %splat
-  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  %cmp = icmp ult <vscale x 8 x i16> %a, splat(i16 27)
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 27)
   ret <vscale x 8 x i16> %res
 }
 
@@ -435,10 +369,8 @@ define <vscale x 8 x i16> @umin_i16_out_of_range(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    umin z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 257, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %cmp = icmp ult <vscale x 8 x i16> %a, %splat
-  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> %splat
+  %cmp = icmp ult <vscale x 8 x i16> %a, splat(i16 257)
+  %res = select <vscale x 8 x i1> %cmp, <vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 257)
   ret <vscale x 8 x i16> %res
 }
 
@@ -447,10 +379,8 @@ define <vscale x 4 x i32> @umin_i32_pos(<vscale x 4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umin z0.s, z0.s, #27
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 27, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %cmp = icmp ult <vscale x 4 x i32> %a, %splat
-  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  %cmp = icmp ult <vscale x 4 x i32> %a, splat(i32 27)
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 27)
   ret <vscale x 4 x i32> %res
 }
 
@@ -462,10 +392,8 @@ define <vscale x 4 x i32> @umin_i32_out_of_range(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    mov z1.s, w8
 ; CHECK-NEXT:    umin z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 257, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %cmp = icmp ult <vscale x 4 x i32> %a, %splat
-  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> %splat
+  %cmp = icmp ult <vscale x 4 x i32> %a, splat(i32 257)
+  %res = select <vscale x 4 x i1> %cmp, <vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 257)
   ret <vscale x 4 x i32> %res
 }
 
@@ -474,10 +402,8 @@ define <vscale x 2 x i64> @umin_i64_pos(<vscale x 2 x i64> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umin z0.d, z0.d, #27
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 27, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %cmp = icmp ult <vscale x 2 x i64> %a, %splat
-  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  %cmp = icmp ult <vscale x 2 x i64> %a, splat(i64 27)
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 27)
   ret <vscale x 2 x i64> %res
 }
 
@@ -488,10 +414,8 @@ define <vscale x 2 x i64> @umin_i64_out_of_range(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    umin z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 65535, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %cmp = icmp ult <vscale x 2 x i64> %a, %splat
-  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> %splat
+  %cmp = icmp ult <vscale x 2 x i64> %a, splat(i64 65535)
+  %res = select <vscale x 2 x i1> %cmp, <vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 65535)
   ret <vscale x 2 x i64> %res
 }
 
@@ -503,9 +427,7 @@ define <vscale x 16 x i8> @mul_i8_neg(<vscale x 16 x i8> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mul z0.b, z0.b, #-17
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 -17, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %res = mul <vscale x 16 x i8> %a, %splat
+  %res = mul <vscale x 16 x i8> %a, splat(i8 -17)
   ret <vscale x 16 x i8> %res
 }
 
@@ -514,9 +436,7 @@ define <vscale x 16 x i8> @mul_i8_pos(<vscale x 16 x i8> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mul z0.b, z0.b, #105
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 105, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %res = mul <vscale x 16 x i8> %a, %splat
+  %res = mul <vscale x 16 x i8> %a, splat(i8 105)
   ret <vscale x 16 x i8> %res
 }
 
@@ -525,9 +445,7 @@ define <vscale x 8 x i16> @mul_i16_neg(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mul z0.h, z0.h, #-17
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 -17, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res = mul <vscale x 8 x i16> %a, %splat
+  %res = mul <vscale x 8 x i16> %a, splat(i16 -17)
   ret <vscale x 8 x i16> %res
 }
 
@@ -536,9 +454,7 @@ define <vscale x 8 x i16> @mul_i16_pos(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mul z0.h, z0.h, #105
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 105, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res = mul <vscale x 8 x i16> %a, %splat
+  %res = mul <vscale x 8 x i16> %a, splat(i16 105)
   ret <vscale x 8 x i16> %res
 }
 
@@ -547,9 +463,7 @@ define <vscale x 4 x i32> @mul_i32_neg(<vscale x 4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mul z0.s, z0.s, #-17
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 -17, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res = mul <vscale x 4 x i32> %a, %splat
+  %res = mul <vscale x 4 x i32> %a, splat(i32 -17)
   ret <vscale x 4 x i32> %res
 }
 
@@ -558,9 +472,7 @@ define <vscale x 4 x i32> @mul_i32_pos(<vscale x 4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mul z0.s, z0.s, #105
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 105, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res = mul <vscale x 4 x i32> %a, %splat
+  %res = mul <vscale x 4 x i32> %a, splat(i32 105)
   ret <vscale x 4 x i32> %res
 }
 
@@ -569,9 +481,7 @@ define <vscale x 2 x i64> @mul_i64_neg(<vscale x 2 x i64> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mul z0.d, z0.d, #-17
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 -17, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res = mul <vscale x 2 x i64> %a, %splat
+  %res = mul <vscale x 2 x i64> %a, splat(i64 -17)
   ret <vscale x 2 x i64> %res
 }
 
@@ -580,9 +490,7 @@ define <vscale x 2 x i64> @mul_i64_pos(<vscale x 2 x i64> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mul z0.d, z0.d, #105
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 105, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res = mul <vscale x 2 x i64> %a, %splat
+  %res = mul <vscale x 2 x i64> %a, splat(i64 105)
   ret <vscale x 2 x i64> %res
 }
 
@@ -593,9 +501,7 @@ define <vscale x 8 x i16> @mul_i16_range(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    mul z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 255, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res = mul <vscale x 8 x i16> %a, %splat
+  %res = mul <vscale x 8 x i16> %a, splat(i16 255)
   ret <vscale x 8 x i16> %res
 }
 
@@ -606,9 +512,7 @@ define <vscale x 4 x i32> @mul_i32_range(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    mul z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 255, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res = mul <vscale x 4 x i32> %a, %splat
+  %res = mul <vscale x 4 x i32> %a, splat(i32 255)
   ret <vscale x 4 x i32> %res
 }
 
@@ -619,9 +523,7 @@ define <vscale x 2 x i64> @mul_i64_range(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mul z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 255, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res = mul <vscale x 2 x i64> %a, %splat
+  %res = mul <vscale x 2 x i64> %a, splat(i64 255)
   ret <vscale x 2 x i64> %res
 }
 
@@ -632,9 +534,7 @@ define <vscale x 16 x i8> @asr_i8(<vscale x 16 x i8> %a){
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    asr z0.b, z0.b, #7
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 7, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %lshr = ashr <vscale x 16 x i8> %a, %splat
+  %lshr = ashr <vscale x 16 x i8> %a, splat(i8 7)
   ret <vscale x 16 x i8> %lshr
 }
 
@@ -643,9 +543,7 @@ define <vscale x 8 x i16> @asr_i16(<vscale x 8 x i16> %a){
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    asr z0.h, z0.h, #15
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 15, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %ashr = ashr <vscale x 8 x i16> %a, %splat
+  %ashr = ashr <vscale x 8 x i16> %a, splat(i16 15)
   ret <vscale x 8 x i16> %ashr
 }
 
@@ -654,9 +552,7 @@ define <vscale x 4 x i32> @asr_i32(<vscale x 4 x i32> %a){
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    asr z0.s, z0.s, #31
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 31, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %ashr = ashr <vscale x 4 x i32> %a, %splat
+  %ashr = ashr <vscale x 4 x i32> %a, splat(i32 31)
   ret <vscale x 4 x i32> %ashr
 }
 
@@ -665,9 +561,7 @@ define <vscale x 2 x i64> @asr_i64(<vscale x 2 x i64> %a){
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    asr z0.d, z0.d, #63
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 63, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %ashr = ashr <vscale x 2 x i64> %a, %splat
+  %ashr = ashr <vscale x 2 x i64> %a, splat(i64 63)
   ret <vscale x 2 x i64> %ashr
 }
 
@@ -678,9 +572,7 @@ define <vscale x 16 x i8> @lsl_i8(<vscale x 16 x i8> %a){
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsl z0.b, z0.b, #7
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 7, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %shl = shl <vscale x 16 x i8> %a, %splat
+  %shl = shl <vscale x 16 x i8> %a, splat(i8 7)
   ret <vscale x 16 x i8> %shl
 }
 
@@ -689,9 +581,7 @@ define <vscale x 8 x i16> @lsl_i16(<vscale x 8 x i16> %a){
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsl z0.h, z0.h, #15
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 15, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %shl = shl <vscale x 8 x i16> %a, %splat
+  %shl = shl <vscale x 8 x i16> %a, splat(i16 15)
   ret <vscale x 8 x i16> %shl
 }
 
@@ -700,9 +590,7 @@ define <vscale x 4 x i32> @lsl_i32(<vscale x 4 x i32> %a){
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsl z0.s, z0.s, #31
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 31, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %shl = shl <vscale x 4 x i32> %a, %splat
+  %shl = shl <vscale x 4 x i32> %a, splat(i32 31)
   ret <vscale x 4 x i32> %shl
 }
 
@@ -711,9 +599,7 @@ define <vscale x 2 x i64> @lsl_i64(<vscale x 2 x i64> %a){
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsl z0.d, z0.d, #63
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 63, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %shl = shl <vscale x 2 x i64> %a, %splat
+  %shl = shl <vscale x 2 x i64> %a, splat(i64 63)
   ret <vscale x 2 x i64> %shl
 }
 
@@ -724,9 +610,7 @@ define <vscale x 16 x i8> @lsr_i8(<vscale x 16 x i8> %a){
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr z0.b, z0.b, #7
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 7, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %lshr = lshr <vscale x 16 x i8> %a, %splat
+  %lshr = lshr <vscale x 16 x i8> %a, splat(i8 7)
   ret <vscale x 16 x i8> %lshr
 }
 
@@ -735,9 +619,7 @@ define <vscale x 8 x i16> @lsr_i16(<vscale x 8 x i16> %a){
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr z0.h, z0.h, #15
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 15, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %lshr = lshr <vscale x 8 x i16> %a, %splat
+  %lshr = lshr <vscale x 8 x i16> %a, splat(i16 15)
   ret <vscale x 8 x i16> %lshr
 }
 
@@ -746,9 +628,7 @@ define <vscale x 4 x i32> @lsr_i32(<vscale x 4 x i32> %a){
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr z0.s, z0.s, #31
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 31, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %lshr = lshr <vscale x 4 x i32> %a, %splat
+  %lshr = lshr <vscale x 4 x i32> %a, splat(i32 31)
   ret <vscale x 4 x i32> %lshr
 }
 
@@ -757,9 +637,7 @@ define <vscale x 2 x i64> @lsr_i64(<vscale x 2 x i64> %a){
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr z0.d, z0.d, #63
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 63, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %lshr = lshr <vscale x 2 x i64> %a, %splat
+  %lshr = lshr <vscale x 2 x i64> %a, splat(i64 63)
   ret <vscale x 2 x i64> %lshr
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve-int-imm.ll b/llvm/test/CodeGen/AArch64/sve-int-imm.ll
index d82e7f8c039cc..4be1abe842008 100644
--- a/llvm/test/CodeGen/AArch64/sve-int-imm.ll
+++ b/llvm/test/CodeGen/AArch64/sve-int-imm.ll
@@ -9,9 +9,7 @@ define <vscale x 16 x i8> @add_i8_low(<vscale x 16 x i8> %a) {
 ; CHECK-LABEL: add_i8_low
 ; CHECK: add  z0.b, z0.b, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 30, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %res =  add <vscale x 16 x i8> %a, %splat
+  %res =  add <vscale x 16 x i8> %a, splat(i8 30)
   ret <vscale x 16 x i8> %res
 }
 
@@ -19,9 +17,7 @@ define <vscale x 8 x i16> @add_i16_low(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: add_i16_low
 ; CHECK: add  z0.h, z0.h, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 30, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res =  add <vscale x 8 x i16> %a, %splat
+  %res =  add <vscale x 8 x i16> %a, splat(i16 30)
   ret <vscale x 8 x i16> %res
 }
 
@@ -29,9 +25,7 @@ define <vscale x 8 x i16> @add_i16_high(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: add_i16_high
 ; CHECK: add  z0.h, z0.h, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 1024, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res =  add <vscale x 8 x i16> %a, %splat
+  %res =  add <vscale x 8 x i16> %a, splat(i16 1024)
   ret <vscale x 8 x i16> %res
 }
 
@@ -39,9 +33,7 @@ define <vscale x 4 x i32> @add_i32_low(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: add_i32_low
 ; CHECK: add  z0.s, z0.s, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 30, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res = add <vscale x 4 x i32> %a, %splat
+  %res = add <vscale x 4 x i32> %a, splat(i32 30)
   ret <vscale x 4 x i32> %res
 }
 
@@ -49,9 +41,7 @@ define <vscale x 4 x i32> @add_i32_high(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: add_i32_high
 ; CHECK: add  z0.s, z0.s, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 1024, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res =  add <vscale x 4 x i32> %a, %splat
+  %res =  add <vscale x 4 x i32> %a, splat(i32 1024)
   ret <vscale x 4 x i32> %res
 }
 
@@ -59,9 +49,7 @@ define <vscale x 2 x i64> @add_i64_low(<vscale x 2 x i64> %a) {
 ; CHECK-LABEL: add_i64_low
 ; CHECK: add  z0.d, z0.d, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 30, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res =  add <vscale x 2 x i64> %a, %splat
+  %res =  add <vscale x 2 x i64> %a, splat(i64 30)
   ret <vscale x 2 x i64> %res
 }
 
@@ -69,9 +57,7 @@ define <vscale x 2 x i64> @add_i64_high(<vscale x 2 x i64> %a) {
 ; CHECK-LABEL: add_i64_high
 ; CHECK: add  z0.d, z0.d, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 1024, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res = add <vscale x 2 x i64> %a, %splat
+  %res = add <vscale x 2 x i64> %a, splat(i64 1024)
   ret <vscale x 2 x i64> %res
 }
 
@@ -79,9 +65,7 @@ define <vscale x 16 x i8> @add_i8_signedness(<vscale x 16 x i8> %a) {
 ; CHECK-LABEL: add_i8_signedness
 ; CHECK: add  z0.b, z0.b, #255
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 255, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %res =  add <vscale x 16 x i8> %a, %splat
+  %res =  add <vscale x 16 x i8> %a, splat(i8 255)
   ret <vscale x 16 x i8> %res
 }
 
@@ -89,9 +73,7 @@ define <vscale x 8 x i16> @add_i16_signedness(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: add_i16_signedness
 ; CHECK: add  z0.h, z0.h, #65280
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 65280, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res =  add <vscale x 8 x i16> %a, %splat
+  %res =  add <vscale x 8 x i16> %a, splat(i16 65280)
   ret <vscale x 8 x i16> %res
 }
 
@@ -100,9 +82,7 @@ define <vscale x 16 x i8> @subr_i8_low(<vscale x 16 x i8> %a) {
 ; CHECK-LABEL: subr_i8_low
 ; CHECK: subr  z0.b, z0.b, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 30, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %res =  sub <vscale x 16 x i8> %splat, %a
+  %res =  sub <vscale x 16 x i8> splat(i8 30), %a
   ret <vscale x 16 x i8> %res
 }
 
@@ -110,9 +90,7 @@ define <vscale x 8 x i16> @subr_i16_low(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: subr_i16_low
 ; CHECK: subr  z0.h, z0.h, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 30, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res =  sub <vscale x 8 x i16> %splat, %a
+  %res =  sub <vscale x 8 x i16> splat(i16 30), %a
   ret <vscale x 8 x i16> %res
 }
 
@@ -120,9 +98,7 @@ define <vscale x 8 x i16> @subr_i16_high(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: subr_i16_high
 ; CHECK: subr  z0.h, z0.h, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 1024, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res =  sub <vscale x 8 x i16> %splat, %a
+  %res =  sub <vscale x 8 x i16> splat(i16 1024), %a
   ret <vscale x 8 x i16> %res
 }
 
@@ -130,9 +106,7 @@ define <vscale x 4 x i32> @subr_i32_low(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: subr_i32_low
 ; CHECK: subr  z0.s, z0.s, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 30, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res =  sub <vscale x 4 x i32> %splat, %a
+  %res =  sub <vscale x 4 x i32> splat(i32 30), %a
   ret <vscale x 4 x i32> %res
 }
 
@@ -140,9 +114,7 @@ define <vscale x 4 x i32> @subr_i32_high(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: subr_i32_high
 ; CHECK: subr  z0.s, z0.s, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 1024, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res =  sub <vscale x 4 x i32> %splat, %a
+  %res =  sub <vscale x 4 x i32> splat(i32 1024), %a
   ret <vscale x 4 x i32> %res
 }
 
@@ -150,9 +122,7 @@ define <vscale x 2 x i64> @subr_i64_low(<vscale x 2 x i64> %a) {
 ; CHECK-LABEL: subr_i64_low
 ; CHECK: subr  z0.d, z0.d, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 30, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res =  sub <vscale x 2 x i64> %splat, %a
+  %res =  sub <vscale x 2 x i64> splat(i64 30), %a
   ret <vscale x 2 x i64> %res
 }
 
@@ -160,9 +130,7 @@ define <vscale x 2 x i64> @subr_i64_high(<vscale x 2 x i64> %a) {
 ; CHECK-LABEL: subr_i64_high
 ; CHECK: subr  z0.d, z0.d, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 1024, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res =  sub <vscale x 2 x i64> %splat, %a
+  %res =  sub <vscale x 2 x i64> splat(i64 1024), %a
   ret <vscale x 2 x i64> %res
 }
 
@@ -171,9 +139,7 @@ define <vscale x 16 x i8> @sub_i8_low(<vscale x 16 x i8> %a) {
 ; CHECK-LABEL: sub_i8_low
 ; CHECK: sub  z0.b, z0.b, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 30, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %res =  sub <vscale x 16 x i8> %a, %splat
+  %res =  sub <vscale x 16 x i8> %a, splat(i8 30)
   ret <vscale x 16 x i8> %res
 }
 
@@ -181,9 +147,7 @@ define <vscale x 8 x i16> @sub_i16_low(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: sub_i16_low
 ; CHECK: sub  z0.h, z0.h, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 30, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res =  sub <vscale x 8 x i16> %a, %splat
+  %res =  sub <vscale x 8 x i16> %a, splat(i16 30)
   ret <vscale x 8 x i16> %res
 }
 
@@ -191,9 +155,7 @@ define <vscale x 8 x i16> @sub_i16_high(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: sub_i16_high
 ; CHECK: sub  z0.h, z0.h, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 1024, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res =  sub <vscale x 8 x i16> %a, %splat
+  %res =  sub <vscale x 8 x i16> %a, splat(i16 1024)
   ret <vscale x 8 x i16> %res
 }
 
@@ -201,9 +163,7 @@ define <vscale x 4 x i32> @sub_i32_low(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: sub_i32_low
 ; CHECK: sub  z0.s, z0.s, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 30, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res = sub <vscale x 4 x i32> %a, %splat
+  %res = sub <vscale x 4 x i32> %a, splat(i32 30)
   ret <vscale x 4 x i32> %res
 }
 
@@ -211,9 +171,7 @@ define <vscale x 4 x i32> @sub_i32_high(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: sub_i32_high
 ; CHECK: sub  z0.s, z0.s, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 1024, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res =  sub <vscale x 4 x i32> %a, %splat
+  %res =  sub <vscale x 4 x i32> %a, splat(i32 1024)
   ret <vscale x 4 x i32> %res
 }
 
@@ -221,9 +179,7 @@ define <vscale x 2 x i64> @sub_i64_low(<vscale x 2 x i64> %a) {
 ; CHECK-LABEL: sub_i64_low
 ; CHECK: sub  z0.d, z0.d, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 30, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res =  sub <vscale x 2 x i64> %a, %splat
+  %res =  sub <vscale x 2 x i64> %a, splat(i64 30)
   ret <vscale x 2 x i64> %res
 }
 
@@ -231,9 +187,7 @@ define <vscale x 2 x i64> @sub_i64_high(<vscale x 2 x i64> %a) {
 ; CHECK-LABEL: sub_i64_high
 ; CHECK: sub  z0.d, z0.d, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 1024, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res = sub <vscale x 2 x i64> %a, %splat
+  %res = sub <vscale x 2 x i64> %a, splat(i64 1024)
   ret <vscale x 2 x i64> %res
 }
 
@@ -242,9 +196,7 @@ define <vscale x 16 x i8> @sqadd_i8_low(<vscale x 16 x i8> %a) {
 ; CHECK-LABEL: sqadd_i8_low
 ; CHECK: sqadd  z0.b, z0.b, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 30, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %res =  call <vscale x 16 x i8> @llvm.sadd.sat.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %splat)
+  %res =  call <vscale x 16 x i8> @llvm.sadd.sat.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> splat(i8 30))
   ret <vscale x 16 x i8> %res
 }
 
@@ -252,9 +204,7 @@ define <vscale x 8 x i16> @sqadd_i16_low(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: sqadd_i16_low
 ; CHECK: sqadd  z0.h, z0.h, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 30, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res =  call <vscale x 8 x i16> @llvm.sadd.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %splat)
+  %res =  call <vscale x 8 x i16> @llvm.sadd.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 30))
   ret <vscale x 8 x i16> %res
 }
 
@@ -262,9 +212,7 @@ define <vscale x 8 x i16> @sqadd_i16_high(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: sqadd_i16_high
 ; CHECK: sqadd  z0.h, z0.h, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 1024, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res =  call <vscale x 8 x i16> @llvm.sadd.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %splat)
+  %res =  call <vscale x 8 x i16> @llvm.sadd.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 1024))
   ret <vscale x 8 x i16> %res
 }
 
@@ -272,9 +220,7 @@ define <vscale x 4 x i32> @sqadd_i32_low(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: sqadd_i32_low
 ; CHECK: sqadd  z0.s, z0.s, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 30, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res =  call <vscale x 4 x i32> @llvm.sadd.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %splat)
+  %res =  call <vscale x 4 x i32> @llvm.sadd.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 30))
   ret <vscale x 4 x i32> %res
 }
 
@@ -282,9 +228,7 @@ define <vscale x 4 x i32> @sqadd_i32_high(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: sqadd_i32_high
 ; CHECK: sqadd  z0.s, z0.s, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 1024, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res =  call <vscale x 4 x i32> @llvm.sadd.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %splat)
+  %res =  call <vscale x 4 x i32> @llvm.sadd.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 1024))
   ret <vscale x 4 x i32> %res
 }
 
@@ -292,9 +236,7 @@ define <vscale x 2 x i64> @sqadd_i64_low(<vscale x 2 x i64> %a) {
 ; CHECK-LABEL: sqadd_i64_low
 ; CHECK: sqadd  z0.d, z0.d, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 30, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res =  call <vscale x 2 x i64> @llvm.sadd.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %splat)
+  %res =  call <vscale x 2 x i64> @llvm.sadd.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 30))
   ret <vscale x 2 x i64> %res
 }
 
@@ -302,9 +244,7 @@ define <vscale x 2 x i64> @sqadd_i64_high(<vscale x 2 x i64> %a) {
 ; CHECK-LABEL: sqadd_i64_high
 ; CHECK: sqadd  z0.d, z0.d, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 1024, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res =  call <vscale x 2 x i64> @llvm.sadd.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %splat)
+  %res =  call <vscale x 2 x i64> @llvm.sadd.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 1024))
   ret <vscale x 2 x i64> %res
 }
 
@@ -313,9 +253,7 @@ define <vscale x 16 x i8> @uqadd_i8_low(<vscale x 16 x i8> %a) {
 ; CHECK-LABEL: uqadd_i8_low
 ; CHECK: uqadd  z0.b, z0.b, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 30, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %res =  call <vscale x 16 x i8> @llvm.uadd.sat.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %splat)
+  %res =  call <vscale x 16 x i8> @llvm.uadd.sat.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> splat(i8 30))
   ret <vscale x 16 x i8> %res
 }
 
@@ -323,9 +261,7 @@ define <vscale x 8 x i16> @uqadd_i16_low(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: uqadd_i16_low
 ; CHECK: uqadd  z0.h, z0.h, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 30, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res =  call <vscale x 8 x i16> @llvm.uadd.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %splat)
+  %res =  call <vscale x 8 x i16> @llvm.uadd.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 30))
   ret <vscale x 8 x i16> %res
 }
 
@@ -333,9 +269,7 @@ define <vscale x 8 x i16> @uqadd_i16_high(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: uqadd_i16_high
 ; CHECK: uqadd  z0.h, z0.h, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 1024, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res =  call <vscale x 8 x i16> @llvm.uadd.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %splat)
+  %res =  call <vscale x 8 x i16> @llvm.uadd.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 1024))
   ret <vscale x 8 x i16> %res
 }
 
@@ -343,9 +277,7 @@ define <vscale x 4 x i32> @uqadd_i32_low(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: uqadd_i32_low
 ; CHECK: uqadd  z0.s, z0.s, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 30, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res =  call <vscale x 4 x i32> @llvm.uadd.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %splat)
+  %res =  call <vscale x 4 x i32> @llvm.uadd.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 30))
   ret <vscale x 4 x i32> %res
 }
 
@@ -353,9 +285,7 @@ define <vscale x 4 x i32> @uqadd_i32_high(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: uqadd_i32_high
 ; CHECK: uqadd  z0.s, z0.s, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 1024, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res =  call <vscale x 4 x i32> @llvm.uadd.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %splat)
+  %res =  call <vscale x 4 x i32> @llvm.uadd.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 1024))
   ret <vscale x 4 x i32> %res
 }
 
@@ -363,9 +293,7 @@ define <vscale x 2 x i64> @uqadd_i64_low(<vscale x 2 x i64> %a) {
 ; CHECK-LABEL: uqadd_i64_low
 ; CHECK: uqadd  z0.d, z0.d, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 30, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res =  call <vscale x 2 x i64> @llvm.uadd.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %splat)
+  %res =  call <vscale x 2 x i64> @llvm.uadd.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 30))
   ret <vscale x 2 x i64> %res
 }
 
@@ -373,9 +301,7 @@ define <vscale x 2 x i64> @uqadd_i64_high(<vscale x 2 x i64> %a) {
 ; CHECK-LABEL: uqadd_i64_high
 ; CHECK: uqadd  z0.d, z0.d, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 1024, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res =  call <vscale x 2 x i64> @llvm.uadd.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %splat)
+  %res =  call <vscale x 2 x i64> @llvm.uadd.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 1024))
   ret <vscale x 2 x i64> %res
 }
 
@@ -384,9 +310,7 @@ define <vscale x 16 x i8> @sqsub_i8_low(<vscale x 16 x i8> %a) {
 ; CHECK-LABEL: sqsub_i8_low
 ; CHECK: sqsub  z0.b, z0.b, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 30, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %res =  call <vscale x 16 x i8> @llvm.ssub.sat.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %splat)
+  %res =  call <vscale x 16 x i8> @llvm.ssub.sat.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> splat(i8 30))
   ret <vscale x 16 x i8> %res
 }
 
@@ -394,9 +318,7 @@ define <vscale x 8 x i16> @sqsub_i16_low(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: sqsub_i16_low
 ; CHECK: sqsub  z0.h, z0.h, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 30, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res =  call <vscale x 8 x i16> @llvm.ssub.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %splat)
+  %res =  call <vscale x 8 x i16> @llvm.ssub.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 30))
   ret <vscale x 8 x i16> %res
 }
 
@@ -404,9 +326,7 @@ define <vscale x 8 x i16> @sqsub_i16_high(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: sqsub_i16_high
 ; CHECK: sqsub  z0.h, z0.h, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 1024, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res =  call <vscale x 8 x i16> @llvm.ssub.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %splat)
+  %res =  call <vscale x 8 x i16> @llvm.ssub.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 1024))
   ret <vscale x 8 x i16> %res
 }
 
@@ -414,9 +334,7 @@ define <vscale x 4 x i32> @sqsub_i32_low(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: sqsub_i32_low
 ; CHECK: sqsub  z0.s, z0.s, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 30, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res =  call <vscale x 4 x i32> @llvm.ssub.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %splat)
+  %res =  call <vscale x 4 x i32> @llvm.ssub.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 30))
   ret <vscale x 4 x i32> %res
 }
 
@@ -424,9 +342,7 @@ define <vscale x 4 x i32> @sqsub_i32_high(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: sqsub_i32_high
 ; CHECK: sqsub  z0.s, z0.s, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 1024, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res =  call <vscale x 4 x i32> @llvm.ssub.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %splat)
+  %res =  call <vscale x 4 x i32> @llvm.ssub.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 1024))
   ret <vscale x 4 x i32> %res
 }
 
@@ -434,9 +350,7 @@ define <vscale x 2 x i64> @sqsub_i64_low(<vscale x 2 x i64> %a) {
 ; CHECK-LABEL: sqsub_i64_low
 ; CHECK: sqsub  z0.d, z0.d, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 30, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res =  call <vscale x 2 x i64> @llvm.ssub.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %splat)
+  %res =  call <vscale x 2 x i64> @llvm.ssub.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 30))
   ret <vscale x 2 x i64> %res
 }
 
@@ -444,9 +358,7 @@ define <vscale x 2 x i64> @sqsub_i64_high(<vscale x 2 x i64> %a) {
 ; CHECK-LABEL: sqsub_i64_high
 ; CHECK: sqsub  z0.d, z0.d, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 1024, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res =  call <vscale x 2 x i64> @llvm.ssub.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %splat)
+  %res =  call <vscale x 2 x i64> @llvm.ssub.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 1024))
   ret <vscale x 2 x i64> %res
 }
 
@@ -455,9 +367,7 @@ define <vscale x 16 x i8> @uqsub_i8_low(<vscale x 16 x i8> %a) {
 ; CHECK-LABEL: uqsub_i8_low
 ; CHECK: uqsub  z0.b, z0.b, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 30, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %res =  call <vscale x 16 x i8> @llvm.usub.sat.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %splat)
+  %res =  call <vscale x 16 x i8> @llvm.usub.sat.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> splat(i8 30))
   ret <vscale x 16 x i8> %res
 }
 
@@ -465,9 +375,7 @@ define <vscale x 8 x i16> @uqsub_i16_low(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: uqsub_i16_low
 ; CHECK: uqsub  z0.h, z0.h, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 30, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res =  call <vscale x 8 x i16> @llvm.usub.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %splat)
+  %res =  call <vscale x 8 x i16> @llvm.usub.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 30))
   ret <vscale x 8 x i16> %res
 }
 
@@ -475,9 +383,7 @@ define <vscale x 8 x i16> @uqsub_i16_high(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: uqsub_i16_high
 ; CHECK: uqsub  z0.h, z0.h, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 1024, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res =  call <vscale x 8 x i16> @llvm.usub.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %splat)
+  %res =  call <vscale x 8 x i16> @llvm.usub.sat.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> splat(i16 1024))
   ret <vscale x 8 x i16> %res
 }
 
@@ -485,9 +391,7 @@ define <vscale x 4 x i32> @uqsub_i32_low(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: uqsub_i32_low
 ; CHECK: uqsub  z0.s, z0.s, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 30, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res =  call <vscale x 4 x i32> @llvm.usub.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %splat)
+  %res =  call <vscale x 4 x i32> @llvm.usub.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 30))
   ret <vscale x 4 x i32> %res
 }
 
@@ -495,9 +399,7 @@ define <vscale x 4 x i32> @uqsub_i32_high(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: uqsub_i32_high
 ; CHECK: uqsub  z0.s, z0.s, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 1024, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res =  call <vscale x 4 x i32> @llvm.usub.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %splat)
+  %res =  call <vscale x 4 x i32> @llvm.usub.sat.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> splat(i32 1024))
   ret <vscale x 4 x i32> %res
 }
 
@@ -505,9 +407,7 @@ define <vscale x 2 x i64> @uqsub_i64_low(<vscale x 2 x i64> %a) {
 ; CHECK-LABEL: uqsub_i64_low
 ; CHECK: uqsub  z0.d, z0.d, #30
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 30, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res =  call <vscale x 2 x i64> @llvm.usub.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %splat)
+  %res =  call <vscale x 2 x i64> @llvm.usub.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 30))
   ret <vscale x 2 x i64> %res
 }
 
@@ -515,9 +415,7 @@ define <vscale x 2 x i64> @uqsub_i64_high(<vscale x 2 x i64> %a) {
 ; CHECK-LABEL: uqsub_i64_high
 ; CHECK: uqsub  z0.d, z0.d, #1024
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 1024, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res =  call <vscale x 2 x i64> @llvm.usub.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %splat)
+  %res =  call <vscale x 2 x i64> @llvm.usub.sat.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> splat(i64 1024))
   ret <vscale x 2 x i64> %res
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve-int-log-imm.ll b/llvm/test/CodeGen/AArch64/sve-int-log-imm.ll
index 52b56d5adb5e2..f67029ed0c09f 100644
--- a/llvm/test/CodeGen/AArch64/sve-int-log-imm.ll
+++ b/llvm/test/CodeGen/AArch64/sve-int-log-imm.ll
@@ -9,9 +9,7 @@ define <vscale x 16 x i8> @orr_i8(<vscale x 16 x i8> %a) {
 ; CHECK-LABEL: orr_i8:
 ; CHECK: orr z0.b, z0.b, #0xf
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 15, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %res = or <vscale x 16 x i8> %a, %splat
+  %res = or <vscale x 16 x i8> %a, splat(i8 15)
   ret <vscale x 16 x i8> %res
 }
 
@@ -19,9 +17,7 @@ define <vscale x 8 x i16> @orr_i16(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: orr_i16:
 ; CHECK: orr z0.h, z0.h, #0xfc07
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 64519, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res = or <vscale x 8 x i16> %a, %splat
+  %res = or <vscale x 8 x i16> %a, splat(i16 64519)
   ret <vscale x 8 x i16> %res
 }
 
@@ -29,9 +25,7 @@ define <vscale x 4 x i32> @orr_i32(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: orr_i32:
 ; CHECK: orr z0.s, z0.s, #0xffff00
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 16776960, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res = or <vscale x 4 x i32> %a, %splat
+  %res = or <vscale x 4 x i32> %a, splat(i32 16776960)
   ret <vscale x 4 x i32> %res
 }
 
@@ -39,9 +33,7 @@ define <vscale x 2 x i64> @orr_i64(<vscale x 2 x i64> %a) {
 ; CHECK-LABEL: orr_i64:
 ; CHECK: orr z0.d, z0.d, #0xfffc000000000000
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 18445618173802708992, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res = or <vscale x 2 x i64> %a, %splat
+  %res = or <vscale x 2 x i64> %a, splat(i64 18445618173802708992)
   ret <vscale x 2 x i64> %res
 }
 
@@ -50,9 +42,7 @@ define <vscale x 16 x i8> @eor_i8(<vscale x 16 x i8> %a) {
 ; CHECK-LABEL: eor_i8:
 ; CHECK: eor z0.b, z0.b, #0xf
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 15, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %res = xor <vscale x 16 x i8> %a, %splat
+  %res = xor <vscale x 16 x i8> %a, splat(i8 15)
   ret <vscale x 16 x i8> %res
 }
 
@@ -60,9 +50,7 @@ define <vscale x 8 x i16> @eor_i16(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: eor_i16:
 ; CHECK: eor z0.h, z0.h, #0xfc07
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 64519, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res = xor <vscale x 8 x i16> %a, %splat
+  %res = xor <vscale x 8 x i16> %a, splat(i16 64519)
   ret <vscale x 8 x i16> %res
 }
 
@@ -70,9 +58,7 @@ define <vscale x 4 x i32> @eor_i32(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: eor_i32:
 ; CHECK: eor z0.s, z0.s, #0xffff00
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 16776960, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res = xor <vscale x 4 x i32> %a, %splat
+  %res = xor <vscale x 4 x i32> %a, splat(i32 16776960)
   ret <vscale x 4 x i32> %res
 }
 
@@ -80,9 +66,7 @@ define <vscale x 2 x i64> @eor_i64(<vscale x 2 x i64> %a) {
 ; CHECK-LABEL: eor_i64:
 ; CHECK: eor z0.d, z0.d, #0xfffc000000000000
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 18445618173802708992, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res = xor <vscale x 2 x i64> %a, %splat
+  %res = xor <vscale x 2 x i64> %a, splat(i64 18445618173802708992)
   ret <vscale x 2 x i64> %res
 }
 
@@ -91,9 +75,7 @@ define <vscale x 16 x i8> @and_i8(<vscale x 16 x i8> %a) {
 ; CHECK-LABEL: and_i8:
 ; CHECK: and z0.b, z0.b, #0xf
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 15, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %res = and <vscale x 16 x i8> %a, %splat
+  %res = and <vscale x 16 x i8> %a, splat(i8 15)
   ret <vscale x 16 x i8> %res
 }
 
@@ -101,9 +83,7 @@ define <vscale x 8 x i16> @and_i16(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: and_i16:
 ; CHECK: and z0.h, z0.h, #0xfc07
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 64519, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res = and <vscale x 8 x i16> %a, %splat
+  %res = and <vscale x 8 x i16> %a, splat(i16 64519)
   ret <vscale x 8 x i16> %res
 }
 
@@ -111,9 +91,7 @@ define <vscale x 4 x i32> @and_i32(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: and_i32:
 ; CHECK: and z0.s, z0.s, #0xffff00
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 16776960, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res = and <vscale x 4 x i32> %a, %splat
+  %res = and <vscale x 4 x i32> %a, splat(i32 16776960)
   ret <vscale x 4 x i32> %res
 }
 
@@ -121,8 +99,6 @@ define <vscale x 2 x i64> @and_i64(<vscale x 2 x i64> %a) {
 ; CHECK-LABEL: and_i64:
 ; CHECK: and z0.d, z0.d, #0xfffc000000000000
 ; CHECK-NEXT: ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 18445618173802708992, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res = and <vscale x 2 x i64> %a, %splat
+  %res = and <vscale x 2 x i64> %a, splat(i64 18445618173802708992)
   ret <vscale x 2 x i64> %res
 }
diff --git a/llvm/test/CodeGen/AArch64/sve-int-mulh-pred.ll b/llvm/test/CodeGen/AArch64/sve-int-mulh-pred.ll
index c636f1172711a..32760caa524ec 100644
--- a/llvm/test/CodeGen/AArch64/sve-int-mulh-pred.ll
+++ b/llvm/test/CodeGen/AArch64/sve-int-mulh-pred.ll
@@ -11,12 +11,10 @@ define <vscale x 16 x i8> @smulh_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b
 ; CHECK-NEXT:    ptrue p0.b
 ; CHECK-NEXT:    smulh z0.b, p0/m, z0.b, z1.b
 ; CHECK-NEXT:    ret
-  %insert = insertelement <vscale x 16 x i16> undef, i16 8, i64 0
-  %splat = shufflevector <vscale x 16 x i16> %insert, <vscale x 16 x i16> undef, <vscale x 16 x i32> zeroinitializer
   %1 = sext <vscale x 16 x i8> %a to <vscale x 16 x i16>
   %2 = sext <vscale x 16 x i8> %b to <vscale x 16 x i16>
   %mul = mul <vscale x 16 x i16> %1, %2
-  %shr = lshr <vscale x 16 x i16> %mul, %splat
+  %shr = lshr <vscale x 16 x i16> %mul, splat(i16 8)
   %tr = trunc <vscale x 16 x i16> %shr to <vscale x 16 x i8>
   ret <vscale x 16 x i8> %tr
 }
@@ -27,12 +25,10 @@ define <vscale x 8 x i16> @smulh_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    smulh z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    ret
-  %insert = insertelement <vscale x 8 x i32> undef, i32 16, i64 0
-  %splat = shufflevector <vscale x 8 x i32> %insert, <vscale x 8 x i32> undef, <vscale x 8 x i32> zeroinitializer
   %1 = sext <vscale x 8 x i16> %a to <vscale x 8 x i32>
   %2 = sext <vscale x 8 x i16> %b to <vscale x 8 x i32>
   %mul = mul <vscale x 8 x i32> %1, %2
-  %shr = lshr <vscale x 8 x i32> %mul, %splat
+  %shr = lshr <vscale x 8 x i32> %mul, splat(i32 16)
   %tr = trunc <vscale x 8 x i32> %shr to <vscale x 8 x i16>
   ret <vscale x 8 x i16> %tr
 }
@@ -43,12 +39,10 @@ define <vscale x 4 x i32> @smulh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    smulh z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ret
-  %insert = insertelement <vscale x 4 x i64> undef, i64 32, i64 0
-  %splat = shufflevector <vscale x 4 x i64> %insert, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
   %1 = sext <vscale x 4 x i32> %a to <vscale x 4 x i64>
   %2 = sext <vscale x 4 x i32> %b to <vscale x 4 x i64>
   %mul = mul <vscale x 4 x i64> %1, %2
-  %shr = lshr <vscale x 4 x i64> %mul, %splat
+  %shr = lshr <vscale x 4 x i64> %mul, splat(i64 32)
   %tr = trunc <vscale x 4 x i64> %shr to <vscale x 4 x i32>
   ret <vscale x 4 x i32> %tr
 }
@@ -59,12 +53,10 @@ define <vscale x 2 x i64> @smulh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    smulh z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    ret
-  %insert = insertelement <vscale x 2 x i128> undef, i128 64, i64 0
-  %splat = shufflevector <vscale x 2 x i128> %insert, <vscale x 2 x i128> undef, <vscale x 2 x i32> zeroinitializer
   %1 = sext <vscale x 2 x i64> %a to <vscale x 2 x i128>
   %2 = sext <vscale x 2 x i64> %b to <vscale x 2 x i128>
   %mul = mul <vscale x 2 x i128> %1, %2
-  %shr = lshr <vscale x 2 x i128> %mul, %splat
+  %shr = lshr <vscale x 2 x i128> %mul, splat(i128 64)
   %tr = trunc <vscale x 2 x i128> %shr to <vscale x 2 x i64>
   ret <vscale x 2 x i64> %tr
 }
@@ -79,12 +71,10 @@ define <vscale x 16 x i8> @umulh_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b
 ; CHECK-NEXT:    ptrue p0.b
 ; CHECK-NEXT:    umulh z0.b, p0/m, z0.b, z1.b
 ; CHECK-NEXT:    ret
-  %insert = insertelement <vscale x 16 x i16> undef, i16 8, i64 0
-  %splat = shufflevector <vscale x 16 x i16> %insert, <vscale x 16 x i16> undef, <vscale x 16 x i32> zeroinitializer
   %1 = zext <vscale x 16 x i8> %a to <vscale x 16 x i16>
   %2 = zext <vscale x 16 x i8> %b to <vscale x 16 x i16>
   %mul = mul <vscale x 16 x i16> %1, %2
-  %shr = lshr <vscale x 16 x i16> %mul, %splat
+  %shr = lshr <vscale x 16 x i16> %mul, splat(i16 8)
   %tr = trunc <vscale x 16 x i16> %shr to <vscale x 16 x i8>
   ret <vscale x 16 x i8> %tr
 }
@@ -95,12 +85,10 @@ define <vscale x 8 x i16> @umulh_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    umulh z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    ret
-  %insert = insertelement <vscale x 8 x i32> undef, i32 16, i64 0
-  %splat = shufflevector <vscale x 8 x i32> %insert, <vscale x 8 x i32> undef, <vscale x 8 x i32> zeroinitializer
   %1 = zext <vscale x 8 x i16> %a to <vscale x 8 x i32>
   %2 = zext <vscale x 8 x i16> %b to <vscale x 8 x i32>
   %mul = mul <vscale x 8 x i32> %1, %2
-  %shr = lshr <vscale x 8 x i32> %mul, %splat
+  %shr = lshr <vscale x 8 x i32> %mul, splat(i32 16)
   %tr = trunc <vscale x 8 x i32> %shr to <vscale x 8 x i16>
   ret <vscale x 8 x i16> %tr
 }
@@ -111,12 +99,10 @@ define <vscale x 4 x i32> @umulh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    umulh z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ret
-  %insert = insertelement <vscale x 4 x i64> undef, i64 32, i64 0
-  %splat = shufflevector <vscale x 4 x i64> %insert, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
   %1 = zext <vscale x 4 x i32> %a to <vscale x 4 x i64>
   %2 = zext <vscale x 4 x i32> %b to <vscale x 4 x i64>
   %mul = mul <vscale x 4 x i64> %1, %2
-  %shr = lshr <vscale x 4 x i64> %mul, %splat
+  %shr = lshr <vscale x 4 x i64> %mul, splat(i64 32)
   %tr = trunc <vscale x 4 x i64> %shr to <vscale x 4 x i32>
   ret <vscale x 4 x i32> %tr
 }
@@ -127,12 +113,10 @@ define <vscale x 2 x i64> @umulh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    umulh z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    ret
-  %insert = insertelement <vscale x 2 x i128> undef, i128 64, i64 0
-  %splat = shufflevector <vscale x 2 x i128> %insert, <vscale x 2 x i128> undef, <vscale x 2 x i32> zeroinitializer
   %1 = zext <vscale x 2 x i64> %a to <vscale x 2 x i128>
   %2 = zext <vscale x 2 x i64> %b to <vscale x 2 x i128>
   %mul = mul <vscale x 2 x i128> %1, %2
-  %shr = lshr <vscale x 2 x i128> %mul, %splat
+  %shr = lshr <vscale x 2 x i128> %mul, splat(i128 64)
   %tr = trunc <vscale x 2 x i128> %shr to <vscale x 2 x i64>
   ret <vscale x 2 x i64> %tr
 }
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-bfloat.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-bfloat.ll
index 7d1e63e23a2ce..7f352041ec587 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-bfloat.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-bfloat.ll
@@ -237,10 +237,19 @@ define <vscale x 4 x float> @bfmmla_f32(<vscale x 4 x float> %a, <vscale x 8 x b
 ; BFCVT
 ;
 
-define <vscale x 8 x bfloat> @fcvt_bf16_f32(<vscale x 8 x bfloat> %a, <vscale x 8 x i1> %pg, <vscale x 4 x float> %b) nounwind {
+define <vscale x 8 x bfloat> @fcvt_bf16_f32(<vscale x 8 x bfloat> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b) nounwind {
 ; CHECK-LABEL: fcvt_bf16_f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    bfcvt z0.h, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32.v2(<vscale x 8 x bfloat> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b)
+  ret <vscale x 8 x bfloat> %out
+}
+
+define <vscale x 8 x bfloat> @fcvt_bf16_f32_deprecated(<vscale x 8 x bfloat> %a, <vscale x 8 x i1> %pg, <vscale x 4 x float> %b) nounwind {
+; CHECK-LABEL: fcvt_bf16_f32_deprecated:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bfcvt z0.h, p0/m, z1.s
 ; CHECK-NEXT:    ret
   %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32(<vscale x 8 x bfloat> %a, <vscale x 8 x i1> %pg, <vscale x 4 x float> %b)
   ret <vscale x 8 x bfloat> %out
@@ -250,10 +259,19 @@ define <vscale x 8 x bfloat> @fcvt_bf16_f32(<vscale x 8 x bfloat> %a, <vscale x
 ; BFCVTNT
 ;
 
-define <vscale x 8 x bfloat> @fcvtnt_bf16_f32(<vscale x 8 x bfloat> %a, <vscale x 8 x i1> %pg, <vscale x 4 x float> %b) nounwind {
+define <vscale x 8 x bfloat> @fcvtnt_bf16_f32(<vscale x 8 x bfloat> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b) nounwind {
 ; CHECK-LABEL: fcvtnt_bf16_f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    bfcvtnt z0.h, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32.v2(<vscale x 8 x bfloat> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b)
+  ret <vscale x 8 x bfloat> %out
+}
+
+define <vscale x 8 x bfloat> @fcvtnt_bf16_f32_deprecated(<vscale x 8 x bfloat> %a, <vscale x 8 x i1> %pg, <vscale x 4 x float> %b) nounwind {
+; CHECK-LABEL: fcvtnt_bf16_f32_deprecated:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bfcvtnt z0.h, p0/m, z1.s
 ; CHECK-NEXT:    ret
   %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32(<vscale x 8 x bfloat> %a, <vscale x 8 x i1> %pg, <vscale x 4 x float> %b)
   ret <vscale x 8 x bfloat> %out
@@ -267,4 +285,6 @@ declare <vscale x 4 x float> @llvm.aarch64.sve.bfmlalt(<vscale x 4 x float>, <vs
 declare <vscale x 4 x float> @llvm.aarch64.sve.bfmlalt.lane.v2(<vscale x 4 x float>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, i32)
 declare <vscale x 4 x float> @llvm.aarch64.sve.bfmmla(<vscale x 4 x float>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
 declare <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32(<vscale x 8 x bfloat>, <vscale x 8 x i1>, <vscale x 4 x float>)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32.v2(<vscale x 8 x bfloat>, <vscale x 4 x i1>, <vscale x 4 x float>)
 declare <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32(<vscale x 8 x bfloat>, <vscale x 8 x i1>, <vscale x 4 x float>)
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32.v2(<vscale x 8 x bfloat>, <vscale x 4 x i1>, <vscale x 4 x float>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-fp-arith-imm.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-fp-arith-imm.ll
index eea6031fbd6b5..e1cb46e0d1dc0 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-fp-arith-imm.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-fp-arith-imm.ll
@@ -8,11 +8,9 @@ define <vscale x 8 x half> @fadd_h_immhalf(<vscale x 8 x i1> %pg, <vscale x 8 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fadd z0.h, p0/m, z0.h, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fadd.nxv8f16(<vscale x 8 x i1> %pg,
-                                                            <vscale x 8 x half> %a,
-                                                            <vscale x 8 x half> %splat)
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> splat(half 0.500000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -23,12 +21,10 @@ define <vscale x 8 x half> @fadd_h_immhalf_zero(<vscale x 8 x i1> %pg, <vscale x
 ; CHECK-NEXT:    movprfx z0.h, p0/z, z0.h
 ; CHECK-NEXT:    fadd z0.h, p0/m, z0.h, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %a_z = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fadd.nxv8f16(<vscale x 8 x i1> %pg,
-                                                            <vscale x 8 x half> %a_z,
-                                                            <vscale x 8 x half> %splat)
+                                                                 <vscale x 8 x half> %a_z,
+                                                                 <vscale x 8 x half> splat(half 0.500000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -37,11 +33,9 @@ define <vscale x 8 x half> @fadd_h_immone(<vscale x 8 x i1> %pg, <vscale x 8 x h
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fadd z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fadd.nxv8f16(<vscale x 8 x i1> %pg,
-                                                            <vscale x 8 x half> %a,
-                                                            <vscale x 8 x half> %splat)
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> splat(half 1.000000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -51,12 +45,10 @@ define <vscale x 8 x half> @fadd_h_immone_zero(<vscale x 8 x i1> %pg, <vscale x
 ; CHECK-NEXT:    movprfx z0.h, p0/z, z0.h
 ; CHECK-NEXT:    fadd z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %a_z = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fadd.nxv8f16(<vscale x 8 x i1> %pg,
-                                                            <vscale x 8 x half> %a_z,
-                                                            <vscale x 8 x half> %splat)
+                                                                 <vscale x 8 x half> %a_z,
+                                                                 <vscale x 8 x half> splat(half 1.000000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -65,11 +57,9 @@ define <vscale x 4 x float> @fadd_s_immhalf(<vscale x 4 x i1> %pg, <vscale x 4 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fadd z0.s, p0/m, z0.s, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fadd.nxv4f32(<vscale x 4 x i1> %pg,
-                                                             <vscale x 4 x float> %a,
-                                                             <vscale x 4 x float> %splat)
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> splat(float 0.500000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -79,12 +69,10 @@ define <vscale x 4 x float> @fadd_s_immhalf_zero(<vscale x 4 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.s, p0/z, z0.s
 ; CHECK-NEXT:    fadd z0.s, p0/m, z0.s, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %a_z = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fadd.nxv4f32(<vscale x 4 x i1> %pg,
-                                                             <vscale x 4 x float> %a_z,
-                                                             <vscale x 4 x float> %splat)
+                                                                  <vscale x 4 x float> %a_z,
+                                                                  <vscale x 4 x float> splat(float 0.500000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -93,11 +81,9 @@ define <vscale x 4 x float> @fadd_s_immone(<vscale x 4 x i1> %pg, <vscale x 4 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fadd z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fadd.nxv4f32(<vscale x 4 x i1> %pg,
-                                                             <vscale x 4 x float> %a,
-                                                             <vscale x 4 x float> %splat)
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> splat(float 1.000000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -107,12 +93,10 @@ define <vscale x 4 x float> @fadd_s_immone_zero(<vscale x 4 x i1> %pg, <vscale x
 ; CHECK-NEXT:    movprfx z0.s, p0/z, z0.s
 ; CHECK-NEXT:    fadd z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %a_z = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fadd.nxv4f32(<vscale x 4 x i1> %pg,
-                                                             <vscale x 4 x float> %a_z,
-                                                             <vscale x 4 x float> %splat)
+                                                                  <vscale x 4 x float> %a_z,
+                                                                  <vscale x 4 x float> splat(float 1.000000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -121,11 +105,9 @@ define <vscale x 2 x double> @fadd_d_immhalf(<vscale x 2 x i1> %pg, <vscale x 2
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fadd z0.d, p0/m, z0.d, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fadd.nxv2f64(<vscale x 2 x i1> %pg,
-                                                              <vscale x 2 x double> %a,
-                                                              <vscale x 2 x double> %splat)
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> splat(double 0.500000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -135,12 +117,10 @@ define <vscale x 2 x double> @fadd_d_immhalf_zero(<vscale x 2 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.d, p0/z, z0.d
 ; CHECK-NEXT:    fadd z0.d, p0/m, z0.d, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %a_z = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fadd.nxv2f64(<vscale x 2 x i1> %pg,
-                                                              <vscale x 2 x double> %a_z,
-                                                              <vscale x 2 x double> %splat)
+                                                                   <vscale x 2 x double> %a_z,
+                                                                   <vscale x 2 x double> splat(double 0.500000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -149,11 +129,9 @@ define <vscale x 2 x double> @fadd_d_immone(<vscale x 2 x i1> %pg, <vscale x 2 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fadd z0.d, p0/m, z0.d, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fadd.nxv2f64(<vscale x 2 x i1> %pg,
-                                                              <vscale x 2 x double> %a,
-                                                              <vscale x 2 x double> %splat)
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> splat(double 1.000000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -163,12 +141,10 @@ define <vscale x 2 x double> @fadd_d_immone_zero(<vscale x 2 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.d, p0/z, z0.d
 ; CHECK-NEXT:    fadd z0.d, p0/m, z0.d, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %a_z = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fadd.nxv2f64(<vscale x 2 x i1> %pg,
-                                                              <vscale x 2 x double> %a_z,
-                                                              <vscale x 2 x double> %splat)
+                                                                   <vscale x 2 x double> %a_z,
+                                                                   <vscale x 2 x double> splat(double 1.000000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -177,11 +153,9 @@ define <vscale x 8 x half> @fmax_h_immzero(<vscale x 8 x i1> %pg, <vscale x 8 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmax z0.h, p0/m, z0.h, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmax.nxv8f16(<vscale x 8 x i1> %pg,
-                                                            <vscale x 8 x half> %a,
-                                                            <vscale x 8 x half> %splat)
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> zeroinitializer)
   ret <vscale x 8 x half> %out
 }
 
@@ -191,12 +165,10 @@ define <vscale x 8 x half> @fmax_h_immzero_zero(<vscale x 8 x i1> %pg, <vscale x
 ; CHECK-NEXT:    movprfx z0.h, p0/z, z0.h
 ; CHECK-NEXT:    fmax z0.h, p0/m, z0.h, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %a_z = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmax.nxv8f16(<vscale x 8 x i1> %pg,
-                                                            <vscale x 8 x half> %a_z,
-                                                            <vscale x 8 x half> %splat)
+                                                                 <vscale x 8 x half> %a_z,
+                                                                 <vscale x 8 x half> zeroinitializer)
   ret <vscale x 8 x half> %out
 }
 
@@ -205,11 +177,9 @@ define <vscale x 8 x half> @fmax_h_immone(<vscale x 8 x i1> %pg, <vscale x 8 x h
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmax z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmax.nxv8f16(<vscale x 8 x i1> %pg,
-                                                            <vscale x 8 x half> %a,
-                                                            <vscale x 8 x half> %splat)
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> splat(half 1.000000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -219,12 +189,10 @@ define <vscale x 8 x half> @fmax_h_immone_zero(<vscale x 8 x i1> %pg, <vscale x
 ; CHECK-NEXT:    movprfx z0.h, p0/z, z0.h
 ; CHECK-NEXT:    fmax z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %a_z = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmax.nxv8f16(<vscale x 8 x i1> %pg,
-                                                            <vscale x 8 x half> %a_z,
-                                                            <vscale x 8 x half> %splat)
+                                                                 <vscale x 8 x half> %a_z,
+                                                                 <vscale x 8 x half> splat(half 1.000000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -233,11 +201,9 @@ define <vscale x 4 x float> @fmax_s_immzero(<vscale x 4 x i1> %pg, <vscale x 4 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmax z0.s, p0/m, z0.s, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmax.nxv4f32(<vscale x 4 x i1> %pg,
-                                                             <vscale x 4 x float> %a,
-                                                             <vscale x 4 x float> %splat)
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> zeroinitializer)
   ret <vscale x 4 x float> %out
 }
 
@@ -247,12 +213,10 @@ define <vscale x 4 x float> @fmax_s_immzero_zero(<vscale x 4 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.s, p0/z, z0.s
 ; CHECK-NEXT:    fmax z0.s, p0/m, z0.s, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %a_z = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmax.nxv4f32(<vscale x 4 x i1> %pg,
-                                                             <vscale x 4 x float> %a_z,
-                                                             <vscale x 4 x float> %splat)
+                                                                  <vscale x 4 x float> %a_z,
+                                                                  <vscale x 4 x float> zeroinitializer)
   ret <vscale x 4 x float> %out
 }
 
@@ -261,11 +225,9 @@ define <vscale x 4 x float> @fmax_s_immone(<vscale x 4 x i1> %pg, <vscale x 4 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmax z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmax.nxv4f32(<vscale x 4 x i1> %pg,
-                                                             <vscale x 4 x float> %a,
-                                                             <vscale x 4 x float> %splat)
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> splat(float 1.000000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -275,12 +237,10 @@ define <vscale x 4 x float> @fmax_s_immone_zero(<vscale x 4 x i1> %pg, <vscale x
 ; CHECK-NEXT:    movprfx z0.s, p0/z, z0.s
 ; CHECK-NEXT:    fmax z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %a_z = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmax.nxv4f32(<vscale x 4 x i1> %pg,
-                                                             <vscale x 4 x float> %a_z,
-                                                             <vscale x 4 x float> %splat)
+                                                                  <vscale x 4 x float> %a_z,
+                                                                  <vscale x 4 x float> splat(float 1.000000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -289,11 +249,9 @@ define <vscale x 2 x double> @fmax_d_immzero(<vscale x 2 x i1> %pg, <vscale x 2
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmax z0.d, p0/m, z0.d, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmax.nxv2f64(<vscale x 2 x i1> %pg,
-                                                              <vscale x 2 x double> %a,
-                                                              <vscale x 2 x double> %splat)
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> zeroinitializer)
   ret <vscale x 2 x double> %out
 }
 
@@ -303,12 +261,10 @@ define <vscale x 2 x double> @fmax_d_immzero_zero(<vscale x 2 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.d, p0/z, z0.d
 ; CHECK-NEXT:    fmax z0.d, p0/m, z0.d, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %a_z = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmax.nxv2f64(<vscale x 2 x i1> %pg,
-                                                              <vscale x 2 x double> %a_z,
-                                                              <vscale x 2 x double> %splat)
+                                                                   <vscale x 2 x double> %a_z,
+                                                                   <vscale x 2 x double> zeroinitializer)
   ret <vscale x 2 x double> %out
 }
 
@@ -317,11 +273,9 @@ define <vscale x 2 x double> @fmax_d_immone(<vscale x 2 x i1> %pg, <vscale x 2 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmax z0.d, p0/m, z0.d, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmax.nxv2f64(<vscale x 2 x i1> %pg,
-                                                              <vscale x 2 x double> %a,
-                                                              <vscale x 2 x double> %splat)
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> splat(double 1.000000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -331,12 +285,10 @@ define <vscale x 2 x double> @fmax_d_immone_zero(<vscale x 2 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.d, p0/z, z0.d
 ; CHECK-NEXT:    fmax z0.d, p0/m, z0.d, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %a_z = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmax.nxv2f64(<vscale x 2 x i1> %pg,
-                                                              <vscale x 2 x double> %a_z,
-                                                              <vscale x 2 x double> %splat)
+                                                                   <vscale x 2 x double> %a_z,
+                                                                   <vscale x 2 x double> splat(double 1.000000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -345,11 +297,9 @@ define <vscale x 8 x half> @fmaxnm_h_immzero(<vscale x 8 x i1> %pg, <vscale x 8
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmaxnm z0.h, p0/m, z0.h, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmaxnm.nxv8f16(<vscale x 8 x i1> %pg,
-                                                              <vscale x 8 x half> %a,
-                                                              <vscale x 8 x half> %splat)
+                                                                   <vscale x 8 x half> %a,
+                                                                   <vscale x 8 x half> zeroinitializer)
   ret <vscale x 8 x half> %out
 }
 
@@ -359,12 +309,10 @@ define <vscale x 8 x half> @fmaxnm_h_immzero_zero(<vscale x 8 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.h, p0/z, z0.h
 ; CHECK-NEXT:    fmaxnm z0.h, p0/m, z0.h, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %a_z = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmaxnm.nxv8f16(<vscale x 8 x i1> %pg,
-                                                              <vscale x 8 x half> %a_z,
-                                                              <vscale x 8 x half> %splat)
+                                                                   <vscale x 8 x half> %a_z,
+                                                                   <vscale x 8 x half> zeroinitializer)
   ret <vscale x 8 x half> %out
 }
 
@@ -373,11 +321,9 @@ define <vscale x 8 x half> @fmaxnm_h_immone(<vscale x 8 x i1> %pg, <vscale x 8 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmaxnm z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmaxnm.nxv8f16(<vscale x 8 x i1> %pg,
-                                                              <vscale x 8 x half> %a,
-                                                              <vscale x 8 x half> %splat)
+                                                                   <vscale x 8 x half> %a,
+                                                                   <vscale x 8 x half> splat(half 1.000000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -387,12 +333,10 @@ define <vscale x 8 x half> @fmaxnm_h_immone_zero(<vscale x 8 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.h, p0/z, z0.h
 ; CHECK-NEXT:    fmaxnm z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %a_z = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmaxnm.nxv8f16(<vscale x 8 x i1> %pg,
-                                                              <vscale x 8 x half> %a_z,
-                                                              <vscale x 8 x half> %splat)
+                                                                   <vscale x 8 x half> %a_z,
+                                                                   <vscale x 8 x half> splat(half 1.000000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -401,11 +345,9 @@ define <vscale x 4 x float> @fmaxnm_s_immzero(<vscale x 4 x i1> %pg, <vscale x 4
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmaxnm z0.s, p0/m, z0.s, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmaxnm.nxv4f32(<vscale x 4 x i1> %pg,
-                                                               <vscale x 4 x float> %a,
-                                                               <vscale x 4 x float> %splat)
+                                                                    <vscale x 4 x float> %a,
+                                                                    <vscale x 4 x float> zeroinitializer)
   ret <vscale x 4 x float> %out
 }
 
@@ -415,12 +357,10 @@ define <vscale x 4 x float> @fmaxnm_s_immzero_zero(<vscale x 4 x i1> %pg, <vscal
 ; CHECK-NEXT:    movprfx z0.s, p0/z, z0.s
 ; CHECK-NEXT:    fmaxnm z0.s, p0/m, z0.s, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %a_z = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmaxnm.nxv4f32(<vscale x 4 x i1> %pg,
-                                                               <vscale x 4 x float> %a_z,
-                                                               <vscale x 4 x float> %splat)
+                                                                    <vscale x 4 x float> %a_z,
+                                                                    <vscale x 4 x float> zeroinitializer)
   ret <vscale x 4 x float> %out
 }
 
@@ -429,11 +369,9 @@ define <vscale x 4 x float> @fmaxnm_s_immone(<vscale x 4 x i1> %pg, <vscale x 4
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmaxnm z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmaxnm.nxv4f32(<vscale x 4 x i1> %pg,
-                                                               <vscale x 4 x float> %a,
-                                                               <vscale x 4 x float> %splat)
+                                                                    <vscale x 4 x float> %a,
+                                                                    <vscale x 4 x float> splat(float 1.000000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -443,12 +381,10 @@ define <vscale x 4 x float> @fmaxnm_s_immone_zero(<vscale x 4 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.s, p0/z, z0.s
 ; CHECK-NEXT:    fmaxnm z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %a_z = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmaxnm.nxv4f32(<vscale x 4 x i1> %pg,
-                                                               <vscale x 4 x float> %a_z,
-                                                               <vscale x 4 x float> %splat)
+                                                                    <vscale x 4 x float> %a_z,
+                                                                    <vscale x 4 x float> splat(float 1.000000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -457,11 +393,9 @@ define <vscale x 2 x double> @fmaxnm_d_immzero(<vscale x 2 x i1> %pg, <vscale x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmaxnm z0.d, p0/m, z0.d, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmaxnm.nxv2f64(<vscale x 2 x i1> %pg,
-                                                                <vscale x 2 x double> %a,
-                                                                <vscale x 2 x double> %splat)
+                                                                     <vscale x 2 x double> %a,
+                                                                     <vscale x 2 x double> zeroinitializer)
   ret <vscale x 2 x double> %out
 }
 
@@ -471,12 +405,10 @@ define <vscale x 2 x double> @fmaxnm_d_immzero_zero(<vscale x 2 x i1> %pg, <vsca
 ; CHECK-NEXT:    movprfx z0.d, p0/z, z0.d
 ; CHECK-NEXT:    fmaxnm z0.d, p0/m, z0.d, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %a_z = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmaxnm.nxv2f64(<vscale x 2 x i1> %pg,
-                                                                <vscale x 2 x double> %a_z,
-                                                                <vscale x 2 x double> %splat)
+                                                                     <vscale x 2 x double> %a_z,
+                                                                     <vscale x 2 x double> zeroinitializer)
   ret <vscale x 2 x double> %out
 }
 
@@ -485,11 +417,9 @@ define <vscale x 2 x double> @fmaxnm_d_immone(<vscale x 2 x i1> %pg, <vscale x 2
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmaxnm z0.d, p0/m, z0.d, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmaxnm.nxv2f64(<vscale x 2 x i1> %pg,
-                                                                <vscale x 2 x double> %a,
-                                                                <vscale x 2 x double> %splat)
+                                                                     <vscale x 2 x double> %a,
+                                                                     <vscale x 2 x double> splat(double 1.000000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -499,12 +429,10 @@ define <vscale x 2 x double> @fmaxnm_d_immone_zero(<vscale x 2 x i1> %pg, <vscal
 ; CHECK-NEXT:    movprfx z0.d, p0/z, z0.d
 ; CHECK-NEXT:    fmaxnm z0.d, p0/m, z0.d, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %a_z = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmaxnm.nxv2f64(<vscale x 2 x i1> %pg,
-                                                                <vscale x 2 x double> %a_z,
-                                                                <vscale x 2 x double> %splat)
+                                                                     <vscale x 2 x double> %a_z,
+                                                                     <vscale x 2 x double> splat(double 1.000000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -513,11 +441,9 @@ define <vscale x 8 x half> @fmin_h_immzero(<vscale x 8 x i1> %pg, <vscale x 8 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmin z0.h, p0/m, z0.h, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmin.nxv8f16(<vscale x 8 x i1> %pg,
-                                                            <vscale x 8 x half> %a,
-                                                            <vscale x 8 x half> %splat)
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> zeroinitializer)
   ret <vscale x 8 x half> %out
 }
 
@@ -527,12 +453,10 @@ define <vscale x 8 x half> @fmin_h_immzero_zero(<vscale x 8 x i1> %pg, <vscale x
 ; CHECK-NEXT:    movprfx z0.h, p0/z, z0.h
 ; CHECK-NEXT:    fmin z0.h, p0/m, z0.h, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %a_z = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmin.nxv8f16(<vscale x 8 x i1> %pg,
-                                                            <vscale x 8 x half> %a_z,
-                                                            <vscale x 8 x half> %splat)
+                                                                 <vscale x 8 x half> %a_z,
+                                                                 <vscale x 8 x half> zeroinitializer)
   ret <vscale x 8 x half> %out
 }
 
@@ -541,11 +465,9 @@ define <vscale x 8 x half> @fmin_h_immone(<vscale x 8 x i1> %pg, <vscale x 8 x h
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmin z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmin.nxv8f16(<vscale x 8 x i1> %pg,
-                                                            <vscale x 8 x half> %a,
-                                                            <vscale x 8 x half> %splat)
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> splat(half 1.000000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -555,12 +477,10 @@ define <vscale x 8 x half> @fmin_h_immone_zero(<vscale x 8 x i1> %pg, <vscale x
 ; CHECK-NEXT:    movprfx z0.h, p0/z, z0.h
 ; CHECK-NEXT:    fmin z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %a_z = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmin.nxv8f16(<vscale x 8 x i1> %pg,
-                                                            <vscale x 8 x half> %a_z,
-                                                            <vscale x 8 x half> %splat)
+                                                                 <vscale x 8 x half> %a_z,
+                                                                 <vscale x 8 x half> splat(half 1.000000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -569,11 +489,9 @@ define <vscale x 4 x float> @fmin_s_immzero(<vscale x 4 x i1> %pg, <vscale x 4 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmin z0.s, p0/m, z0.s, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmin.nxv4f32(<vscale x 4 x i1> %pg,
-                                                             <vscale x 4 x float> %a,
-                                                             <vscale x 4 x float> %splat)
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> zeroinitializer)
   ret <vscale x 4 x float> %out
 }
 
@@ -583,12 +501,10 @@ define <vscale x 4 x float> @fmin_s_immzero_zero(<vscale x 4 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.s, p0/z, z0.s
 ; CHECK-NEXT:    fmin z0.s, p0/m, z0.s, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %a_z = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmin.nxv4f32(<vscale x 4 x i1> %pg,
-                                                             <vscale x 4 x float> %a_z,
-                                                             <vscale x 4 x float> %splat)
+                                                                  <vscale x 4 x float> %a_z,
+                                                                  <vscale x 4 x float> zeroinitializer)
   ret <vscale x 4 x float> %out
 }
 
@@ -597,11 +513,9 @@ define <vscale x 4 x float> @fmin_s_immone(<vscale x 4 x i1> %pg, <vscale x 4 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmin z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmin.nxv4f32(<vscale x 4 x i1> %pg,
-                                                             <vscale x 4 x float> %a,
-                                                             <vscale x 4 x float> %splat)
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> splat(float 1.000000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -611,12 +525,10 @@ define <vscale x 4 x float> @fmin_s_immone_zero(<vscale x 4 x i1> %pg, <vscale x
 ; CHECK-NEXT:    movprfx z0.s, p0/z, z0.s
 ; CHECK-NEXT:    fmin z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %a_z = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmin.nxv4f32(<vscale x 4 x i1> %pg,
-                                                             <vscale x 4 x float> %a_z,
-                                                             <vscale x 4 x float> %splat)
+                                                                  <vscale x 4 x float> %a_z,
+                                                                  <vscale x 4 x float> splat(float 1.000000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -625,11 +537,9 @@ define <vscale x 2 x double> @fmin_d_immzero(<vscale x 2 x i1> %pg, <vscale x 2
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmin z0.d, p0/m, z0.d, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmin.nxv2f64(<vscale x 2 x i1> %pg,
-                                                              <vscale x 2 x double> %a,
-                                                              <vscale x 2 x double> %splat)
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> zeroinitializer)
   ret <vscale x 2 x double> %out
 }
 
@@ -639,12 +549,10 @@ define <vscale x 2 x double> @fmin_d_immzero_zero(<vscale x 2 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.d, p0/z, z0.d
 ; CHECK-NEXT:    fmin z0.d, p0/m, z0.d, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %a_z = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmin.nxv2f64(<vscale x 2 x i1> %pg,
-                                                              <vscale x 2 x double> %a_z,
-                                                              <vscale x 2 x double> %splat)
+                                                                   <vscale x 2 x double> %a_z,
+                                                                   <vscale x 2 x double> zeroinitializer)
   ret <vscale x 2 x double> %out
 }
 
@@ -653,11 +561,9 @@ define <vscale x 2 x double> @fmin_d_immone(<vscale x 2 x i1> %pg, <vscale x 2 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmin z0.d, p0/m, z0.d, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmin.nxv2f64(<vscale x 2 x i1> %pg,
-                                                              <vscale x 2 x double> %a,
-                                                              <vscale x 2 x double> %splat)
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> splat(double 1.000000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -667,12 +573,10 @@ define <vscale x 2 x double> @fmin_d_immone_zero(<vscale x 2 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.d, p0/z, z0.d
 ; CHECK-NEXT:    fmin z0.d, p0/m, z0.d, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %a_z = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmin.nxv2f64(<vscale x 2 x i1> %pg,
-                                                              <vscale x 2 x double> %a_z,
-                                                              <vscale x 2 x double> %splat)
+                                                                   <vscale x 2 x double> %a_z,
+                                                                   <vscale x 2 x double> splat(double 1.000000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -681,11 +585,9 @@ define <vscale x 8 x half> @fminnm_h_immzero(<vscale x 8 x i1> %pg, <vscale x 8
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fminnm z0.h, p0/m, z0.h, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fminnm.nxv8f16(<vscale x 8 x i1> %pg,
-                                                              <vscale x 8 x half> %a,
-                                                              <vscale x 8 x half> %splat)
+                                                                   <vscale x 8 x half> %a,
+                                                                   <vscale x 8 x half> zeroinitializer)
   ret <vscale x 8 x half> %out
 }
 
@@ -695,12 +597,10 @@ define <vscale x 8 x half> @fminnm_h_immzero_zero(<vscale x 8 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.h, p0/z, z0.h
 ; CHECK-NEXT:    fminnm z0.h, p0/m, z0.h, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %a_z = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fminnm.nxv8f16(<vscale x 8 x i1> %pg,
-                                                              <vscale x 8 x half> %a_z,
-                                                              <vscale x 8 x half> %splat)
+                                                                   <vscale x 8 x half> %a_z,
+                                                                   <vscale x 8 x half> zeroinitializer)
   ret <vscale x 8 x half> %out
 }
 
@@ -709,11 +609,9 @@ define <vscale x 8 x half> @fminnm_h_immone(<vscale x 8 x i1> %pg, <vscale x 8 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fminnm z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fminnm.nxv8f16(<vscale x 8 x i1> %pg,
-                                                              <vscale x 8 x half> %a,
-                                                              <vscale x 8 x half> %splat)
+                                                                   <vscale x 8 x half> %a,
+                                                                   <vscale x 8 x half> splat(half 1.000000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -723,12 +621,10 @@ define <vscale x 8 x half> @fminnm_h_immone_zero(<vscale x 8 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.h, p0/z, z0.h
 ; CHECK-NEXT:    fminnm z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %a_z = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fminnm.nxv8f16(<vscale x 8 x i1> %pg,
-                                                              <vscale x 8 x half> %a_z,
-                                                              <vscale x 8 x half> %splat)
+                                                                   <vscale x 8 x half> %a_z,
+                                                                   <vscale x 8 x half> splat(half 1.000000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -737,11 +633,9 @@ define <vscale x 4 x float> @fminnm_s_immzero(<vscale x 4 x i1> %pg, <vscale x 4
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fminnm z0.s, p0/m, z0.s, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fminnm.nxv4f32(<vscale x 4 x i1> %pg,
-                                                               <vscale x 4 x float> %a,
-                                                               <vscale x 4 x float> %splat)
+                                                                    <vscale x 4 x float> %a,
+                                                                    <vscale x 4 x float> zeroinitializer)
   ret <vscale x 4 x float> %out
 }
 
@@ -751,12 +645,10 @@ define <vscale x 4 x float> @fminnm_s_immzero_zero(<vscale x 4 x i1> %pg, <vscal
 ; CHECK-NEXT:    movprfx z0.s, p0/z, z0.s
 ; CHECK-NEXT:    fminnm z0.s, p0/m, z0.s, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %a_z = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fminnm.nxv4f32(<vscale x 4 x i1> %pg,
-                                                               <vscale x 4 x float> %a_z,
-                                                               <vscale x 4 x float> %splat)
+                                                                    <vscale x 4 x float> %a_z,
+                                                                    <vscale x 4 x float> zeroinitializer)
   ret <vscale x 4 x float> %out
 }
 
@@ -765,11 +657,9 @@ define <vscale x 4 x float> @fminnm_s_immone(<vscale x 4 x i1> %pg, <vscale x 4
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fminnm z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fminnm.nxv4f32(<vscale x 4 x i1> %pg,
-                                                               <vscale x 4 x float> %a,
-                                                               <vscale x 4 x float> %splat)
+                                                                    <vscale x 4 x float> %a,
+                                                                    <vscale x 4 x float> splat(float 1.000000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -779,12 +669,10 @@ define <vscale x 4 x float> @fminnm_s_immone_zero(<vscale x 4 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.s, p0/z, z0.s
 ; CHECK-NEXT:    fminnm z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %a_z = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fminnm.nxv4f32(<vscale x 4 x i1> %pg,
-                                                               <vscale x 4 x float> %a_z,
-                                                               <vscale x 4 x float> %splat)
+                                                                    <vscale x 4 x float> %a_z,
+                                                                    <vscale x 4 x float> splat(float 1.000000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -793,11 +681,9 @@ define <vscale x 2 x double> @fminnm_d_immzero(<vscale x 2 x i1> %pg, <vscale x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fminnm z0.d, p0/m, z0.d, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fminnm.nxv2f64(<vscale x 2 x i1> %pg,
-                                                                <vscale x 2 x double> %a,
-                                                                <vscale x 2 x double> %splat)
+                                                                     <vscale x 2 x double> %a,
+                                                                     <vscale x 2 x double> zeroinitializer)
   ret <vscale x 2 x double> %out
 }
 
@@ -807,12 +693,10 @@ define <vscale x 2 x double> @fminnm_d_immzero_zero(<vscale x 2 x i1> %pg, <vsca
 ; CHECK-NEXT:    movprfx z0.d, p0/z, z0.d
 ; CHECK-NEXT:    fminnm z0.d, p0/m, z0.d, #0.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %a_z = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fminnm.nxv2f64(<vscale x 2 x i1> %pg,
-                                                                <vscale x 2 x double> %a_z,
-                                                                <vscale x 2 x double> %splat)
+                                                                     <vscale x 2 x double> %a_z,
+                                                                     <vscale x 2 x double> zeroinitializer)
   ret <vscale x 2 x double> %out
 }
 
@@ -821,11 +705,9 @@ define <vscale x 2 x double> @fminnm_d_immone(<vscale x 2 x i1> %pg, <vscale x 2
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fminnm z0.d, p0/m, z0.d, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fminnm.nxv2f64(<vscale x 2 x i1> %pg,
-                                                                <vscale x 2 x double> %a,
-                                                                <vscale x 2 x double> %splat)
+                                                                     <vscale x 2 x double> %a,
+                                                                     <vscale x 2 x double> splat(double 1.000000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -835,12 +717,10 @@ define <vscale x 2 x double> @fminnm_d_immone_zero(<vscale x 2 x i1> %pg, <vscal
 ; CHECK-NEXT:    movprfx z0.d, p0/z, z0.d
 ; CHECK-NEXT:    fminnm z0.d, p0/m, z0.d, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %a_z = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fminnm.nxv2f64(<vscale x 2 x i1> %pg,
-                                                                <vscale x 2 x double> %a_z,
-                                                                <vscale x 2 x double> %splat)
+                                                                     <vscale x 2 x double> %a_z,
+                                                                     <vscale x 2 x double> splat(double 1.000000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -849,11 +729,9 @@ define <vscale x 8 x half> @fmul_h_immhalf(<vscale x 8 x i1> %pg, <vscale x 8 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmul z0.h, p0/m, z0.h, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmul.nxv8f16(<vscale x 8 x i1> %pg,
-                                                            <vscale x 8 x half> %a,
-                                                            <vscale x 8 x half> %splat)
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> splat(half 0.500000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -863,12 +741,10 @@ define <vscale x 8 x half> @fmul_h_immhalf_zero(<vscale x 8 x i1> %pg, <vscale x
 ; CHECK-NEXT:    movprfx z0.h, p0/z, z0.h
 ; CHECK-NEXT:    fmul z0.h, p0/m, z0.h, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %a_z = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmul.nxv8f16(<vscale x 8 x i1> %pg,
-                                                            <vscale x 8 x half> %a_z,
-                                                            <vscale x 8 x half> %splat)
+                                                                 <vscale x 8 x half> %a_z,
+                                                                 <vscale x 8 x half> splat(half 0.500000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -877,11 +753,9 @@ define <vscale x 8 x half> @fmul_h_immtwo(<vscale x 8 x i1> %pg, <vscale x 8 x h
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmul z0.h, p0/m, z0.h, #2.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 2.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmul.nxv8f16(<vscale x 8 x i1> %pg,
-                                                            <vscale x 8 x half> %a,
-                                                            <vscale x 8 x half> %splat)
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> splat(half 2.000000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -891,12 +765,10 @@ define <vscale x 8 x half> @fmul_h_immtwo_zero(<vscale x 8 x i1> %pg, <vscale x
 ; CHECK-NEXT:    movprfx z0.h, p0/z, z0.h
 ; CHECK-NEXT:    fmul z0.h, p0/m, z0.h, #2.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 2.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %a_z = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fmul.nxv8f16(<vscale x 8 x i1> %pg,
-                                                            <vscale x 8 x half> %a_z,
-                                                            <vscale x 8 x half> %splat)
+                                                                 <vscale x 8 x half> %a_z,
+                                                                 <vscale x 8 x half> splat(half 2.000000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -905,11 +777,9 @@ define <vscale x 4 x float> @fmul_s_immhalf(<vscale x 4 x i1> %pg, <vscale x 4 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmul z0.s, p0/m, z0.s, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmul.nxv4f32(<vscale x 4 x i1> %pg,
-                                                             <vscale x 4 x float> %a,
-                                                             <vscale x 4 x float> %splat)
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> splat(float 0.500000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -919,12 +789,10 @@ define <vscale x 4 x float> @fmul_s_immhalf_zero(<vscale x 4 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.s, p0/z, z0.s
 ; CHECK-NEXT:    fmul z0.s, p0/m, z0.s, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %a_z = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmul.nxv4f32(<vscale x 4 x i1> %pg,
-                                                             <vscale x 4 x float> %a_z,
-                                                             <vscale x 4 x float> %splat)
+                                                                  <vscale x 4 x float> %a_z,
+                                                                  <vscale x 4 x float> splat(float 0.500000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -933,11 +801,9 @@ define <vscale x 4 x float> @fmul_s_immtwo(<vscale x 4 x i1> %pg, <vscale x 4 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmul z0.s, p0/m, z0.s, #2.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 2.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmul.nxv4f32(<vscale x 4 x i1> %pg,
-                                                             <vscale x 4 x float> %a,
-                                                             <vscale x 4 x float> %splat)
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> splat(float 2.000000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -947,12 +813,10 @@ define <vscale x 4 x float> @fmul_s_immtwo_zero(<vscale x 4 x i1> %pg, <vscale x
 ; CHECK-NEXT:    movprfx z0.s, p0/z, z0.s
 ; CHECK-NEXT:    fmul z0.s, p0/m, z0.s, #2.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 2.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %a_z = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fmul.nxv4f32(<vscale x 4 x i1> %pg,
-                                                             <vscale x 4 x float> %a_z,
-                                                             <vscale x 4 x float> %splat)
+                                                                  <vscale x 4 x float> %a_z,
+                                                                  <vscale x 4 x float> splat(float 2.000000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -961,11 +825,9 @@ define <vscale x 2 x double> @fmul_d_immhalf(<vscale x 2 x i1> %pg, <vscale x 2
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmul z0.d, p0/m, z0.d, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmul.nxv2f64(<vscale x 2 x i1> %pg,
-                                                              <vscale x 2 x double> %a,
-                                                              <vscale x 2 x double> %splat)
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> splat(double 0.500000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -975,12 +837,10 @@ define <vscale x 2 x double> @fmul_d_immhalf_zero(<vscale x 2 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.d, p0/z, z0.d
 ; CHECK-NEXT:    fmul z0.d, p0/m, z0.d, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %a_z = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmul.nxv2f64(<vscale x 2 x i1> %pg,
-                                                              <vscale x 2 x double> %a_z,
-                                                              <vscale x 2 x double> %splat)
+                                                                   <vscale x 2 x double> %a_z,
+                                                                   <vscale x 2 x double> splat(double 0.500000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -989,11 +849,9 @@ define <vscale x 2 x double> @fmul_d_immtwo(<vscale x 2 x i1> %pg, <vscale x 2 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmul z0.d, p0/m, z0.d, #2.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 2.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmul.nxv2f64(<vscale x 2 x i1> %pg,
-                                                              <vscale x 2 x double> %a,
-                                                              <vscale x 2 x double> %splat)
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> splat(double 2.000000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -1003,12 +861,10 @@ define <vscale x 2 x double> @fmul_d_immtwo_zero(<vscale x 2 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.d, p0/z, z0.d
 ; CHECK-NEXT:    fmul z0.d, p0/m, z0.d, #2.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 2.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %a_z = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fmul.nxv2f64(<vscale x 2 x i1> %pg,
-                                                              <vscale x 2 x double> %a_z,
-                                                              <vscale x 2 x double> %splat)
+                                                                   <vscale x 2 x double> %a_z,
+                                                                   <vscale x 2 x double> splat(double 2.000000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -1017,11 +873,9 @@ define <vscale x 8 x half> @fsub_h_immhalf(<vscale x 8 x i1> %pg, <vscale x 8 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fsub z0.h, p0/m, z0.h, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fsub.nxv8f16(<vscale x 8 x i1> %pg,
-                                                            <vscale x 8 x half> %a,
-                                                            <vscale x 8 x half> %splat)
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> splat(half 0.500000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -1031,12 +885,10 @@ define <vscale x 8 x half> @fsub_h_immhalf_zero(<vscale x 8 x i1> %pg, <vscale x
 ; CHECK-NEXT:    movprfx z0.h, p0/z, z0.h
 ; CHECK-NEXT:    fsub z0.h, p0/m, z0.h, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %a_z = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fsub.nxv8f16(<vscale x 8 x i1> %pg,
-                                                            <vscale x 8 x half> %a_z,
-                                                            <vscale x 8 x half> %splat)
+                                                                 <vscale x 8 x half> %a_z,
+                                                                 <vscale x 8 x half> splat(half 0.500000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -1045,11 +897,9 @@ define <vscale x 8 x half> @fsub_h_immone(<vscale x 8 x i1> %pg, <vscale x 8 x h
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fsub z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fsub.nxv8f16(<vscale x 8 x i1> %pg,
-                                                            <vscale x 8 x half> %a,
-                                                            <vscale x 8 x half> %splat)
+                                                                 <vscale x 8 x half> %a,
+                                                                 <vscale x 8 x half> splat(half 1.000000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -1059,12 +909,10 @@ define <vscale x 8 x half> @fsub_h_immone_zero(<vscale x 8 x i1> %pg, <vscale x
 ; CHECK-NEXT:    movprfx z0.h, p0/z, z0.h
 ; CHECK-NEXT:    fsub z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %a_z = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fsub.nxv8f16(<vscale x 8 x i1> %pg,
-                                                            <vscale x 8 x half> %a_z,
-                                                            <vscale x 8 x half> %splat)
+                                                                 <vscale x 8 x half> %a_z,
+                                                                 <vscale x 8 x half> splat(half 1.000000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -1073,11 +921,9 @@ define <vscale x 4 x float> @fsub_s_immhalf(<vscale x 4 x i1> %pg, <vscale x 4 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fsub z0.s, p0/m, z0.s, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fsub.nxv4f32(<vscale x 4 x i1> %pg,
-                                                             <vscale x 4 x float> %a,
-                                                             <vscale x 4 x float> %splat)
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> splat(float 0.500000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -1087,12 +933,10 @@ define <vscale x 4 x float> @fsub_s_immhalf_zero(<vscale x 4 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.s, p0/z, z0.s
 ; CHECK-NEXT:    fsub z0.s, p0/m, z0.s, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %a_z = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fsub.nxv4f32(<vscale x 4 x i1> %pg,
-                                                             <vscale x 4 x float> %a_z,
-                                                             <vscale x 4 x float> %splat)
+                                                                  <vscale x 4 x float> %a_z,
+                                                                  <vscale x 4 x float> splat(float 0.500000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -1101,11 +945,9 @@ define <vscale x 4 x float> @fsub_s_immone(<vscale x 4 x i1> %pg, <vscale x 4 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fsub z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fsub.nxv4f32(<vscale x 4 x i1> %pg,
-                                                             <vscale x 4 x float> %a,
-                                                             <vscale x 4 x float> %splat)
+                                                                  <vscale x 4 x float> %a,
+                                                                  <vscale x 4 x float> splat(float 1.000000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -1115,12 +957,10 @@ define <vscale x 4 x float> @fsub_s_immone_zero(<vscale x 4 x i1> %pg, <vscale x
 ; CHECK-NEXT:    movprfx z0.s, p0/z, z0.s
 ; CHECK-NEXT:    fsub z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %a_z = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fsub.nxv4f32(<vscale x 4 x i1> %pg,
-                                                             <vscale x 4 x float> %a_z,
-                                                             <vscale x 4 x float> %splat)
+                                                                  <vscale x 4 x float> %a_z,
+                                                                  <vscale x 4 x float> splat(float 1.000000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -1129,11 +969,9 @@ define <vscale x 2 x double> @fsub_d_immhalf(<vscale x 2 x i1> %pg, <vscale x 2
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fsub z0.d, p0/m, z0.d, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fsub.nxv2f64(<vscale x 2 x i1> %pg,
-                                                              <vscale x 2 x double> %a,
-                                                              <vscale x 2 x double> %splat)
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> splat(double 0.500000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -1143,12 +981,10 @@ define <vscale x 2 x double> @fsub_d_immhalf_zero(<vscale x 2 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.d, p0/z, z0.d
 ; CHECK-NEXT:    fsub z0.d, p0/m, z0.d, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %a_z = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fsub.nxv2f64(<vscale x 2 x i1> %pg,
-                                                              <vscale x 2 x double> %a_z,
-                                                              <vscale x 2 x double> %splat)
+                                                                   <vscale x 2 x double> %a_z,
+                                                                   <vscale x 2 x double> splat(double 0.500000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -1157,11 +993,9 @@ define <vscale x 2 x double> @fsub_d_immone(<vscale x 2 x i1> %pg, <vscale x 2 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fsub z0.d, p0/m, z0.d, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fsub.nxv2f64(<vscale x 2 x i1> %pg,
-                                                              <vscale x 2 x double> %a,
-                                                              <vscale x 2 x double> %splat)
+                                                                   <vscale x 2 x double> %a,
+                                                                   <vscale x 2 x double> splat(double 1.000000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -1171,12 +1005,10 @@ define <vscale x 2 x double> @fsub_d_immone_zero(<vscale x 2 x i1> %pg, <vscale
 ; CHECK-NEXT:    movprfx z0.d, p0/z, z0.d
 ; CHECK-NEXT:    fsub z0.d, p0/m, z0.d, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %a_z = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fsub.nxv2f64(<vscale x 2 x i1> %pg,
-                                                              <vscale x 2 x double> %a_z,
-                                                              <vscale x 2 x double> %splat)
+                                                                   <vscale x 2 x double> %a_z,
+                                                                   <vscale x 2 x double> splat(double 1.000000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -1186,12 +1018,10 @@ define <vscale x 8 x half> @fsubr_h_immhalf(<vscale x 8 x i1> %pg, <vscale x 8 x
 ; CHECK-NEXT:    movprfx z0.h, p0/z, z0.h
 ; CHECK-NEXT:    fsubr z0.h, p0/m, z0.h, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %a_z = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fsubr.nxv8f16(<vscale x 8 x i1> %pg,
-                                                             <vscale x 8 x half> %a_z,
-                                                             <vscale x 8 x half> %splat)
+                                                                  <vscale x 8 x half> %a_z,
+                                                                  <vscale x 8 x half> splat(half 0.500000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -1201,12 +1031,10 @@ define <vscale x 8 x half> @fsubr_h_immone(<vscale x 8 x i1> %pg, <vscale x 8 x
 ; CHECK-NEXT:    movprfx z0.h, p0/z, z0.h
 ; CHECK-NEXT:    fsubr z0.h, p0/m, z0.h, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x half> undef, half 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 8 x half> %elt, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
   %a_z = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> zeroinitializer
   %out = call <vscale x 8 x half> @llvm.aarch64.sve.fsubr.nxv8f16(<vscale x 8 x i1> %pg,
-                                                             <vscale x 8 x half> %a_z,
-                                                             <vscale x 8 x half> %splat)
+                                                                  <vscale x 8 x half> %a_z,
+                                                                  <vscale x 8 x half> splat(half 1.000000e+00))
   ret <vscale x 8 x half> %out
 }
 
@@ -1216,12 +1044,10 @@ define <vscale x 4 x float> @fsubr_s_immhalf(<vscale x 4 x i1> %pg, <vscale x 4
 ; CHECK-NEXT:    movprfx z0.s, p0/z, z0.s
 ; CHECK-NEXT:    fsubr z0.s, p0/m, z0.s, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %a_z = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fsubr.nxv4f32(<vscale x 4 x i1> %pg,
-                                                              <vscale x 4 x float> %a_z,
-                                                              <vscale x 4 x float> %splat)
+                                                                   <vscale x 4 x float> %a_z,
+                                                                   <vscale x 4 x float> splat(float 0.500000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -1231,12 +1057,10 @@ define <vscale x 4 x float> @fsubr_s_immone(<vscale x 4 x i1> %pg, <vscale x 4 x
 ; CHECK-NEXT:    movprfx z0.s, p0/z, z0.s
 ; CHECK-NEXT:    fsubr z0.s, p0/m, z0.s, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x float> undef, float 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 4 x float> %elt, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
   %a_z = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> zeroinitializer
   %out = call <vscale x 4 x float> @llvm.aarch64.sve.fsubr.nxv4f32(<vscale x 4 x i1> %pg,
-                                                              <vscale x 4 x float> %a_z,
-                                                              <vscale x 4 x float> %splat)
+                                                                   <vscale x 4 x float> %a_z,
+                                                                   <vscale x 4 x float> splat(float 1.000000e+00))
   ret <vscale x 4 x float> %out
 }
 
@@ -1246,12 +1070,10 @@ define <vscale x 2 x double> @fsubr_d_immhalf(<vscale x 2 x i1> %pg, <vscale x 2
 ; CHECK-NEXT:    movprfx z0.d, p0/z, z0.d
 ; CHECK-NEXT:    fsubr z0.d, p0/m, z0.d, #0.5
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 0.500000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %a_z = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fsubr.nxv2f64(<vscale x 2 x i1> %pg,
-                                                               <vscale x 2 x double> %a_z,
-                                                               <vscale x 2 x double> %splat)
+                                                                    <vscale x 2 x double> %a_z,
+                                                                    <vscale x 2 x double> splat(double 0.500000e+00))
   ret <vscale x 2 x double> %out
 }
 
@@ -1261,18 +1083,13 @@ define <vscale x 2 x double> @fsubr_d_immone(<vscale x 2 x i1> %pg, <vscale x 2
 ; CHECK-NEXT:    movprfx z0.d, p0/z, z0.d
 ; CHECK-NEXT:    fsubr z0.d, p0/m, z0.d, #1.0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x double> undef, double 1.000000e+00, i32 0
-  %splat = shufflevector <vscale x 2 x double> %elt, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
   %a_z = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> zeroinitializer
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fsubr.nxv2f64(<vscale x 2 x i1> %pg,
-                                                               <vscale x 2 x double> %a_z,
-                                                               <vscale x 2 x double> %splat)
+                                                                    <vscale x 2 x double> %a_z,
+                                                                    <vscale x 2 x double> splat(double 1.000000e+00))
   ret <vscale x 2 x double> %out
 }
 
-
-;; Arithmetic intrinsic declarations
-
 declare <vscale x 8 x half> @llvm.aarch64.sve.fadd.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
 declare <vscale x 4 x float> @llvm.aarch64.sve.fadd.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.fadd.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-index.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-index.ll
index bc94c087ef5fe..4d4b1b67bbafc 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-index.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-index.ll
@@ -58,11 +58,9 @@ define <vscale x 8 x i16> @index_ii_range_combine(i16 %a) {
 ; CHECK-NEXT:    index z0.h, #0, #8
 ; CHECK-NEXT:    orr z0.h, z0.h, #0x2
 ; CHECK-NEXT:    ret
-  %val = insertelement <vscale x 8 x i16> poison, i16 2, i32 0
-  %val1 = shufflevector <vscale x 8 x i16> %val, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
   %val2 = call <vscale x 8 x i16> @llvm.aarch64.sve.index.nxv8i16(i16 0, i16 2)
-  %val3 = shl <vscale x 8 x i16> %val2, %val1
-  %out = add <vscale x 8 x i16> %val3, %val1
+  %val3 = shl <vscale x 8 x i16> %val2, splat(i16 2)
+  %out = add <vscale x 8 x i16> %val3, splat(i16 2) 
   ret <vscale x 8 x i16> %out
 }
 
@@ -121,10 +119,8 @@ define <vscale x 4 x i32> @index_ir_range_combine(i32 %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.s, #0, w0
 ; CHECK-NEXT:    ret
-  %val = insertelement <vscale x 4 x i32> poison, i32 2, i32 0
-  %val1 = shufflevector <vscale x 4 x i32> %val, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
   %tmp = call <vscale x 4 x i32> @llvm.aarch64.sve.index.nxv4i32(i32 2, i32 1)
-  %tmp1 = sub <vscale x 4 x i32> %tmp, %val1
+  %tmp1 = sub <vscale x 4 x i32> %tmp, splat(i32 2) 
   %val2 = insertelement <vscale x 4 x i32> poison, i32 %a, i32 0
   %val3 = shufflevector <vscale x 4 x i32> %val2, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
   %out = mul <vscale x 4 x i32> %tmp1, %val3
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm-zero.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm-zero.ll
index 6593978b03d41..b22ddcb0c3010 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm-zero.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm-zero.ll
@@ -9,9 +9,7 @@ define <vscale x 16 x i8> @asr_i8(<vscale x 16 x i8> %a, <vscale x 16 x i1> %pg)
 ; CHECK-NEXT:    asr z0.b, p0/m, z0.b, #8
 ; CHECK-NEXT:    ret
   %vsel = select <vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> zeroinitializer
-  %ele = insertelement <vscale x 16 x i8> poison, i8 8, i32 0
-  %shuffle = shufflevector <vscale x 16 x i8> %ele, <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
-  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.asr.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %vsel, <vscale x 16 x i8> %shuffle)
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.asr.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %vsel, <vscale x 16 x i8> splat(i8 8))
   ret <vscale x 16 x i8> %res
 }
 
@@ -22,9 +20,7 @@ define <vscale x 8 x i16> @asr_i16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg)
 ; CHECK-NEXT:    asr z0.h, p0/m, z0.h, #16
 ; CHECK-NEXT:    ret
   %vsel = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> zeroinitializer
-  %ele = insertelement <vscale x 8 x i16> poison, i16 16, i32 0
-  %shuffle = shufflevector <vscale x 8 x i16> %ele, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
-  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.asr.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %vsel, <vscale x 8 x i16> %shuffle)
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.asr.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %vsel, <vscale x 8 x i16> splat(i16 16))
   ret <vscale x 8 x i16> %res
 }
 
@@ -35,9 +31,7 @@ define <vscale x 4 x i32> @asr_i32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg)
 ; CHECK-NEXT:    asr z0.s, p0/m, z0.s, #32
 ; CHECK-NEXT:    ret
   %vsel = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> zeroinitializer
-  %ele = insertelement <vscale x 4 x i32> poison, i32 32, i32 0
-  %shuffle = shufflevector <vscale x 4 x i32> %ele, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.asr.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %vsel, <vscale x 4 x i32> %shuffle)
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.asr.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %vsel, <vscale x 4 x i32> splat(i32 32))
   ret <vscale x 4 x i32> %res
 }
 
@@ -48,9 +42,7 @@ define <vscale x 2 x i64> @asr_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg)
 ; CHECK-NEXT:    asr z0.d, p0/m, z0.d, #64
 ; CHECK-NEXT:    ret
   %vsel = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> zeroinitializer
-  %ele = insertelement <vscale x 2 x i64> poison, i64 64, i32 0
-  %shuffle = shufflevector <vscale x 2 x i64> %ele, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.asr.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %vsel, <vscale x 2 x i64> %shuffle)
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.asr.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %vsel, <vscale x 2 x i64> splat(i64 64))
   ret <vscale x 2 x i64> %res
 }
 
@@ -62,9 +54,7 @@ define <vscale x 16 x i8> @lsl_i8(<vscale x 16 x i8> %a, <vscale x 16 x i1> %pg)
 ; CHECK-NEXT:    lsl z0.b, p0/m, z0.b, #7
 ; CHECK-NEXT:    ret
   %vsel = select <vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> zeroinitializer
-  %ele = insertelement <vscale x 16 x i8> poison, i8 7, i32 0
-  %shuffle = shufflevector <vscale x 16 x i8> %ele, <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
-  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.lsl.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %vsel, <vscale x 16 x i8> %shuffle)
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.lsl.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %vsel, <vscale x 16 x i8> splat(i8 7))
   ret <vscale x 16 x i8> %res
 }
 
@@ -75,9 +65,7 @@ define <vscale x 8 x i16> @lsl_i16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg)
 ; CHECK-NEXT:    lsl z0.h, p0/m, z0.h, #15
 ; CHECK-NEXT:    ret
   %vsel = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> zeroinitializer
-  %ele = insertelement <vscale x 8 x i16> poison, i16 15, i32 0
-  %shuffle = shufflevector <vscale x 8 x i16> %ele, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
-  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.lsl.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %vsel, <vscale x 8 x i16> %shuffle)
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.lsl.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %vsel, <vscale x 8 x i16> splat(i16 15))
   ret <vscale x 8 x i16> %res
 }
 
@@ -88,9 +76,7 @@ define <vscale x 4 x i32> @lsl_i32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg)
 ; CHECK-NEXT:    lsl z0.s, p0/m, z0.s, #31
 ; CHECK-NEXT:    ret
   %vsel = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> zeroinitializer
-  %ele = insertelement <vscale x 4 x i32> poison, i32 31, i32 0
-  %shuffle = shufflevector <vscale x 4 x i32> %ele, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.lsl.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %vsel, <vscale x 4 x i32> %shuffle)
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.lsl.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %vsel, <vscale x 4 x i32> splat(i32 31))
   ret <vscale x 4 x i32> %res
 }
 
@@ -101,9 +87,7 @@ define <vscale x 2 x i64> @lsl_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg)
 ; CHECK-NEXT:    lsl z0.d, p0/m, z0.d, #63
 ; CHECK-NEXT:    ret
   %vsel = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> zeroinitializer
-  %ele = insertelement <vscale x 2 x i64> poison, i64 63, i32 0
-  %shuffle = shufflevector <vscale x 2 x i64> %ele, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.lsl.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %vsel, <vscale x 2 x i64> %shuffle)
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.lsl.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %vsel, <vscale x 2 x i64> splat(i64 63))
   ret <vscale x 2 x i64> %res
 }
 
@@ -115,9 +99,7 @@ define <vscale x 16 x i8> @lsr_i8(<vscale x 16 x i8> %a, <vscale x 16 x i1> %pg)
 ; CHECK-NEXT:    lsr z0.b, p0/m, z0.b, #8
 ; CHECK-NEXT:    ret
   %vsel = select <vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> zeroinitializer
-  %ele = insertelement <vscale x 16 x i8> poison, i8 8, i32 0
-  %shuffle = shufflevector <vscale x 16 x i8> %ele, <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
-  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.lsr.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %vsel, <vscale x 16 x i8> %shuffle)
+  %res = call <vscale x 16 x i8> @llvm.aarch64.sve.lsr.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %vsel, <vscale x 16 x i8> splat(i8 8))
   ret <vscale x 16 x i8> %res
 }
 
@@ -128,9 +110,7 @@ define <vscale x 8 x i16> @lsr_i16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg)
 ; CHECK-NEXT:    lsr z0.h, p0/m, z0.h, #16
 ; CHECK-NEXT:    ret
   %vsel = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> zeroinitializer
-  %ele = insertelement <vscale x 8 x i16> poison, i16 16, i32 0
-  %shuffle = shufflevector <vscale x 8 x i16> %ele, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
-  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.lsr.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %vsel, <vscale x 8 x i16> %shuffle)
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.lsr.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %vsel, <vscale x 8 x i16> splat(i16 16))
   ret <vscale x 8 x i16> %res
 }
 
@@ -141,9 +121,7 @@ define <vscale x 4 x i32> @lsr_i32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg)
 ; CHECK-NEXT:    lsr z0.s, p0/m, z0.s, #32
 ; CHECK-NEXT:    ret
   %vsel = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> zeroinitializer
-  %ele = insertelement <vscale x 4 x i32> poison, i32 32, i32 0
-  %shuffle = shufflevector <vscale x 4 x i32> %ele, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.lsr.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %vsel, <vscale x 4 x i32> %shuffle)
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.lsr.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %vsel, <vscale x 4 x i32> splat(i32 32))
   ret <vscale x 4 x i32> %res
 }
 
@@ -154,9 +132,7 @@ define <vscale x 2 x i64> @lsr_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg)
 ; CHECK-NEXT:    lsr z0.d, p0/m, z0.d, #64
 ; CHECK-NEXT:    ret
   %vsel = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> zeroinitializer
-  %ele = insertelement <vscale x 2 x i64> poison, i64 64, i32 0
-  %shuffle = shufflevector <vscale x 2 x i64> %ele, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.lsr.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %vsel, <vscale x 2 x i64> %shuffle)
+  %res = call <vscale x 2 x i64> @llvm.aarch64.sve.lsr.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %vsel, <vscale x 2 x i64> splat(i64 64))
   ret <vscale x 2 x i64> %res
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm.ll
index 5648e8244e6ec..73a2292b183ba 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm.ll
@@ -9,11 +9,9 @@ define <vscale x 16 x i8> @add_i8(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    add z0.b, z0.b, #127 // =0x7f
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-  %elt = insertelement <vscale x 16 x i8> undef, i8 127, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.add.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %splat)
+                                                                 <vscale x 16 x i8> splat(i8 127))
   ret <vscale x 16 x i8> %out
 }
 
@@ -23,11 +21,9 @@ define <vscale x 8 x i16> @add_i16(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    add z0.h, z0.h, #127 // =0x7f
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 127, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.add.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %splat)
+                                                                 <vscale x 8 x i16> splat(i16 127))
   ret <vscale x 8 x i16> %out
 }
 
@@ -38,11 +34,9 @@ define <vscale x 8 x i16> @add_i16_out_of_range(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    add z0.h, z0.h, z1.h
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 257, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.add.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %splat)
+                                                                 <vscale x 8 x i16> splat(i16 257))
   ret <vscale x 8 x i16> %out
 }
 
@@ -52,11 +46,9 @@ define <vscale x 4 x i32> @add_i32(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    add z0.s, z0.s, #127 // =0x7f
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 127, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.add.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %splat)
+                                                                 <vscale x 4 x i32> splat(i32 127))
   ret <vscale x 4 x i32> %out
 }
 
@@ -68,11 +60,9 @@ define <vscale x 4 x i32> @add_i32_out_of_range(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    add z0.s, z0.s, z1.s
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 257, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.add.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %splat)
+                                                                 <vscale x 4 x i32> splat(i32 257))
   ret <vscale x 4 x i32> %out
 }
 
@@ -82,11 +72,9 @@ define <vscale x 2 x i64> @add_i64(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    add z0.d, z0.d, #127 // =0x7f
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 127, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.add.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %splat)
+                                                                 <vscale x 2 x i64> splat(i64 127))
   ret <vscale x 2 x i64> %out
 }
 
@@ -98,11 +86,9 @@ define <vscale x 2 x i64> @add_i64_out_of_range(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    add z0.d, z0.d, z1.d
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 257, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.add.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %splat)
+                                                                 <vscale x 2 x i64> splat(i64 257))
   ret <vscale x 2 x i64> %out
 }
 
@@ -114,11 +100,9 @@ define <vscale x 16 x i8> @sub_i8(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    sub z0.b, z0.b, #127 // =0x7f
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-  %elt = insertelement <vscale x 16 x i8> undef, i8 127, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sub.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %splat)
+                                                                 <vscale x 16 x i8> splat(i8 127))
   ret <vscale x 16 x i8> %out
 }
 
@@ -128,11 +112,9 @@ define <vscale x 8 x i16> @sub_i16(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    sub z0.h, z0.h, #127 // =0x7f
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 127, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sub.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %splat)
+                                                                 <vscale x 8 x i16> splat(i16 127))
   ret <vscale x 8 x i16> %out
 }
 
@@ -143,11 +125,9 @@ define <vscale x 8 x i16> @sub_i16_out_of_range(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    sub z0.h, z0.h, z1.h
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 257, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sub.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %splat)
+                                                                 <vscale x 8 x i16> splat(i16 257))
   ret <vscale x 8 x i16> %out
 }
 
@@ -157,11 +137,9 @@ define <vscale x 4 x i32> @sub_i32(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    sub z0.s, z0.s, #127 // =0x7f
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 127, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sub.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %splat)
+                                                                 <vscale x 4 x i32> splat(i32 127))
   ret <vscale x 4 x i32> %out
 }
 
@@ -173,11 +151,9 @@ define <vscale x 4 x i32> @sub_i32_out_of_range(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    sub z0.s, z0.s, z1.s
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 257, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sub.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %splat)
+                                                                 <vscale x 4 x i32> splat(i32 257))
   ret <vscale x 4 x i32> %out
 }
 
@@ -187,11 +163,9 @@ define <vscale x 2 x i64> @sub_i64(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    sub z0.d, z0.d, #127 // =0x7f
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 127, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sub.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %splat)
+                                                                 <vscale x 2 x i64> splat(i64 127))
   ret <vscale x 2 x i64> %out
 }
 
@@ -203,11 +177,9 @@ define <vscale x 2 x i64> @sub_i64_out_of_range(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    sub z0.d, z0.d, z1.d
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 257, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sub.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %splat)
+                                                                 <vscale x 2 x i64> splat(i64 257))
   ret <vscale x 2 x i64> %out
 }
 
@@ -269,11 +241,9 @@ define <vscale x 16 x i8> @subr_i8(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    subr z0.b, z0.b, #127 // =0x7f
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-  %elt = insertelement <vscale x 16 x i8> undef, i8 127, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.subr.nxv16i8(<vscale x 16 x i1> %pg,
                                                                 <vscale x 16 x i8> %a,
-                                                                <vscale x 16 x i8> %splat)
+                                                                <vscale x 16 x i8> splat(i8 127))
   ret <vscale x 16 x i8> %out
 }
 
@@ -283,11 +253,9 @@ define <vscale x 8 x i16> @subr_i16(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    subr z0.h, z0.h, #127 // =0x7f
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 127, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.subr.nxv8i16(<vscale x 8 x i1> %pg,
                                                                 <vscale x 8 x i16> %a,
-                                                                <vscale x 8 x i16> %splat)
+                                                                <vscale x 8 x i16> splat(i16 127))
   ret <vscale x 8 x i16> %out
 }
 
@@ -298,11 +266,9 @@ define <vscale x 8 x i16> @subr_i16_out_of_range(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    sub z0.h, z1.h, z0.h
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 257, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.subr.nxv8i16(<vscale x 8 x i1> %pg,
                                                                 <vscale x 8 x i16> %a,
-                                                                <vscale x 8 x i16> %splat)
+                                                                <vscale x 8 x i16> splat(i16 257))
   ret <vscale x 8 x i16> %out
 }
 
@@ -312,11 +278,9 @@ define <vscale x 4 x i32> @subr_i32(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    subr z0.s, z0.s, #127 // =0x7f
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 127, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.subr.nxv4i32(<vscale x 4 x i1> %pg,
                                                                 <vscale x 4 x i32> %a,
-                                                                <vscale x 4 x i32> %splat)
+                                                                <vscale x 4 x i32> splat(i32 127))
   ret <vscale x 4 x i32> %out
 }
 
@@ -328,11 +292,9 @@ define <vscale x 4 x i32> @subr_i32_out_of_range(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    sub z0.s, z1.s, z0.s
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 257, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.subr.nxv4i32(<vscale x 4 x i1> %pg,
                                                                 <vscale x 4 x i32> %a,
-                                                                <vscale x 4 x i32> %splat)
+                                                                <vscale x 4 x i32> splat(i32 257))
   ret <vscale x 4 x i32> %out
 }
 
@@ -342,11 +304,9 @@ define <vscale x 2 x i64> @subr_i64(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    subr z0.d, z0.d, #127 // =0x7f
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 127, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.subr.nxv2i64(<vscale x 2 x i1> %pg,
                                                                 <vscale x 2 x i64> %a,
-                                                                <vscale x 2 x i64> %splat)
+                                                                <vscale x 2 x i64> splat(i64 127))
   ret <vscale x 2 x i64> %out
 }
 
@@ -358,11 +318,9 @@ define <vscale x 2 x i64> @subr_i64_out_of_range(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    sub z0.d, z1.d, z0.d
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 257, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.subr.nxv2i64(<vscale x 2 x i1> %pg,
                                                                 <vscale x 2 x i64> %a,
-                                                                <vscale x 2 x i64> %splat)
+                                                                <vscale x 2 x i64> splat(i64 257))
   ret <vscale x 2 x i64> %out
 }
 
@@ -424,11 +382,9 @@ define <vscale x 16 x i8> @smax_i8(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    smax z0.b, z0.b, #-128
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-  %elt = insertelement <vscale x 16 x i8> undef, i8 -128, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.smax.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                   <vscale x 16 x i8> %a,
-                                                                  <vscale x 16 x i8> %splat)
+                                                                  <vscale x 16 x i8> splat(i8 -128))
   ret <vscale x 16 x i8> %out
 }
 
@@ -438,11 +394,9 @@ define <vscale x 8 x i16> @smax_i16(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    smax z0.h, z0.h, #127
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 127, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.smax.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                   <vscale x 8 x i16> %a,
-                                                                  <vscale x 8 x i16> %splat)
+                                                                  <vscale x 8 x i16> splat(i16 127))
   ret <vscale x 8 x i16> %out
 }
 
@@ -455,11 +409,9 @@ define <vscale x 8 x i16> @smax_i16_out_of_range(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    smax z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 129, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.smax.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                   <vscale x 8 x i16> %a,
-                                                                  <vscale x 8 x i16> %splat)
+                                                                  <vscale x 8 x i16> splat(i16 129))
   ret <vscale x 8 x i16> %out
 }
 
@@ -469,11 +421,9 @@ define <vscale x 4 x i32> @smax_i32(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    smax z0.s, z0.s, #-128
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 -128, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.smax.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                   <vscale x 4 x i32> %a,
-                                                                  <vscale x 4 x i32> %splat)
+                                                                  <vscale x 4 x i32> splat(i32 -128))
   ret <vscale x 4 x i32> %out
 }
 
@@ -485,11 +435,9 @@ define <vscale x 4 x i32> @smax_i32_out_of_range(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    smax z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 -129, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.smax.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                   <vscale x 4 x i32> %a,
-                                                                  <vscale x 4 x i32> %splat)
+                                                                  <vscale x 4 x i32> splat(i32 -129))
   ret <vscale x 4 x i32> %out
 }
 
@@ -499,11 +447,9 @@ define <vscale x 2 x i64> @smax_i64(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    smax z0.d, z0.d, #127
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 127, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.smax.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                   <vscale x 2 x i64> %a,
-                                                                  <vscale x 2 x i64> %splat)
+                                                                  <vscale x 2 x i64> splat(i64 127))
   ret <vscale x 2 x i64> %out
 }
 
@@ -515,11 +461,9 @@ define <vscale x 2 x i64> @smax_i64_out_of_range(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    smax z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 65535, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.smax.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                   <vscale x 2 x i64> %a,
-                                                                  <vscale x 2 x i64> %splat)
+                                                                  <vscale x 2 x i64> splat(i64 65535))
   ret <vscale x 2 x i64> %out
 }
 
@@ -581,11 +525,9 @@ define <vscale x 16 x i8> @smin_i8(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    smin z0.b, z0.b, #127
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-  %elt = insertelement <vscale x 16 x i8> undef, i8 127, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.smin.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                   <vscale x 16 x i8> %a,
-                                                                  <vscale x 16 x i8> %splat)
+                                                                  <vscale x 16 x i8> splat(i8 127))
   ret <vscale x 16 x i8> %out
 }
 
@@ -595,11 +537,9 @@ define <vscale x 8 x i16> @smin_i16(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    smin z0.h, z0.h, #-128
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 -128, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.smin.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                   <vscale x 8 x i16> %a,
-                                                                  <vscale x 8 x i16> %splat)
+                                                                  <vscale x 8 x i16> splat(i16 -128))
   ret <vscale x 8 x i16> %out
 }
 
@@ -611,11 +551,9 @@ define <vscale x 8 x i16> @smin_i16_out_of_range(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    smin z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 -129, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.smin.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                   <vscale x 8 x i16> %a,
-                                                                  <vscale x 8 x i16> %splat)
+                                                                  <vscale x 8 x i16> splat(i16 -129))
   ret <vscale x 8 x i16> %out
 }
 
@@ -625,11 +563,9 @@ define <vscale x 4 x i32> @smin_i32(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    smin z0.s, z0.s, #127
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 127, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.smin.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                   <vscale x 4 x i32> %a,
-                                                                  <vscale x 4 x i32> %splat)
+                                                                  <vscale x 4 x i32> splat(i32 127))
   ret <vscale x 4 x i32> %out
 }
 
@@ -642,11 +578,9 @@ define <vscale x 4 x i32> @smin_i32_out_of_range(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    smin z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 257, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.smin.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                   <vscale x 4 x i32> %a,
-                                                                  <vscale x 4 x i32> %splat)
+                                                                  <vscale x 4 x i32> splat(i32 257))
   ret <vscale x 4 x i32> %out
 }
 
@@ -657,11 +591,9 @@ define <vscale x 2 x i64> @smin_i64(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    smin z0.d, z0.d, #-128
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 -128, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.smin.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                   <vscale x 2 x i64> %a,
-                                                                  <vscale x 2 x i64> %splat)
+                                                                  <vscale x 2 x i64> splat(i64 -128))
   ret <vscale x 2 x i64> %out
 }
 
@@ -673,11 +605,9 @@ define <vscale x 2 x i64> @smin_i64_out_of_range(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    smin z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 -256, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.smin.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                   <vscale x 2 x i64> %a,
-                                                                  <vscale x 2 x i64> %splat)
+                                                                  <vscale x 2 x i64> splat(i64 -256))
   ret <vscale x 2 x i64> %out
 }
 
@@ -739,11 +669,9 @@ define <vscale x 16 x i8> @umax_i8(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    umax z0.b, z0.b, #0
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-  %elt = insertelement <vscale x 16 x i8> undef, i8 0, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.umax.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                   <vscale x 16 x i8> %a,
-                                                                  <vscale x 16 x i8> %splat)
+                                                                  <vscale x 16 x i8> zeroinitializer)
   ret <vscale x 16 x i8> %out
 }
 
@@ -753,11 +681,9 @@ define <vscale x 8 x i16> @umax_i16(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    umax z0.h, z0.h, #255
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 255, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.umax.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                   <vscale x 8 x i16> %a,
-                                                                  <vscale x 8 x i16> %splat)
+                                                                  <vscale x 8 x i16> splat(i16 255))
   ret <vscale x 8 x i16> %out
 }
 
@@ -769,11 +695,9 @@ define <vscale x 8 x i16> @umax_i16_out_of_range(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    umax z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 257, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.umax.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                   <vscale x 8 x i16> %a,
-                                                                  <vscale x 8 x i16> %splat)
+                                                                  <vscale x 8 x i16> splat(i16 257))
   ret <vscale x 8 x i16> %out
 }
 
@@ -783,11 +707,9 @@ define <vscale x 4 x i32> @umax_i32(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    umax z0.s, z0.s, #0
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 0, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.umax.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                   <vscale x 4 x i32> %a,
-                                                                  <vscale x 4 x i32> %splat)
+                                                                  <vscale x 4 x i32> zeroinitializer)
   ret <vscale x 4 x i32> %out
 }
 
@@ -800,11 +722,9 @@ define <vscale x 4 x i32> @umax_i32_out_of_range(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    umax z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 257, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.umax.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                   <vscale x 4 x i32> %a,
-                                                                  <vscale x 4 x i32> %splat)
+                                                                  <vscale x 4 x i32> splat(i32 257))
   ret <vscale x 4 x i32> %out
 }
 
@@ -814,11 +734,9 @@ define <vscale x 2 x i64> @umax_i64(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    umax z0.d, z0.d, #255
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 255, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.umax.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                   <vscale x 2 x i64> %a,
-                                                                  <vscale x 2 x i64> %splat)
+                                                                  <vscale x 2 x i64> splat(i64 255))
   ret <vscale x 2 x i64> %out
 }
 
@@ -830,11 +748,9 @@ define <vscale x 2 x i64> @umax_i64_out_of_range(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    umax z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 65535, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.umax.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                   <vscale x 2 x i64> %a,
-                                                                  <vscale x 2 x i64> %splat)
+                                                                  <vscale x 2 x i64> splat(i64 65535))
   ret <vscale x 2 x i64> %out
 }
 
@@ -896,11 +812,9 @@ define <vscale x 16 x i8> @umin_i8(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    umin z0.b, z0.b, #255
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-  %elt = insertelement <vscale x 16 x i8> undef, i8 255, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.umin.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                   <vscale x 16 x i8> %a,
-                                                                  <vscale x 16 x i8> %splat)
+                                                                  <vscale x 16 x i8> splat(i8 255))
   ret <vscale x 16 x i8> %out
 }
 
@@ -910,11 +824,9 @@ define <vscale x 8 x i16> @umin_i16(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    umin z0.h, z0.h, #0
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.umin.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                   <vscale x 8 x i16> %a,
-                                                                  <vscale x 8 x i16> %splat)
+                                                                  <vscale x 8 x i16> zeroinitializer)
   ret <vscale x 8 x i16> %out
 }
 
@@ -926,11 +838,9 @@ define <vscale x 8 x i16> @umin_i16_out_of_range(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    umin z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 257, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.umin.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                   <vscale x 8 x i16> %a,
-                                                                  <vscale x 8 x i16> %splat)
+                                                                  <vscale x 8 x i16> splat(i16 257))
   ret <vscale x 8 x i16> %out
 }
 
@@ -940,11 +850,9 @@ define <vscale x 4 x i32> @umin_i32(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    umin z0.s, z0.s, #255
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 255, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.umin.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                   <vscale x 4 x i32> %a,
-                                                                  <vscale x 4 x i32> %splat)
+                                                                  <vscale x 4 x i32> splat(i32 255))
   ret <vscale x 4 x i32> %out
 }
 
@@ -957,11 +865,9 @@ define <vscale x 4 x i32> @umin_i32_out_of_range(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    umin z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 257, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.umin.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                   <vscale x 4 x i32> %a,
-                                                                  <vscale x 4 x i32> %splat)
+                                                                  <vscale x 4 x i32> splat(i32 257))
   ret <vscale x 4 x i32> %out
 }
 
@@ -971,11 +877,9 @@ define <vscale x 2 x i64> @umin_i64(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    umin z0.d, z0.d, #0
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 0, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.umin.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                   <vscale x 2 x i64> %a,
-                                                                  <vscale x 2 x i64> %splat)
+                                                                  <vscale x 2 x i64> zeroinitializer)
   ret <vscale x 2 x i64> %out
 }
 
@@ -987,11 +891,9 @@ define <vscale x 2 x i64> @umin_i64_out_of_range(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    umin z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 65535, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.umin.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                   <vscale x 2 x i64> %a,
-                                                                  <vscale x 2 x i64> %splat)
+                                                                  <vscale x 2 x i64> splat(i64 65535))
   ret <vscale x 2 x i64> %out
 }
 
@@ -1052,10 +954,8 @@ define <vscale x 16 x i8> @sqadd_b_lowimm(<vscale x 16 x i8> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqadd z0.b, z0.b, #27 // =0x1b
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 27, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sqadd.x.nxv16i8(<vscale x 16 x i8> %a,
-                                                                   <vscale x 16 x i8> %splat)
+                                                                   <vscale x 16 x i8> splat(i8 27))
   ret <vscale x 16 x i8> %out
 }
 
@@ -1065,10 +965,8 @@ define <vscale x 16 x i8> @sqadd_b_negimm(<vscale x 16 x i8> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqsub z0.b, z0.b, #128 // =0x80
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 -128, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sqadd.x.nxv16i8(<vscale x 16 x i8> %a,
-                                                                   <vscale x 16 x i8> %splat)
+                                                                   <vscale x 16 x i8> splat(i8 -128))
   ret <vscale x 16 x i8> %out
 }
 
@@ -1077,10 +975,8 @@ define <vscale x 8 x i16> @sqadd_h_lowimm(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqadd z0.h, z0.h, #43 // =0x2b
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 43, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqadd.x.nxv8i16(<vscale x 8 x i16> %a,
-                                                                   <vscale x 8 x i16> %splat)
+                                                                   <vscale x 8 x i16> splat(i16 43))
   ret <vscale x 8 x i16> %out
 }
 
@@ -1089,10 +985,8 @@ define <vscale x 8 x i16> @sqadd_h_highimm(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqadd z0.h, z0.h, #2048 // =0x800
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 2048, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqadd.x.nxv8i16(<vscale x 8 x i16> %a,
-                                                                   <vscale x 8 x i16> %splat)
+                                                                   <vscale x 8 x i16> splat(i16 2048))
   ret <vscale x 8 x i16> %out
 }
 
@@ -1102,10 +996,8 @@ define <vscale x 8 x i16> @sqadd_h_negimm(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqsub z0.h, z0.h, #1 // =0x1
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 -1, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqadd.x.nxv8i16(<vscale x 8 x i16> %a,
-                                                                   <vscale x 8 x i16> %splat)
+                                                                   <vscale x 8 x i16> splat(i16 -1))
   ret <vscale x 8 x i16> %out
 }
 
@@ -1114,10 +1006,8 @@ define <vscale x 4 x i32> @sqadd_s_lowimm(<vscale x 4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqadd z0.s, z0.s, #1 // =0x1
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 1, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqadd.x.nxv4i32(<vscale x 4 x i32> %a,
-                                                                   <vscale x 4 x i32> %splat)
+                                                                   <vscale x 4 x i32> splat(i32 1))
   ret <vscale x 4 x i32> %out
 }
 
@@ -1126,10 +1016,8 @@ define <vscale x 4 x i32> @sqadd_s_highimm(<vscale x 4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqadd z0.s, z0.s, #8192 // =0x2000
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 8192, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqadd.x.nxv4i32(<vscale x 4 x i32> %a,
-                                                                   <vscale x 4 x i32> %splat)
+                                                                   <vscale x 4 x i32> splat(i32 8192))
   ret <vscale x 4 x i32> %out
 }
 
@@ -1139,10 +1027,8 @@ define <vscale x 4 x i32> @sqadd_s_negimm(<vscale x 4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqsub z0.s, z0.s, #65280 // =0xff00
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 -65280, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqadd.x.nxv4i32(<vscale x 4 x i32> %a,
-                                                                   <vscale x 4 x i32> %splat)
+                                                                   <vscale x 4 x i32> splat(i32 -65280))
   ret <vscale x 4 x i32> %out
 }
 
@@ -1151,10 +1037,8 @@ define <vscale x 2 x i64> @sqadd_d_lowimm(<vscale x 2 x i64> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqadd z0.d, z0.d, #255 // =0xff
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 255, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sqadd.x.nxv2i64(<vscale x 2 x i64> %a,
-                                                                   <vscale x 2 x i64> %splat)
+                                                                   <vscale x 2 x i64> splat(i64 255))
   ret <vscale x 2 x i64> %out
 }
 
@@ -1163,10 +1047,8 @@ define <vscale x 2 x i64> @sqadd_d_highimm(<vscale x 2 x i64> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqadd z0.d, z0.d, #65280 // =0xff00
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 65280, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sqadd.x.nxv2i64(<vscale x 2 x i64> %a,
-                                                                   <vscale x 2 x i64> %splat)
+                                                                   <vscale x 2 x i64> splat(i64 65280))
   ret <vscale x 2 x i64> %out
 }
 
@@ -1176,10 +1058,8 @@ define <vscale x 2 x i64> @sqadd_d_negimm(<vscale x 2 x i64> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqsub z0.d, z0.d, #3840 // =0xf00
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 -3840, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sqadd.x.nxv2i64(<vscale x 2 x i64> %a,
-                                                                   <vscale x 2 x i64> %splat)
+                                                                   <vscale x 2 x i64> splat(i64 -3840))
   ret <vscale x 2 x i64> %out
 }
 
@@ -1190,10 +1070,8 @@ define <vscale x 16 x i8> @sqsub_b_lowimm(<vscale x 16 x i8> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqsub z0.b, z0.b, #27 // =0x1b
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 27, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sqsub.x.nxv16i8(<vscale x 16 x i8> %a,
-                                                                   <vscale x 16 x i8> %splat)
+                                                                   <vscale x 16 x i8> splat(i8 27))
   ret <vscale x 16 x i8> %out
 }
 
@@ -1203,10 +1081,8 @@ define <vscale x 16 x i8> @sqsub_b_negimm(<vscale x 16 x i8> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqadd z0.b, z0.b, #1 // =0x1
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 -1, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sqsub.x.nxv16i8(<vscale x 16 x i8> %a,
-                                                                   <vscale x 16 x i8> %splat)
+                                                                   <vscale x 16 x i8> splat(i8 -1))
   ret <vscale x 16 x i8> %out
 }
 
@@ -1215,10 +1091,8 @@ define <vscale x 8 x i16> @sqsub_h_lowimm(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqsub z0.h, z0.h, #43 // =0x2b
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 43, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqsub.x.nxv8i16(<vscale x 8 x i16> %a,
-                                                                   <vscale x 8 x i16> %splat)
+                                                                   <vscale x 8 x i16> splat(i16 43))
   ret <vscale x 8 x i16> %out
 }
 
@@ -1227,10 +1101,8 @@ define <vscale x 8 x i16> @sqsub_h_highimm(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqsub z0.h, z0.h, #2048 // =0x800
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 2048, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqsub.x.nxv8i16(<vscale x 8 x i16> %a,
-                                                                   <vscale x 8 x i16> %splat)
+                                                                   <vscale x 8 x i16> splat(i16 2048))
   ret <vscale x 8 x i16> %out
 }
 
@@ -1240,10 +1112,8 @@ define <vscale x 8 x i16> @sqsub_h_negimm(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqadd z0.h, z0.h, #128 // =0x80
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 -128, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqsub.x.nxv8i16(<vscale x 8 x i16> %a,
-                                                                   <vscale x 8 x i16> %splat)
+                                                                   <vscale x 8 x i16> splat(i16 -128))
   ret <vscale x 8 x i16> %out
 }
 
@@ -1252,10 +1122,8 @@ define <vscale x 4 x i32> @sqsub_s_lowimm(<vscale x 4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqsub z0.s, z0.s, #1 // =0x1
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 1, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqsub.x.nxv4i32(<vscale x 4 x i32> %a,
-                                                                   <vscale x 4 x i32> %splat)
+                                                                   <vscale x 4 x i32> splat(i32 1))
   ret <vscale x 4 x i32> %out
 }
 
@@ -1264,10 +1132,8 @@ define <vscale x 4 x i32> @sqsub_s_highimm(<vscale x 4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqsub z0.s, z0.s, #8192 // =0x2000
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 8192, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqsub.x.nxv4i32(<vscale x 4 x i32> %a,
-                                                                   <vscale x 4 x i32> %splat)
+                                                                   <vscale x 4 x i32> splat(i32 8192))
   ret <vscale x 4 x i32> %out
 }
 
@@ -1277,10 +1143,8 @@ define <vscale x 4 x i32> @sqsub_s_negimm(<vscale x 4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqadd z0.s, z0.s, #32768 // =0x8000
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 -32768, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqsub.x.nxv4i32(<vscale x 4 x i32> %a,
-                                                                   <vscale x 4 x i32> %splat)
+                                                                   <vscale x 4 x i32> splat(i32 -32768))
   ret <vscale x 4 x i32> %out
 }
 
@@ -1289,10 +1153,8 @@ define <vscale x 2 x i64> @sqsub_d_lowimm(<vscale x 2 x i64> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqsub z0.d, z0.d, #255 // =0xff
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 255, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sqsub.x.nxv2i64(<vscale x 2 x i64> %a,
-                                                                   <vscale x 2 x i64> %splat)
+                                                                   <vscale x 2 x i64> splat(i64 255))
   ret <vscale x 2 x i64> %out
 }
 
@@ -1301,10 +1163,8 @@ define <vscale x 2 x i64> @sqsub_d_highimm(<vscale x 2 x i64> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqsub z0.d, z0.d, #65280 // =0xff00
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 65280, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sqsub.x.nxv2i64(<vscale x 2 x i64> %a,
-                                                                   <vscale x 2 x i64> %splat)
+                                                                   <vscale x 2 x i64> splat(i64 65280))
   ret <vscale x 2 x i64> %out
 }
 
@@ -1314,10 +1174,8 @@ define <vscale x 2 x i64> @sqsub_d_negimm(<vscale x 2 x i64> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqadd z0.d, z0.d, #57344 // =0xe000
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 -57344, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sqsub.x.nxv2i64(<vscale x 2 x i64> %a,
-                                                                   <vscale x 2 x i64> %splat)
+                                                                   <vscale x 2 x i64> splat(i64 -57344))
   ret <vscale x 2 x i64> %out
 }
 
@@ -1328,10 +1186,8 @@ define <vscale x 16 x i8> @uqadd_b_lowimm(<vscale x 16 x i8> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uqadd z0.b, z0.b, #27 // =0x1b
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 27, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.uqadd.x.nxv16i8(<vscale x 16 x i8> %a,
-                                                                   <vscale x 16 x i8> %splat)
+                                                                   <vscale x 16 x i8> splat(i8 27))
   ret <vscale x 16 x i8> %out
 }
 
@@ -1340,10 +1196,8 @@ define <vscale x 8 x i16> @uqadd_h_lowimm(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uqadd z0.h, z0.h, #43 // =0x2b
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 43, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uqadd.x.nxv8i16(<vscale x 8 x i16> %a,
-                                                                   <vscale x 8 x i16> %splat)
+                                                                   <vscale x 8 x i16> splat(i16 43))
   ret <vscale x 8 x i16> %out
 }
 
@@ -1352,10 +1206,8 @@ define <vscale x 8 x i16> @uqadd_h_highimm(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uqadd z0.h, z0.h, #2048 // =0x800
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 2048, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uqadd.x.nxv8i16(<vscale x 8 x i16> %a,
-                                                                   <vscale x 8 x i16> %splat)
+                                                                   <vscale x 8 x i16> splat(i16 2048))
   ret <vscale x 8 x i16> %out
 }
 
@@ -1364,10 +1216,8 @@ define <vscale x 4 x i32> @uqadd_s_lowimm(<vscale x 4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uqadd z0.s, z0.s, #1 // =0x1
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 1, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uqadd.x.nxv4i32(<vscale x 4 x i32> %a,
-                                                                   <vscale x 4 x i32> %splat)
+                                                                   <vscale x 4 x i32> splat(i32 1))
   ret <vscale x 4 x i32> %out
 }
 
@@ -1376,10 +1226,8 @@ define <vscale x 4 x i32> @uqadd_s_highimm(<vscale x 4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uqadd z0.s, z0.s, #8192 // =0x2000
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 8192, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uqadd.x.nxv4i32(<vscale x 4 x i32> %a,
-                                                                   <vscale x 4 x i32> %splat)
+                                                                   <vscale x 4 x i32> splat(i32 8192))
   ret <vscale x 4 x i32> %out
 }
 
@@ -1388,10 +1236,8 @@ define <vscale x 2 x i64> @uqadd_d_lowimm(<vscale x 2 x i64> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uqadd z0.d, z0.d, #255 // =0xff
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 255, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uqadd.x.nxv2i64(<vscale x 2 x i64> %a,
-                                                                   <vscale x 2 x i64> %splat)
+                                                                   <vscale x 2 x i64> splat(i64 255))
   ret <vscale x 2 x i64> %out
 }
 
@@ -1400,10 +1246,8 @@ define <vscale x 2 x i64> @uqadd_d_highimm(<vscale x 2 x i64> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uqadd z0.d, z0.d, #65280 // =0xff00
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 65280, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uqadd.x.nxv2i64(<vscale x 2 x i64> %a,
-                                                                   <vscale x 2 x i64> %splat)
+                                                                   <vscale x 2 x i64> splat(i64 65280))
   ret <vscale x 2 x i64> %out
 }
 
@@ -1414,10 +1258,8 @@ define <vscale x 16 x i8> @uqsub_b_lowimm(<vscale x 16 x i8> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uqsub z0.b, z0.b, #27 // =0x1b
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 27, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.uqsub.x.nxv16i8(<vscale x 16 x i8> %a,
-                                                                   <vscale x 16 x i8> %splat)
+                                                                   <vscale x 16 x i8> splat(i8 27))
   ret <vscale x 16 x i8> %out
 }
 
@@ -1426,10 +1268,8 @@ define <vscale x 8 x i16> @uqsub_h_lowimm(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uqsub z0.h, z0.h, #43 // =0x2b
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 43, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uqsub.x.nxv8i16(<vscale x 8 x i16> %a,
-                                                                   <vscale x 8 x i16> %splat)
+                                                                   <vscale x 8 x i16> splat(i16 43))
   ret <vscale x 8 x i16> %out
 }
 
@@ -1438,10 +1278,8 @@ define <vscale x 8 x i16> @uqsub_h_highimm(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uqsub z0.h, z0.h, #2048 // =0x800
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 2048, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uqsub.x.nxv8i16(<vscale x 8 x i16> %a,
-                                                                   <vscale x 8 x i16> %splat)
+                                                                   <vscale x 8 x i16> splat(i16 2048))
   ret <vscale x 8 x i16> %out
 }
 
@@ -1450,10 +1288,8 @@ define <vscale x 4 x i32> @uqsub_s_lowimm(<vscale x 4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uqsub z0.s, z0.s, #1 // =0x1
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 1, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uqsub.x.nxv4i32(<vscale x 4 x i32> %a,
-                                                                   <vscale x 4 x i32> %splat)
+                                                                   <vscale x 4 x i32> splat(i32 1))
   ret <vscale x 4 x i32> %out
 }
 
@@ -1462,10 +1298,8 @@ define <vscale x 4 x i32> @uqsub_s_highimm(<vscale x 4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uqsub z0.s, z0.s, #8192 // =0x2000
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 8192, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uqsub.x.nxv4i32(<vscale x 4 x i32> %a,
-                                                                   <vscale x 4 x i32> %splat)
+                                                                   <vscale x 4 x i32> splat(i32 8192))
   ret <vscale x 4 x i32> %out
 }
 
@@ -1474,10 +1308,8 @@ define <vscale x 2 x i64> @uqsub_d_lowimm(<vscale x 2 x i64> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uqsub z0.d, z0.d, #255 // =0xff
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 255, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uqsub.x.nxv2i64(<vscale x 2 x i64> %a,
-                                                                   <vscale x 2 x i64> %splat)
+                                                                   <vscale x 2 x i64> splat(i64 255))
   ret <vscale x 2 x i64> %out
 }
 
@@ -1486,10 +1318,8 @@ define <vscale x 2 x i64> @uqsub_d_highimm(<vscale x 2 x i64> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uqsub z0.d, z0.d, #65280 // =0xff00
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 65280, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uqsub.x.nxv2i64(<vscale x 2 x i64> %a,
-                                                                   <vscale x 2 x i64> %splat)
+                                                                   <vscale x 2 x i64> splat(i64 65280))
   ret <vscale x 2 x i64> %out
 }
 
@@ -1500,11 +1330,9 @@ define <vscale x 16 x i8> @asr_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a)
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    asr z0.b, p0/m, z0.b, #8
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 9, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.asr.nxv16i8(<vscale x 16 x i1> %pg,
                                                                <vscale x 16 x i8> %a,
-                                                               <vscale x 16 x i8> %splat)
+                                                               <vscale x 16 x i8> splat(i8 9))
   ret <vscale x 16 x i8> %out
 }
 
@@ -1514,11 +1342,9 @@ define <vscale x 16 x i8> @asr_i8_all_active(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    asr z0.b, z0.b, #8
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-  %elt = insertelement <vscale x 16 x i8> undef, i8 8, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.asr.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %splat)
+                                                                 <vscale x 16 x i8> splat(i8 8))
   ret <vscale x 16 x i8> %out
 }
 
@@ -1540,11 +1366,9 @@ define <vscale x 8 x i16> @asr_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a)
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    asr z0.h, p0/m, z0.h, #16
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 17, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.asr.nxv8i16(<vscale x 8 x i1> %pg,
                                                                <vscale x 8 x i16> %a,
-                                                               <vscale x 8 x i16> %splat)
+                                                               <vscale x 8 x i16> splat(i16 17))
   ret <vscale x 8 x i16> %out
 }
 
@@ -1554,11 +1378,9 @@ define <vscale x 8 x i16> @asr_i16_all_active(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    asr z0.h, z0.h, #16
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 16, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.asr.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %splat)
+                                                                 <vscale x 8 x i16> splat(i16 16))
   ret <vscale x 8 x i16> %out
 }
 
@@ -1580,11 +1402,9 @@ define <vscale x 4 x i32> @asr_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a)
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    asr z0.s, p0/m, z0.s, #32
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 33, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.asr.nxv4i32(<vscale x 4 x i1> %pg,
                                                                <vscale x 4 x i32> %a,
-                                                               <vscale x 4 x i32> %splat)
+                                                               <vscale x 4 x i32> splat(i32 33))
   ret <vscale x 4 x i32> %out
 }
 
@@ -1594,11 +1414,9 @@ define <vscale x 4 x i32> @asr_i32_all_active(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    asr z0.s, z0.s, #32
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 32, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.asr.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %splat)
+                                                                 <vscale x 4 x i32> splat(i32 32))
   ret <vscale x 4 x i32> %out
 }
 
@@ -1620,11 +1438,9 @@ define <vscale x 2 x i64> @asr_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a)
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    asr z0.d, p0/m, z0.d, #64
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 65, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.asr.nxv2i64(<vscale x 2 x i1> %pg,
                                                                <vscale x 2 x i64> %a,
-                                                               <vscale x 2 x i64> %splat)
+                                                               <vscale x 2 x i64> splat(i64 65))
   ret <vscale x 2 x i64> %out
 }
 
@@ -1634,11 +1450,9 @@ define <vscale x 2 x i64> @asr_i64_all_active(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    asr z0.d, z0.d, #64
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 64, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.asr.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %splat)
+                                                                 <vscale x 2 x i64> splat(i64 64))
   ret <vscale x 2 x i64> %out
 }
 
@@ -1662,11 +1476,9 @@ define <vscale x 16 x i8> @lsl_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a)
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsl z0.b, p0/m, z0.b, #7
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 7, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.lsl.nxv16i8(<vscale x 16 x i1> %pg,
                                                                <vscale x 16 x i8> %a,
-                                                               <vscale x 16 x i8> %splat)
+                                                               <vscale x 16 x i8> splat(i8 7))
   ret <vscale x 16 x i8> %out
 }
 
@@ -1676,11 +1488,9 @@ define <vscale x 16 x i8> @lsl_i8_all_active(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    lsl z0.b, z0.b, #7
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-  %elt = insertelement <vscale x 16 x i8> undef, i8 7, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.lsl.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %splat)
+                                                                 <vscale x 16 x i8> splat(i8 7))
   ret <vscale x 16 x i8> %out
 }
 
@@ -1691,11 +1501,9 @@ define <vscale x 16 x i8> @lsl_i8_too_big(<vscale x 16 x i1> %pg, <vscale x 16 x
 ; CHECK-NEXT:    mov z1.b, #8 // =0x8
 ; CHECK-NEXT:    lsl z0.b, p0/m, z0.b, z1.b
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 8, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.lsl.nxv16i8(<vscale x 16 x i1> %pg,
                                                                <vscale x 16 x i8> %a,
-                                                               <vscale x 16 x i8> %splat)
+                                                               <vscale x 16 x i8> splat(i8 8))
   ret <vscale x 16 x i8> %out
 }
 
@@ -1715,11 +1523,9 @@ define <vscale x 8 x i16> @lsl_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a)
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsl z0.h, p0/m, z0.h, #15
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 15, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.lsl.nxv8i16(<vscale x 8 x i1> %pg,
                                                                <vscale x 8 x i16> %a,
-                                                               <vscale x 8 x i16> %splat)
+                                                               <vscale x 8 x i16> splat(i16 15))
   ret <vscale x 8 x i16> %out
 }
 
@@ -1729,11 +1535,9 @@ define <vscale x 8 x i16> @lsl_i16_all_active(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    lsl z0.h, z0.h, #15
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 15, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.lsl.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %splat)
+                                                                 <vscale x 8 x i16> splat(i16 15))
   ret <vscale x 8 x i16> %out
 }
 
@@ -1744,11 +1548,9 @@ define <vscale x 8 x i16> @lsl_i16_too_big(<vscale x 8 x i1> %pg, <vscale x 8 x
 ; CHECK-NEXT:    mov z1.h, #16 // =0x10
 ; CHECK-NEXT:    lsl z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 16, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.lsl.nxv8i16(<vscale x 8 x i1> %pg,
                                                                <vscale x 8 x i16> %a,
-                                                               <vscale x 8 x i16> %splat)
+                                                               <vscale x 8 x i16> splat(i16 16))
   ret <vscale x 8 x i16> %out
 }
 
@@ -1768,11 +1570,9 @@ define <vscale x 4 x i32> @lsl_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a)
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsl z0.s, p0/m, z0.s, #31
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 31, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.lsl.nxv4i32(<vscale x 4 x i1> %pg,
                                                                <vscale x 4 x i32> %a,
-                                                               <vscale x 4 x i32> %splat)
+                                                               <vscale x 4 x i32> splat(i32 31))
   ret <vscale x 4 x i32> %out
 }
 
@@ -1782,11 +1582,9 @@ define <vscale x 4 x i32> @lsl_i32_all_active(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    lsl z0.s, z0.s, #31
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 31, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.lsl.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %splat)
+                                                                 <vscale x 4 x i32> splat(i32 31))
   ret <vscale x 4 x i32> %out
 }
 
@@ -1797,11 +1595,9 @@ define <vscale x 4 x i32> @lsl_i32_too_big(<vscale x 4 x i1> %pg, <vscale x 4 x
 ; CHECK-NEXT:    mov z1.s, #32 // =0x20
 ; CHECK-NEXT:    lsl z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 32, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.lsl.nxv4i32(<vscale x 4 x i1> %pg,
                                                                <vscale x 4 x i32> %a,
-                                                               <vscale x 4 x i32> %splat)
+                                                               <vscale x 4 x i32> splat(i32 32))
   ret <vscale x 4 x i32> %out
 }
 
@@ -1821,11 +1617,9 @@ define <vscale x 2 x i64> @lsl_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a)
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsl z0.d, p0/m, z0.d, #63
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 63, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.lsl.nxv2i64(<vscale x 2 x i1> %pg,
                                                                <vscale x 2 x i64> %a,
-                                                               <vscale x 2 x i64> %splat)
+                                                               <vscale x 2 x i64> splat(i64 63))
   ret <vscale x 2 x i64> %out
 }
 
@@ -1835,11 +1629,9 @@ define <vscale x 2 x i64> @lsl_i64_all_active(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    lsl z0.d, z0.d, #63
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 63, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.lsl.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %splat)
+                                                                 <vscale x 2 x i64> splat(i64 63))
   ret <vscale x 2 x i64> %out
 }
 
@@ -1850,11 +1642,9 @@ define <vscale x 2 x i64> @lsl_i64_too_big(<vscale x 2 x i1> %pg, <vscale x 2 x
 ; CHECK-NEXT:    mov z1.d, #64 // =0x40
 ; CHECK-NEXT:    lsl z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 64, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.lsl.nxv2i64(<vscale x 2 x i1> %pg,
                                                                <vscale x 2 x i64> %a,
-                                                               <vscale x 2 x i64> %splat)
+                                                               <vscale x 2 x i64> splat(i64 64))
   ret <vscale x 2 x i64> %out
 }
 
@@ -1876,11 +1666,9 @@ define <vscale x 16 x i8> @lsr_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a)
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr z0.b, p0/m, z0.b, #8
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 16 x i8> undef, i8 9, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.lsr.nxv16i8(<vscale x 16 x i1> %pg,
                                                                <vscale x 16 x i8> %a,
-                                                               <vscale x 16 x i8> %splat)
+                                                               <vscale x 16 x i8> splat(i8 9))
   ret <vscale x 16 x i8> %out
 }
 
@@ -1890,11 +1678,9 @@ define <vscale x 16 x i8> @lsr_i8_all_active(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    lsr z0.b, z0.b, #8
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-  %elt = insertelement <vscale x 16 x i8> undef, i8 8, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.lsr.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %splat)
+                                                                 <vscale x 16 x i8> splat(i8 8))
   ret <vscale x 16 x i8> %out
 }
 
@@ -1916,11 +1702,9 @@ define <vscale x 8 x i16> @lsr_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a)
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr z0.h, p0/m, z0.h, #16
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 17, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.lsr.nxv8i16(<vscale x 8 x i1> %pg,
                                                                <vscale x 8 x i16> %a,
-                                                               <vscale x 8 x i16> %splat)
+                                                               <vscale x 8 x i16> splat(i16 17))
   ret <vscale x 8 x i16> %out
 }
 
@@ -1930,11 +1714,9 @@ define <vscale x 8 x i16> @lsr_i16_all_active(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    lsr z0.h, z0.h, #16
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 16, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.lsr.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %splat)
+                                                                 <vscale x 8 x i16> splat(i16 16))
   ret <vscale x 8 x i16> %out
 }
 
@@ -1956,11 +1738,9 @@ define <vscale x 4 x i32> @lsr_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a)
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr z0.s, p0/m, z0.s, #32
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 33, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.lsr.nxv4i32(<vscale x 4 x i1> %pg,
                                                                <vscale x 4 x i32> %a,
-                                                               <vscale x 4 x i32> %splat)
+                                                               <vscale x 4 x i32> splat(i32 33))
   ret <vscale x 4 x i32> %out
 }
 
@@ -1970,11 +1750,9 @@ define <vscale x 4 x i32> @lsr_i32_all_active(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    lsr z0.s, z0.s, #32
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 32, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.lsr.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %splat)
+                                                                 <vscale x 4 x i32> splat(i32 32))
   ret <vscale x 4 x i32> %out
 }
 
@@ -1996,11 +1774,9 @@ define <vscale x 2 x i64> @lsr_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a)
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr z0.d, p0/m, z0.d, #64
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 65, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.lsr.nxv2i64(<vscale x 2 x i1> %pg,
                                                                <vscale x 2 x i64> %a,
-                                                               <vscale x 2 x i64> %splat)
+                                                               <vscale x 2 x i64> splat(i64 65))
   ret <vscale x 2 x i64> %out
 }
 
@@ -2010,11 +1786,9 @@ define <vscale x 2 x i64> @lsr_i64_all_active(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    lsr z0.d, z0.d, #64
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 64, i64 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.lsr.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %splat)
+                                                                 <vscale x 2 x i64> splat(i64 64))
   ret <vscale x 2 x i64> %out
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-undef.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-undef.ll
index 929f8f5f54e01..3c001a87083d4 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-undef.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-undef.ll
@@ -61,11 +61,9 @@ define <vscale x 16 x i8> @add_imm_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add z0.b, z0.b, #3 // =0x3
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 16 x i8> undef, i8 3, i32 0
-  %imm.splat = shufflevector <vscale x 16 x i8> %imm, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.add.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %imm.splat)
+                                                                 <vscale x 16 x i8> splat(i8 3))
   ret <vscale x 16 x i8> %out
 }
 
@@ -74,11 +72,9 @@ define <vscale x 8 x i16> @add_imm_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add z0.h, z0.h, #4 // =0x4
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 8 x i16> undef, i16 4, i32 0
-  %imm.splat = shufflevector <vscale x 8 x i16> %imm, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.add.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %imm.splat)
+                                                                 <vscale x 8 x i16> splat(i16 4))
   ret <vscale x 8 x i16> %out
 }
 
@@ -87,11 +83,9 @@ define <vscale x 4 x i32> @add_imm_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add z0.s, z0.s, #5 // =0x5
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 4 x i32> undef, i32 5, i32 0
-  %imm.splat = shufflevector <vscale x 4 x i32> %imm, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.add.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %imm.splat)
+                                                                 <vscale x 4 x i32> splat(i32 5))
   ret <vscale x 4 x i32> %out
 }
 
@@ -100,11 +94,9 @@ define <vscale x 2 x i64> @add_imm_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add z0.d, z0.d, #6 // =0x6
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 2 x i64> undef, i64 6, i32 0
-  %imm.splat = shufflevector <vscale x 2 x i64> %imm, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.add.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %imm.splat)
+                                                                 <vscale x 2 x i64> splat(i64 6))
   ret <vscale x 2 x i64> %out
 }
 
@@ -289,11 +281,9 @@ define <vscale x 16 x i8> @mul_imm_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mul z0.b, z0.b, #3
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 16 x i8> undef, i8 3, i32 0
-  %imm.splat = shufflevector <vscale x 16 x i8> %imm, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.mul.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %imm.splat)
+                                                                 <vscale x 16 x i8> splat(i8 3))
   ret <vscale x 16 x i8> %out
 }
 
@@ -302,11 +292,9 @@ define <vscale x 8 x i16> @mul_imm_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mul z0.h, z0.h, #4
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 8 x i16> undef, i16 4, i32 0
-  %imm.splat = shufflevector <vscale x 8 x i16> %imm, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.mul.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %imm.splat)
+                                                                 <vscale x 8 x i16> splat(i16 4))
   ret <vscale x 8 x i16> %out
 }
 
@@ -315,11 +303,9 @@ define <vscale x 4 x i32> @mul_imm_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mul z0.s, z0.s, #5
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 4 x i32> undef, i32 5, i32 0
-  %imm.splat = shufflevector <vscale x 4 x i32> %imm, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.mul.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %imm.splat)
+                                                                 <vscale x 4 x i32> splat(i32 5))
   ret <vscale x 4 x i32> %out
 }
 
@@ -328,11 +314,9 @@ define <vscale x 2 x i64> @mul_imm_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mul z0.d, z0.d, #6
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 2 x i64> undef, i64 6, i32 0
-  %imm.splat = shufflevector <vscale x 2 x i64> %imm, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.mul.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %imm.splat)
+                                                                 <vscale x 2 x i64> splat(i64 6))
   ret <vscale x 2 x i64> %out
 }
 
@@ -497,11 +481,9 @@ define <vscale x 16 x i8> @smax_imm_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smax z0.b, z0.b, #3
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 16 x i8> undef, i8 3, i32 0
-  %imm.splat = shufflevector <vscale x 16 x i8> %imm, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.smax.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                   <vscale x 16 x i8> %a,
-                                                                  <vscale x 16 x i8> %imm.splat)
+                                                                  <vscale x 16 x i8> splat(i8 3))
   ret <vscale x 16 x i8> %out
 }
 
@@ -510,11 +492,9 @@ define <vscale x 8 x i16> @smax_imm_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smax z0.h, z0.h, #4
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 8 x i16> undef, i16 4, i32 0
-  %imm.splat = shufflevector <vscale x 8 x i16> %imm, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.smax.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                   <vscale x 8 x i16> %a,
-                                                                  <vscale x 8 x i16> %imm.splat)
+                                                                  <vscale x 8 x i16> splat(i16 4))
   ret <vscale x 8 x i16> %out
 }
 
@@ -523,11 +503,9 @@ define <vscale x 4 x i32> @smax_imm_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smax z0.s, z0.s, #5
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 4 x i32> undef, i32 5, i32 0
-  %imm.splat = shufflevector <vscale x 4 x i32> %imm, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.smax.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                   <vscale x 4 x i32> %a,
-                                                                  <vscale x 4 x i32> %imm.splat)
+                                                                  <vscale x 4 x i32> splat(i32 5))
   ret <vscale x 4 x i32> %out
 }
 
@@ -536,11 +514,9 @@ define <vscale x 2 x i64> @smax_imm_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smax z0.d, z0.d, #6
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 2 x i64> undef, i64 6, i32 0
-  %imm.splat = shufflevector <vscale x 2 x i64> %imm, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.smax.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                   <vscale x 2 x i64> %a,
-                                                                  <vscale x 2 x i64> %imm.splat)
+                                                                  <vscale x 2 x i64> splat(i64 6))
   ret <vscale x 2 x i64> %out
 }
 
@@ -601,11 +577,9 @@ define <vscale x 16 x i8> @smin_imm_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smin z0.b, z0.b, #3
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 16 x i8> undef, i8 3, i32 0
-  %imm.splat = shufflevector <vscale x 16 x i8> %imm, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.smin.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                   <vscale x 16 x i8> %a,
-                                                                  <vscale x 16 x i8> %imm.splat)
+                                                                  <vscale x 16 x i8> splat(i8 3))
   ret <vscale x 16 x i8> %out
 }
 
@@ -614,11 +588,9 @@ define <vscale x 8 x i16> @smin_imm_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smin z0.h, z0.h, #4
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 8 x i16> undef, i16 4, i32 0
-  %imm.splat = shufflevector <vscale x 8 x i16> %imm, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.smin.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                   <vscale x 8 x i16> %a,
-                                                                  <vscale x 8 x i16> %imm.splat)
+                                                                  <vscale x 8 x i16> splat(i16 4))
   ret <vscale x 8 x i16> %out
 }
 
@@ -627,11 +599,9 @@ define <vscale x 4 x i32> @smin_imm_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smin z0.s, z0.s, #5
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 4 x i32> undef, i32 5, i32 0
-  %imm.splat = shufflevector <vscale x 4 x i32> %imm, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.smin.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                   <vscale x 4 x i32> %a,
-                                                                  <vscale x 4 x i32> %imm.splat)
+                                                                  <vscale x 4 x i32> splat(i32 5))
   ret <vscale x 4 x i32> %out
 }
 
@@ -640,11 +610,9 @@ define <vscale x 2 x i64> @smin_imm_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smin z0.d, z0.d, #6
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 2 x i64> undef, i64 6, i32 0
-  %imm.splat = shufflevector <vscale x 2 x i64> %imm, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.smin.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                   <vscale x 2 x i64> %a,
-                                                                  <vscale x 2 x i64> %imm.splat)
+                                                                  <vscale x 2 x i64> splat(i64 6))
   ret <vscale x 2 x i64> %out
 }
 
@@ -773,11 +741,9 @@ define <vscale x 16 x i8> @sub_imm_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub z0.b, z0.b, #3 // =0x3
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 16 x i8> undef, i8 3, i32 0
-  %imm.splat = shufflevector <vscale x 16 x i8> %imm, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sub.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %imm.splat)
+                                                                 <vscale x 16 x i8> splat(i8 3))
   ret <vscale x 16 x i8> %out
 }
 
@@ -786,11 +752,9 @@ define <vscale x 8 x i16> @sub_imm_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub z0.h, z0.h, #4 // =0x4
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 8 x i16> undef, i16 4, i32 0
-  %imm.splat = shufflevector <vscale x 8 x i16> %imm, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sub.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %imm.splat)
+                                                                 <vscale x 8 x i16> splat(i16 4))
   ret <vscale x 8 x i16> %out
 }
 
@@ -799,11 +763,9 @@ define <vscale x 4 x i32> @sub_imm_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub z0.s, z0.s, #5 // =0x5
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 4 x i32> undef, i32 5, i32 0
-  %imm.splat = shufflevector <vscale x 4 x i32> %imm, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sub.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %imm.splat)
+                                                                 <vscale x 4 x i32> splat(i32 5))
   ret <vscale x 4 x i32> %out
 }
 
@@ -812,11 +774,9 @@ define <vscale x 2 x i64> @sub_imm_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub z0.d, z0.d, #6 // =0x6
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 2 x i64> undef, i64 6, i32 0
-  %imm.splat = shufflevector <vscale x 2 x i64> %imm, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sub.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %imm.splat)
+                                                                 <vscale x 2 x i64> splat(i64 6))
   ret <vscale x 2 x i64> %out
 }
 
@@ -877,10 +837,8 @@ define <vscale x 16 x i8> @subr_imm_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    subr z0.b, z0.b, #3 // =0x3
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 16 x i8> undef, i8 3, i32 0
-  %imm.splat = shufflevector <vscale x 16 x i8> %imm, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sub.u.nxv16i8(<vscale x 16 x i1> %pg,
-                                                                 <vscale x 16 x i8> %imm.splat,
+                                                                 <vscale x 16 x i8> splat(i8 3),
                                                                  <vscale x 16 x i8> %a)
   ret <vscale x 16 x i8> %out
 }
@@ -890,10 +848,8 @@ define <vscale x 8 x i16> @subr_imm_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    subr z0.h, z0.h, #4 // =0x4
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 8 x i16> undef, i16 4, i32 0
-  %imm.splat = shufflevector <vscale x 8 x i16> %imm, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sub.u.nxv8i16(<vscale x 8 x i1> %pg,
-                                                                 <vscale x 8 x i16> %imm.splat,
+                                                                 <vscale x 8 x i16> splat(i16 4),
                                                                  <vscale x 8 x i16> %a)
   ret <vscale x 8 x i16> %out
 }
@@ -903,10 +859,8 @@ define <vscale x 4 x i32> @subr_imm_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    subr z0.s, z0.s, #5 // =0x5
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 4 x i32> undef, i32 5, i32 0
-  %imm.splat = shufflevector <vscale x 4 x i32> %imm, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sub.u.nxv4i32(<vscale x 4 x i1> %pg,
-                                                                 <vscale x 4 x i32> %imm.splat,
+                                                                 <vscale x 4 x i32> splat(i32 5),
                                                                  <vscale x 4 x i32> %a)
   ret <vscale x 4 x i32> %out
 }
@@ -916,10 +870,8 @@ define <vscale x 2 x i64> @subr_imm_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    subr z0.d, z0.d, #6 // =0x6
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 2 x i64> undef, i64 6, i32 0
-  %imm.splat = shufflevector <vscale x 2 x i64> %imm, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sub.u.nxv2i64(<vscale x 2 x i1> %pg,
-                                                                 <vscale x 2 x i64> %imm.splat,
+                                                                 <vscale x 2 x i64> splat(i64 6),
                                                                  <vscale x 2 x i64> %a)
   ret <vscale x 2 x i64> %out
 }
@@ -1085,11 +1037,9 @@ define <vscale x 16 x i8> @umax_imm_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umax z0.b, z0.b, #3
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 16 x i8> undef, i8 3, i32 0
-  %imm.splat = shufflevector <vscale x 16 x i8> %imm, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.umax.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                   <vscale x 16 x i8> %a,
-                                                                  <vscale x 16 x i8> %imm.splat)
+                                                                  <vscale x 16 x i8> splat(i8 3))
   ret <vscale x 16 x i8> %out
 }
 
@@ -1098,11 +1048,9 @@ define <vscale x 8 x i16> @umax_imm_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umax z0.h, z0.h, #4
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 8 x i16> undef, i16 4, i32 0
-  %imm.splat = shufflevector <vscale x 8 x i16> %imm, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.umax.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                   <vscale x 8 x i16> %a,
-                                                                  <vscale x 8 x i16> %imm.splat)
+                                                                  <vscale x 8 x i16> splat(i16 4))
   ret <vscale x 8 x i16> %out
 }
 
@@ -1111,11 +1059,9 @@ define <vscale x 4 x i32> @umax_imm_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umax z0.s, z0.s, #5
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 4 x i32> undef, i32 5, i32 0
-  %imm.splat = shufflevector <vscale x 4 x i32> %imm, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.umax.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                   <vscale x 4 x i32> %a,
-                                                                  <vscale x 4 x i32> %imm.splat)
+                                                                  <vscale x 4 x i32> splat(i32 5))
   ret <vscale x 4 x i32> %out
 }
 
@@ -1124,11 +1070,9 @@ define <vscale x 2 x i64> @umax_imm_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umax z0.d, z0.d, #6
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 2 x i64> undef, i64 6, i32 0
-  %imm.splat = shufflevector <vscale x 2 x i64> %imm, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.umax.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                   <vscale x 2 x i64> %a,
-                                                                  <vscale x 2 x i64> %imm.splat)
+                                                                  <vscale x 2 x i64> splat(i64 6))
   ret <vscale x 2 x i64> %out
 }
 
@@ -1189,11 +1133,9 @@ define <vscale x 16 x i8> @umin_imm_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umin z0.b, z0.b, #3
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 16 x i8> undef, i8 3, i32 0
-  %imm.splat = shufflevector <vscale x 16 x i8> %imm, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.umin.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                   <vscale x 16 x i8> %a,
-                                                                  <vscale x 16 x i8> %imm.splat)
+                                                                  <vscale x 16 x i8> splat(i8 3))
   ret <vscale x 16 x i8> %out
 }
 
@@ -1202,11 +1144,9 @@ define <vscale x 8 x i16> @umin_imm_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umin z0.h, z0.h, #4
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 8 x i16> undef, i16 4, i32 0
-  %imm.splat = shufflevector <vscale x 8 x i16> %imm, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.umin.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                   <vscale x 8 x i16> %a,
-                                                                  <vscale x 8 x i16> %imm.splat)
+                                                                  <vscale x 8 x i16> splat(i16 4))
   ret <vscale x 8 x i16> %out
 }
 
@@ -1215,11 +1155,9 @@ define <vscale x 4 x i32> @umin_imm_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umin z0.s, z0.s, #5
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 4 x i32> undef, i32 5, i32 0
-  %imm.splat = shufflevector <vscale x 4 x i32> %imm, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.umin.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                   <vscale x 4 x i32> %a,
-                                                                  <vscale x 4 x i32> %imm.splat)
+                                                                  <vscale x 4 x i32> splat(i32 5))
   ret <vscale x 4 x i32> %out
 }
 
@@ -1228,11 +1166,9 @@ define <vscale x 2 x i64> @umin_imm_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umin z0.d, z0.d, #6
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 2 x i64> undef, i64 6, i32 0
-  %imm.splat = shufflevector <vscale x 2 x i64> %imm, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.umin.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                   <vscale x 2 x i64> %a,
-                                                                  <vscale x 2 x i64> %imm.splat)
+                                                                  <vscale x 2 x i64> splat(i64 6))
   ret <vscale x 2 x i64> %out
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-compares-with-imm.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-compares-with-imm.ll
index cefa3f9d825a3..bb72f0506690b 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-compares-with-imm.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-compares-with-imm.ll
@@ -15,9 +15,7 @@ define <vscale x 16 x i1> @ir_cmpeq_b(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    ptrue p0.b
 ; CHECK-NEXT:    cmpeq p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %out = icmp eq <vscale x 16 x i8> %a, %splat
+  %out = icmp eq <vscale x 16 x i8> %a, splat(i8 4)
   ret <vscale x 16 x i1> %out
 }
 
@@ -26,11 +24,9 @@ define <vscale x 16 x i1> @int_cmpeq_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpeq p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpeq.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %splat)
+                                                                 <vscale x 16 x i8> splat(i8 4))
   ret <vscale x 16 x i1> %out
 }
 
@@ -39,11 +35,9 @@ define <vscale x 16 x i1> @wide_cmpeq_b(<vscale x 16 x i1> %pg, <vscale x 16 x i
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpeq p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                       <vscale x 16 x i8> %a,
-                                                                      <vscale x 2 x i64> %splat)
+                                                                      <vscale x 2 x i64> splat(i64 4))
   ret <vscale x 16 x i1> %out
 }
 
@@ -53,9 +47,7 @@ define <vscale x 8 x i1> @ir_cmpeq_h(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    cmpeq p0.h, p0/z, z0.h, #-16
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %out = icmp eq <vscale x 8 x i16> %a, %splat
+  %out = icmp eq <vscale x 8 x i16> %a, splat(i16 -16)
   ret <vscale x 8 x i1> %out
 }
 
@@ -64,11 +56,9 @@ define <vscale x 8 x i1> @int_cmpeq_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpeq p0.h, p0/z, z0.h, #-16
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpeq.nxv8i16(<vscale x 8 x i1> %pg,
                                                                 <vscale x 8 x i16> %a,
-                                                                <vscale x 8 x i16> %splat)
+                                                                <vscale x 8 x i16> splat(i16 -16))
   ret <vscale x 8 x i1> %out
 }
 
@@ -77,11 +67,9 @@ define <vscale x 8 x i1> @wide_cmpeq_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpeq p0.h, p0/z, z0.h, #-16
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 -16, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv8i16(<vscale x 8 x i1> %pg,
                                                                      <vscale x 8 x i16> %a,
-                                                                     <vscale x 2 x i64> %splat)
+                                                                     <vscale x 2 x i64> splat(i64 -16))
   ret <vscale x 8 x i1> %out
 }
 
@@ -91,9 +79,7 @@ define <vscale x 4 x i1> @ir_cmpeq_s(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    cmpeq p0.s, p0/z, z0.s, #15
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %out = icmp eq <vscale x 4 x i32> %a, %splat
+  %out = icmp eq <vscale x 4 x i32> %a, splat(i32 15)
   ret <vscale x 4 x i1> %out
 }
 
@@ -102,11 +88,9 @@ define <vscale x 4 x i1> @int_cmpeq_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpeq p0.s, p0/z, z0.s, #15
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpeq.nxv4i32(<vscale x 4 x i1> %pg,
                                                                 <vscale x 4 x i32> %a,
-                                                                <vscale x 4 x i32> %splat)
+                                                                <vscale x 4 x i32> splat(i32 15))
   ret <vscale x 4 x i1> %out
 }
 
@@ -115,11 +99,9 @@ define <vscale x 4 x i1> @wide_cmpeq_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpeq p0.s, p0/z, z0.s, #15
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 15, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                      <vscale x 4 x i32> %a,
-                                                                     <vscale x 2 x i64> %splat)
+                                                                     <vscale x 2 x i64> splat(i64 15))
   ret <vscale x 4 x i1> %out
 }
 
@@ -129,9 +111,7 @@ define <vscale x 2 x i1> @ir_cmpeq_d(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    cmpeq p0.d, p0/z, z0.d, #0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %out = icmp eq <vscale x 2 x i64> %a, %splat
+  %out = icmp eq <vscale x 2 x i64> %a, zeroinitializer
   ret <vscale x 2 x i1> %out
 }
 
@@ -140,11 +120,9 @@ define <vscale x 2 x i1> @int_cmpeq_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpeq p0.d, p0/z, z0.d, #0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpeq.nxv2i64(<vscale x 2 x i1> %pg,
                                                                 <vscale x 2 x i64> %a,
-                                                                <vscale x 2 x i64> %splat)
+                                                                <vscale x 2 x i64> zeroinitializer)
   ret <vscale x 2 x i1> %out
 }
 
@@ -158,9 +136,7 @@ define <vscale x 16 x i1> @ir_cmpge_b(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    ptrue p0.b
 ; CHECK-NEXT:    cmpge p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %out = icmp sge <vscale x 16 x i8> %a, %splat
+  %out = icmp sge <vscale x 16 x i8> %a, splat(i8 4)
   ret <vscale x 16 x i1> %out
 }
 
@@ -169,11 +145,9 @@ define <vscale x 16 x i1> @int_cmpge_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpge p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %splat)
+                                                                 <vscale x 16 x i8> splat(i8 4))
   ret <vscale x 16 x i1> %out
 }
 
@@ -182,11 +156,9 @@ define <vscale x 16 x i1> @wide_cmpge_b(<vscale x 16 x i1> %pg, <vscale x 16 x i
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpge p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                       <vscale x 16 x i8> %a,
-                                                                      <vscale x 2 x i64> %splat)
+                                                                      <vscale x 2 x i64> splat(i64 4))
   ret <vscale x 16 x i1> %out
 }
 
@@ -196,9 +168,7 @@ define <vscale x 8 x i1> @ir_cmpge_h(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    cmpge p0.h, p0/z, z0.h, #-16
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %out = icmp sge <vscale x 8 x i16> %a, %splat
+  %out = icmp sge <vscale x 8 x i16> %a, splat(i16 -16)
   ret <vscale x 8 x i1> %out
 }
 
@@ -207,11 +177,9 @@ define <vscale x 8 x i1> @int_cmpge_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpge p0.h, p0/z, z0.h, #-16
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.nxv8i16(<vscale x 8 x i1> %pg,
                                                                 <vscale x 8 x i16> %a,
-                                                                <vscale x 8 x i16> %splat)
+                                                                <vscale x 8 x i16> splat(i16 -16))
   ret <vscale x 8 x i1> %out
 }
 
@@ -220,11 +188,9 @@ define <vscale x 8 x i1> @wide_cmpge_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpge p0.h, p0/z, z0.h, #-16
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 -16, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.wide.nxv8i16(<vscale x 8 x i1> %pg,
                                                                      <vscale x 8 x i16> %a,
-                                                                     <vscale x 2 x i64> %splat)
+                                                                     <vscale x 2 x i64> splat(i64 -16))
   ret <vscale x 8 x i1> %out
 }
 
@@ -234,9 +200,7 @@ define <vscale x 4 x i1> @ir_cmpge_s(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    cmpge p0.s, p0/z, z0.s, #15
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %out = icmp sge <vscale x 4 x i32> %a, %splat
+  %out = icmp sge <vscale x 4 x i32> %a, splat(i32 15)
   ret <vscale x 4 x i1> %out
 }
 
@@ -245,11 +209,9 @@ define <vscale x 4 x i1> @int_cmpge_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpge p0.s, p0/z, z0.s, #15
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.nxv4i32(<vscale x 4 x i1> %pg,
                                                                 <vscale x 4 x i32> %a,
-                                                                <vscale x 4 x i32> %splat)
+                                                                <vscale x 4 x i32> splat(i32 15))
   ret <vscale x 4 x i1> %out
 }
 
@@ -258,11 +220,9 @@ define <vscale x 4 x i1> @wide_cmpge_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpge p0.s, p0/z, z0.s, #15
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 15, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                      <vscale x 4 x i32> %a,
-                                                                     <vscale x 2 x i64> %splat)
+                                                                     <vscale x 2 x i64> splat(i64 15))
   ret <vscale x 4 x i1> %out
 }
 
@@ -272,9 +232,7 @@ define <vscale x 2 x i1> @ir_cmpge_d(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    cmpge p0.d, p0/z, z0.d, #0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %out = icmp sge <vscale x 2 x i64> %a, %splat
+  %out = icmp sge <vscale x 2 x i64> %a, zeroinitializer
   ret <vscale x 2 x i1> %out
 }
 
@@ -283,11 +241,9 @@ define <vscale x 2 x i1> @int_cmpge_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpge p0.d, p0/z, z0.d, #0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpge.nxv2i64(<vscale x 2 x i1> %pg,
                                                                 <vscale x 2 x i64> %a,
-                                                                <vscale x 2 x i64> %splat)
+                                                                <vscale x 2 x i64> zeroinitializer)
   ret <vscale x 2 x i1> %out
 }
 
@@ -301,9 +257,7 @@ define <vscale x 16 x i1> @ir_cmpgt_b(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    ptrue p0.b
 ; CHECK-NEXT:    cmpgt p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %out = icmp sgt <vscale x 16 x i8> %a, %splat
+  %out = icmp sgt <vscale x 16 x i8> %a, splat(i8 4)
   ret <vscale x 16 x i1> %out
 }
 
@@ -312,11 +266,9 @@ define <vscale x 16 x i1> @int_cmpgt_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpgt p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %splat)
+                                                                 <vscale x 16 x i8> splat(i8 4))
   ret <vscale x 16 x i1> %out
 }
 
@@ -325,11 +277,9 @@ define <vscale x 16 x i1> @wide_cmpgt_b(<vscale x 16 x i1> %pg, <vscale x 16 x i
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpgt p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                       <vscale x 16 x i8> %a,
-                                                                      <vscale x 2 x i64> %splat)
+                                                                      <vscale x 2 x i64> splat(i64 4))
   ret <vscale x 16 x i1> %out
 }
 
@@ -339,9 +289,7 @@ define <vscale x 8 x i1> @ir_cmpgt_h(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    cmpgt p0.h, p0/z, z0.h, #-16
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %out = icmp sgt <vscale x 8 x i16> %a, %splat
+  %out = icmp sgt <vscale x 8 x i16> %a, splat(i16 -16)
   ret <vscale x 8 x i1> %out
 }
 
@@ -350,11 +298,9 @@ define <vscale x 8 x i1> @int_cmpgt_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpgt p0.h, p0/z, z0.h, #-16
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.nxv8i16(<vscale x 8 x i1> %pg,
                                                                 <vscale x 8 x i16> %a,
-                                                                <vscale x 8 x i16> %splat)
+                                                                <vscale x 8 x i16> splat(i16 -16))
   ret <vscale x 8 x i1> %out
 }
 
@@ -363,11 +309,9 @@ define <vscale x 8 x i1> @wide_cmpgt_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpgt p0.h, p0/z, z0.h, #-16
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 -16, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv8i16(<vscale x 8 x i1> %pg,
                                                                      <vscale x 8 x i16> %a,
-                                                                     <vscale x 2 x i64> %splat)
+                                                                     <vscale x 2 x i64> splat(i64 -16))
   ret <vscale x 8 x i1> %out
 }
 
@@ -377,9 +321,7 @@ define <vscale x 4 x i1> @ir_cmpgt_s(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    cmpgt p0.s, p0/z, z0.s, #15
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %out = icmp sgt <vscale x 4 x i32> %a, %splat
+  %out = icmp sgt <vscale x 4 x i32> %a, splat(i32 15)
   ret <vscale x 4 x i1> %out
 }
 
@@ -388,11 +330,9 @@ define <vscale x 4 x i1> @int_cmpgt_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpgt p0.s, p0/z, z0.s, #15
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.nxv4i32(<vscale x 4 x i1> %pg,
                                                                 <vscale x 4 x i32> %a,
-                                                                <vscale x 4 x i32> %splat)
+                                                                <vscale x 4 x i32> splat(i32 15))
   ret <vscale x 4 x i1> %out
 }
 
@@ -401,11 +341,9 @@ define <vscale x 4 x i1> @wide_cmpgt_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpgt p0.s, p0/z, z0.s, #15
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 15, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                      <vscale x 4 x i32> %a,
-                                                                     <vscale x 2 x i64> %splat)
+                                                                     <vscale x 2 x i64> splat(i64 15))
   ret <vscale x 4 x i1> %out
 }
 
@@ -415,9 +353,7 @@ define <vscale x 2 x i1> @ir_cmpgt_d(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    cmpgt p0.d, p0/z, z0.d, #0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %out = icmp sgt <vscale x 2 x i64> %a, %splat
+  %out = icmp sgt <vscale x 2 x i64> %a, zeroinitializer
   ret <vscale x 2 x i1> %out
 }
 
@@ -426,11 +362,9 @@ define <vscale x 2 x i1> @int_cmpgt_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpgt p0.d, p0/z, z0.d, #0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpgt.nxv2i64(<vscale x 2 x i1> %pg,
                                                                 <vscale x 2 x i64> %a,
-                                                                <vscale x 2 x i64> %splat)
+                                                                <vscale x 2 x i64> zeroinitializer)
   ret <vscale x 2 x i1> %out
 }
 
@@ -444,9 +378,7 @@ define <vscale x 16 x i1> @ir_cmple_b(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    ptrue p0.b
 ; CHECK-NEXT:    cmple p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %out = icmp sle <vscale x 16 x i8> %a, %splat
+  %out = icmp sle <vscale x 16 x i8> %a, splat(i8 4)
   ret <vscale x 16 x i1> %out
 }
 
@@ -455,10 +387,8 @@ define <vscale x 16 x i1> @int_cmple_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmple p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.nxv16i8(<vscale x 16 x i1> %pg,
-                                                                 <vscale x 16 x i8> %splat,
+                                                                 <vscale x 16 x i8> splat(i8 4),
                                                                  <vscale x 16 x i8> %a)
   ret <vscale x 16 x i1> %out
 }
@@ -468,11 +398,9 @@ define <vscale x 16 x i1> @wide_cmple_b(<vscale x 16 x i1> %pg, <vscale x 16 x i
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmple p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmple.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                       <vscale x 16 x i8> %a,
-                                                                      <vscale x 2 x i64> %splat)
+                                                                      <vscale x 2 x i64> splat(i64 4))
   ret <vscale x 16 x i1> %out
 }
 
@@ -482,9 +410,7 @@ define <vscale x 8 x i1> @ir_cmple_h(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    cmple p0.h, p0/z, z0.h, #-16
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %out = icmp sle <vscale x 8 x i16> %a, %splat
+  %out = icmp sle <vscale x 8 x i16> %a, splat(i16 -16)
   ret <vscale x 8 x i1> %out
 }
 
@@ -493,10 +419,8 @@ define <vscale x 8 x i1> @int_cmple_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmple p0.h, p0/z, z0.h, #-16
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.nxv8i16(<vscale x 8 x i1> %pg,
-                                                                <vscale x 8 x i16> %splat,
+                                                                <vscale x 8 x i16> splat(i16 -16),
                                                                 <vscale x 8 x i16> %a)
   ret <vscale x 8 x i1> %out
 }
@@ -506,11 +430,9 @@ define <vscale x 8 x i1> @wide_cmple_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmple p0.h, p0/z, z0.h, #-16
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 -16, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmple.wide.nxv8i16(<vscale x 8 x i1> %pg,
                                                                      <vscale x 8 x i16> %a,
-                                                                     <vscale x 2 x i64> %splat)
+                                                                     <vscale x 2 x i64> splat(i64 -16))
   ret <vscale x 8 x i1> %out
 }
 
@@ -520,9 +442,7 @@ define <vscale x 4 x i1> @ir_cmple_s(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    cmple p0.s, p0/z, z0.s, #15
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %out = icmp sle <vscale x 4 x i32> %a, %splat
+  %out = icmp sle <vscale x 4 x i32> %a, splat(i32 15)
   ret <vscale x 4 x i1> %out
 }
 
@@ -531,10 +451,8 @@ define <vscale x 4 x i1> @int_cmple_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmple p0.s, p0/z, z0.s, #15
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.nxv4i32(<vscale x 4 x i1> %pg,
-                                                                <vscale x 4 x i32> %splat,
+                                                                <vscale x 4 x i32> splat(i32 15),
                                                                 <vscale x 4 x i32> %a)
   ret <vscale x 4 x i1> %out
 }
@@ -544,11 +462,9 @@ define <vscale x 4 x i1> @wide_cmple_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmple p0.s, p0/z, z0.s, #15
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 15, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmple.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                      <vscale x 4 x i32> %a,
-                                                                     <vscale x 2 x i64> %splat)
+                                                                     <vscale x 2 x i64> splat(i64 15))
   ret <vscale x 4 x i1> %out
 }
 
@@ -558,9 +474,7 @@ define <vscale x 2 x i1> @ir_cmple_d(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    cmple p0.d, p0/z, z0.d, #0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %out = icmp sle <vscale x 2 x i64> %a, %splat
+  %out = icmp sle <vscale x 2 x i64> %a, zeroinitializer
   ret <vscale x 2 x i1> %out
 }
 
@@ -569,10 +483,8 @@ define <vscale x 2 x i1> @int_cmple_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmple p0.d, p0/z, z0.d, #0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpge.nxv2i64(<vscale x 2 x i1> %pg,
-                                                                <vscale x 2 x i64> %splat,
+                                                                <vscale x 2 x i64> zeroinitializer,
                                                                 <vscale x 2 x i64> %a)
   ret <vscale x 2 x i1> %out
 }
@@ -587,9 +499,7 @@ define <vscale x 16 x i1> @ir_cmplt_b(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    ptrue p0.b
 ; CHECK-NEXT:    cmplt p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %out = icmp slt <vscale x 16 x i8> %a, %splat
+  %out = icmp slt <vscale x 16 x i8> %a, splat(i8 4)
   ret <vscale x 16 x i1> %out
 }
 
@@ -598,10 +508,8 @@ define <vscale x 16 x i1> @int_cmplt_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmplt p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.nxv16i8(<vscale x 16 x i1> %pg,
-                                                                 <vscale x 16 x i8> %splat,
+                                                                 <vscale x 16 x i8> splat(i8 4),
                                                                  <vscale x 16 x i8> %a)
   ret <vscale x 16 x i1> %out
 }
@@ -611,11 +519,9 @@ define <vscale x 16 x i1> @wide_cmplt_b(<vscale x 16 x i1> %pg, <vscale x 16 x i
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmplt p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmplt.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                       <vscale x 16 x i8> %a,
-                                                                      <vscale x 2 x i64> %splat)
+                                                                      <vscale x 2 x i64> splat(i64 4))
   ret <vscale x 16 x i1> %out
 }
 
@@ -625,9 +531,7 @@ define <vscale x 8 x i1> @ir_cmplt_h(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    cmplt p0.h, p0/z, z0.h, #-16
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %out = icmp slt <vscale x 8 x i16> %a, %splat
+  %out = icmp slt <vscale x 8 x i16> %a, splat(i16 -16)
   ret <vscale x 8 x i1> %out
 }
 
@@ -636,10 +540,8 @@ define <vscale x 8 x i1> @int_cmplt_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmplt p0.h, p0/z, z0.h, #-16
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.nxv8i16(<vscale x 8 x i1> %pg,
-                                                                <vscale x 8 x i16> %splat,
+                                                                <vscale x 8 x i16> splat(i16 -16),
                                                                 <vscale x 8 x i16> %a)
   ret <vscale x 8 x i1> %out
 }
@@ -649,11 +551,9 @@ define <vscale x 8 x i1> @wide_cmplt_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmplt p0.h, p0/z, z0.h, #-16
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 -16, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmplt.wide.nxv8i16(<vscale x 8 x i1> %pg,
                                                                      <vscale x 8 x i16> %a,
-                                                                     <vscale x 2 x i64> %splat)
+                                                                     <vscale x 2 x i64> splat(i64 -16))
   ret <vscale x 8 x i1> %out
 }
 
@@ -663,9 +563,7 @@ define <vscale x 4 x i1> @ir_cmplt_s(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    cmplt p0.s, p0/z, z0.s, #15
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %out = icmp slt <vscale x 4 x i32> %a, %splat
+  %out = icmp slt <vscale x 4 x i32> %a, splat(i32 15)
   ret <vscale x 4 x i1> %out
 }
 
@@ -674,10 +572,8 @@ define <vscale x 4 x i1> @int_cmplt_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmplt p0.s, p0/z, z0.s, #15
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.nxv4i32(<vscale x 4 x i1> %pg,
-                                                                <vscale x 4 x i32> %splat,
+                                                                <vscale x 4 x i32> splat(i32 15),
                                                                 <vscale x 4 x i32> %a)
   ret <vscale x 4 x i1> %out
 }
@@ -687,11 +583,9 @@ define <vscale x 4 x i1> @wide_cmplt_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmplt p0.s, p0/z, z0.s, #15
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 15, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmplt.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                      <vscale x 4 x i32> %a,
-                                                                     <vscale x 2 x i64> %splat)
+                                                                     <vscale x 2 x i64> splat(i64 15))
   ret <vscale x 4 x i1> %out
 }
 
@@ -701,9 +595,7 @@ define <vscale x 2 x i1> @ir_cmplt_d(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    cmplt p0.d, p0/z, z0.d, #0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %out = icmp slt <vscale x 2 x i64> %a, %splat
+  %out = icmp slt <vscale x 2 x i64> %a, zeroinitializer
   ret <vscale x 2 x i1> %out
 }
 
@@ -712,10 +604,8 @@ define <vscale x 2 x i1> @int_cmplt_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmplt p0.d, p0/z, z0.d, #0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpgt.nxv2i64(<vscale x 2 x i1> %pg,
-                                                                <vscale x 2 x i64> %splat,
+                                                                <vscale x 2 x i64> zeroinitializer,
                                                                 <vscale x 2 x i64> %a)
   ret <vscale x 2 x i1> %out
 }
@@ -730,9 +620,7 @@ define <vscale x 16 x i1> @ir_cmpne_b(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    ptrue p0.b
 ; CHECK-NEXT:    cmpne p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %out = icmp ne <vscale x 16 x i8> %a, %splat
+  %out = icmp ne <vscale x 16 x i8> %a, splat(i8 4)
   ret <vscale x 16 x i1> %out
 }
 
@@ -741,11 +629,9 @@ define <vscale x 16 x i1> @int_cmpne_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpne p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpne.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %splat)
+                                                                 <vscale x 16 x i8> splat(i8 4))
   ret <vscale x 16 x i1> %out
 }
 
@@ -754,11 +640,9 @@ define <vscale x 16 x i1> @wide_cmpne_b(<vscale x 16 x i1> %pg, <vscale x 16 x i
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpne p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpne.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                       <vscale x 16 x i8> %a,
-                                                                      <vscale x 2 x i64> %splat)
+                                                                      <vscale x 2 x i64> splat(i64 4))
   ret <vscale x 16 x i1> %out
 }
 
@@ -768,9 +652,7 @@ define <vscale x 8 x i1> @ir_cmpne_h(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, #-16
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %out = icmp ne <vscale x 8 x i16> %a, %splat
+  %out = icmp ne <vscale x 8 x i16> %a, splat(i16 -16)
   ret <vscale x 8 x i1> %out
 }
 
@@ -779,11 +661,9 @@ define <vscale x 8 x i1> @int_cmpne_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, #-16
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x i16> undef, i16 -16, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpne.nxv8i16(<vscale x 8 x i1> %pg,
                                                                 <vscale x 8 x i16> %a,
-                                                                <vscale x 8 x i16> %splat)
+                                                                <vscale x 8 x i16> splat(i16 -16))
   ret <vscale x 8 x i1> %out
 }
 
@@ -792,11 +672,9 @@ define <vscale x 8 x i1> @wide_cmpne_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, #-16
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 -16, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpne.wide.nxv8i16(<vscale x 8 x i1> %pg,
                                                                      <vscale x 8 x i16> %a,
-                                                                     <vscale x 2 x i64> %splat)
+                                                                     <vscale x 2 x i64> splat(i64 -16))
   ret <vscale x 8 x i1> %out
 }
 
@@ -806,9 +684,7 @@ define <vscale x 4 x i1> @ir_cmpne_s(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    cmpne p0.s, p0/z, z0.s, #15
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %out = icmp ne <vscale x 4 x i32> %a, %splat
+  %out = icmp ne <vscale x 4 x i32> %a, splat(i32 15)
   ret <vscale x 4 x i1> %out
 }
 
@@ -817,11 +693,9 @@ define <vscale x 4 x i1> @int_cmpne_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpne p0.s, p0/z, z0.s, #15
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x i32> undef, i32 15, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpne.nxv4i32(<vscale x 4 x i1> %pg,
                                                                 <vscale x 4 x i32> %a,
-                                                                <vscale x 4 x i32> %splat)
+                                                                <vscale x 4 x i32> splat(i32 15))
   ret <vscale x 4 x i1> %out
 }
 
@@ -830,11 +704,9 @@ define <vscale x 4 x i1> @wide_cmpne_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpne p0.s, p0/z, z0.s, #15
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 15, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpne.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                      <vscale x 4 x i32> %a,
-                                                                     <vscale x 2 x i64> %splat)
+                                                                     <vscale x 2 x i64> splat(i64 15))
   ret <vscale x 4 x i1> %out
 }
 
@@ -844,9 +716,7 @@ define <vscale x 2 x i1> @ir_cmpne_d(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    cmpne p0.d, p0/z, z0.d, #0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %out = icmp ne <vscale x 2 x i64> %a, %splat
+  %out = icmp ne <vscale x 2 x i64> %a, zeroinitializer
   ret <vscale x 2 x i1> %out
 }
 
@@ -855,11 +725,9 @@ define <vscale x 2 x i1> @int_cmpne_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpne p0.d, p0/z, z0.d, #0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpne.nxv2i64(<vscale x 2 x i1> %pg,
                                                                 <vscale x 2 x i64> %a,
-                                                                <vscale x 2 x i64> %splat)
+                                                                <vscale x 2 x i64> zeroinitializer)
   ret <vscale x 2 x i1> %out
 }
 
@@ -877,9 +745,7 @@ define <vscale x 16 x i1> @ir_cmphi_b(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    ptrue p0.b
 ; CHECK-NEXT:    cmphi p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %out = icmp ugt <vscale x 16 x i8> %a, %splat
+  %out = icmp ugt <vscale x 16 x i8> %a, splat(i8 4)
   ret <vscale x 16 x i1> %out
 }
 
@@ -888,11 +754,9 @@ define <vscale x 16 x i1> @int_cmphi_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmphi p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %splat)
+                                                                 <vscale x 16 x i8> splat(i8 4))
   ret <vscale x 16 x i1> %out
 }
 
@@ -901,11 +765,9 @@ define <vscale x 16 x i1> @wide_cmphi_b(<vscale x 16 x i1> %pg, <vscale x 16 x i
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmphi p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                       <vscale x 16 x i8> %a,
-                                                                      <vscale x 2 x i64> %splat)
+                                                                      <vscale x 2 x i64> splat(i64 4))
   ret <vscale x 16 x i1> %out
 }
 
@@ -915,9 +777,7 @@ define <vscale x 8 x i1> @ir_cmphi_h(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, #0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %out = icmp ugt <vscale x 8 x i16> %a, %splat
+  %out = icmp ugt <vscale x 8 x i16> %a, zeroinitializer
   ret <vscale x 8 x i1> %out
 }
 
@@ -926,11 +786,9 @@ define <vscale x 8 x i1> @int_cmphi_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmphi p0.h, p0/z, z0.h, #0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.nxv8i16(<vscale x 8 x i1> %pg,
                                                                 <vscale x 8 x i16> %a,
-                                                                <vscale x 8 x i16> %splat)
+                                                                <vscale x 8 x i16> zeroinitializer)
   ret <vscale x 8 x i1> %out
 }
 
@@ -939,11 +797,9 @@ define <vscale x 8 x i1> @wide_cmphi_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmphi p0.h, p0/z, z0.h, #0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.wide.nxv8i16(<vscale x 8 x i1> %pg,
                                                                      <vscale x 8 x i16> %a,
-                                                                     <vscale x 2 x i64> %splat)
+                                                                     <vscale x 2 x i64> zeroinitializer)
   ret <vscale x 8 x i1> %out
 }
 
@@ -953,9 +809,7 @@ define <vscale x 4 x i1> @ir_cmphi_s(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    cmphi p0.s, p0/z, z0.s, #68
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %out = icmp ugt <vscale x 4 x i32> %a, %splat
+  %out = icmp ugt <vscale x 4 x i32> %a, splat(i32 68)
   ret <vscale x 4 x i1> %out
 }
 
@@ -964,11 +818,9 @@ define <vscale x 4 x i1> @int_cmphi_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmphi p0.s, p0/z, z0.s, #68
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.nxv4i32(<vscale x 4 x i1> %pg,
                                                                 <vscale x 4 x i32> %a,
-                                                                <vscale x 4 x i32> %splat)
+                                                                <vscale x 4 x i32> splat(i32 68))
   ret <vscale x 4 x i1> %out
 }
 
@@ -977,11 +829,9 @@ define <vscale x 4 x i1> @wide_cmphi_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmphi p0.s, p0/z, z0.s, #68
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 68, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                      <vscale x 4 x i32> %a,
-                                                                     <vscale x 2 x i64> %splat)
+                                                                     <vscale x 2 x i64> splat(i64 68))
   ret <vscale x 4 x i1> %out
 }
 
@@ -991,9 +841,7 @@ define <vscale x 2 x i1> @ir_cmphi_d(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    cmphi p0.d, p0/z, z0.d, #127
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %out = icmp ugt <vscale x 2 x i64> %a, %splat
+  %out = icmp ugt <vscale x 2 x i64> %a, splat(i64 127)
   ret <vscale x 2 x i1> %out
 }
 
@@ -1002,11 +850,9 @@ define <vscale x 2 x i1> @int_cmphi_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmphi p0.d, p0/z, z0.d, #127
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmphi.nxv2i64(<vscale x 2 x i1> %pg,
                                                                 <vscale x 2 x i64> %a,
-                                                                <vscale x 2 x i64> %splat)
+                                                                <vscale x 2 x i64> splat(i64 127))
   ret <vscale x 2 x i1> %out
 }
 
@@ -1020,9 +866,7 @@ define <vscale x 16 x i1> @ir_cmphs_b(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    ptrue p0.b
 ; CHECK-NEXT:    cmphs p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %out = icmp uge <vscale x 16 x i8> %a, %splat
+  %out = icmp uge <vscale x 16 x i8> %a, splat(i8 4)
   ret <vscale x 16 x i1> %out
 }
 
@@ -1031,11 +875,9 @@ define <vscale x 16 x i1> @int_cmphs_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmphs p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %splat)
+                                                                 <vscale x 16 x i8> splat(i8 4))
   ret <vscale x 16 x i1> %out
 }
 
@@ -1044,11 +886,9 @@ define <vscale x 16 x i1> @wide_cmphs_b(<vscale x 16 x i1> %pg, <vscale x 16 x i
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmphs p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                       <vscale x 16 x i8> %a,
-                                                                      <vscale x 2 x i64> %splat)
+                                                                      <vscale x 2 x i64> splat(i64 4))
   ret <vscale x 16 x i1> %out
 }
 
@@ -1057,9 +897,7 @@ define <vscale x 8 x i1> @ir_cmphs_h(<vscale x 8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %out = icmp uge <vscale x 8 x i16> %a, %splat
+  %out = icmp uge <vscale x 8 x i16> %a, zeroinitializer
   ret <vscale x 8 x i1> %out
 }
 
@@ -1068,11 +906,9 @@ define <vscale x 8 x i1> @int_cmphs_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmphs p0.h, p0/z, z0.h, #0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.nxv8i16(<vscale x 8 x i1> %pg,
                                                                 <vscale x 8 x i16> %a,
-                                                                <vscale x 8 x i16> %splat)
+                                                                <vscale x 8 x i16> zeroinitializer)
   ret <vscale x 8 x i1> %out
 }
 
@@ -1081,11 +917,9 @@ define <vscale x 8 x i1> @wide_cmphs_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmphs p0.h, p0/z, z0.h, #0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.wide.nxv8i16(<vscale x 8 x i1> %pg,
                                                                      <vscale x 8 x i16> %a,
-                                                                     <vscale x 2 x i64> %splat)
+                                                                     <vscale x 2 x i64> zeroinitializer)
   ret <vscale x 8 x i1> %out
 }
 
@@ -1095,9 +929,7 @@ define <vscale x 4 x i1> @ir_cmphs_s(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    cmphs p0.s, p0/z, z0.s, #68
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %out = icmp uge <vscale x 4 x i32> %a, %splat
+  %out = icmp uge <vscale x 4 x i32> %a, splat(i32 68)
   ret <vscale x 4 x i1> %out
 }
 
@@ -1106,11 +938,9 @@ define <vscale x 4 x i1> @int_cmphs_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmphs p0.s, p0/z, z0.s, #68
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.nxv4i32(<vscale x 4 x i1> %pg,
                                                                 <vscale x 4 x i32> %a,
-                                                                <vscale x 4 x i32> %splat)
+                                                                <vscale x 4 x i32> splat(i32 68))
   ret <vscale x 4 x i1> %out
 }
 
@@ -1119,11 +949,9 @@ define <vscale x 4 x i1> @wide_cmphs_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmphs p0.s, p0/z, z0.s, #68
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 68, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                      <vscale x 4 x i32> %a,
-                                                                     <vscale x 2 x i64> %splat)
+                                                                     <vscale x 2 x i64> splat(i64 68))
   ret <vscale x 4 x i1> %out
 }
 
@@ -1133,9 +961,7 @@ define <vscale x 2 x i1> @ir_cmphs_d(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    cmphs p0.d, p0/z, z0.d, #127
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %out = icmp uge <vscale x 2 x i64> %a, %splat
+  %out = icmp uge <vscale x 2 x i64> %a, splat(i64 127)
   ret <vscale x 2 x i1> %out
 }
 
@@ -1144,11 +970,9 @@ define <vscale x 2 x i1> @int_cmphs_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmphs p0.d, p0/z, z0.d, #127
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmphs.nxv2i64(<vscale x 2 x i1> %pg,
                                                                 <vscale x 2 x i64> %a,
-                                                                <vscale x 2 x i64> %splat)
+                                                                <vscale x 2 x i64> splat(i64 127))
   ret <vscale x 2 x i1> %out
 }
 
@@ -1162,9 +986,7 @@ define <vscale x 16 x i1> @ir_cmplo_b(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    ptrue p0.b
 ; CHECK-NEXT:    cmplo p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %out = icmp ult <vscale x 16 x i8> %a, %splat
+  %out = icmp ult <vscale x 16 x i8> %a, splat(i8 4)
   ret <vscale x 16 x i1> %out
 }
 
@@ -1173,10 +995,8 @@ define <vscale x 16 x i1> @int_cmplo_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmplo p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.nxv16i8(<vscale x 16 x i1> %pg,
-                                                                 <vscale x 16 x i8> %splat,
+                                                                 <vscale x 16 x i8> splat(i8 4),
                                                                  <vscale x 16 x i8> %a)
   ret <vscale x 16 x i1> %out
 }
@@ -1186,11 +1006,9 @@ define <vscale x 16 x i1> @wide_cmplo_b(<vscale x 16 x i1> %pg, <vscale x 16 x i
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmplo p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmplo.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                       <vscale x 16 x i8> %a,
-                                                                      <vscale x 2 x i64> %splat)
+                                                                      <vscale x 2 x i64> splat(i64 4))
   ret <vscale x 16 x i1> %out
 }
 
@@ -1200,9 +1018,7 @@ define <vscale x 8 x i1> @ir_cmplo_h(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    cmplo p0.h, p0/z, z0.h, #2
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x i16> undef, i16 2, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %out = icmp ult <vscale x 8 x i16> %a, %splat
+  %out = icmp ult <vscale x 8 x i16> %a, splat(i16 2)
   ret <vscale x 8 x i1> %out
 }
 
@@ -1211,10 +1027,8 @@ define <vscale x 8 x i1> @int_cmplo_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmplo p0.h, p0/z, z0.h, #3
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x i16> undef, i16 3, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.nxv8i16(<vscale x 8 x i1> %pg,
-                                                                <vscale x 8 x i16> %splat,
+                                                                <vscale x 8 x i16> splat(i16 3),
                                                                 <vscale x 8 x i16> %a)
   ret <vscale x 8 x i1> %out
 }
@@ -1224,11 +1038,9 @@ define <vscale x 8 x i1> @wide_cmplo_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmplo p0.h, p0/z, z0.h, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmplo.wide.nxv8i16(<vscale x 8 x i1> %pg,
                                                                      <vscale x 8 x i16> %a,
-                                                                     <vscale x 2 x i64> %splat)
+                                                                     <vscale x 2 x i64> splat(i64 4))
   ret <vscale x 8 x i1> %out
 }
 
@@ -1238,9 +1050,7 @@ define <vscale x 4 x i1> @ir_cmplo_s(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    cmplo p0.s, p0/z, z0.s, #68
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %out = icmp ult <vscale x 4 x i32> %a, %splat
+  %out = icmp ult <vscale x 4 x i32> %a, splat(i32 68)
   ret <vscale x 4 x i1> %out
 }
 
@@ -1249,10 +1059,8 @@ define <vscale x 4 x i1> @int_cmplo_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmplo p0.s, p0/z, z0.s, #68
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.nxv4i32(<vscale x 4 x i1> %pg,
-                                                                <vscale x 4 x i32> %splat,
+                                                                <vscale x 4 x i32> splat(i32 68),
                                                                 <vscale x 4 x i32> %a)
   ret <vscale x 4 x i1> %out
 }
@@ -1262,11 +1070,9 @@ define <vscale x 4 x i1> @wide_cmplo_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmplo p0.s, p0/z, z0.s, #68
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 68, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmplo.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                      <vscale x 4 x i32> %a,
-                                                                     <vscale x 2 x i64> %splat)
+                                                                     <vscale x 2 x i64> splat(i64 68))
   ret <vscale x 4 x i1> %out
 }
 
@@ -1276,9 +1082,7 @@ define <vscale x 2 x i1> @ir_cmplo_d(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    cmplo p0.d, p0/z, z0.d, #127
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %out = icmp ult <vscale x 2 x i64> %a, %splat
+  %out = icmp ult <vscale x 2 x i64> %a, splat(i64 127)
   ret <vscale x 2 x i1> %out
 }
 
@@ -1287,10 +1091,8 @@ define <vscale x 2 x i1> @int_cmplo_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmplo p0.d, p0/z, z0.d, #127
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmphi.nxv2i64(<vscale x 2 x i1> %pg,
-                                                                <vscale x 2 x i64> %splat,
+                                                                <vscale x 2 x i64> splat(i64 127),
                                                                 <vscale x 2 x i64> %a)
   ret <vscale x 2 x i1> %out
 }
@@ -1305,9 +1107,7 @@ define <vscale x 16 x i1> @ir_cmpls_b(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    ptrue p0.b
 ; CHECK-NEXT:    cmpls p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  %out = icmp ule <vscale x 16 x i8> %a, %splat
+  %out = icmp ule <vscale x 16 x i8> %a, splat(i8 4)
   ret <vscale x 16 x i1> %out
 }
 
@@ -1316,10 +1116,8 @@ define <vscale x 16 x i1> @int_cmpls_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpls p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 16 x i8> undef, i8 4, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.nxv16i8(<vscale x 16 x i1> %pg,
-                                                                 <vscale x 16 x i8> %splat,
+                                                                 <vscale x 16 x i8> splat(i8 4),
                                                                  <vscale x 16 x i8> %a)
   ret <vscale x 16 x i1> %out
 }
@@ -1329,11 +1127,9 @@ define <vscale x 16 x i1> @wide_cmpls_b(<vscale x 16 x i1> %pg, <vscale x 16 x i
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpls p0.b, p0/z, z0.b, #4
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 4, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpls.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                       <vscale x 16 x i8> %a,
-                                                                      <vscale x 2 x i64> %splat)
+                                                                      <vscale x 2 x i64> splat(i64 4))
   ret <vscale x 16 x i1> %out
 }
 
@@ -1343,9 +1139,7 @@ define <vscale x 8 x i1> @ir_cmpls_h(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    cmpls p0.h, p0/z, z0.h, #0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %out = icmp ule <vscale x 8 x i16> %a, %splat
+  %out = icmp ule <vscale x 8 x i16> %a, zeroinitializer
   ret <vscale x 8 x i1> %out
 }
 
@@ -1354,10 +1148,8 @@ define <vscale x 8 x i1> @int_cmpls_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpls p0.h, p0/z, z0.h, #0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 8 x i16> undef, i16 0, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.nxv8i16(<vscale x 8 x i1> %pg,
-                                                                <vscale x 8 x i16> %splat,
+                                                                <vscale x 8 x i16> zeroinitializer,
                                                                 <vscale x 8 x i16> %a)
   ret <vscale x 8 x i1> %out
 }
@@ -1367,11 +1159,9 @@ define <vscale x 8 x i1> @wide_cmpls_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpls p0.h, p0/z, z0.h, #0
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 0, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpls.wide.nxv8i16(<vscale x 8 x i1> %pg,
                                                                      <vscale x 8 x i16> %a,
-                                                                     <vscale x 2 x i64> %splat)
+                                                                     <vscale x 2 x i64> zeroinitializer)
   ret <vscale x 8 x i1> %out
 }
 
@@ -1381,9 +1171,7 @@ define <vscale x 4 x i1> @ir_cmpls_s(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    cmpls p0.s, p0/z, z0.s, #68
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %out = icmp ule <vscale x 4 x i32> %a, %splat
+  %out = icmp ule <vscale x 4 x i32> %a, splat(i32 68)
   ret <vscale x 4 x i1> %out
 }
 
@@ -1392,10 +1180,8 @@ define <vscale x 4 x i1> @int_cmpls_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpls p0.s, p0/z, z0.s, #68
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 4 x i32> undef, i32 68, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.nxv4i32(<vscale x 4 x i1> %pg,
-                                                                <vscale x 4 x i32> %splat,
+                                                                <vscale x 4 x i32> splat(i32 68),
                                                                 <vscale x 4 x i32> %a)
   ret <vscale x 4 x i1> %out
 }
@@ -1405,11 +1191,9 @@ define <vscale x 4 x i1> @wide_cmpls_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpls p0.s, p0/z, z0.s, #68
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 68, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpls.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                      <vscale x 4 x i32> %a,
-                                                                     <vscale x 2 x i64> %splat)
+                                                                     <vscale x 2 x i64> splat(i64 68))
   ret <vscale x 4 x i1> %out
 }
 
@@ -1419,9 +1203,7 @@ define <vscale x 2 x i1> @ir_cmpls_d(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    cmpls p0.d, p0/z, z0.d, #127
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %out = icmp ule <vscale x 2 x i64> %a, %splat
+  %out = icmp ule <vscale x 2 x i64> %a, splat(i64 127)
   ret <vscale x 2 x i1> %out
 }
 
@@ -1430,10 +1212,8 @@ define <vscale x 2 x i1> @int_cmpls_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmpls p0.d, p0/z, z0.d, #127
 ; CHECK-NEXT:    ret
-  %elt   = insertelement <vscale x 2 x i64> undef, i64 127, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmphs.nxv2i64(<vscale x 2 x i1> %pg,
-                                                                <vscale x 2 x i64> %splat,
+                                                                <vscale x 2 x i64> splat(i64 127),
                                                                 <vscale x 2 x i64> %a)
   ret <vscale x 2 x i1> %out
 }
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-logical-undef.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-logical-undef.ll
index 07413c5a1617d..d3f4f89b60af3 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-logical-undef.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-logical-undef.ll
@@ -60,11 +60,9 @@ define <vscale x 16 x i8> @and_imm_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and z0.b, z0.b, #0x3
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 16 x i8> undef, i8 3, i32 0
-  %imm.splat = shufflevector <vscale x 16 x i8> %imm, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.and.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %imm.splat)
+                                                                 <vscale x 16 x i8> splat(i8 3))
   ret <vscale x 16 x i8> %out
 }
 
@@ -73,11 +71,9 @@ define <vscale x 8 x i16> @and_imm_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and z0.h, z0.h, #0x4
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 8 x i16> undef, i16 4, i32 0
-  %imm.splat = shufflevector <vscale x 8 x i16> %imm, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.and.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %imm.splat)
+                                                                 <vscale x 8 x i16> splat(i16 4))
   ret <vscale x 8 x i16> %out
 }
 
@@ -86,11 +82,9 @@ define <vscale x 4 x i32> @and_imm_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and z0.s, z0.s, #0x10
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 4 x i32> undef, i32 16, i32 0
-  %imm.splat = shufflevector <vscale x 4 x i32> %imm, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %imm.splat)
+                                                                 <vscale x 4 x i32> splat(i32 16))
   ret <vscale x 4 x i32> %out
 }
 
@@ -99,11 +93,9 @@ define <vscale x 2 x i64> @and_imm_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and z0.d, z0.d, #0x20
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 2 x i64> undef, i64 32, i32 0
-  %imm.splat = shufflevector <vscale x 2 x i64> %imm, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.and.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %imm.splat)
+                                                                 <vscale x 2 x i64> splat(i64 32))
   ret <vscale x 2 x i64> %out
 }
 
@@ -164,11 +156,9 @@ define <vscale x 16 x i8> @eor_imm_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    eor z0.b, z0.b, #0x7
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 16 x i8> undef, i8 7, i32 0
-  %imm.splat = shufflevector <vscale x 16 x i8> %imm, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.eor.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %imm.splat)
+                                                                 <vscale x 16 x i8> splat(i8 7))
   ret <vscale x 16 x i8> %out
 }
 
@@ -177,11 +167,9 @@ define <vscale x 8 x i16> @eor_imm_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    eor z0.h, z0.h, #0x8
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 8 x i16> undef, i16 8, i32 0
-  %imm.splat = shufflevector <vscale x 8 x i16> %imm, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.eor.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %imm.splat)
+                                                                 <vscale x 8 x i16> splat(i16 8))
   ret <vscale x 8 x i16> %out
 }
 
@@ -190,11 +178,9 @@ define <vscale x 4 x i32> @eor_imm_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    eor z0.s, z0.s, #0x10
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 4 x i32> undef, i32 16, i32 0
-  %imm.splat = shufflevector <vscale x 4 x i32> %imm, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %imm.splat)
+                                                                 <vscale x 4 x i32> splat(i32 16))
   ret <vscale x 4 x i32> %out
 }
 
@@ -203,11 +189,9 @@ define <vscale x 2 x i64> @eor_imm_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    eor z0.d, z0.d, #0x20
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 2 x i64> undef, i64 32, i32 0
-  %imm.splat = shufflevector <vscale x 2 x i64> %imm, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.eor.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %imm.splat)
+                                                                 <vscale x 2 x i64> splat(i64 32))
   ret <vscale x 2 x i64> %out
 }
 
@@ -268,11 +252,9 @@ define <vscale x 16 x i8> @orr_imm_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    orr z0.b, z0.b, #0x8
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 16 x i8> undef, i8 8, i32 0
-  %imm.splat = shufflevector <vscale x 16 x i8> %imm, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.orr.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %imm.splat)
+                                                                 <vscale x 16 x i8> splat(i8 8))
   ret <vscale x 16 x i8> %out
 }
 
@@ -281,11 +263,9 @@ define <vscale x 8 x i16> @orr_imm_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    orr z0.h, z0.h, #0xc
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 8 x i16> undef, i16 12, i32 0
-  %imm.splat = shufflevector <vscale x 8 x i16> %imm, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.orr.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %imm.splat)
+                                                                 <vscale x 8 x i16> splat(i16 12))
   ret <vscale x 8 x i16> %out
 }
 
@@ -294,11 +274,9 @@ define <vscale x 4 x i32> @orr_imm_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    orr z0.s, z0.s, #0x10
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 4 x i32> undef, i32 16, i32 0
-  %imm.splat = shufflevector <vscale x 4 x i32> %imm, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.orr.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %imm.splat)
+                                                                 <vscale x 4 x i32> splat(i32 16))
   ret <vscale x 4 x i32> %out
 }
 
@@ -307,11 +285,9 @@ define <vscale x 2 x i64> @orr_imm_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    orr z0.d, z0.d, #0x20
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 2 x i64> undef, i64 32, i32 0
-  %imm.splat = shufflevector <vscale x 2 x i64> %imm, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.orr.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %imm.splat)
+                                                                 <vscale x 2 x i64> splat(i64 32))
   ret <vscale x 2 x i64> %out
 }
 
@@ -372,11 +348,9 @@ define <vscale x 16 x i8> @bic_imm_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and z0.b, z0.b, #0xf8
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 16 x i8> undef, i8 7, i32 0
-  %imm.splat = shufflevector <vscale x 16 x i8> %imm, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.bic.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %imm.splat)
+                                                                 <vscale x 16 x i8> splat(i8 7))
   ret <vscale x 16 x i8> %out
 }
 
@@ -385,11 +359,9 @@ define <vscale x 8 x i16> @bic_imm_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and z0.h, z0.h, #0xfff7
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 8 x i16> undef, i16 8, i32 0
-  %imm.splat = shufflevector <vscale x 8 x i16> %imm, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.bic.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %imm.splat)
+                                                                 <vscale x 8 x i16> splat(i16 8))
   ret <vscale x 8 x i16> %out
 }
 
@@ -398,11 +370,9 @@ define <vscale x 4 x i32> @bic_imm_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and z0.s, z0.s, #0xffffffef
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 4 x i32> undef, i32 16, i32 0
-  %imm.splat = shufflevector <vscale x 4 x i32> %imm, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.bic.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %imm.splat)
+                                                                 <vscale x 4 x i32> splat(i32 16))
   ret <vscale x 4 x i32> %out
 }
 
@@ -411,11 +381,9 @@ define <vscale x 2 x i64> @bic_imm_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and z0.d, z0.d, #0xffffffffffffffdf
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 2 x i64> undef, i64 32, i32 0
-  %imm.splat = shufflevector <vscale x 2 x i64> %imm, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.bic.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %imm.splat)
+                                                                 <vscale x 2 x i64> splat(i64 32))
   ret <vscale x 2 x i64> %out
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-reinterpret-no-streaming.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-reinterpret-no-streaming.ll
index bc5cdb48fef61..3986a5a79d57d 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-reinterpret-no-streaming.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-reinterpret-no-streaming.ll
@@ -9,9 +9,7 @@ define <vscale x 16 x i1> @reinterpret_bool_from_splat() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 2 x i1> undef, i1 1, i32 0
-  %splat = shufflevector <vscale x 2 x i1> %ins, <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer
-  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1> %splat)
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1> splat(i1 true))
   ret <vscale x 16 x i1> %out
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-shifts-undef.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-shifts-undef.ll
index 70c2e5298c897..370eea26450f3 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-shifts-undef.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-shifts-undef.ll
@@ -60,11 +60,9 @@ define <vscale x 16 x i8> @asr_imm_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    asr z0.b, z0.b, #3
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 16 x i8> undef, i8 3, i32 0
-  %imm.splat = shufflevector <vscale x 16 x i8> %imm, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.asr.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %imm.splat)
+                                                                 <vscale x 16 x i8> splat(i8 3))
   ret <vscale x 16 x i8> %out
 }
 
@@ -73,11 +71,9 @@ define <vscale x 8 x i16> @asr_imm_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    asr z0.h, z0.h, #4
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 8 x i16> undef, i16 4, i32 0
-  %imm.splat = shufflevector <vscale x 8 x i16> %imm, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.asr.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %imm.splat)
+                                                                 <vscale x 8 x i16> splat(i16 4))
   ret <vscale x 8 x i16> %out
 }
 
@@ -86,11 +82,9 @@ define <vscale x 4 x i32> @asr_imm_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    asr z0.s, z0.s, #5
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 4 x i32> undef, i32 5, i32 0
-  %imm.splat = shufflevector <vscale x 4 x i32> %imm, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.asr.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %imm.splat)
+                                                                 <vscale x 4 x i32> splat(i32 5))
   ret <vscale x 4 x i32> %out
 }
 
@@ -99,11 +93,9 @@ define <vscale x 2 x i64> @asr_imm_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    asr z0.d, z0.d, #6
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 2 x i64> undef, i64 6, i32 0
-  %imm.splat = shufflevector <vscale x 2 x i64> %imm, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.asr.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %imm.splat)
+                                                                 <vscale x 2 x i64> splat(i64 6))
   ret <vscale x 2 x i64> %out
 }
 
@@ -164,11 +156,9 @@ define <vscale x 16 x i8> @lsl_imm_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsl z0.b, z0.b, #7
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 16 x i8> undef, i8 7, i32 0
-  %imm.splat = shufflevector <vscale x 16 x i8> %imm, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.lsl.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %imm.splat)
+                                                                 <vscale x 16 x i8> splat(i8 7))
   ret <vscale x 16 x i8> %out
 }
 
@@ -177,11 +167,9 @@ define <vscale x 8 x i16> @lsl_imm_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsl z0.h, z0.h, #8
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 8 x i16> undef, i16 8, i32 0
-  %imm.splat = shufflevector <vscale x 8 x i16> %imm, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.lsl.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %imm.splat)
+                                                                 <vscale x 8 x i16> splat(i16 8))
   ret <vscale x 8 x i16> %out
 }
 
@@ -190,11 +178,9 @@ define <vscale x 4 x i32> @lsl_imm_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsl z0.s, z0.s, #9
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 4 x i32> undef, i32 9, i32 0
-  %imm.splat = shufflevector <vscale x 4 x i32> %imm, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.lsl.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %imm.splat)
+                                                                 <vscale x 4 x i32> splat(i32 9))
   ret <vscale x 4 x i32> %out
 }
 
@@ -203,11 +189,9 @@ define <vscale x 2 x i64> @lsl_imm_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsl z0.d, z0.d, #10
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 2 x i64> undef, i64 10, i32 0
-  %imm.splat = shufflevector <vscale x 2 x i64> %imm, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.lsl.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %imm.splat)
+                                                                 <vscale x 2 x i64> splat(i64 10))
   ret <vscale x 2 x i64> %out
 }
 
@@ -268,11 +252,9 @@ define <vscale x 16 x i8> @lsr_imm_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr z0.b, z0.b, #8
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 16 x i8> undef, i8 8, i32 0
-  %imm.splat = shufflevector <vscale x 16 x i8> %imm, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.lsr.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %imm.splat)
+                                                                 <vscale x 16 x i8> splat(i8 8))
   ret <vscale x 16 x i8> %out
 }
 
@@ -281,11 +263,9 @@ define <vscale x 8 x i16> @lsr_imm_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr z0.h, z0.h, #12
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 8 x i16> undef, i16 12, i32 0
-  %imm.splat = shufflevector <vscale x 8 x i16> %imm, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.lsr.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %imm.splat)
+                                                                 <vscale x 8 x i16> splat(i16 12))
   ret <vscale x 8 x i16> %out
 }
 
@@ -294,11 +274,9 @@ define <vscale x 4 x i32> @lsr_imm_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr z0.s, z0.s, #13
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 4 x i32> undef, i32 13, i32 0
-  %imm.splat = shufflevector <vscale x 4 x i32> %imm, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.lsr.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %imm.splat)
+                                                                 <vscale x 4 x i32> splat(i32 13))
   ret <vscale x 4 x i32> %out
 }
 
@@ -307,11 +285,9 @@ define <vscale x 2 x i64> @lsr_imm_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr z0.d, z0.d, #14
 ; CHECK-NEXT:    ret
-  %imm = insertelement <vscale x 2 x i64> undef, i64 14, i32 0
-  %imm.splat = shufflevector <vscale x 2 x i64> %imm, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.lsr.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %imm.splat)
+                                                                 <vscale x 2 x i64> splat(i64 14))
   ret <vscale x 2 x i64> %out
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-while.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-while.ll
index ba7826aaae773..8380d5ca55cf2 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-while.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-while.ll
@@ -81,7 +81,7 @@ define <vscale x 2 x i1> @whilele_d_xx(i64 %a, i64 %b) {
 define <vscale x 2 x i1> @whilele_d_ii_dont_fold_to_ptrue_larger_than_minvec() {
 ; CHECK-LABEL: whilele_d_ii_dont_fold_to_ptrue_larger_than_minvec:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #3
+; CHECK-NEXT:    mov w8, #3 // =0x3
 ; CHECK-NEXT:    whilele p0.d, xzr, x8
 ; CHECK-NEXT:    ret
   %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilele.nxv2i1.i64(i64 0, i64 3)
@@ -90,21 +90,19 @@ define <vscale x 2 x i1> @whilele_d_ii_dont_fold_to_ptrue_larger_than_minvec() {
 
 define <vscale x 16 x i1> @whilele_b_ii() {
 ; CHECK-LABEL: whilele_b_ii:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.b, vl6
 ; CHECK-NEXT:    ret
-entry:
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilele.nxv16i1.i64(i64 -2, i64 3)
   ret <vscale x 16 x i1> %out
 }
 
 define <vscale x 16 x i1> @whilele_b_ii_dont_fold_to_ptrue_nonexistent_vl9() {
 ; CHECK-LABEL: whilele_b_ii_dont_fold_to_ptrue_nonexistent_vl9:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #9
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #9 // =0x9
 ; CHECK-NEXT:    whilele p0.b, xzr, x8
 ; CHECK-NEXT:    ret
-entry:
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilele.nxv16i1.i64(i64 0, i64 9)
   ret <vscale x 16 x i1> %out
 }
@@ -120,24 +118,22 @@ define <vscale x 16 x i1> @whilele_b_vl_maximum() vscale_range(16, 16) {
 
 define <vscale x 16 x i1> @whilele_b_ii_dont_fold_to_ptrue_overflow() {
 ; CHECK-LABEL: whilele_b_ii_dont_fold_to_ptrue_overflow:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #2
-; CHECK-NEXT:    mov w9, #2147483647
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #2 // =0x2
+; CHECK-NEXT:    mov w9, #2147483647 // =0x7fffffff
 ; CHECK-NEXT:    movk w8, #32768, lsl #16
 ; CHECK-NEXT:    whilele p0.b, w9, w8
 ; CHECK-NEXT:    ret
-entry:
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilele.nxv16i1.i32(i32 2147483647, i32 -2147483646)
   ret <vscale x 16 x i1> %out
 }
 
 define <vscale x 16 x i1> @whilele_b_ii_dont_fold_to_ptrue_increment_overflow() {
 ; CHECK-LABEL: whilele_b_ii_dont_fold_to_ptrue_increment_overflow:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #2147483647
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #2147483647 // =0x7fffffff
 ; CHECK-NEXT:    whilele p0.b, wzr, w8
 ; CHECK-NEXT:    ret
-entry:
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilele.nxv16i1.i32(i32 0, i32 2147483647)
   ret <vscale x 16 x i1> %out
 }
@@ -221,7 +217,7 @@ define <vscale x 2 x i1> @whilelo_d_xx(i64 %a, i64 %b) {
 define <vscale x 2 x i1> @whilelo_d_ii_dont_fold_to_ptrue_larger_than_minvec() {
 ; CHECK-LABEL: whilelo_d_ii_dont_fold_to_ptrue_larger_than_minvec:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #3
+; CHECK-NEXT:    mov w8, #3 // =0x3
 ; CHECK-NEXT:    whilelo p0.d, xzr, x8
 ; CHECK-NEXT:    ret
   %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilelo.nxv2i1.i64(i64 0, i64 3)
@@ -230,21 +226,19 @@ define <vscale x 2 x i1> @whilelo_d_ii_dont_fold_to_ptrue_larger_than_minvec() {
 
 define <vscale x 16 x i1> @whilelo_b_ii() {
 ; CHECK-LABEL: whilelo_b_ii:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.b, vl6
 ; CHECK-NEXT:    ret
-entry:
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilelo.nxv16i1.i64(i64 2, i64 8)
   ret <vscale x 16 x i1> %out
 }
 
 define <vscale x 16 x i1> @whilelo_b_ii_dont_fold_to_ptrue_nonexistent_vl9() {
 ; CHECK-LABEL: whilelo_b_ii_dont_fold_to_ptrue_nonexistent_vl9:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #9
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #9 // =0x9
 ; CHECK-NEXT:    whilelo p0.b, xzr, x8
 ; CHECK-NEXT:    ret
-entry:
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilelo.nxv16i1.i64(i64 0, i64 9)
   ret <vscale x 16 x i1> %out
 }
@@ -260,12 +254,11 @@ define <vscale x 16 x i1> @whilelo_b_vl_maximum() vscale_range(16, 16) {
 
 define <vscale x 16 x i1> @whilelo_b_ii_dont_fold_to_ptrue_overflow() {
 ; CHECK-LABEL: whilelo_b_ii_dont_fold_to_ptrue_overflow:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #6
-; CHECK-NEXT:    mov w9, #-1
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #6 // =0x6
+; CHECK-NEXT:    mov w9, #-1 // =0xffffffff
 ; CHECK-NEXT:    whilelo p0.b, w9, w8
 ; CHECK-NEXT:    ret
-entry:
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilelo.nxv16i1.i32(i32 4294967295, i32 6)
   ret <vscale x 16 x i1> %out
 }
@@ -349,7 +342,7 @@ define <vscale x 2 x i1> @whilels_d_xx(i64 %a, i64 %b) {
 define <vscale x 2 x i1> @whilels_d_ii_dont_fold_to_ptrue_larger_than_minvec() {
 ; CHECK-LABEL: whilels_d_ii_dont_fold_to_ptrue_larger_than_minvec:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #3
+; CHECK-NEXT:    mov w8, #3 // =0x3
 ; CHECK-NEXT:    whilels p0.d, xzr, x8
 ; CHECK-NEXT:    ret
   %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilels.nxv2i1.i64(i64 0, i64 3)
@@ -358,21 +351,19 @@ define <vscale x 2 x i1> @whilels_d_ii_dont_fold_to_ptrue_larger_than_minvec() {
 
 define <vscale x 16 x i1> @whilels_b_ii() {
 ; CHECK-LABEL: whilels_b_ii:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.b, vl7
 ; CHECK-NEXT:    ret
-entry:
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilels.nxv16i1.i64(i64 2, i64 8)
   ret <vscale x 16 x i1> %out
 }
 
 define <vscale x 16 x i1> @whilels_b_ii_dont_fold_to_ptrue_nonexistent_vl9() {
 ; CHECK-LABEL: whilels_b_ii_dont_fold_to_ptrue_nonexistent_vl9:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #9
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #9 // =0x9
 ; CHECK-NEXT:    whilels p0.b, xzr, x8
 ; CHECK-NEXT:    ret
-entry:
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilels.nxv16i1.i64(i64 0, i64 9)
   ret <vscale x 16 x i1> %out
 }
@@ -388,23 +379,21 @@ define <vscale x 16 x i1> @whilels_b_ii_vl_maximum() vscale_range(16, 16) {
 
 define <vscale x 16 x i1> @whilels_b_ii_dont_fold_to_ptrue_overflow() {
 ; CHECK-LABEL: whilels_b_ii_dont_fold_to_ptrue_overflow:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #6
-; CHECK-NEXT:    mov w9, #-1
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #6 // =0x6
+; CHECK-NEXT:    mov w9, #-1 // =0xffffffff
 ; CHECK-NEXT:    whilels p0.b, w9, w8
 ; CHECK-NEXT:    ret
-entry:
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilels.nxv16i1.i32(i32 4294967295, i32 6)
   ret <vscale x 16 x i1> %out
 }
 
 define <vscale x 16 x i1> @whilels_b_ii_dont_fold_to_ptrue_increment_overflow() {
 ; CHECK-LABEL: whilels_b_ii_dont_fold_to_ptrue_increment_overflow:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #-1
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #-1 // =0xffffffff
 ; CHECK-NEXT:    whilels p0.b, wzr, w8
 ; CHECK-NEXT:    ret
-entry:
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilels.nxv16i1.i32(i32 0, i32 4294967295)
   ret <vscale x 16 x i1> %out
 }
@@ -488,7 +477,7 @@ define <vscale x 2 x i1> @whilelt_d_xx(i64 %a, i64 %b) {
 define <vscale x 2 x i1> @whilelt_d_ii_dont_fold_to_ptrue_larger_than_minvec() {
 ; CHECK-LABEL: whilelt_d_ii_dont_fold_to_ptrue_larger_than_minvec:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #3
+; CHECK-NEXT:    mov w8, #3 // =0x3
 ; CHECK-NEXT:    whilelt p0.d, xzr, x8
 ; CHECK-NEXT:    ret
   %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilelt.nxv2i1.i64(i64 0, i64 3)
@@ -497,21 +486,19 @@ define <vscale x 2 x i1> @whilelt_d_ii_dont_fold_to_ptrue_larger_than_minvec() {
 
 define <vscale x 16 x i1> @whilelt_b_ii() {
 ; CHECK-LABEL: whilelt_b_ii:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.b, vl5
 ; CHECK-NEXT:    ret
-entry:
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilelt.nxv16i1.i64(i64 -2, i64 3)
   ret <vscale x 16 x i1> %out
 }
 
 define <vscale x 16 x i1> @whilelt_b_ii_dont_fold_to_ptrue_nonexistent_vl9() {
 ; CHECK-LABEL: whilelt_b_ii_dont_fold_to_ptrue_nonexistent_vl9:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #9
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #9 // =0x9
 ; CHECK-NEXT:    whilelt p0.b, xzr, x8
 ; CHECK-NEXT:    ret
-entry:
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilelt.nxv16i1.i64(i64 0, i64 9)
   ret <vscale x 16 x i1> %out
 }
@@ -527,13 +514,12 @@ define <vscale x 16 x i1> @whilelt_b_ii_vl_maximum() vscale_range(16, 16) {
 
 define <vscale x 16 x i1> @whilelt_b_ii_dont_fold_to_ptrue_overflow() {
 ; CHECK-LABEL: whilelt_b_ii_dont_fold_to_ptrue_overflow:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #2
-; CHECK-NEXT:    mov w9, #2147483647
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #2 // =0x2
+; CHECK-NEXT:    mov w9, #2147483647 // =0x7fffffff
 ; CHECK-NEXT:    movk w8, #32768, lsl #16
 ; CHECK-NEXT:    whilelt p0.b, w9, w8
 ; CHECK-NEXT:    ret
-entry:
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilelt.nxv16i1.i32(i32 2147483647, i32 -2147483646)
   ret <vscale x 16 x i1> %out
 }
diff --git a/llvm/test/CodeGen/AArch64/sve-knownbits.ll b/llvm/test/CodeGen/AArch64/sve-knownbits.ll
index ac391948d6bd4..c22d18c7e2ede 100644
--- a/llvm/test/CodeGen/AArch64/sve-knownbits.ll
+++ b/llvm/test/CodeGen/AArch64/sve-knownbits.ll
@@ -18,9 +18,7 @@ define <vscale x 4 x i32> @asrlsr(<vscale x 4 x i64> %va) {
 ; CHECK-NEXT:    lsr z0.d, z0.d, #15
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z1.s
 ; CHECK-NEXT:    ret
-  %head = insertelement <vscale x 4 x i32> poison, i32 15, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %head, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-  %vb = zext <vscale x 4 x i32> %splat to <vscale x 4 x i64>
+  %vb = zext <vscale x 4 x i32> splat(i32 15) to <vscale x 4 x i64>
   %x = ashr <vscale x 4 x i64> %va, %vb
   %y = trunc <vscale x 4 x i64> %x to <vscale x 4 x i32>
   ret <vscale x 4 x i32> %y
diff --git a/llvm/test/CodeGen/AArch64/sve-splat-one-and-ptrue.ll b/llvm/test/CodeGen/AArch64/sve-splat-one-and-ptrue.ll
index a517eff05b078..a409865138e0d 100644
--- a/llvm/test/CodeGen/AArch64/sve-splat-one-and-ptrue.ll
+++ b/llvm/test/CodeGen/AArch64/sve-splat-one-and-ptrue.ll
@@ -7,58 +7,52 @@ target triple = "aarch64-unknown-linux-gnu"
 
 define <vscale x 16 x i1> @fold_away_ptrue_and_ptrue() #0 {
 ; CHECK-LABEL: fold_away_ptrue_and_ptrue:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    ret
-entry:
-  %0 = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %1 = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> %0)
-  %2 = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-  %and = and <vscale x 16 x i1> %2, %1
+  %1 = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %2 = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> %1)
+  %3 = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
+  %and = and <vscale x 16 x i1> %3, %2
   ret <vscale x 16 x i1> %and
 }
 
 define <vscale x 16 x i1> @fold_away_ptrue_and_splat_predicate() #0 {
 ; CHECK-LABEL: fold_away_ptrue_and_splat_predicate:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    ret
-entry:
-  %ins = insertelement <vscale x 4 x i1> undef, i1 1, i32 0
-  %splat = shufflevector <vscale x 4 x i1> %ins, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer
-  %0 = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> %splat)
-  %1 = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-  %and = and <vscale x 16 x i1> %0, %1
+  %1 = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> splat(i1 true))
+  %2 = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
+  %and = and <vscale x 16 x i1> %1, %2
   ret <vscale x 16 x i1> %and
 }
 
 ; Ensure that one AND operation remain for inactive lanes zeroing with 2 x i1 type (llvm.aarch64.sve.convert.to.svbool.nxv2i1).
 define <vscale x 16 x i1> @fold_away_ptrue_and_convert_to() #0 {
 ; CHECK-LABEL: fold_away_ptrue_and_convert_to:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    ptrue p1.d
 ; CHECK-NEXT:    and p0.b, p1/z, p1.b, p0.b
 ; CHECK-NEXT:    ret
-entry:
-  %0 = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %1 = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> %0)
-  %2 = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-  %3 = call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> %2)
-  %4 = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1> %3)
-  %and = and <vscale x 16 x i1> %4, %1
+  %1 = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %2 = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> %1)
+  %3 = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
+  %4 = call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> %3)
+  %5 = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1> %4)
+  %and = and <vscale x 16 x i1> %5, %2
   ret <vscale x 16 x i1> %and
 }
 
 define <vscale x 16 x i1> @fold_away_two_similar() #0 {
 ; CHECK-LABEL: fold_away_two_similar:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.b
 ; CHECK-NEXT:    ret
-entry:
-  %0 = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
   %1 = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-  %and = and <vscale x 16 x i1> %0, %1
+  %2 = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
+  %and = and <vscale x 16 x i1> %1, %2
   ret <vscale x 16 x i1> %and
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve-splat-sext.ll b/llvm/test/CodeGen/AArch64/sve-splat-sext.ll
index f689aa6469a25..467af905a4d84 100644
--- a/llvm/test/CodeGen/AArch64/sve-splat-sext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-splat-sext.ll
@@ -6,9 +6,7 @@ define <vscale x 8 x i16> @sext_splat_v8i16_128() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z0.h, #-128 // =0xffffffffffffff80
 ; CHECK-NEXT:    ret
-  %i = insertelement <vscale x 8 x i16> poison, i16 128, i32 0
-  %s = shufflevector <vscale x 8 x i16> %i, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
-  %a = shl <vscale x 8 x i16> %s, splat (i16 8)
+  %a = shl <vscale x 8 x i16> splat(i16 128), splat (i16 8)
   %b = ashr <vscale x 8 x i16> %a, splat (i16 8)
   ret <vscale x 8 x i16> %b
 }
@@ -20,9 +18,7 @@ define <vscale x 8 x i1> @sext_icmp_splat_v8i16_128(<vscale x 8 x i8> %d) {
 ; CHECK-NEXT:    sxtb z0.h, p0/m, z0.h
 ; CHECK-NEXT:    cmpgt p0.h, p0/z, z0.h, #-1
 ; CHECK-NEXT:    ret
-  %i = insertelement <vscale x 8 x i8> poison, i8 128, i32 0
-  %s = shufflevector <vscale x 8 x i8> %i, <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer
-  %c = icmp ugt <vscale x 8 x i8> %s, %d
+  %c = icmp ugt <vscale x 8 x i8> splat(i8 128), %d
   ret <vscale x 8 x i1> %c
 }
 
@@ -33,8 +29,6 @@ define <vscale x 4 x i1> @sext_icmp_splat_v4i16_128(<vscale x 4 x i8> %d) {
 ; CHECK-NEXT:    sxtb z0.s, p0/m, z0.s
 ; CHECK-NEXT:    cmpgt p0.s, p0/z, z0.s, #-1
 ; CHECK-NEXT:    ret
-  %i = insertelement <vscale x 4 x i8> poison, i8 128, i32 0
-  %s = shufflevector <vscale x 4 x i8> %i, <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
-  %c = icmp ugt <vscale x 4 x i8> %s, %d
+  %c = icmp ugt <vscale x 4 x i8> splat(i8 128), %d
   ret <vscale x 4 x i1> %c
 }
diff --git a/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-imm.ll b/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-imm.ll
index 3273e6b384f63..a0bfc7034a386 100644
--- a/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-imm.ll
+++ b/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-imm.ll
@@ -109,9 +109,7 @@ define void @store_nxv2f32(ptr %out) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    st1w { z0.d }, p0, [x0]
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 2 x float> undef, float 1.0, i32 0
-  %splat = shufflevector <vscale x 2 x float> %ins, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
-  store <vscale x 2 x float> %splat, ptr %out
+  store <vscale x 2 x float> splat(float 1.0), ptr %out
   ret void
 }
 
@@ -122,9 +120,7 @@ define void @store_nxv4f16(ptr %out) {
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    st1h { z0.s }, p0, [x0]
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 4 x half> undef, half 1.0, i32 0
-  %splat = shufflevector <vscale x 4 x half> %ins, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
-  store <vscale x 4 x half> %splat, ptr %out
+  store <vscale x 4 x half> splat(half 1.0), ptr %out
   ret void
 }
 
@@ -139,9 +135,7 @@ define void @store_nxv6f32(ptr %out) {
 ; CHECK-NEXT:    st1w { z0.d }, p0, [x0, #2, mul vl]
 ; CHECK-NEXT:    st1w { z0.s }, p1, [x0]
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 6 x float> undef, float 1.0, i32 0
-  %splat = shufflevector <vscale x 6 x float> %ins, <vscale x 6 x float> undef, <vscale x 6 x i32> zeroinitializer
-  store <vscale x 6 x float> %splat, ptr %out
+  store <vscale x 6 x float> splat(float 1.0), ptr %out
   ret void
 }
 
@@ -154,8 +148,6 @@ define void @store_nxv12f16(ptr %out) {
 ; CHECK-NEXT:    st1h { z0.s }, p0, [x0, #2, mul vl]
 ; CHECK-NEXT:    st1h { z0.h }, p1, [x0]
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 12 x half> undef, half 1.0, i32 0
-  %splat = shufflevector <vscale x 12 x half> %ins, <vscale x 12 x half> undef, <vscale x 12 x i32> zeroinitializer
-  store <vscale x 12 x half> %splat, ptr %out
+  store <vscale x 12 x half> splat(half 1.0), ptr %out
   ret void
 }
diff --git a/llvm/test/CodeGen/AArch64/sve-stepvector.ll b/llvm/test/CodeGen/AArch64/sve-stepvector.ll
index f79ec00a19e6f..91c4659997789 100644
--- a/llvm/test/CodeGen/AArch64/sve-stepvector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-stepvector.ll
@@ -5,75 +5,69 @@
 
 define <vscale x 2 x i64> @stepvector_nxv2i64() {
 ; CHECK-LABEL: stepvector_nxv2i64:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.d, #0, #1
 ; CHECK-NEXT:    ret
-entry:
-  %0 = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-  ret <vscale x 2 x i64> %0
+  %1 = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+  ret <vscale x 2 x i64> %1
 }
 
 define <vscale x 4 x i32> @stepvector_nxv4i32() {
 ; CHECK-LABEL: stepvector_nxv4i32:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.s, #0, #1
 ; CHECK-NEXT:    ret
-entry:
-  %0 = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-  ret <vscale x 4 x i32> %0
+  %1 = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+  ret <vscale x 4 x i32> %1
 }
 
 define <vscale x 8 x i16> @stepvector_nxv8i16() {
 ; CHECK-LABEL: stepvector_nxv8i16:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.h, #0, #1
 ; CHECK-NEXT:    ret
-entry:
-  %0 = call <vscale x 8 x i16> @llvm.stepvector.nxv8i16()
-  ret <vscale x 8 x i16> %0
+  %1 = call <vscale x 8 x i16> @llvm.stepvector.nxv8i16()
+  ret <vscale x 8 x i16> %1
 }
 
 define <vscale x 16 x i8> @stepvector_nxv16i8() {
 ; CHECK-LABEL: stepvector_nxv16i8:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.b, #0, #1
 ; CHECK-NEXT:    ret
-entry:
-  %0 = call <vscale x 16 x i8> @llvm.stepvector.nxv16i8()
-  ret <vscale x 16 x i8> %0
+  %1 = call <vscale x 16 x i8> @llvm.stepvector.nxv16i8()
+  ret <vscale x 16 x i8> %1
 }
 
 ; ILLEGAL INTEGER TYPES
 
 define <vscale x 6 x i64> @stepvector_nxv6i64() {
 ; CHECK-LABEL: stepvector_nxv6i64:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.d, #0, #1
 ; CHECK-NEXT:    mov z2.d, z0.d
 ; CHECK-NEXT:    mov z1.d, z0.d
 ; CHECK-NEXT:    incd z1.d
 ; CHECK-NEXT:    incd z2.d, all, mul #2
 ; CHECK-NEXT:    ret
-entry:
-  %0 = call <vscale x 6 x i64> @llvm.stepvector.nxv6i64()
-  ret <vscale x 6 x i64> %0
+  %1 = call <vscale x 6 x i64> @llvm.stepvector.nxv6i64()
+  ret <vscale x 6 x i64> %1
 }
 
 define <vscale x 4 x i64> @stepvector_nxv4i64() {
 ; CHECK-LABEL: stepvector_nxv4i64:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.d, #0, #1
 ; CHECK-NEXT:    mov z1.d, z0.d
 ; CHECK-NEXT:    incd z1.d
 ; CHECK-NEXT:    ret
-entry:
-  %0 = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-  ret <vscale x 4 x i64> %0
+  %1 = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+  ret <vscale x 4 x i64> %1
 }
 
 define <vscale x 16 x i32> @stepvector_nxv16i32() {
 ; CHECK-LABEL: stepvector_nxv16i32:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.s, #0, #1
 ; CHECK-NEXT:    mov z1.d, z0.d
 ; CHECK-NEXT:    mov z2.d, z0.d
@@ -82,231 +76,201 @@ define <vscale x 16 x i32> @stepvector_nxv16i32() {
 ; CHECK-NEXT:    mov z3.d, z1.d
 ; CHECK-NEXT:    incw z3.s, all, mul #2
 ; CHECK-NEXT:    ret
-entry:
-  %0 = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
-  ret <vscale x 16 x i32> %0
+  %1 = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+  ret <vscale x 16 x i32> %1
 }
 
 define <vscale x 3 x i32> @stepvector_nxv3i32() {
 ; CHECK-LABEL: stepvector_nxv3i32:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.s, #0, #1
 ; CHECK-NEXT:    ret
-entry:
-  %0 = call <vscale x 3 x i32> @llvm.stepvector.nxv3i32()
-  ret <vscale x 3 x i32> %0
+  %1 = call <vscale x 3 x i32> @llvm.stepvector.nxv3i32()
+  ret <vscale x 3 x i32> %1
 }
 
 define <vscale x 2 x i32> @stepvector_nxv2i32() {
 ; CHECK-LABEL: stepvector_nxv2i32:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.d, #0, #1
 ; CHECK-NEXT:    ret
-entry:
-  %0 = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
-  ret <vscale x 2 x i32> %0
+  %1 = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+  ret <vscale x 2 x i32> %1
 }
 
 define <vscale x 4 x i16> @stepvector_nxv4i16() {
 ; CHECK-LABEL: stepvector_nxv4i16:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.s, #0, #1
 ; CHECK-NEXT:    ret
-entry:
-  %0 = call <vscale x 4 x i16> @llvm.stepvector.nxv4i16()
-  ret <vscale x 4 x i16> %0
+  %1 = call <vscale x 4 x i16> @llvm.stepvector.nxv4i16()
+  ret <vscale x 4 x i16> %1
 }
 
 define <vscale x 8 x i8> @stepvector_nxv8i8() {
 ; CHECK-LABEL: stepvector_nxv8i8:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.h, #0, #1
 ; CHECK-NEXT:    ret
-entry:
-  %0 = call <vscale x 8 x i8> @llvm.stepvector.nxv8i8()
-  ret <vscale x 8 x i8> %0
+  %1 = call <vscale x 8 x i8> @llvm.stepvector.nxv8i8()
+  ret <vscale x 8 x i8> %1
 }
 
 define <vscale x 8 x i8> @add_stepvector_nxv8i8() {
 ; CHECK-LABEL: add_stepvector_nxv8i8:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.h, #0, #2
 ; CHECK-NEXT:    ret
-entry:
-  %0 = call <vscale x 8 x i8> @llvm.stepvector.nxv8i8()
   %1 = call <vscale x 8 x i8> @llvm.stepvector.nxv8i8()
-  %2 = add <vscale x 8 x i8> %0, %1
-  ret <vscale x 8 x i8> %2
+  %2 = call <vscale x 8 x i8> @llvm.stepvector.nxv8i8()
+  %3 = add <vscale x 8 x i8> %1, %2
+  ret <vscale x 8 x i8> %3
 }
 
 define <vscale x 8 x i8> @add_stepvector_nxv8i8_1(<vscale x 8 x i8> %p) {
 ; CHECK-LABEL: add_stepvector_nxv8i8_1:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z1.h, #0, #2
 ; CHECK-NEXT:    add z0.h, z0.h, z1.h
 ; CHECK-NEXT:    ret
-entry:
-  %0 = call <vscale x 8 x i8> @llvm.stepvector.nxv8i8()
-  %1 = add <vscale x 8 x i8> %p, %0
-  %2 = call <vscale x 8 x i8> @llvm.stepvector.nxv8i8()
-  %3 = add <vscale x 8 x i8> %1, %2
-  ret <vscale x 8 x i8> %3
+  %1 = call <vscale x 8 x i8> @llvm.stepvector.nxv8i8()
+  %2 = add <vscale x 8 x i8> %p, %1
+  %3 = call <vscale x 8 x i8> @llvm.stepvector.nxv8i8()
+  %4 = add <vscale x 8 x i8> %2, %3
+  ret <vscale x 8 x i8> %4
 }
 
 define <vscale x 8 x i8> @add_stepvector_nxv8i8_2() {
 ; CHECK-LABEL: add_stepvector_nxv8i8_2:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.h, #2, #1
 ; CHECK-NEXT:    ret
-entry:
-  %0 = insertelement <vscale x 8 x i8> poison, i8 2, i32 0
-  %1 = shufflevector <vscale x 8 x i8> %0, <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer
   %2 = call <vscale x 8 x i8> @llvm.stepvector.nxv8i8()
-  %3 = add <vscale x 8 x i8> %2, %1
+  %3 = add <vscale x 8 x i8> %2, splat(i8 2)
   ret <vscale x 8 x i8> %3
 }
 
 define <vscale x 8 x i8> @add_stepvector_nxv8i8_2_commutative() {
 ; CHECK-LABEL: add_stepvector_nxv8i8_2_commutative:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.h, #2, #1
 ; CHECK-NEXT:    ret
-entry:
-  %0 = insertelement <vscale x 8 x i8> poison, i8 2, i32 0
-  %1 = shufflevector <vscale x 8 x i8> %0, <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer
   %2 = call <vscale x 8 x i8> @llvm.stepvector.nxv8i8()
-  %3 = add <vscale x 8 x i8> %1, %2
+  %3 = add <vscale x 8 x i8> splat(i8 2), %2
   ret <vscale x 8 x i8> %3
 }
 
 define <vscale x 8 x i16> @add_stepvector_nxv8i16_1(i16 %data) {
 ; CHECK-LABEL: add_stepvector_nxv8i16_1:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.h, w0, #1
 ; CHECK-NEXT:    ret
-entry:
-  %0 = insertelement <vscale x 8 x i16> poison, i16 %data, i32 0
-  %1 = shufflevector <vscale x 8 x i16> %0, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
-  %2 = call <vscale x 8 x i16> @llvm.stepvector.nxv8i16()
-  %3 = add <vscale x 8 x i16> %2, %1
-  ret <vscale x 8 x i16> %3
+  %1 = insertelement <vscale x 8 x i16> poison, i16 %data, i32 0
+  %2 = shufflevector <vscale x 8 x i16> %1, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
+  %3 = call <vscale x 8 x i16> @llvm.stepvector.nxv8i16()
+  %4 = add <vscale x 8 x i16> %3, %2
+  ret <vscale x 8 x i16> %4
 }
 
 define <vscale x 4 x i32> @add_stepvector_nxv4i32_1(i32 %data) {
 ; CHECK-LABEL: add_stepvector_nxv4i32_1:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.s, w0, #1
 ; CHECK-NEXT:    ret
-entry:
-  %0 = insertelement <vscale x 4 x i32> poison, i32 %data, i32 0
-  %1 = shufflevector <vscale x 4 x i32> %0, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-  %2 = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-  %3 = add <vscale x 4 x i32> %2, %1
-  ret <vscale x 4 x i32> %3
+  %1 = insertelement <vscale x 4 x i32> poison, i32 %data, i32 0
+  %2 = shufflevector <vscale x 4 x i32> %1, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+  %3 = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+  %4 = add <vscale x 4 x i32> %3, %2
+  ret <vscale x 4 x i32> %4
 }
 
 define <vscale x 4 x i32> @multiple_use_stepvector_nxv4i32_1(i32 %data) {
 ; CHECK-LABEL: multiple_use_stepvector_nxv4i32_1:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.s, w0, #1
 ; CHECK-NEXT:    mov z1.s, w0
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    mul z1.s, p0/m, z1.s, z0.s
 ; CHECK-NEXT:    sub z0.s, z1.s, z0.s
 ; CHECK-NEXT:    ret
-entry:
-  %0 = insertelement <vscale x 4 x i32> poison, i32 %data, i32 0
-  %1 = shufflevector <vscale x 4 x i32> %0, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-  %2 = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-  %3 = add <vscale x 4 x i32> %2, %1
-  %4 = mul <vscale x 4 x i32> %1, %3
-  %5 = sub <vscale x 4 x i32> %4, %3
-  ret <vscale x 4 x i32> %5
+  %1 = insertelement <vscale x 4 x i32> poison, i32 %data, i32 0
+  %2 = shufflevector <vscale x 4 x i32> %1, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+  %3 = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+  %4 = add <vscale x 4 x i32> %3, %2
+  %5 = mul <vscale x 4 x i32> %2, %4
+  %6 = sub <vscale x 4 x i32> %5, %4
+  ret <vscale x 4 x i32> %6
 }
 
 define <vscale x 2 x i64> @add_stepvector_nxv2i64_1(i64 %data) {
 ; CHECK-LABEL: add_stepvector_nxv2i64_1:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.d, x0, #1
 ; CHECK-NEXT:    ret
-entry:
-  %0 = insertelement <vscale x 2 x i64> poison, i64 %data, i32 0
-  %1 = shufflevector <vscale x 2 x i64> %0, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-  %2 = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-  %3 = add <vscale x 2 x i64> %1, %2
-  ret <vscale x 2 x i64> %3
+  %1 = insertelement <vscale x 2 x i64> poison, i64 %data, i32 0
+  %2 = shufflevector <vscale x 2 x i64> %1, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+  %3 = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+  %4 = add <vscale x 2 x i64> %2, %3
+  ret <vscale x 2 x i64> %4
 }
 
 define <vscale x 2 x i64> @multiple_use_stepvector_nxv2i64_1(i64 %data) {
 ; CHECK-LABEL: multiple_use_stepvector_nxv2i64_1:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.d, #0, #1
 ; CHECK-NEXT:    mov z1.d, x0
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    add z1.d, z0.d, z1.d
 ; CHECK-NEXT:    mul z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    ret
-entry:
-  %0 = insertelement <vscale x 2 x i64> poison, i64 %data, i32 0
-  %1 = shufflevector <vscale x 2 x i64> %0, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-  %2 = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-  %3 = add <vscale x 2 x i64> %1, %2
-  %4 = mul <vscale x 2 x i64> %3, %2
-  ret <vscale x 2 x i64> %4
+  %1 = insertelement <vscale x 2 x i64> poison, i64 %data, i32 0
+  %2 = shufflevector <vscale x 2 x i64> %1, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+  %3 = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+  %4 = add <vscale x 2 x i64> %2, %3
+  %5 = mul <vscale x 2 x i64> %4, %3
+  ret <vscale x 2 x i64> %5
 }
 
 define <vscale x 8 x i8> @mul_stepvector_nxv8i8() {
 ; CHECK-LABEL: mul_stepvector_nxv8i8:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.h, #0, #2
 ; CHECK-NEXT:    ret
-entry:
-  %0 = insertelement <vscale x 8 x i8> poison, i8 2, i32 0
-  %1 = shufflevector <vscale x 8 x i8> %0, <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer
   %2 = call <vscale x 8 x i8> @llvm.stepvector.nxv8i8()
-  %3 = mul <vscale x 8 x i8> %2, %1
+  %3 = mul <vscale x 8 x i8> %2, splat(i8 2)
   ret <vscale x 8 x i8> %3
 }
 
 define <vscale x 2 x i64> @mul_stepvector_nxv2i64() {
 ; CHECK-LABEL: mul_stepvector_nxv2i64:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #2222 // =0x8ae
 ; CHECK-NEXT:    index z0.d, #0, x8
 ; CHECK-NEXT:    ret
-entry:
-  %0 = insertelement <vscale x 2 x i64> poison, i64 2222, i32 0
-  %1 = shufflevector <vscale x 2 x i64> %0, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
   %2 = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-  %3 = mul <vscale x 2 x i64> %2, %1
+  %3 = mul <vscale x 2 x i64> %2, splat(i64 2222)
   ret <vscale x 2 x i64> %3
 }
 
 define <vscale x 2 x i64> @mul_stepvector_bigconst_nxv2i64() {
 ; CHECK-LABEL: mul_stepvector_bigconst_nxv2i64:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov x8, #146028888064 // =0x2200000000
 ; CHECK-NEXT:    index z0.d, #0, x8
 ; CHECK-NEXT:    ret
-entry:
-  %0 = insertelement <vscale x 2 x i64> poison, i64 146028888064, i32 0
-  %1 = shufflevector <vscale x 2 x i64> %0, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
   %2 = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-  %3 = mul <vscale x 2 x i64> %2, %1
+  %3 = mul <vscale x 2 x i64> %2, splat(i64 146028888064)
   ret <vscale x 2 x i64> %3
 }
 
 define <vscale x 2 x i64> @mul_add_stepvector_nxv2i64(i64 %x) {
 ; CHECK-LABEL: mul_add_stepvector_nxv2i64:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #2222 // =0x8ae
 ; CHECK-NEXT:    index z0.d, x0, x8
 ; CHECK-NEXT:    ret
-entry:
-  %0 = insertelement <vscale x 2 x i64> poison, i64 2222, i32 0
-  %1 = shufflevector <vscale x 2 x i64> %0, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
   %2 = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-  %3 = mul <vscale x 2 x i64> %2, %1
+  %3 = mul <vscale x 2 x i64> %2, splat(i64 2222)
   %4 = insertelement <vscale x 2 x i64> poison, i64 %x, i32 0
   %5 = shufflevector <vscale x 2 x i64> %4, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
   %6 = add <vscale x 2 x i64> %3, %5
@@ -315,31 +279,27 @@ entry:
 
 define <vscale x 2 x i64> @mul_add_stepvector_nxv2i64_commutative(i64 %x, i64 %y) {
 ; CHECK-LABEL: mul_add_stepvector_nxv2i64_commutative:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.d, x0, x1
 ; CHECK-NEXT:    ret
-entry:
-  %0 = insertelement <vscale x 2 x i64> poison, i64 %y, i32 0
-  %1 = shufflevector <vscale x 2 x i64> %0, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-  %2 = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-  %3 = mul <vscale x 2 x i64> %1, %2
-  %4 = insertelement <vscale x 2 x i64> poison, i64 %x, i32 0
-  %5 = shufflevector <vscale x 2 x i64> %4, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-  %6 = add <vscale x 2 x i64> %5, %3
-  ret <vscale x 2 x i64> %6
+  %1 = insertelement <vscale x 2 x i64> poison, i64 %y, i32 0
+  %2 = shufflevector <vscale x 2 x i64> %1, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+  %3 = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+  %4 = mul <vscale x 2 x i64> %2, %3
+  %5 = insertelement <vscale x 2 x i64> poison, i64 %x, i32 0
+  %6 = shufflevector <vscale x 2 x i64> %5, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+  %7 = add <vscale x 2 x i64> %6, %4
+  ret <vscale x 2 x i64> %7
 }
 
 define <vscale x 2 x i64> @mul_add_stepvector_bigconst_nxv2i64(i64 %x) {
 ; CHECK-LABEL: mul_add_stepvector_bigconst_nxv2i64:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov x8, #146028888064 // =0x2200000000
 ; CHECK-NEXT:    index z0.d, x0, x8
 ; CHECK-NEXT:    ret
-entry:
-  %0 = insertelement <vscale x 2 x i64> poison, i64 146028888064, i32 0
-  %1 = shufflevector <vscale x 2 x i64> %0, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
   %2 = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-  %3 = mul <vscale x 2 x i64> %2, %1
+  %3 = mul <vscale x 2 x i64> %2, splat(i64 146028888064)
   %4 = insertelement <vscale x 2 x i64> poison, i64 %x, i32 0
   %5 = shufflevector <vscale x 2 x i64> %4, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
   %6 = add <vscale x 2 x i64> %3, %5
@@ -348,82 +308,69 @@ entry:
 
 define <vscale x 2 x i64> @mul_mul_add_stepvector_nxv2i64(i64 %x, i64 %y) {
 ; CHECK-LABEL: mul_mul_add_stepvector_nxv2i64:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add x8, x0, x0, lsl #1
 ; CHECK-NEXT:    index z0.d, x1, x8
 ; CHECK-NEXT:    ret
-entry:
   %xmul = mul i64 %x, 3
-  %0 = insertelement <vscale x 2 x i64> poison, i64 %xmul, i32 0
-  %1 = shufflevector <vscale x 2 x i64> %0, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-  %2 = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-  %3 = mul <vscale x 2 x i64> %2, %1
-  %4 = insertelement <vscale x 2 x i64> poison, i64 %y, i32 0
-  %5 = shufflevector <vscale x 2 x i64> %4, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-  %6 = add <vscale x 2 x i64> %3, %5
-  ret <vscale x 2 x i64> %6
+  %1 = insertelement <vscale x 2 x i64> poison, i64 %xmul, i32 0
+  %2 = shufflevector <vscale x 2 x i64> %1, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+  %3 = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+  %4 = mul <vscale x 2 x i64> %3, %2
+  %5 = insertelement <vscale x 2 x i64> poison, i64 %y, i32 0
+  %6 = shufflevector <vscale x 2 x i64> %5, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+  %7 = add <vscale x 2 x i64> %4, %6
+  ret <vscale x 2 x i64> %7
 }
 
 define <vscale x 8 x i8> @shl_stepvector_nxv8i8() {
 ; CHECK-LABEL: shl_stepvector_nxv8i8:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.h, #0, #4
 ; CHECK-NEXT:    ret
-entry:
-  %0 = insertelement <vscale x 8 x i8> poison, i8 2, i32 0
-  %1 = shufflevector <vscale x 8 x i8> %0, <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer
   %2 = call <vscale x 8 x i8> @llvm.stepvector.nxv8i8()
-  %3 = shl <vscale x 8 x i8> %2, %1
+  %3 = shl <vscale x 8 x i8> %2, splat(i8 2)
   ret <vscale x 8 x i8> %3
 }
 
 define <vscale x 8 x i16> @sub_multiple_use_stepvector_nxv8i16() {
 ; CHECK-LABEL: sub_multiple_use_stepvector_nxv8i16:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.h, #0, #1
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    mov z1.d, z0.d
 ; CHECK-NEXT:    subr z1.h, z1.h, #2 // =0x2
 ; CHECK-NEXT:    lsl z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    ret
-entry:
-  %0 = insertelement <vscale x 8 x i16> poison, i16 2, i32 0
-  %1 = shufflevector <vscale x 8 x i16> %0, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
   %2 = call <vscale x 8 x i16> @llvm.stepvector.nxv8i16()
-  %3 = sub <vscale x 8 x i16> %1, %2
+  %3 = sub <vscale x 8 x i16> splat(i16 2), %2
   %4 = shl <vscale x 8 x i16> %2, %3
   ret <vscale x 8 x i16> %4
 }
 
 define <vscale x 8 x i16> @sub_stepvector_nxv8i16() {
 ; CHECK-LABEL: sub_stepvector_nxv8i16:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.h, #2, #-1
 ; CHECK-NEXT:    ret
-entry:
-  %0 = insertelement <vscale x 8 x i16> poison, i16 2, i32 0
-  %1 = shufflevector <vscale x 8 x i16> %0, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
   %2 = call <vscale x 8 x i16> @llvm.stepvector.nxv8i16()
-  %3 = sub <vscale x 8 x i16> %1, %2
+  %3 = sub <vscale x 8 x i16> splat(i16 2), %2
   ret <vscale x 8 x i16> %3
 }
 
 define <vscale x 8 x i8> @promote_sub_stepvector_nxv8i8() {
 ; CHECK-LABEL: promote_sub_stepvector_nxv8i8:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    index z0.h, #2, #-1
 ; CHECK-NEXT:    ret
-entry:
-  %0 = insertelement <vscale x 8 x i8> poison, i8 2, i32 0
-  %1 = shufflevector <vscale x 8 x i8> %0, <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer
   %2 = call <vscale x 8 x i8> @llvm.stepvector.nxv8i8()
-  %3 = sub <vscale x 8 x i8> %1, %2
+  %3 = sub <vscale x 8 x i8> splat(i8 2), %2
   ret <vscale x 8 x i8> %3
 }
 
 define <vscale x 16 x i32> @split_sub_stepvector_nxv16i32() {
 ; CHECK-LABEL: split_sub_stepvector_nxv16i32:
-; CHECK:       // %bb.0: // %entry
+; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cntw x8
 ; CHECK-NEXT:    index z0.s, #0, #-1
 ; CHECK-NEXT:    neg x8, x8
@@ -435,10 +382,9 @@ define <vscale x 16 x i32> @split_sub_stepvector_nxv16i32() {
 ; CHECK-NEXT:    add z2.s, z0.s, z3.s
 ; CHECK-NEXT:    add z3.s, z1.s, z3.s
 ; CHECK-NEXT:    ret
-entry:
-  %0 = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
-  %1 = sub <vscale x 16 x i32> zeroinitializer, %0
-  ret <vscale x 16 x i32> %1
+  %1 = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+  %2 = sub <vscale x 16 x i32> zeroinitializer, %1
+  ret <vscale x 16 x i32> %2
 }
 
 declare <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
diff --git a/llvm/test/CodeGen/AArch64/sve-vecreduce-fold.ll b/llvm/test/CodeGen/AArch64/sve-vecreduce-fold.ll
index 0bdaefdfc2a3f..c309ad6ac5cce 100644
--- a/llvm/test/CodeGen/AArch64/sve-vecreduce-fold.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vecreduce-fold.ll
@@ -56,9 +56,7 @@ define i1 @reduce_and_insert_subvec_into_ones(<vscale x 4 x i1> %in) {
 ; CHECK-NEXT:    nots p0.b, p1/z, p0.b
 ; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
-  %allones.ins = insertelement <vscale x 16 x i1> poison, i1 1, i32 0
-  %allones = shufflevector <vscale x 16 x i1> %allones.ins,  <vscale x 16 x i1> poison,  <vscale x 16 x i32> zeroinitializer
-  %t = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv4i1(<vscale x 16 x i1> %allones, <vscale x 4 x i1> %in, i64 0)
+  %t = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv4i1(<vscale x 16 x i1> splat(i1 true), <vscale x 4 x i1> %in, i64 0)
   %res = call i1 @llvm.vector.reduce.and.nxv16i1(<vscale x 16 x i1> %t)
   ret i1 %res
 }
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-splat.ll b/llvm/test/CodeGen/AArch64/sve-vector-splat.ll
index 9253d5ab4531a..8dd433b6f23c6 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-splat.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-splat.ll
@@ -48,9 +48,7 @@ define <vscale x 16 x i8> @sve_splat_16xi8_imm() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z0.b, #1 // =0x1
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 16 x i8> undef, i8 1, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %ins, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
-  ret <vscale x 16 x i8> %splat
+  ret <vscale x 16 x i8> splat(i8 1)
 }
 
 define <vscale x 8 x i16> @sve_splat_8xi16_dup_imm() {
@@ -58,9 +56,7 @@ define <vscale x 8 x i16> @sve_splat_8xi16_dup_imm() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z0.h, #1 // =0x1
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 8 x i16> undef, i16 1, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  ret <vscale x 8 x i16> %splat
+  ret <vscale x 8 x i16> splat(i16 1)
 }
 
 define <vscale x 8 x i16> @sve_splat_8xi16_dupm_imm() {
@@ -68,9 +64,7 @@ define <vscale x 8 x i16> @sve_splat_8xi16_dupm_imm() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z0.h, #16256 // =0x3f80
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 8 x i16> undef, i16 16256, i32 0 ; 0x3f80
-  %splat = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  ret <vscale x 8 x i16> %splat
+  ret <vscale x 8 x i16> splat(i16 16256)
 }
 
 define <vscale x 4 x i32> @sve_splat_4xi32_dup_imm() {
@@ -78,9 +72,7 @@ define <vscale x 4 x i32> @sve_splat_4xi32_dup_imm() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z0.s, #1 // =0x1
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 4 x i32> undef, i32 1, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  ret <vscale x 4 x i32> %splat
+  ret <vscale x 4 x i32> splat(i32 1)
 }
 
 define <vscale x 4 x i32> @sve_splat_4xi32_dupm_imm() {
@@ -88,9 +80,7 @@ define <vscale x 4 x i32> @sve_splat_4xi32_dupm_imm() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z0.s, #0xff0000
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 4 x i32> undef, i32 16711680, i32 0 ; 0xff0000
-  %splat = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  ret <vscale x 4 x i32> %splat
+  ret <vscale x 4 x i32> splat(i32 16711680)
 }
 
 define <vscale x 2 x i64> @sve_splat_2xi64_dup_imm() {
@@ -98,9 +88,7 @@ define <vscale x 2 x i64> @sve_splat_2xi64_dup_imm() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z0.d, #1 // =0x1
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 2 x i64> undef, i64 1, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  ret <vscale x 2 x i64> %splat
+  ret <vscale x 2 x i64> splat(i64 1)
 }
 
 define <vscale x 2 x i64> @sve_splat_2xi64_dupm_imm() {
@@ -108,9 +96,7 @@ define <vscale x 2 x i64> @sve_splat_2xi64_dupm_imm() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z0.d, #0xffff00000000
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 2 x i64> undef, i64 281470681743360, i32 0 ; 0xffff00000000
-  %splat = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  ret <vscale x 2 x i64> %splat
+  ret <vscale x 2 x i64> splat(i64 281470681743360)
 }
 
 ;; Promote splats of smaller illegal integer vector types
@@ -151,9 +137,7 @@ define <vscale x 8 x i8> @sve_splat_8xi8_imm() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z0.h, #255 // =0xff
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 8 x i8> undef, i8 -1, i32 0
-  %splat = shufflevector <vscale x 8 x i8> %ins, <vscale x 8 x i8> undef, <vscale x 8 x i32> zeroinitializer
-  ret <vscale x 8 x i8> %splat
+  ret <vscale x 8 x i8> splat(i8 -1)
 }
 
 define <vscale x 2 x i16> @sve_splat_2xi16(i16 %val) {
@@ -182,9 +166,7 @@ define <vscale x 4 x i16> @sve_splat_4xi16_imm() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z0.s, #65535 // =0xffff
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 4 x i16> undef, i16 -1, i32 0
-  %splat = shufflevector <vscale x 4 x i16> %ins, <vscale x 4 x i16> undef, <vscale x 4 x i32> zeroinitializer
-  ret <vscale x 4 x i16> %splat
+  ret <vscale x 4 x i16> splat(i16 -1)
 }
 
 define <vscale x 2 x i32> @sve_splat_2xi32(i32 %val) {
@@ -203,9 +185,7 @@ define <vscale x 2 x i32> @sve_splat_2xi32_imm() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z0.d, #0xffffffff
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 2 x i32> undef, i32 -1, i32 0
-  %splat = shufflevector <vscale x 2 x i32> %ins, <vscale x 2 x i32> undef, <vscale x 2 x i32> zeroinitializer
-  ret <vscale x 2 x i32> %splat
+  ret <vscale x 2 x i32> splat(i32 -1)
 }
 
 ;; Widen/split splats of wide vector types.
@@ -459,9 +439,7 @@ define <vscale x 8 x half> @splat_nxv8f16_imm() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmov z0.h, #1.00000000
 ; CHECK-NEXT:    ret
-  %1 = insertelement <vscale x 8 x half> undef, half 1.0, i32 0
-  %2 = shufflevector <vscale x 8 x half> %1, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
-  ret <vscale x 8 x half> %2
+  ret <vscale x 8 x half> splat(half 1.0)
 }
 
 define <vscale x 4 x half> @splat_nxv4f16_imm() {
@@ -469,9 +447,7 @@ define <vscale x 4 x half> @splat_nxv4f16_imm() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmov z0.h, #1.00000000
 ; CHECK-NEXT:    ret
-  %1 = insertelement <vscale x 4 x half> undef, half 1.0, i32 0
-  %2 = shufflevector <vscale x 4 x half> %1, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
-  ret <vscale x 4 x half> %2
+  ret <vscale x 4 x half> splat(half 1.0)
 }
 
 define <vscale x 2 x half> @splat_nxv2f16_imm() {
@@ -479,9 +455,7 @@ define <vscale x 2 x half> @splat_nxv2f16_imm() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmov z0.h, #1.00000000
 ; CHECK-NEXT:    ret
-  %1 = insertelement <vscale x 2 x half> undef, half 1.0, i32 0
-  %2 = shufflevector <vscale x 2 x half> %1, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
-  ret <vscale x 2 x half> %2
+  ret <vscale x 2 x half> splat(half 1.0)
 }
 
 define <vscale x 4 x float> @splat_nxv4f32_imm() {
@@ -489,9 +463,7 @@ define <vscale x 4 x float> @splat_nxv4f32_imm() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmov z0.s, #1.00000000
 ; CHECK-NEXT:    ret
-  %1 = insertelement <vscale x 4 x float> undef, float 1.0, i32 0
-  %2 = shufflevector <vscale x 4 x float> %1, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
-  ret <vscale x 4 x float> %2
+  ret <vscale x 4 x float> splat(float 1.0)
 }
 
 define <vscale x 2 x float> @splat_nxv2f32_imm() {
@@ -499,9 +471,7 @@ define <vscale x 2 x float> @splat_nxv2f32_imm() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmov z0.s, #1.00000000
 ; CHECK-NEXT:    ret
-  %1 = insertelement <vscale x 2 x float> undef, float 1.0, i32 0
-  %2 = shufflevector <vscale x 2 x float> %1, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
-  ret <vscale x 2 x float> %2
+  ret <vscale x 2 x float> splat(float 1.0)
 }
 
 define <vscale x 2 x double> @splat_nxv2f64_imm() {
@@ -509,9 +479,7 @@ define <vscale x 2 x double> @splat_nxv2f64_imm() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmov z0.d, #1.00000000
 ; CHECK-NEXT:    ret
-  %1 = insertelement <vscale x 2 x double> undef, double 1.0, i32 0
-  %2 = shufflevector <vscale x 2 x double> %1, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
-  ret <vscale x 2 x double> %2
+  ret <vscale x 2 x double> splat(double 1.0)
 }
 
 define <vscale x 4 x i32> @splat_nxv4i32_fold(<vscale x 4 x i32> %x) {
@@ -539,9 +507,7 @@ define <vscale x 2 x float> @splat_nxv2f32_fmov_fold() {
 ; CHECK-NEXT:    mov w8, #1109917696 // =0x42280000
 ; CHECK-NEXT:    mov z0.s, w8
 ; CHECK-NEXT:    ret
-  %1 = insertelement <vscale x 2 x float> undef, float 4.200000e+01, i32 0
-  %2 = shufflevector <vscale x 2 x float> %1, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
-  ret <vscale x 2 x float> %2
+  ret <vscale x 2 x float> splat(float 4.200000e+01)
 }
 
 define <vscale x 4 x float> @splat_nxv4f32_fmov_fold() {
@@ -550,9 +516,7 @@ define <vscale x 4 x float> @splat_nxv4f32_fmov_fold() {
 ; CHECK-NEXT:    mov w8, #1109917696 // =0x42280000
 ; CHECK-NEXT:    mov z0.s, w8
 ; CHECK-NEXT:    ret
-  %1 = insertelement <vscale x 4 x float> undef, float 4.200000e+01, i32 0
-  %2 = shufflevector <vscale x 4 x float> %1, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
-  ret <vscale x 4 x float> %2
+  ret <vscale x 4 x float> splat(float 4.200000e+01)
 }
 
 define <vscale x 2 x double> @splat_nxv2f64_fmov_fold() {
@@ -561,9 +525,7 @@ define <vscale x 2 x double> @splat_nxv2f64_fmov_fold() {
 ; CHECK-NEXT:    mov x8, #4631107791820423168 // =0x4045000000000000
 ; CHECK-NEXT:    mov z0.d, x8
 ; CHECK-NEXT:    ret
-  %1 = insertelement <vscale x 2 x double> undef, double 4.200000e+01, i32 0
-  %2 = shufflevector <vscale x 2 x double> %1, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
-  ret <vscale x 2 x double> %2
+  ret <vscale x 2 x double> splat(double 4.200000e+01)
 }
 
 ; Splat of float constants not representable as a single immediate.
@@ -575,9 +537,7 @@ define <vscale x 2 x float> @splat_nxv2f32_imm_out_of_range() {
 ; CHECK-NEXT:    movk w8, #16469, lsl #16
 ; CHECK-NEXT:    mov z0.s, w8
 ; CHECK-NEXT:    ret
-  %1 = insertelement <vscale x 2 x float> undef, float 3.3299999237060546875, i32 0
-  %2 = shufflevector <vscale x 2 x float> %1, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
-  ret <vscale x 2 x float> %2
+  ret <vscale x 2 x float> splat(float 3.3299999237060546875)
 }
 
 define <vscale x 4 x float> @splat_nxv4f32_imm_out_of_range() {
@@ -587,9 +547,7 @@ define <vscale x 4 x float> @splat_nxv4f32_imm_out_of_range() {
 ; CHECK-NEXT:    movk w8, #16469, lsl #16
 ; CHECK-NEXT:    mov z0.s, w8
 ; CHECK-NEXT:    ret
-  %1 = insertelement <vscale x 4 x float> undef, float 3.3299999237060546875, i32 0
-  %2 = shufflevector <vscale x 4 x float> %1, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
-  ret <vscale x 4 x float> %2
+  ret <vscale x 4 x float> splat(float 3.3299999237060546875)
 }
 
 define <vscale x 2 x double> @splat_nxv2f64_imm_out_of_range() {
@@ -600,9 +558,7 @@ define <vscale x 2 x double> @splat_nxv2f64_imm_out_of_range() {
 ; CHECK-NEXT:    add x8, x8, :lo12:.LCPI57_0
 ; CHECK-NEXT:    ld1rd { z0.d }, p0/z, [x8]
 ; CHECK-NEXT:    ret
-  %1 = insertelement <vscale x 2 x double> undef, double 3.33, i32 0
-  %2 = shufflevector <vscale x 2 x double> %1, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
-  ret <vscale x 2 x double> %2
+  ret <vscale x 2 x double> splat(double 3.33)
 }
 
 ; Splat for predicates
@@ -613,10 +569,7 @@ define <vscale x 2 x i1> @sve_splat_i1_allactive() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 2 x i1> undef, i1 1, i32 0
-  %splat = shufflevector <vscale x 2 x i1> %ins, <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer
-  ret <vscale x 2 x i1> %splat
+  ret <vscale x 2 x i1> splat(i1 true)
 }
 
-; +bf16 is required for the bfloat version.
-attributes #0 = { "target-features"="+sve,+bf16" }
+attributes #0 = { "target-features"="+sve" }
diff --git a/llvm/test/CodeGen/AArch64/sve2-int-mul.ll b/llvm/test/CodeGen/AArch64/sve2-int-mul.ll
index 800888b7e6cb9..2bf3d05eff06a 100644
--- a/llvm/test/CodeGen/AArch64/sve2-int-mul.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-int-mul.ll
@@ -10,9 +10,7 @@ define <vscale x 8 x i16> @mul_i16_imm(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    mov z1.h, #255 // =0xff
 ; CHECK-NEXT:    mul z0.h, z0.h, z1.h
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 255, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res = mul <vscale x 8 x i16> %a, %splat
+  %res = mul <vscale x 8 x i16> %a, splat(i16 255)
   ret <vscale x 8 x i16> %res
 }
 
@@ -23,9 +21,7 @@ define <vscale x 8 x i16> @mul_i16_imm_neg(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    mov z1.h, w8
 ; CHECK-NEXT:    mul z0.h, z0.h, z1.h
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 8 x i16> undef, i16 -200, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
-  %res = mul <vscale x 8 x i16> %a, %splat
+  %res = mul <vscale x 8 x i16> %a, splat(i16 -200)
   ret <vscale x 8 x i16> %res
 }
 
@@ -35,9 +31,7 @@ define <vscale x 4 x i32> @mul_i32_imm(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    mov z1.s, #255 // =0xff
 ; CHECK-NEXT:    mul z0.s, z0.s, z1.s
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 255, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res = mul <vscale x 4 x i32> %a, %splat
+  %res = mul <vscale x 4 x i32> %a, splat(i32 255)
   ret <vscale x 4 x i32> %res
 }
 
@@ -48,9 +42,7 @@ define <vscale x 4 x i32> @mul_i32_imm_neg(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    mov z1.s, w8
 ; CHECK-NEXT:    mul z0.s, z0.s, z1.s
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 4 x i32> undef, i32 -200, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
-  %res = mul <vscale x 4 x i32> %a, %splat
+  %res = mul <vscale x 4 x i32> %a, splat(i32 -200)
   ret <vscale x 4 x i32> %res
 }
 
@@ -60,9 +52,7 @@ define <vscale x 2 x i64> @mul_i64_imm(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    mov z1.d, #255 // =0xff
 ; CHECK-NEXT:    mul z0.d, z0.d, z1.d
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 255, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res = mul <vscale x 2 x i64> %a, %splat
+  %res = mul <vscale x 2 x i64> %a, splat(i64 255)
   ret <vscale x 2 x i64> %res
 }
 
@@ -73,51 +63,45 @@ define <vscale x 2 x i64> @mul_i64_imm_neg(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    mov z1.d, x8
 ; CHECK-NEXT:    mul z0.d, z0.d, z1.d
 ; CHECK-NEXT:    ret
-  %elt = insertelement <vscale x 2 x i64> undef, i64 -200, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
-  %res = mul <vscale x 2 x i64> %a, %splat
+  %res = mul <vscale x 2 x i64> %a, splat(i64 -200)
   ret <vscale x 2 x i64> %res
 }
 
 ;
 ; MUL (vector, unpredicated)
 ;
-define <vscale x 16 x i8> @mul_i8(<vscale x 16 x i8> %a,
+define <vscale x 16 x i8> @mul_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: mul_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mul z0.b, z0.b, z1.b
 ; CHECK-NEXT:    ret
-                                  <vscale x 16 x i8> %b) {
   %res = mul <vscale x 16 x i8> %a, %b
   ret <vscale x 16 x i8> %res
 }
 
-define <vscale x 8 x i16> @mul_i16(<vscale x 8 x i16> %a,
+define <vscale x 8 x i16> @mul_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
 ; CHECK-LABEL: mul_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mul z0.h, z0.h, z1.h
 ; CHECK-NEXT:    ret
-                                  <vscale x 8 x i16> %b) {
   %res = mul <vscale x 8 x i16> %a, %b
   ret <vscale x 8 x i16> %res
 }
 
-define <vscale x 4 x i32> @mul_i32(<vscale x 4 x i32> %a,
+define <vscale x 4 x i32> @mul_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: mul_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mul z0.s, z0.s, z1.s
 ; CHECK-NEXT:    ret
-                                  <vscale x 4 x i32> %b) {
   %res = mul <vscale x 4 x i32> %a, %b
   ret <vscale x 4 x i32> %res
 }
 
-define <vscale x 2 x i64> @mul_i64(<vscale x 2 x i64> %a,
+define <vscale x 2 x i64> @mul_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
 ; CHECK-LABEL: mul_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mul z0.d, z0.d, z1.d
 ; CHECK-NEXT:    ret
-                                  <vscale x 2 x i64> %b) {
   %res = mul <vscale x 2 x i64> %a, %b
   ret <vscale x 2 x i64> %res
 }
@@ -125,48 +109,44 @@ define <vscale x 2 x i64> @mul_i64(<vscale x 2 x i64> %a,
 ;
 ; SMULH (vector, unpredicated)
 ;
-define <vscale x 16 x i8> @smulh_i8(<vscale x 16 x i8> %a,
+define <vscale x 16 x i8> @smulh_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: smulh_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smulh z0.b, z0.b, z1.b
 ; CHECK-NEXT:    ret
-                                    <vscale x 16 x i8> %b) {
   %sel = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
   %res = call <vscale x 16 x i8> @llvm.aarch64.sve.smulh.u.nxv16i8(<vscale x 16 x i1> %sel, <vscale x 16 x i8> %a,
                                                                    <vscale x 16 x i8> %b)
   ret <vscale x 16 x i8> %res
 }
 
-define <vscale x 8 x i16> @smulh_i16(<vscale x 8 x i16> %a,
+define <vscale x 8 x i16> @smulh_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
 ; CHECK-LABEL: smulh_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smulh z0.h, z0.h, z1.h
 ; CHECK-NEXT:    ret
-                                     <vscale x 8 x i16> %b) {
   %sel = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
   %res = call <vscale x 8 x i16> @llvm.aarch64.sve.smulh.u.nxv8i16(<vscale x 8 x i1> %sel, <vscale x 8 x i16> %a,
                                                                    <vscale x 8 x i16> %b)
   ret <vscale x 8 x i16> %res
 }
 
-define <vscale x 4 x i32> @smulh_i32(<vscale x 4 x i32> %a,
+define <vscale x 4 x i32> @smulh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: smulh_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smulh z0.s, z0.s, z1.s
 ; CHECK-NEXT:    ret
-                                     <vscale x 4 x i32> %b) {
   %sel = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
   %res = call <vscale x 4 x i32> @llvm.aarch64.sve.smulh.u.nxv4i32(<vscale x 4 x i1> %sel, <vscale x 4 x i32> %a,
                                                                    <vscale x 4 x i32> %b)
   ret <vscale x 4 x i32> %res
 }
 
-define <vscale x 2 x i64> @smulh_i64(<vscale x 2 x i64> %a,
+define <vscale x 2 x i64> @smulh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
 ; CHECK-LABEL: smulh_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smulh z0.d, z0.d, z1.d
 ; CHECK-NEXT:    ret
-                                     <vscale x 2 x i64> %b) {
   %sel = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
   %res = call <vscale x 2 x i64> @llvm.aarch64.sve.smulh.u.nxv2i64(<vscale x 2 x i1> %sel, <vscale x 2 x i64> %a,
                                                                    <vscale x 2 x i64> %b)
@@ -176,48 +156,44 @@ define <vscale x 2 x i64> @smulh_i64(<vscale x 2 x i64> %a,
 ;
 ; UMULH (vector, unpredicated)
 ;
-define <vscale x 16 x i8> @umulh_i8(<vscale x 16 x i8> %a,
+define <vscale x 16 x i8> @umulh_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: umulh_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umulh z0.b, z0.b, z1.b
 ; CHECK-NEXT:    ret
-                                    <vscale x 16 x i8> %b) {
   %sel = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
   %res = call <vscale x 16 x i8> @llvm.aarch64.sve.umulh.u.nxv16i8(<vscale x 16 x i1> %sel, <vscale x 16 x i8> %a,
                                                                    <vscale x 16 x i8> %b)
   ret <vscale x 16 x i8> %res
 }
 
-define <vscale x 8 x i16> @umulh_i16(<vscale x 8 x i16> %a,
+define <vscale x 8 x i16> @umulh_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
 ; CHECK-LABEL: umulh_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umulh z0.h, z0.h, z1.h
 ; CHECK-NEXT:    ret
-                                     <vscale x 8 x i16> %b) {
   %sel = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
   %res = call <vscale x 8 x i16> @llvm.aarch64.sve.umulh.u.nxv8i16(<vscale x 8 x i1> %sel, <vscale x 8 x i16> %a,
                                                                    <vscale x 8 x i16> %b)
   ret <vscale x 8 x i16> %res
 }
 
-define <vscale x 4 x i32> @umulh_i32(<vscale x 4 x i32> %a,
+define <vscale x 4 x i32> @umulh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: umulh_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umulh z0.s, z0.s, z1.s
 ; CHECK-NEXT:    ret
-                                     <vscale x 4 x i32> %b) {
   %sel = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
   %res = call <vscale x 4 x i32> @llvm.aarch64.sve.umulh.u.nxv4i32(<vscale x 4 x i1> %sel, <vscale x 4 x i32> %a,
                                                                    <vscale x 4 x i32> %b)
   ret <vscale x 4 x i32> %res
 }
 
-define <vscale x 2 x i64> @umulh_i64(<vscale x 2 x i64> %a,
+define <vscale x 2 x i64> @umulh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
 ; CHECK-LABEL: umulh_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umulh z0.d, z0.d, z1.d
 ; CHECK-NEXT:    ret
-                                     <vscale x 2 x i64> %b) {
   %sel = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
   %res = call <vscale x 2 x i64> @llvm.aarch64.sve.umulh.u.nxv2i64(<vscale x 2 x i1> %sel, <vscale x 2 x i64> %a,
                                                                    <vscale x 2 x i64> %b)
@@ -227,12 +203,11 @@ define <vscale x 2 x i64> @umulh_i64(<vscale x 2 x i64> %a,
 ;
 ; PMUL (vector, unpredicated)
 ;
-define <vscale x 16 x i8> @pmul_i8(<vscale x 16 x i8> %a,
+define <vscale x 16 x i8> @pmul_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: pmul_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    pmul z0.b, z0.b, z1.b
 ; CHECK-NEXT:    ret
-                                   <vscale x 16 x i8> %b) {
   %res = call <vscale x 16 x i8> @llvm.aarch64.sve.pmul.nxv16i8(<vscale x 16 x i8> %a,
                                                                 <vscale x 16 x i8> %b)
   ret <vscale x 16 x i8> %res
@@ -241,45 +216,41 @@ define <vscale x 16 x i8> @pmul_i8(<vscale x 16 x i8> %a,
 ;
 ; SQDMULH (vector, unpredicated)
 ;
-define <vscale x 16 x i8> @sqdmulh_i8(<vscale x 16 x i8> %a,
+define <vscale x 16 x i8> @sqdmulh_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: sqdmulh_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqdmulh z0.b, z0.b, z1.b
 ; CHECK-NEXT:    ret
-                                      <vscale x 16 x i8> %b) {
   %res = call <vscale x 16 x i8> @llvm.aarch64.sve.sqdmulh.nxv16i8(<vscale x 16 x i8> %a,
                                                                    <vscale x 16 x i8> %b)
   ret <vscale x 16 x i8> %res
 }
 
-define <vscale x 8 x i16> @sqdmulh_i16(<vscale x 8 x i16> %a,
+define <vscale x 8 x i16> @sqdmulh_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
 ; CHECK-LABEL: sqdmulh_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqdmulh z0.h, z0.h, z1.h
 ; CHECK-NEXT:    ret
-                                       <vscale x 8 x i16> %b) {
   %res = call <vscale x 8 x i16> @llvm.aarch64.sve.sqdmulh.nxv8i16(<vscale x 8 x i16> %a,
                                                                    <vscale x 8 x i16> %b)
   ret <vscale x 8 x i16> %res
 }
 
-define <vscale x 4 x i32> @sqdmulh_i32(<vscale x 4 x i32> %a,
+define <vscale x 4 x i32> @sqdmulh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: sqdmulh_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqdmulh z0.s, z0.s, z1.s
 ; CHECK-NEXT:    ret
-                                       <vscale x 4 x i32> %b) {
   %res = call <vscale x 4 x i32> @llvm.aarch64.sve.sqdmulh.nxv4i32(<vscale x 4 x i32> %a,
                                                                    <vscale x 4 x i32> %b)
   ret <vscale x 4 x i32> %res
 }
 
-define <vscale x 2 x i64> @sqdmulh_i64(<vscale x 2 x i64> %a,
+define <vscale x 2 x i64> @sqdmulh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
 ; CHECK-LABEL: sqdmulh_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqdmulh z0.d, z0.d, z1.d
 ; CHECK-NEXT:    ret
-                                       <vscale x 2 x i64> %b) {
   %res = call <vscale x 2 x i64> @llvm.aarch64.sve.sqdmulh.nxv2i64(<vscale x 2 x i64> %a,
                                                                    <vscale x 2 x i64> %b)
   ret <vscale x 2 x i64> %res
@@ -288,45 +259,41 @@ define <vscale x 2 x i64> @sqdmulh_i64(<vscale x 2 x i64> %a,
 ;
 ; SQRDMULH (vector, unpredicated)
 ;
-define <vscale x 16 x i8> @sqrdmulh_i8(<vscale x 16 x i8> %a,
+define <vscale x 16 x i8> @sqrdmulh_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: sqrdmulh_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqrdmulh z0.b, z0.b, z1.b
 ; CHECK-NEXT:    ret
-                                       <vscale x 16 x i8> %b) {
   %res = call <vscale x 16 x i8> @llvm.aarch64.sve.sqrdmulh.nxv16i8(<vscale x 16 x i8> %a,
                                                                     <vscale x 16 x i8> %b)
   ret <vscale x 16 x i8> %res
 }
 
-define <vscale x 8 x i16> @sqrdmulh_i16(<vscale x 8 x i16> %a,
+define <vscale x 8 x i16> @sqrdmulh_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
 ; CHECK-LABEL: sqrdmulh_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqrdmulh z0.h, z0.h, z1.h
 ; CHECK-NEXT:    ret
-                                        <vscale x 8 x i16> %b) {
   %res = call <vscale x 8 x i16> @llvm.aarch64.sve.sqrdmulh.nxv8i16(<vscale x 8 x i16> %a,
                                                                     <vscale x 8 x i16> %b)
   ret <vscale x 8 x i16> %res
 }
 
-define <vscale x 4 x i32> @sqrdmulh_i32(<vscale x 4 x i32> %a,
+define <vscale x 4 x i32> @sqrdmulh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: sqrdmulh_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqrdmulh z0.s, z0.s, z1.s
 ; CHECK-NEXT:    ret
-                                        <vscale x 4 x i32> %b) {
   %res = call <vscale x 4 x i32> @llvm.aarch64.sve.sqrdmulh.nxv4i32(<vscale x 4 x i32> %a,
                                                                     <vscale x 4 x i32> %b)
   ret <vscale x 4 x i32> %res
 }
 
-define <vscale x 2 x i64> @sqrdmulh_i64(<vscale x 2 x i64> %a,
+define <vscale x 2 x i64> @sqrdmulh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
 ; CHECK-LABEL: sqrdmulh_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sqrdmulh z0.d, z0.d, z1.d
 ; CHECK-NEXT:    ret
-                                        <vscale x 2 x i64> %b) {
   %res = call <vscale x 2 x i64> @llvm.aarch64.sve.sqrdmulh.nxv2i64(<vscale x 2 x i64> %a,
                                                                     <vscale x 2 x i64> %b)
   ret <vscale x 2 x i64> %res
diff --git a/llvm/test/CodeGen/AArch64/sve2-int-mulh.ll b/llvm/test/CodeGen/AArch64/sve2-int-mulh.ll
index c929a3faffabf..bcf76d5b13d62 100644
--- a/llvm/test/CodeGen/AArch64/sve2-int-mulh.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-int-mulh.ll
@@ -10,12 +10,10 @@ define <vscale x 16 x i8> @smulh_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smulh z0.b, z0.b, z1.b
 ; CHECK-NEXT:    ret
-  %insert = insertelement <vscale x 16 x i16> undef, i16 8, i64 0
-  %splat = shufflevector <vscale x 16 x i16> %insert, <vscale x 16 x i16> undef, <vscale x 16 x i32> zeroinitializer
   %1 = sext <vscale x 16 x i8> %a to <vscale x 16 x i16>
   %2 = sext <vscale x 16 x i8> %b to <vscale x 16 x i16>
   %mul = mul <vscale x 16 x i16> %1, %2
-  %shr = lshr <vscale x 16 x i16> %mul, %splat
+  %shr = lshr <vscale x 16 x i16> %mul, splat(i16 8)
   %tr = trunc <vscale x 16 x i16> %shr to <vscale x 16 x i8>
   ret <vscale x 16 x i8> %tr
 }
@@ -25,12 +23,10 @@ define <vscale x 8 x i16> @smulh_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smulh z0.h, z0.h, z1.h
 ; CHECK-NEXT:    ret
-  %insert = insertelement <vscale x 8 x i32> undef, i32 16, i64 0
-  %splat = shufflevector <vscale x 8 x i32> %insert, <vscale x 8 x i32> undef, <vscale x 8 x i32> zeroinitializer
   %1 = sext <vscale x 8 x i16> %a to <vscale x 8 x i32>
   %2 = sext <vscale x 8 x i16> %b to <vscale x 8 x i32>
   %mul = mul <vscale x 8 x i32> %1, %2
-  %shr = lshr <vscale x 8 x i32> %mul, %splat
+  %shr = lshr <vscale x 8 x i32> %mul, splat(i32 16)
   %tr = trunc <vscale x 8 x i32> %shr to <vscale x 8 x i16>
   ret <vscale x 8 x i16> %tr
 }
@@ -40,12 +36,10 @@ define <vscale x 4 x i32> @smulh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smulh z0.s, z0.s, z1.s
 ; CHECK-NEXT:    ret
-  %insert = insertelement <vscale x 4 x i64> undef, i64 32, i64 0
-  %splat = shufflevector <vscale x 4 x i64> %insert, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
   %1 = sext <vscale x 4 x i32> %a to <vscale x 4 x i64>
   %2 = sext <vscale x 4 x i32> %b to <vscale x 4 x i64>
   %mul = mul <vscale x 4 x i64> %1, %2
-  %shr = lshr <vscale x 4 x i64> %mul, %splat
+  %shr = lshr <vscale x 4 x i64> %mul, splat(i64 32)
   %tr = trunc <vscale x 4 x i64> %shr to <vscale x 4 x i32>
   ret <vscale x 4 x i32> %tr
 }
@@ -55,12 +49,10 @@ define <vscale x 2 x i64> @smulh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smulh z0.d, z0.d, z1.d
 ; CHECK-NEXT:    ret
-  %insert = insertelement <vscale x 2 x i128> undef, i128 64, i64 0
-  %splat = shufflevector <vscale x 2 x i128> %insert, <vscale x 2 x i128> undef, <vscale x 2 x i32> zeroinitializer
   %1 = sext <vscale x 2 x i64> %a to <vscale x 2 x i128>
   %2 = sext <vscale x 2 x i64> %b to <vscale x 2 x i128>
   %mul = mul <vscale x 2 x i128> %1, %2
-  %shr = lshr <vscale x 2 x i128> %mul, %splat
+  %shr = lshr <vscale x 2 x i128> %mul, splat(i128 64)
   %tr = trunc <vscale x 2 x i128> %shr to <vscale x 2 x i64>
   ret <vscale x 2 x i64> %tr
 }
@@ -74,12 +66,10 @@ define <vscale x 16 x i8> @umulh_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umulh z0.b, z0.b, z1.b
 ; CHECK-NEXT:    ret
-  %insert = insertelement <vscale x 16 x i16> undef, i16 8, i64 0
-  %splat = shufflevector <vscale x 16 x i16> %insert, <vscale x 16 x i16> undef, <vscale x 16 x i32> zeroinitializer
   %1 = zext <vscale x 16 x i8> %a to <vscale x 16 x i16>
   %2 = zext <vscale x 16 x i8> %b to <vscale x 16 x i16>
   %mul = mul <vscale x 16 x i16> %1, %2
-  %shr = lshr <vscale x 16 x i16> %mul, %splat
+  %shr = lshr <vscale x 16 x i16> %mul, splat(i16 8)
   %tr = trunc <vscale x 16 x i16> %shr to <vscale x 16 x i8>
   ret <vscale x 16 x i8> %tr
 }
@@ -89,12 +79,10 @@ define <vscale x 8 x i16> @umulh_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umulh z0.h, z0.h, z1.h
 ; CHECK-NEXT:    ret
-  %insert = insertelement <vscale x 8 x i32> undef, i32 16, i64 0
-  %splat = shufflevector <vscale x 8 x i32> %insert, <vscale x 8 x i32> undef, <vscale x 8 x i32> zeroinitializer
   %1 = zext <vscale x 8 x i16> %a to <vscale x 8 x i32>
   %2 = zext <vscale x 8 x i16> %b to <vscale x 8 x i32>
   %mul = mul <vscale x 8 x i32> %1, %2
-  %shr = lshr <vscale x 8 x i32> %mul, %splat
+  %shr = lshr <vscale x 8 x i32> %mul, splat(i32 16)
   %tr = trunc <vscale x 8 x i32> %shr to <vscale x 8 x i16>
   ret <vscale x 8 x i16> %tr
 }
@@ -104,12 +92,10 @@ define <vscale x 4 x i32> @umulh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umulh z0.s, z0.s, z1.s
 ; CHECK-NEXT:    ret
-  %insert = insertelement <vscale x 4 x i64> undef, i64 32, i64 0
-  %splat = shufflevector <vscale x 4 x i64> %insert, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
   %1 = zext <vscale x 4 x i32> %a to <vscale x 4 x i64>
   %2 = zext <vscale x 4 x i32> %b to <vscale x 4 x i64>
   %mul = mul <vscale x 4 x i64> %1, %2
-  %shr = lshr <vscale x 4 x i64> %mul, %splat
+  %shr = lshr <vscale x 4 x i64> %mul, splat(i64 32)
   %tr = trunc <vscale x 4 x i64> %shr to <vscale x 4 x i32>
   ret <vscale x 4 x i32> %tr
 }
@@ -119,12 +105,10 @@ define <vscale x 2 x i64> @umulh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umulh z0.d, z0.d, z1.d
 ; CHECK-NEXT:    ret
-  %insert = insertelement <vscale x 2 x i128> undef, i128 64, i64 0
-  %splat = shufflevector <vscale x 2 x i128> %insert, <vscale x 2 x i128> undef, <vscale x 2 x i32> zeroinitializer
   %1 = zext <vscale x 2 x i64> %a to <vscale x 2 x i128>
   %2 = zext <vscale x 2 x i64> %b to <vscale x 2 x i128>
   %mul = mul <vscale x 2 x i128> %1, %2
-  %shr = lshr <vscale x 2 x i128> %mul, %splat
+  %shr = lshr <vscale x 2 x i128> %mul, splat(i128 64)
   %tr = trunc <vscale x 2 x i128> %shr to <vscale x 2 x i64>
   ret <vscale x 2 x i64> %tr
 }
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-int-arith-imm.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-int-arith-imm.ll
index 500973d053f5b..20fdd7f0bba4e 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-int-arith-imm.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-int-arith-imm.ll
@@ -9,11 +9,9 @@ define <vscale x 16 x i8> @sqadd_b_lowimm(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    sqadd z0.b, z0.b, #27 // =0x1b
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-  %elt = insertelement <vscale x 16 x i8> undef, i8 27, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sqadd.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %splat)
+                                                                 <vscale x 16 x i8> splat(i8 27))
   ret <vscale x 16 x i8> %out
 }
 
@@ -23,11 +21,9 @@ define <vscale x 8 x i16> @sqadd_h_lowimm(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    sqadd z0.h, z0.h, #43 // =0x2b
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 43, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqadd.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %splat)
+                                                                 <vscale x 8 x i16> splat(i16 43))
   ret <vscale x 8 x i16> %out
 }
 
@@ -37,11 +33,9 @@ define <vscale x 8 x i16> @sqadd_h_highimm(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    sqadd z0.h, z0.h, #2048 // =0x800
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 2048, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqadd.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %splat)
+                                                                 <vscale x 8 x i16> splat(i16 2048))
   ret <vscale x 8 x i16> %out
 }
 
@@ -51,11 +45,9 @@ define <vscale x 4 x i32> @sqadd_s_lowimm(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    sqadd z0.s, z0.s, #1 // =0x1
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 1, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqadd.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %splat)
+                                                                 <vscale x 4 x i32> splat(i32 1))
   ret <vscale x 4 x i32> %out
 }
 
@@ -65,11 +57,9 @@ define <vscale x 4 x i32> @sqadd_s_highimm(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    sqadd z0.s, z0.s, #8192 // =0x2000
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 8192, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqadd.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %splat)
+                                                                 <vscale x 4 x i32> splat(i32 8192))
   ret <vscale x 4 x i32> %out
 }
 
@@ -79,11 +69,9 @@ define <vscale x 2 x i64> @sqadd_d_lowimm(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    sqadd z0.d, z0.d, #255 // =0xff
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 255, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sqadd.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %splat)
+                                                                 <vscale x 2 x i64> splat(i64 255))
   ret <vscale x 2 x i64> %out
 }
 
@@ -93,11 +81,9 @@ define <vscale x 2 x i64> @sqadd_d_highimm(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    sqadd z0.d, z0.d, #65280 // =0xff00
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 65280, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sqadd.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %splat)
+                                                                 <vscale x 2 x i64> splat(i64 65280))
   ret <vscale x 2 x i64> %out
 }
 
@@ -109,11 +95,9 @@ define <vscale x 16 x i8> @sqsub_b_lowimm(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    sqsub z0.b, z0.b, #27 // =0x1b
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-  %elt = insertelement <vscale x 16 x i8> undef, i8 27, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sqsub.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                    <vscale x 16 x i8> %a,
-                                                                   <vscale x 16 x i8> %splat)
+                                                                   <vscale x 16 x i8> splat(i8 27))
   ret <vscale x 16 x i8> %out
 }
 
@@ -123,11 +107,9 @@ define <vscale x 8 x i16> @sqsub_h_lowimm(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    sqsub z0.h, z0.h, #43 // =0x2b
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 43, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqsub.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                    <vscale x 8 x i16> %a,
-                                                                   <vscale x 8 x i16> %splat)
+                                                                   <vscale x 8 x i16> splat(i16 43))
   ret <vscale x 8 x i16> %out
 }
 
@@ -137,11 +119,9 @@ define <vscale x 8 x i16> @sqsub_h_highimm(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    sqsub z0.h, z0.h, #2048 // =0x800
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 2048, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqsub.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                    <vscale x 8 x i16> %a,
-                                                                   <vscale x 8 x i16> %splat)
+                                                                   <vscale x 8 x i16> splat(i16 2048))
   ret <vscale x 8 x i16> %out
 }
 
@@ -151,11 +131,9 @@ define <vscale x 4 x i32> @sqsub_s_lowimm(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    sqsub z0.s, z0.s, #1 // =0x1
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 1, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqsub.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                    <vscale x 4 x i32> %a,
-                                                                   <vscale x 4 x i32> %splat)
+                                                                   <vscale x 4 x i32> splat(i32 1))
   ret <vscale x 4 x i32> %out
 }
 
@@ -165,11 +143,9 @@ define <vscale x 4 x i32> @sqsub_s_highimm(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    sqsub z0.s, z0.s, #8192 // =0x2000
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 8192, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqsub.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                    <vscale x 4 x i32> %a,
-                                                                   <vscale x 4 x i32> %splat)
+                                                                   <vscale x 4 x i32> splat(i32 8192))
   ret <vscale x 4 x i32> %out
 }
 
@@ -179,11 +155,9 @@ define <vscale x 2 x i64> @sqsub_d_lowimm(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    sqsub z0.d, z0.d, #255 // =0xff
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 255, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sqsub.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                    <vscale x 2 x i64> %a,
-                                                                   <vscale x 2 x i64> %splat)
+                                                                   <vscale x 2 x i64> splat(i64 255))
   ret <vscale x 2 x i64> %out
 }
 
@@ -193,11 +167,9 @@ define <vscale x 2 x i64> @sqsub_d_highimm(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    sqsub z0.d, z0.d, #65280 // =0xff00
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 65280, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sqsub.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                    <vscale x 2 x i64> %a,
-                                                                   <vscale x 2 x i64> %splat)
+                                                                   <vscale x 2 x i64> splat(i64 65280))
   ret <vscale x 2 x i64> %out
 }
 
@@ -209,11 +181,9 @@ define <vscale x 16 x i8> @uqadd_b_lowimm(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    uqadd z0.b, z0.b, #27 // =0x1b
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-  %elt = insertelement <vscale x 16 x i8> undef, i8 27, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.uqadd.nxv16i8(<vscale x 16 x i1> %pg,
                                                                  <vscale x 16 x i8> %a,
-                                                                 <vscale x 16 x i8> %splat)
+                                                                 <vscale x 16 x i8> splat(i8 27))
   ret <vscale x 16 x i8> %out
 }
 
@@ -223,11 +193,9 @@ define <vscale x 8 x i16> @uqadd_h_lowimm(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    uqadd z0.h, z0.h, #43 // =0x2b
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 43, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uqadd.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %splat)
+                                                                 <vscale x 8 x i16> splat(i16 43))
   ret <vscale x 8 x i16> %out
 }
 
@@ -237,11 +205,9 @@ define <vscale x 8 x i16> @uqadd_h_highimm(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    uqadd z0.h, z0.h, #2048 // =0x800
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 2048, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uqadd.nxv8i16(<vscale x 8 x i1> %pg,
                                                                  <vscale x 8 x i16> %a,
-                                                                 <vscale x 8 x i16> %splat)
+                                                                 <vscale x 8 x i16> splat(i16 2048))
   ret <vscale x 8 x i16> %out
 }
 
@@ -251,11 +217,9 @@ define <vscale x 4 x i32> @uqadd_s_lowimm(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    uqadd z0.s, z0.s, #1 // =0x1
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 1, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uqadd.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %splat)
+                                                                 <vscale x 4 x i32> splat(i32 1))
   ret <vscale x 4 x i32> %out
 }
 
@@ -265,11 +229,9 @@ define <vscale x 4 x i32> @uqadd_s_highimm(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    uqadd z0.s, z0.s, #8192 // =0x2000
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 8192, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uqadd.nxv4i32(<vscale x 4 x i1> %pg,
                                                                  <vscale x 4 x i32> %a,
-                                                                 <vscale x 4 x i32> %splat)
+                                                                 <vscale x 4 x i32> splat(i32 8192))
   ret <vscale x 4 x i32> %out
 }
 
@@ -279,11 +241,9 @@ define <vscale x 2 x i64> @uqadd_d_lowimm(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    uqadd z0.d, z0.d, #255 // =0xff
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 255, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uqadd.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %splat)
+                                                                 <vscale x 2 x i64> splat(i64 255))
   ret <vscale x 2 x i64> %out
 }
 
@@ -293,11 +253,9 @@ define <vscale x 2 x i64> @uqadd_d_highimm(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    uqadd z0.d, z0.d, #65280 // =0xff00
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 65280, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uqadd.nxv2i64(<vscale x 2 x i1> %pg,
                                                                  <vscale x 2 x i64> %a,
-                                                                 <vscale x 2 x i64> %splat)
+                                                                 <vscale x 2 x i64> splat(i64 65280))
   ret <vscale x 2 x i64> %out
 }
 
@@ -309,11 +267,9 @@ define <vscale x 16 x i8> @uqsub_b_lowimm(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    uqsub z0.b, z0.b, #27 // =0x1b
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-  %elt = insertelement <vscale x 16 x i8> undef, i8 27, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
   %out = call <vscale x 16 x i8> @llvm.aarch64.sve.uqsub.u.nxv16i8(<vscale x 16 x i1> %pg,
                                                                    <vscale x 16 x i8> %a,
-                                                                   <vscale x 16 x i8> %splat)
+                                                                   <vscale x 16 x i8> splat(i8 27))
   ret <vscale x 16 x i8> %out
 }
 
@@ -323,11 +279,9 @@ define <vscale x 8 x i16> @uqsub_h_lowimm(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    uqsub z0.h, z0.h, #43 // =0x2b
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 43, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uqsub.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                    <vscale x 8 x i16> %a,
-                                                                   <vscale x 8 x i16> %splat)
+                                                                   <vscale x 8 x i16> splat(i16 43))
   ret <vscale x 8 x i16> %out
 }
 
@@ -337,11 +291,9 @@ define <vscale x 8 x i16> @uqsub_h_highimm(<vscale x 8 x i16> %a) {
 ; CHECK-NEXT:    uqsub z0.h, z0.h, #2048 // =0x800
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %elt = insertelement <vscale x 8 x i16> undef, i16 2048, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
   %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uqsub.u.nxv8i16(<vscale x 8 x i1> %pg,
                                                                    <vscale x 8 x i16> %a,
-                                                                   <vscale x 8 x i16> %splat)
+                                                                   <vscale x 8 x i16> splat(i16 2048))
   ret <vscale x 8 x i16> %out
 }
 
@@ -351,11 +303,9 @@ define <vscale x 4 x i32> @uqsub_s_lowimm(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    uqsub z0.s, z0.s, #1 // =0x1
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 1, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uqsub.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                    <vscale x 4 x i32> %a,
-                                                                   <vscale x 4 x i32> %splat)
+                                                                   <vscale x 4 x i32> splat(i32 1))
   ret <vscale x 4 x i32> %out
 }
 
@@ -365,11 +315,9 @@ define <vscale x 4 x i32> @uqsub_s_highimm(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    uqsub z0.s, z0.s, #8192 // =0x2000
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %elt = insertelement <vscale x 4 x i32> undef, i32 8192, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uqsub.u.nxv4i32(<vscale x 4 x i1> %pg,
                                                                    <vscale x 4 x i32> %a,
-                                                                   <vscale x 4 x i32> %splat)
+                                                                   <vscale x 4 x i32> splat(i32 8192))
   ret <vscale x 4 x i32> %out
 }
 
@@ -379,11 +327,9 @@ define <vscale x 2 x i64> @uqsub_d_lowimm(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    uqsub z0.d, z0.d, #255 // =0xff
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 255, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uqsub.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                    <vscale x 2 x i64> %a,
-                                                                   <vscale x 2 x i64> %splat)
+                                                                   <vscale x 2 x i64> splat(i64 255))
   ret <vscale x 2 x i64> %out
 }
 
@@ -393,11 +339,9 @@ define <vscale x 2 x i64> @uqsub_d_highimm(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    uqsub z0.d, z0.d, #65280 // =0xff00
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %elt = insertelement <vscale x 2 x i64> undef, i64 65280, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
   %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uqsub.u.nxv2i64(<vscale x 2 x i1> %pg,
                                                                    <vscale x 2 x i64> %a,
-                                                                   <vscale x 2 x i64> %splat)
+                                                                   <vscale x 2 x i64> splat(i64 65280))
   ret <vscale x 2 x i64> %out
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve2-sra.ll b/llvm/test/CodeGen/AArch64/sve2-sra.ll
index 3de49c9b8768b..eafcd60bc1605 100644
--- a/llvm/test/CodeGen/AArch64/sve2-sra.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-sra.ll
@@ -10,9 +10,7 @@ define <vscale x 16 x i8> @usra_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    usra z0.b, z1.b, #1
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 16 x i8> poison, i8 1, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %ins, <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
-  %shift = lshr <vscale x 16 x i8> %b, %splat
+  %shift = lshr <vscale x 16 x i8> %b, splat(i8 1)
   %add = add <vscale x 16 x i8> %a, %shift
   ret <vscale x 16 x i8> %add
 }
@@ -22,9 +20,7 @@ define <vscale x 8 x i16> @usra_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    usra z0.h, z1.h, #2
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 8 x i16> poison, i16 2, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
-  %shift = lshr <vscale x 8 x i16> %b, %splat
+  %shift = lshr <vscale x 8 x i16> %b, splat(i16 2)
   %add = add <vscale x 8 x i16> %a, %shift
   ret <vscale x 8 x i16> %add
 }
@@ -34,9 +30,7 @@ define <vscale x 4 x i32> @usra_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    usra z0.s, z1.s, #3
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 4 x i32> poison, i32 3, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-  %shift = lshr <vscale x 4 x i32> %b, %splat
+  %shift = lshr <vscale x 4 x i32> %b, splat(i32 3)
   %add = add <vscale x 4 x i32> %a, %shift
   ret <vscale x 4 x i32> %add
 }
@@ -46,9 +40,7 @@ define <vscale x 2 x i64> @usra_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    usra z0.d, z1.d, #4
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 2 x i64> poison, i64 4, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-  %shift = lshr <vscale x 2 x i64> %b, %splat
+  %shift = lshr <vscale x 2 x i64> %b, splat(i64 4)
   %add = add <vscale x 2 x i64> %a, %shift
   ret <vscale x 2 x i64> %add
 }
@@ -59,9 +51,7 @@ define <vscale x 16 x i8> @usra_intr_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8
 ; CHECK-NEXT:    usra z0.b, z1.b, #1
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-  %ins = insertelement <vscale x 16 x i8> poison, i8 1, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %ins, <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
-  %shift = call <vscale x 16 x i8> @llvm.aarch64.sve.lsr.u.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %b, <vscale x 16 x i8> %splat)
+  %shift = call <vscale x 16 x i8> @llvm.aarch64.sve.lsr.u.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %b, <vscale x 16 x i8> splat(i8 1))
   %add = add <vscale x 16 x i8> %a, %shift
   ret <vscale x 16 x i8> %add
 }
@@ -72,9 +62,7 @@ define <vscale x 8 x i16> @usra_intr_i16(<vscale x 8 x i16> %a, <vscale x 8 x i1
 ; CHECK-NEXT:    usra z0.h, z1.h, #2
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %ins = insertelement <vscale x 8 x i16> poison, i16 2, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
-  %shift = call <vscale x 8 x i16> @llvm.aarch64.sve.lsr.u.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %b, <vscale x 8 x i16> %splat)
+  %shift = call <vscale x 8 x i16> @llvm.aarch64.sve.lsr.u.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %b, <vscale x 8 x i16> splat(i16 2))
   %add = add <vscale x 8 x i16> %a, %shift
   ret <vscale x 8 x i16> %add
 }
@@ -85,9 +73,7 @@ define <vscale x 4 x i32> @usra_intr_i32(<vscale x 4 x i32> %a, <vscale x 4 x i3
 ; CHECK-NEXT:    usra z0.s, z1.s, #3
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %ins = insertelement <vscale x 4 x i32> poison, i32 3, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-  %shift = call <vscale x 4 x i32> @llvm.aarch64.sve.lsr.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %b, <vscale x 4 x i32> %splat)
+  %shift = call <vscale x 4 x i32> @llvm.aarch64.sve.lsr.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %b, <vscale x 4 x i32> splat(i32 3))
   %add = add <vscale x 4 x i32> %a, %shift
   ret <vscale x 4 x i32> %add
 }
@@ -98,9 +84,7 @@ define <vscale x 2 x i64> @usra_intr_i64(<vscale x 2 x i64> %a, <vscale x 2 x i6
 ; CHECK-NEXT:    usra z0.d, z1.d, #4
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %ins = insertelement <vscale x 2 x i64> poison, i64 4, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-  %shift = call <vscale x 2 x i64> @llvm.aarch64.sve.lsr.u.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %b, <vscale x 2 x i64> %splat)
+  %shift = call <vscale x 2 x i64> @llvm.aarch64.sve.lsr.u.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %b, <vscale x 2 x i64> splat(i64 4))
   %add = add <vscale x 2 x i64> %a, %shift
   ret <vscale x 2 x i64> %add
 }
@@ -110,9 +94,7 @@ define <vscale x 16 x i8> @usra_intr_u_i8(<vscale x 16 x i1> %pg, <vscale x 16 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    usra z0.b, z1.b, #1
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 16 x i8> poison, i8 1, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %ins, <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
-  %shift = call <vscale x 16 x i8> @llvm.aarch64.sve.lsr.u.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %b, <vscale x 16 x i8> %splat)
+  %shift = call <vscale x 16 x i8> @llvm.aarch64.sve.lsr.u.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %b, <vscale x 16 x i8> splat(i8 1))
   %add = add <vscale x 16 x i8> %a, %shift
   ret <vscale x 16 x i8> %add
 }
@@ -122,9 +104,7 @@ define <vscale x 8 x i16> @usra_intr_u_i16(<vscale x 8 x i1> %pg, <vscale x 8 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    usra z0.h, z1.h, #2
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 8 x i16> poison, i16 2, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
-  %shift = call <vscale x 8 x i16> @llvm.aarch64.sve.lsr.u.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %b, <vscale x 8 x i16> %splat)
+  %shift = call <vscale x 8 x i16> @llvm.aarch64.sve.lsr.u.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %b, <vscale x 8 x i16> splat(i16 2))
   %add = add <vscale x 8 x i16> %a, %shift
   ret <vscale x 8 x i16> %add
 }
@@ -134,9 +114,7 @@ define <vscale x 4 x i32> @usra_intr_u_i32(<vscale x 4 x i1> %pg, <vscale x 4 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    usra z0.s, z1.s, #3
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 4 x i32> poison, i32 3, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-  %shift = call <vscale x 4 x i32> @llvm.aarch64.sve.lsr.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %b, <vscale x 4 x i32> %splat)
+  %shift = call <vscale x 4 x i32> @llvm.aarch64.sve.lsr.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %b, <vscale x 4 x i32> splat(i32 3))
   %add = add <vscale x 4 x i32> %a, %shift
   ret <vscale x 4 x i32> %add
 }
@@ -146,9 +124,7 @@ define <vscale x 2 x i64> @usra_intr_u_i64(<vscale x 2 x i1> %pg, <vscale x 2 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    usra z0.d, z1.d, #4
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 2 x i64> poison, i64 4, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-  %shift = call <vscale x 2 x i64> @llvm.aarch64.sve.lsr.u.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %b, <vscale x 2 x i64> %splat)
+  %shift = call <vscale x 2 x i64> @llvm.aarch64.sve.lsr.u.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %b, <vscale x 2 x i64> splat(i64 4))
   %add = add <vscale x 2 x i64> %a, %shift
   ret <vscale x 2 x i64> %add
 }
@@ -160,9 +136,7 @@ define <vscale x 16 x i8> @ssra_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ssra z0.b, z1.b, #1
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 16 x i8> poison, i8 1, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %ins, <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
-  %shift = ashr <vscale x 16 x i8> %b, %splat
+  %shift = ashr <vscale x 16 x i8> %b, splat(i8 1)
   %add = add <vscale x 16 x i8> %a, %shift
   ret <vscale x 16 x i8> %add
 }
@@ -172,9 +146,7 @@ define <vscale x 8 x i16> @ssra_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ssra z0.h, z1.h, #2
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 8 x i16> poison, i16 2, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
-  %shift = ashr <vscale x 8 x i16> %b, %splat
+  %shift = ashr <vscale x 8 x i16> %b, splat(i16 2)
   %add = add <vscale x 8 x i16> %a, %shift
   ret <vscale x 8 x i16> %add
 }
@@ -184,9 +156,7 @@ define <vscale x 4 x i32> @ssra_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ssra z0.s, z1.s, #3
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 4 x i32> poison, i32 3, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-  %shift = ashr <vscale x 4 x i32> %b, %splat
+  %shift = ashr <vscale x 4 x i32> %b, splat(i32 3)
   %add = add <vscale x 4 x i32> %a, %shift
   ret <vscale x 4 x i32> %add
 }
@@ -196,9 +166,7 @@ define <vscale x 2 x i64> @ssra_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ssra z0.d, z1.d, #4
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 2 x i64> poison, i64 4, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-  %shift = ashr <vscale x 2 x i64> %b, %splat
+  %shift = ashr <vscale x 2 x i64> %b, splat(i64 4)
   %add = add <vscale x 2 x i64> %a, %shift
   ret <vscale x 2 x i64> %add
 }
@@ -209,9 +177,7 @@ define <vscale x 16 x i8> @ssra_intr_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8
 ; CHECK-NEXT:    ssra z0.b, z1.b, #1
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
-  %ins = insertelement <vscale x 16 x i8> poison, i8 1, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %ins, <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
-  %shift = call <vscale x 16 x i8> @llvm.aarch64.sve.asr.u.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %b, <vscale x 16 x i8> %splat)
+  %shift = call <vscale x 16 x i8> @llvm.aarch64.sve.asr.u.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %b, <vscale x 16 x i8> splat(i8 1))
   %add = add <vscale x 16 x i8> %a, %shift
   ret <vscale x 16 x i8> %add
 }
@@ -222,9 +188,7 @@ define <vscale x 8 x i16> @ssra_intr_i16(<vscale x 8 x i16> %a, <vscale x 8 x i1
 ; CHECK-NEXT:    ssra z0.h, z1.h, #2
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %ins = insertelement <vscale x 8 x i16> poison, i16 2, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
-  %shift = call <vscale x 8 x i16> @llvm.aarch64.sve.asr.u.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %b, <vscale x 8 x i16> %splat)
+  %shift = call <vscale x 8 x i16> @llvm.aarch64.sve.asr.u.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %b, <vscale x 8 x i16> splat(i16 2))
   %add = add <vscale x 8 x i16> %a, %shift
   ret <vscale x 8 x i16> %add
 }
@@ -235,9 +199,7 @@ define <vscale x 4 x i32> @ssra_intr_i32(<vscale x 4 x i32> %a, <vscale x 4 x i3
 ; CHECK-NEXT:    ssra z0.s, z1.s, #3
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-  %ins = insertelement <vscale x 4 x i32> poison, i32 3, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-  %shift = call <vscale x 4 x i32> @llvm.aarch64.sve.asr.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %b, <vscale x 4 x i32> %splat)
+  %shift = call <vscale x 4 x i32> @llvm.aarch64.sve.asr.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %b, <vscale x 4 x i32> splat(i32 3))
   %add = add <vscale x 4 x i32> %a, %shift
   ret <vscale x 4 x i32> %add
 }
@@ -248,9 +210,7 @@ define <vscale x 2 x i64> @ssra_intr_i64(<vscale x 2 x i64> %a, <vscale x 2 x i6
 ; CHECK-NEXT:    ssra z0.d, z1.d, #4
 ; CHECK-NEXT:    ret
   %pg = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-  %ins = insertelement <vscale x 2 x i64> poison, i64 4, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-  %shift = call <vscale x 2 x i64> @llvm.aarch64.sve.asr.u.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %b, <vscale x 2 x i64> %splat)
+  %shift = call <vscale x 2 x i64> @llvm.aarch64.sve.asr.u.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %b, <vscale x 2 x i64> splat(i64 4))
   %add = add <vscale x 2 x i64> %a, %shift
   ret <vscale x 2 x i64> %add
 }
@@ -260,9 +220,7 @@ define <vscale x 16 x i8> @ssra_intr_u_i8(<vscale x 16 x i1> %pg, <vscale x 16 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ssra z0.b, z1.b, #1
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 16 x i8> poison, i8 1, i32 0
-  %splat = shufflevector <vscale x 16 x i8> %ins, <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
-  %shift = call <vscale x 16 x i8> @llvm.aarch64.sve.asr.u.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %b, <vscale x 16 x i8> %splat)
+  %shift = call <vscale x 16 x i8> @llvm.aarch64.sve.asr.u.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %b, <vscale x 16 x i8> splat(i8 1))
   %add = add <vscale x 16 x i8> %a, %shift
   ret <vscale x 16 x i8> %add
 }
@@ -272,9 +230,7 @@ define <vscale x 8 x i16> @ssra_intr_u_i16(<vscale x 8 x i1> %pg, <vscale x 8 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ssra z0.h, z1.h, #2
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 8 x i16> poison, i16 2, i32 0
-  %splat = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
-  %shift = call <vscale x 8 x i16> @llvm.aarch64.sve.asr.u.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %b, <vscale x 8 x i16> %splat)
+  %shift = call <vscale x 8 x i16> @llvm.aarch64.sve.asr.u.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %b, <vscale x 8 x i16> splat(i16 2))
   %add = add <vscale x 8 x i16> %a, %shift
   ret <vscale x 8 x i16> %add
 }
@@ -284,9 +240,7 @@ define <vscale x 4 x i32> @ssra_intr_u_i32(<vscale x 4 x i1> %pg, <vscale x 4 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ssra z0.s, z1.s, #3
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 4 x i32> poison, i32 3, i32 0
-  %splat = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-  %shift = call <vscale x 4 x i32> @llvm.aarch64.sve.asr.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %b, <vscale x 4 x i32> %splat)
+  %shift = call <vscale x 4 x i32> @llvm.aarch64.sve.asr.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %b, <vscale x 4 x i32> splat(i32 3))
   %add = add <vscale x 4 x i32> %a, %shift
   ret <vscale x 4 x i32> %add
 }
@@ -296,9 +250,7 @@ define <vscale x 2 x i64> @ssra_intr_u_i64(<vscale x 2 x i1> %pg, <vscale x 2 x
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ssra z0.d, z1.d, #4
 ; CHECK-NEXT:    ret
-  %ins = insertelement <vscale x 2 x i64> poison, i64 4, i32 0
-  %splat = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-  %shift = call <vscale x 2 x i64> @llvm.aarch64.sve.asr.u.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %b, <vscale x 2 x i64> %splat)
+  %shift = call <vscale x 2 x i64> @llvm.aarch64.sve.asr.u.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %b, <vscale x 2 x i64> splat(i64 4))
   %add = add <vscale x 2 x i64> %a, %shift
   ret <vscale x 2 x i64> %add
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-load-store-pointers.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-load-store-pointers.ll
new file mode 100644
index 0000000000000..091c9f143ce7e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-load-store-pointers.ll
@@ -0,0 +1,301 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs -stop-after=instruction-select < %s | FileCheck --check-prefix=GFX9 %s
+
+define ptr @buffer_load_p0(ptr addrspace(8) inreg %buf) {
+  ; GFX9-LABEL: name: buffer_load_p0
+  ; GFX9: bb.1 (%ir-block.0):
+  ; GFX9-NEXT:   liveins: $sgpr6, $sgpr7, $sgpr16, $sgpr17
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX9-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; GFX9-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr16
+  ; GFX9-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr17
+  ; GFX9-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX9-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX9-NEXT:   [[BUFFER_LOAD_DWORDX2_OFFSET:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_DWORDX2_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s64) from %ir.buf, align 1, addrspace 8)
+  ; GFX9-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_OFFSET]].sub0
+  ; GFX9-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_OFFSET]].sub1
+  ; GFX9-NEXT:   $vgpr0 = COPY [[COPY4]]
+  ; GFX9-NEXT:   $vgpr1 = COPY [[COPY5]]
+  ; GFX9-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1
+  %ret = call ptr @llvm.amdgcn.raw.ptr.buffer.load.p0(ptr addrspace(8) inreg %buf, i32 0, i32 0, i32 0)
+  ret ptr %ret
+}
+
+define void @buffer_store_p0(ptr %data, ptr addrspace(8) inreg %buf) {
+  ; GFX9-LABEL: name: buffer_store_p0
+  ; GFX9: bb.1 (%ir-block.0):
+  ; GFX9-NEXT:   liveins: $sgpr6, $sgpr7, $sgpr16, $sgpr17, $vgpr0, $vgpr1
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX9-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX9-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX9-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX9-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; GFX9-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr16
+  ; GFX9-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr17
+  ; GFX9-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX9-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX9-NEXT:   BUFFER_STORE_DWORDX2_OFFSET_exact [[REG_SEQUENCE]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s64) into %ir.buf, align 1, addrspace 8)
+  ; GFX9-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.ptr.buffer.store.p0(ptr %data, ptr addrspace(8) inreg %buf, i32 0, i32 0, i32 0)
+  ret void
+}
+
+define ptr addrspace(1) @buffer_load_p1(ptr addrspace(8) inreg %buf) {
+  ; GFX9-LABEL: name: buffer_load_p1
+  ; GFX9: bb.1 (%ir-block.0):
+  ; GFX9-NEXT:   liveins: $sgpr6, $sgpr7, $sgpr16, $sgpr17
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX9-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; GFX9-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr16
+  ; GFX9-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr17
+  ; GFX9-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX9-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX9-NEXT:   [[BUFFER_LOAD_DWORDX2_OFFSET:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_DWORDX2_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s64) from %ir.buf, align 1, addrspace 8)
+  ; GFX9-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_OFFSET]].sub0
+  ; GFX9-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_OFFSET]].sub1
+  ; GFX9-NEXT:   $vgpr0 = COPY [[COPY4]]
+  ; GFX9-NEXT:   $vgpr1 = COPY [[COPY5]]
+  ; GFX9-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1
+  %ret = call ptr addrspace(1) @llvm.amdgcn.raw.ptr.buffer.load.p1(ptr addrspace(8) inreg %buf, i32 0, i32 0, i32 0)
+  ret ptr addrspace(1) %ret
+}
+
+define void @buffer_store_p1(ptr addrspace(1) %data, ptr addrspace(8) inreg %buf) {
+  ; GFX9-LABEL: name: buffer_store_p1
+  ; GFX9: bb.1 (%ir-block.0):
+  ; GFX9-NEXT:   liveins: $sgpr6, $sgpr7, $sgpr16, $sgpr17, $vgpr0, $vgpr1
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX9-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX9-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX9-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX9-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; GFX9-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr16
+  ; GFX9-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr17
+  ; GFX9-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX9-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX9-NEXT:   BUFFER_STORE_DWORDX2_OFFSET_exact [[REG_SEQUENCE]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s64) into %ir.buf, align 1, addrspace 8)
+  ; GFX9-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.ptr.buffer.store.p1(ptr addrspace(1) %data, ptr addrspace(8) inreg %buf, i32 0, i32 0, i32 0)
+  ret void
+}
+
+define ptr addrspace(4) @buffer_load_p4(ptr addrspace(8) inreg %buf) {
+  ; GFX9-LABEL: name: buffer_load_p4
+  ; GFX9: bb.1 (%ir-block.0):
+  ; GFX9-NEXT:   liveins: $sgpr6, $sgpr7, $sgpr16, $sgpr17
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX9-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; GFX9-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr16
+  ; GFX9-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr17
+  ; GFX9-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX9-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX9-NEXT:   [[BUFFER_LOAD_DWORDX2_OFFSET:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_DWORDX2_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s64) from %ir.buf, align 1, addrspace 8)
+  ; GFX9-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_OFFSET]].sub0
+  ; GFX9-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_OFFSET]].sub1
+  ; GFX9-NEXT:   $vgpr0 = COPY [[COPY4]]
+  ; GFX9-NEXT:   $vgpr1 = COPY [[COPY5]]
+  ; GFX9-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1
+  %ret = call ptr addrspace(4) @llvm.amdgcn.raw.ptr.buffer.load.p4(ptr addrspace(8) inreg %buf, i32 0, i32 0, i32 0)
+  ret ptr addrspace(4) %ret
+}
+
+define void @buffer_store_p4(ptr addrspace(4) %data, ptr addrspace(8) inreg %buf) {
+  ; GFX9-LABEL: name: buffer_store_p4
+  ; GFX9: bb.1 (%ir-block.0):
+  ; GFX9-NEXT:   liveins: $sgpr6, $sgpr7, $sgpr16, $sgpr17, $vgpr0, $vgpr1
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX9-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX9-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX9-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX9-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; GFX9-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr16
+  ; GFX9-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr17
+  ; GFX9-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX9-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX9-NEXT:   BUFFER_STORE_DWORDX2_OFFSET_exact [[REG_SEQUENCE]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s64) into %ir.buf, align 1, addrspace 8)
+  ; GFX9-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.ptr.buffer.store.p4(ptr addrspace(4) %data, ptr addrspace(8) inreg %buf, i32 0, i32 0, i32 0)
+  ret void
+}
+
+define ptr addrspace(5) @buffer_load_p5(ptr addrspace(8) inreg %buf) {
+  ; GFX9-LABEL: name: buffer_load_p5
+  ; GFX9: bb.1 (%ir-block.0):
+  ; GFX9-NEXT:   liveins: $sgpr6, $sgpr7, $sgpr16, $sgpr17
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX9-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; GFX9-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr16
+  ; GFX9-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr17
+  ; GFX9-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX9-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX9-NEXT:   [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.buf, align 1, addrspace 8)
+  ; GFX9-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
+  ; GFX9-NEXT:   SI_RETURN implicit $vgpr0
+  %ret = call ptr addrspace(5) @llvm.amdgcn.raw.ptr.buffer.load.p5(ptr addrspace(8) inreg %buf, i32 0, i32 0, i32 0)
+  ret ptr addrspace(5) %ret
+}
+
+define void @buffer_store_p5(ptr addrspace(5) %data, ptr addrspace(8) inreg %buf) {
+  ; GFX9-LABEL: name: buffer_store_p5
+  ; GFX9: bb.1 (%ir-block.0):
+  ; GFX9-NEXT:   liveins: $sgpr6, $sgpr7, $sgpr16, $sgpr17, $vgpr0
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX9-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX9-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; GFX9-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr16
+  ; GFX9-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr17
+  ; GFX9-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX9-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX9-NEXT:   BUFFER_STORE_DWORD_OFFSET_exact [[COPY]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32) into %ir.buf, align 1, addrspace 8)
+  ; GFX9-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.ptr.buffer.store.p5(ptr addrspace(5) %data, ptr addrspace(8) inreg %buf, i32 0, i32 0, i32 0)
+  ret void
+}
+
+define <2 x ptr addrspace(1)> @buffer_load_v2p1(ptr addrspace(8) inreg %buf) {
+  ; GFX9-LABEL: name: buffer_load_v2p1
+  ; GFX9: bb.1 (%ir-block.0):
+  ; GFX9-NEXT:   liveins: $sgpr6, $sgpr7, $sgpr16, $sgpr17
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX9-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; GFX9-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr16
+  ; GFX9-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr17
+  ; GFX9-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX9-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX9-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s64>) from %ir.buf, align 1, addrspace 8)
+  ; GFX9-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFSET]].sub0
+  ; GFX9-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFSET]].sub1
+  ; GFX9-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFSET]].sub2
+  ; GFX9-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFSET]].sub3
+  ; GFX9-NEXT:   $vgpr0 = COPY [[COPY4]]
+  ; GFX9-NEXT:   $vgpr1 = COPY [[COPY5]]
+  ; GFX9-NEXT:   $vgpr2 = COPY [[COPY6]]
+  ; GFX9-NEXT:   $vgpr3 = COPY [[COPY7]]
+  ; GFX9-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  %ret = call <2 x ptr addrspace(1)> @llvm.amdgcn.raw.ptr.buffer.load.v2p1(ptr addrspace(8) inreg %buf, i32 0, i32 0, i32 0)
+  ret <2 x ptr addrspace(1)> %ret
+}
+
+define void @buffer_store_v2p5(<2 x ptr addrspace(1)> %data, ptr addrspace(8) inreg %buf) {
+  ; GFX9-LABEL: name: buffer_store_v2p5
+  ; GFX9: bb.1 (%ir-block.0):
+  ; GFX9-NEXT:   liveins: $sgpr6, $sgpr7, $sgpr16, $sgpr17, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX9-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX9-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX9-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX9-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX9-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX9-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+  ; GFX9-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX9-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; GFX9-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr16
+  ; GFX9-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr17
+  ; GFX9-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX9-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX9-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact [[REG_SEQUENCE2]], [[REG_SEQUENCE3]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s64>) into %ir.buf, align 1, addrspace 8)
+  ; GFX9-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> %data, ptr addrspace(8) inreg %buf, i32 0, i32 0, i32 0)
+  ret void
+}
+
+define <3 x ptr addrspace(5)> @buffer_load_v3p5(ptr addrspace(8) inreg %buf) {
+  ; GFX9-LABEL: name: buffer_load_v3p5
+  ; GFX9: bb.1 (%ir-block.0):
+  ; GFX9-NEXT:   liveins: $sgpr6, $sgpr7, $sgpr16, $sgpr17
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX9-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; GFX9-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr16
+  ; GFX9-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr17
+  ; GFX9-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX9-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX9-NEXT:   [[BUFFER_LOAD_DWORDX3_OFFSET:%[0-9]+]]:vreg_96_align2 = BUFFER_LOAD_DWORDX3_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>) from %ir.buf, align 1, addrspace 8)
+  ; GFX9-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_OFFSET]].sub0
+  ; GFX9-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_OFFSET]].sub1
+  ; GFX9-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_OFFSET]].sub2
+  ; GFX9-NEXT:   $vgpr0 = COPY [[COPY4]]
+  ; GFX9-NEXT:   $vgpr1 = COPY [[COPY5]]
+  ; GFX9-NEXT:   $vgpr2 = COPY [[COPY6]]
+  ; GFX9-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
+  %ret = call <3 x ptr addrspace(5)> @llvm.amdgcn.raw.ptr.buffer.load.v3p5(ptr addrspace(8) inreg %buf, i32 0, i32 0, i32 0)
+  ret <3 x ptr addrspace(5)> %ret
+}
+
+define void @buffer_store_v3p5(<3 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %buf) {
+  ; GFX9-LABEL: name: buffer_store_v3p5
+  ; GFX9: bb.1 (%ir-block.0):
+  ; GFX9-NEXT:   liveins: $sgpr6, $sgpr7, $sgpr16, $sgpr17, $vgpr0, $vgpr1, $vgpr2
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX9-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX9-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX9-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_96_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2
+  ; GFX9-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX9-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; GFX9-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr16
+  ; GFX9-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr17
+  ; GFX9-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX9-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY6]], %subreg.sub3
+  ; GFX9-NEXT:   BUFFER_STORE_DWORDX3_OFFSET_exact [[REG_SEQUENCE]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x s32>) into %ir.buf, align 1, addrspace 8)
+  ; GFX9-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.ptr.buffer.store.v3p5(<3 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %buf, i32 0, i32 0, i32 0)
+  ret void
+}
+
+define <4 x ptr addrspace(5)> @buffer_load_v4p5(ptr addrspace(8) inreg %buf) {
+  ; GFX9-LABEL: name: buffer_load_v4p5
+  ; GFX9: bb.1 (%ir-block.0):
+  ; GFX9-NEXT:   liveins: $sgpr6, $sgpr7, $sgpr16, $sgpr17
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX9-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; GFX9-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr16
+  ; GFX9-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr17
+  ; GFX9-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX9-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX9-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>) from %ir.buf, align 1, addrspace 8)
+  ; GFX9-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFSET]].sub0
+  ; GFX9-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFSET]].sub1
+  ; GFX9-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFSET]].sub2
+  ; GFX9-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFSET]].sub3
+  ; GFX9-NEXT:   $vgpr0 = COPY [[COPY4]]
+  ; GFX9-NEXT:   $vgpr1 = COPY [[COPY5]]
+  ; GFX9-NEXT:   $vgpr2 = COPY [[COPY6]]
+  ; GFX9-NEXT:   $vgpr3 = COPY [[COPY7]]
+  ; GFX9-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  %ret = call <4 x ptr addrspace(5)> @llvm.amdgcn.raw.ptr.buffer.load.v4p5(ptr addrspace(8) inreg %buf, i32 0, i32 0, i32 0)
+  ret <4 x ptr addrspace(5)> %ret
+}
+
+define void @buffer_store_v4p5(<4 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %buf) {
+  ; GFX9-LABEL: name: buffer_store_v4p5
+  ; GFX9: bb.1 (%ir-block.0):
+  ; GFX9-NEXT:   liveins: $sgpr6, $sgpr7, $sgpr16, $sgpr17, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX9-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX9-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX9-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX9-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX9-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX9-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; GFX9-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr16
+  ; GFX9-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr17
+  ; GFX9-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX9-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX9-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact [[REG_SEQUENCE]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>) into %ir.buf, align 1, addrspace 8)
+  ; GFX9-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.ptr.buffer.store.v4p5(<4 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %buf, i32 0, i32 0, i32 0)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch.ll
index 688146a6000e7..ce528467cd35b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch.ll
@@ -177,15 +177,13 @@ define amdgpu_kernel void @store_load_vindex_kernel() {
 ; GFX9-LABEL: store_load_vindex_kernel:
 ; GFX9:       ; %bb.0: ; %bb
 ; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s6, s11
-; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
-; GFX9-NEXT:    v_sub_u32_e32 v0, 0, v0
 ; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s7, 0
-; GFX9-NEXT:    v_add_u32_e32 v1, 0, v1
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 15
-; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT:    v_sub_u32_e32 v0, 0, v0
 ; GFX9-NEXT:    scratch_store_dword v1, v2, off
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v0, 0, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX9-NEXT:    scratch_load_dword v0, v0, off offset:124 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_endpgm
@@ -200,8 +198,6 @@ define amdgpu_kernel void @store_load_vindex_kernel() {
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 15
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, 0, v0
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 0, v1
 ; GFX10-NEXT:    scratch_store_dword v0, v2, off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dword v0, v1, off offset:124 glc dlc
@@ -212,12 +208,11 @@ define amdgpu_kernel void @store_load_vindex_kernel() {
 ; GFX940:       ; %bb.0: ; %bb
 ; GFX940-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
-; GFX940-NEXT:    v_sub_u32_e32 v0, 0, v0
 ; GFX940-NEXT:    v_mov_b32_e32 v2, 15
-; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX940-NEXT:    v_sub_u32_e32 v0, 0, v0
 ; GFX940-NEXT:    scratch_store_dword v1, v2, off sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-NEXT:    v_add_u32_e32 v0, 0, v0
+; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:124 sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    s_endpgm
@@ -232,7 +227,6 @@ define amdgpu_kernel void @store_load_vindex_kernel() {
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
 ; GFX11-NEXT:    scratch_store_b32 v0, v2, off dlc
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT:    v_add_nc_u32_e32 v1, 0, v1
 ; GFX11-NEXT:    scratch_load_b32 v0, v1, off offset:124 glc dlc
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_endpgm
@@ -253,15 +247,13 @@ define amdgpu_kernel void @store_load_vindex_kernel() {
 ; UNALIGNED_GFX9-LABEL: store_load_vindex_kernel:
 ; UNALIGNED_GFX9:       ; %bb.0: ; %bb
 ; UNALIGNED_GFX9-NEXT:    s_add_u32 flat_scratch_lo, s6, s11
-; UNALIGNED_GFX9-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
-; UNALIGNED_GFX9-NEXT:    v_sub_u32_e32 v0, 0, v0
 ; UNALIGNED_GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s7, 0
-; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v1, 0, v1
+; UNALIGNED_GFX9-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
 ; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v2, 15
-; UNALIGNED_GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; UNALIGNED_GFX9-NEXT:    v_sub_u32_e32 v0, 0, v0
 ; UNALIGNED_GFX9-NEXT:    scratch_store_dword v1, v2, off
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
-; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v0, 0, v0
+; UNALIGNED_GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; UNALIGNED_GFX9-NEXT:    scratch_load_dword v0, v0, off offset:124 glc
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX9-NEXT:    s_endpgm
@@ -276,8 +268,6 @@ define amdgpu_kernel void @store_load_vindex_kernel() {
 ; UNALIGNED_GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v2, 15
 ; UNALIGNED_GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
-; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v0, 0, v0
-; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v1, 0, v1
 ; UNALIGNED_GFX10-NEXT:    scratch_store_dword v0, v2, off
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; UNALIGNED_GFX10-NEXT:    scratch_load_dword v0, v1, off offset:124 glc dlc
@@ -288,12 +278,11 @@ define amdgpu_kernel void @store_load_vindex_kernel() {
 ; UNALIGNED_GFX940:       ; %bb.0: ; %bb
 ; UNALIGNED_GFX940-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; UNALIGNED_GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
-; UNALIGNED_GFX940-NEXT:    v_sub_u32_e32 v0, 0, v0
 ; UNALIGNED_GFX940-NEXT:    v_mov_b32_e32 v2, 15
-; UNALIGNED_GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; UNALIGNED_GFX940-NEXT:    v_sub_u32_e32 v0, 0, v0
 ; UNALIGNED_GFX940-NEXT:    scratch_store_dword v1, v2, off sc0 sc1
 ; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
-; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v0, 0, v0
+; UNALIGNED_GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; UNALIGNED_GFX940-NEXT:    scratch_load_dword v0, v0, off offset:124 sc0 sc1
 ; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX940-NEXT:    s_endpgm
@@ -308,7 +297,6 @@ define amdgpu_kernel void @store_load_vindex_kernel() {
 ; UNALIGNED_GFX11-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
 ; UNALIGNED_GFX11-NEXT:    scratch_store_b32 v0, v2, off dlc
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v1, 0, v1
 ; UNALIGNED_GFX11-NEXT:    scratch_load_b32 v0, v1, off offset:124 glc dlc
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX11-NEXT:    s_endpgm
@@ -987,15 +975,15 @@ define void @store_load_vindex_small_offset_foo(i32 %idx) {
 ; GFX9-NEXT:    scratch_load_dword v1, off, s32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
-; GFX9-NEXT:    s_add_i32 s0, s32, 0x100
 ; GFX9-NEXT:    v_and_b32_e32 v0, 15, v0
-; GFX9-NEXT:    v_add_u32_e32 v1, s0, v1
-; GFX9-NEXT:    v_mov_b32_e32 v2, 15
+; GFX9-NEXT:    v_add_u32_e32 v1, s32, v1
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-NEXT:    s_add_i32 s0, s32, 0x100
+; GFX9-NEXT:    v_add_u32_e32 v1, 0x100, v1
+; GFX9-NEXT:    v_mov_b32_e32 v2, 15
+; GFX9-NEXT:    v_add_u32_e32 v0, s32, v0
 ; GFX9-NEXT:    scratch_store_dword v1, v2, off
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v0, s0, v0
+; GFX9-NEXT:    v_add_u32_e32 v0, 0x100, v0
 ; GFX9-NEXT:    scratch_load_dword v0, v0, off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -1005,14 +993,14 @@ define void @store_load_vindex_small_offset_foo(i32 %idx) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_and_b32_e32 v1, 15, v0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    s_add_i32 s0, s32, 0x100
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 15
 ; GFX10-NEXT:    scratch_load_dword v3, off, s32 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, s0, v0
-; GFX10-NEXT:    s_add_i32 s0, s32, 0x100
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, s0, v1
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, s32, v0
+; GFX10-NEXT:    v_add_nc_u32_e32 v1, s32, v1
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, 0x100, v0
+; GFX10-NEXT:    v_add_nc_u32_e32 v1, 0x100, v1
 ; GFX10-NEXT:    scratch_store_dword v0, v2, off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dword v0, v1, off glc dlc
@@ -1025,8 +1013,8 @@ define void @store_load_vindex_small_offset_foo(i32 %idx) {
 ; GFX940-NEXT:    scratch_load_dword v1, off, s32 sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
-; GFX940-NEXT:    s_add_i32 s0, s32, 0x100
-; GFX940-NEXT:    v_add_u32_e32 v1, s0, v1
+; GFX940-NEXT:    v_add_u32_e32 v1, s32, v1
+; GFX940-NEXT:    v_add_u32_e32 v1, 0x100, v1
 ; GFX940-NEXT:    v_mov_b32_e32 v2, 15
 ; GFX940-NEXT:    v_and_b32_e32 v0, 15, v0
 ; GFX940-NEXT:    scratch_store_dword v1, v2, off sc0 sc1
@@ -1041,11 +1029,12 @@ define void @store_load_vindex_small_offset_foo(i32 %idx) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_dual_mov_b32 v2, 15 :: v_dual_lshlrev_b32 v1, 2, v0
 ; GFX11-NEXT:    v_and_b32_e32 v0, 15, v0
-; GFX11-NEXT:    s_add_i32 s0, s32, 0x100
 ; GFX11-NEXT:    scratch_load_b32 v3, off, s32 glc dlc
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_add_nc_u32_e32 v1, s0, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, s32, v1
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 0x100, v1
 ; GFX11-NEXT:    scratch_store_b32 v1, v2, off dlc
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    scratch_load_b32 v0, v0, s32 offset:256 glc dlc
@@ -1077,15 +1066,15 @@ define void @store_load_vindex_small_offset_foo(i32 %idx) {
 ; UNALIGNED_GFX9-NEXT:    scratch_load_dword v1, off, s32 glc
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX9-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
-; UNALIGNED_GFX9-NEXT:    s_add_i32 s0, s32, 0x100
 ; UNALIGNED_GFX9-NEXT:    v_and_b32_e32 v0, 15, v0
-; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v1, s0, v1
-; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v2, 15
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v1, s32, v1
 ; UNALIGNED_GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; UNALIGNED_GFX9-NEXT:    s_add_i32 s0, s32, 0x100
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v1, 0x100, v1
+; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v2, 15
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v0, s32, v0
 ; UNALIGNED_GFX9-NEXT:    scratch_store_dword v1, v2, off
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
-; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v0, s0, v0
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v0, 0x100, v0
 ; UNALIGNED_GFX9-NEXT:    scratch_load_dword v0, v0, off glc
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -1095,14 +1084,14 @@ define void @store_load_vindex_small_offset_foo(i32 %idx) {
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; UNALIGNED_GFX10-NEXT:    v_and_b32_e32 v1, 15, v0
 ; UNALIGNED_GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; UNALIGNED_GFX10-NEXT:    s_add_i32 s0, s32, 0x100
 ; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v2, 15
 ; UNALIGNED_GFX10-NEXT:    scratch_load_dword v3, off, s32 glc dlc
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
-; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v0, s0, v0
-; UNALIGNED_GFX10-NEXT:    s_add_i32 s0, s32, 0x100
-; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v1, s0, v1
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v0, s32, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v1, s32, v1
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v0, 0x100, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v1, 0x100, v1
 ; UNALIGNED_GFX10-NEXT:    scratch_store_dword v0, v2, off
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; UNALIGNED_GFX10-NEXT:    scratch_load_dword v0, v1, off glc dlc
@@ -1115,8 +1104,8 @@ define void @store_load_vindex_small_offset_foo(i32 %idx) {
 ; UNALIGNED_GFX940-NEXT:    scratch_load_dword v1, off, s32 sc0 sc1
 ; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
-; UNALIGNED_GFX940-NEXT:    s_add_i32 s0, s32, 0x100
-; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v1, s0, v1
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v1, s32, v1
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v1, 0x100, v1
 ; UNALIGNED_GFX940-NEXT:    v_mov_b32_e32 v2, 15
 ; UNALIGNED_GFX940-NEXT:    v_and_b32_e32 v0, 15, v0
 ; UNALIGNED_GFX940-NEXT:    scratch_store_dword v1, v2, off sc0 sc1
@@ -1131,11 +1120,12 @@ define void @store_load_vindex_small_offset_foo(i32 %idx) {
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; UNALIGNED_GFX11-NEXT:    v_dual_mov_b32 v2, 15 :: v_dual_lshlrev_b32 v1, 2, v0
 ; UNALIGNED_GFX11-NEXT:    v_and_b32_e32 v0, 15, v0
-; UNALIGNED_GFX11-NEXT:    s_add_i32 s0, s32, 0x100
 ; UNALIGNED_GFX11-NEXT:    scratch_load_b32 v3, off, s32 glc dlc
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
-; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v1, s0, v1
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v1, s32, v1
 ; UNALIGNED_GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; UNALIGNED_GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v1, 0x100, v1
 ; UNALIGNED_GFX11-NEXT:    scratch_store_b32 v1, v2, off dlc
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; UNALIGNED_GFX11-NEXT:    scratch_load_b32 v0, v0, s32 offset:256 glc dlc
@@ -1578,15 +1568,15 @@ define void @store_load_vindex_large_offset_foo(i32 %idx) {
 ; GFX9-NEXT:    scratch_load_dword v1, off, s32 offset:4 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
-; GFX9-NEXT:    s_add_i32 s0, s32, 0x4004
 ; GFX9-NEXT:    v_and_b32_e32 v0, 15, v0
-; GFX9-NEXT:    v_add_u32_e32 v1, s0, v1
-; GFX9-NEXT:    v_mov_b32_e32 v2, 15
+; GFX9-NEXT:    v_add_u32_e32 v1, s32, v1
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-NEXT:    s_add_i32 s0, s32, 0x4004
+; GFX9-NEXT:    v_add_u32_e32 v1, 0x4004, v1
+; GFX9-NEXT:    v_mov_b32_e32 v2, 15
+; GFX9-NEXT:    v_add_u32_e32 v0, s32, v0
 ; GFX9-NEXT:    scratch_store_dword v1, v2, off
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v0, s0, v0
+; GFX9-NEXT:    v_add_u32_e32 v0, 0x4004, v0
 ; GFX9-NEXT:    scratch_load_dword v0, v0, off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -1596,14 +1586,14 @@ define void @store_load_vindex_large_offset_foo(i32 %idx) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_and_b32_e32 v1, 15, v0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    s_add_i32 s0, s32, 0x4004
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 15
 ; GFX10-NEXT:    scratch_load_dword v3, off, s32 offset:4 glc dlc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, s0, v0
-; GFX10-NEXT:    s_add_i32 s0, s32, 0x4004
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, s0, v1
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, s32, v0
+; GFX10-NEXT:    v_add_nc_u32_e32 v1, s32, v1
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, 0x4004, v0
+; GFX10-NEXT:    v_add_nc_u32_e32 v1, 0x4004, v1
 ; GFX10-NEXT:    scratch_store_dword v0, v2, off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dword v0, v1, off glc dlc
@@ -1616,8 +1606,8 @@ define void @store_load_vindex_large_offset_foo(i32 %idx) {
 ; GFX940-NEXT:    scratch_load_dword v1, off, s32 offset:4 sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
-; GFX940-NEXT:    s_add_i32 s0, s32, 0x4004
-; GFX940-NEXT:    v_add_u32_e32 v1, s0, v1
+; GFX940-NEXT:    v_add_u32_e32 v1, s32, v1
+; GFX940-NEXT:    v_add_u32_e32 v1, 0x4004, v1
 ; GFX940-NEXT:    v_mov_b32_e32 v2, 15
 ; GFX940-NEXT:    v_and_b32_e32 v0, 15, v0
 ; GFX940-NEXT:    scratch_store_dword v1, v2, off sc0 sc1
@@ -1636,9 +1626,10 @@ define void @store_load_vindex_large_offset_foo(i32 %idx) {
 ; GFX11-NEXT:    s_add_i32 s0, s32, 0x4004
 ; GFX11-NEXT:    scratch_load_b32 v3, off, s32 offset:4 glc dlc
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_add_nc_u32_e32 v1, s0, v1
-; GFX11-NEXT:    s_add_i32 s0, s32, 0x4004
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, s32, v1
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 0x4004, v1
 ; GFX11-NEXT:    scratch_store_b32 v1, v2, off dlc
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    scratch_load_b32 v0, v0, s0 glc dlc
@@ -1670,15 +1661,15 @@ define void @store_load_vindex_large_offset_foo(i32 %idx) {
 ; UNALIGNED_GFX9-NEXT:    scratch_load_dword v1, off, s32 offset:4 glc
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX9-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
-; UNALIGNED_GFX9-NEXT:    s_add_i32 s0, s32, 0x4004
 ; UNALIGNED_GFX9-NEXT:    v_and_b32_e32 v0, 15, v0
-; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v1, s0, v1
-; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v2, 15
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v1, s32, v1
 ; UNALIGNED_GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; UNALIGNED_GFX9-NEXT:    s_add_i32 s0, s32, 0x4004
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v1, 0x4004, v1
+; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v2, 15
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v0, s32, v0
 ; UNALIGNED_GFX9-NEXT:    scratch_store_dword v1, v2, off
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
-; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v0, s0, v0
+; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v0, 0x4004, v0
 ; UNALIGNED_GFX9-NEXT:    scratch_load_dword v0, v0, off glc
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -1688,14 +1679,14 @@ define void @store_load_vindex_large_offset_foo(i32 %idx) {
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; UNALIGNED_GFX10-NEXT:    v_and_b32_e32 v1, 15, v0
 ; UNALIGNED_GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; UNALIGNED_GFX10-NEXT:    s_add_i32 s0, s32, 0x4004
 ; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v2, 15
 ; UNALIGNED_GFX10-NEXT:    scratch_load_dword v3, off, s32 offset:4 glc dlc
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
-; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v0, s0, v0
-; UNALIGNED_GFX10-NEXT:    s_add_i32 s0, s32, 0x4004
-; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v1, s0, v1
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v0, s32, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v1, s32, v1
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v0, 0x4004, v0
+; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v1, 0x4004, v1
 ; UNALIGNED_GFX10-NEXT:    scratch_store_dword v0, v2, off
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; UNALIGNED_GFX10-NEXT:    scratch_load_dword v0, v1, off glc dlc
@@ -1708,8 +1699,8 @@ define void @store_load_vindex_large_offset_foo(i32 %idx) {
 ; UNALIGNED_GFX940-NEXT:    scratch_load_dword v1, off, s32 offset:4 sc0 sc1
 ; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX940-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
-; UNALIGNED_GFX940-NEXT:    s_add_i32 s0, s32, 0x4004
-; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v1, s0, v1
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v1, s32, v1
+; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v1, 0x4004, v1
 ; UNALIGNED_GFX940-NEXT:    v_mov_b32_e32 v2, 15
 ; UNALIGNED_GFX940-NEXT:    v_and_b32_e32 v0, 15, v0
 ; UNALIGNED_GFX940-NEXT:    scratch_store_dword v1, v2, off sc0 sc1
@@ -1728,9 +1719,10 @@ define void @store_load_vindex_large_offset_foo(i32 %idx) {
 ; UNALIGNED_GFX11-NEXT:    s_add_i32 s0, s32, 0x4004
 ; UNALIGNED_GFX11-NEXT:    scratch_load_b32 v3, off, s32 offset:4 glc dlc
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0)
-; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v1, s0, v1
-; UNALIGNED_GFX11-NEXT:    s_add_i32 s0, s32, 0x4004
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v1, s32, v1
 ; UNALIGNED_GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; UNALIGNED_GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v1, 0x4004, v1
 ; UNALIGNED_GFX11-NEXT:    scratch_store_b32 v1, v2, off dlc
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; UNALIGNED_GFX11-NEXT:    scratch_load_b32 v0, v0, s0 glc dlc
@@ -2109,7 +2101,6 @@ define amdgpu_kernel void @store_load_vidx_sidx_offset(i32 %sidx) {
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
-; GFX9-NEXT:    v_add_u32_e32 v0, 0, v0
 ; GFX9-NEXT:    scratch_store_dword v0, v1, off offset:1024
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    scratch_load_dword v0, v0, off offset:1024 glc
@@ -2126,7 +2117,6 @@ define amdgpu_kernel void @store_load_vidx_sidx_offset(i32 %sidx) {
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, 0, v0
 ; GFX10-NEXT:    scratch_store_dword v0, v1, off offset:1024
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dword v0, v0, off offset:1024 glc dlc
@@ -2140,7 +2130,6 @@ define amdgpu_kernel void @store_load_vidx_sidx_offset(i32 %sidx) {
 ; GFX940-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX940-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
-; GFX940-NEXT:    v_add_u32_e32 v0, 0, v0
 ; GFX940-NEXT:    scratch_store_dword v0, v1, off offset:1024 sc0 sc1
 ; GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; GFX940-NEXT:    scratch_load_dword v0, v0, off offset:1024 sc0 sc1
@@ -2152,9 +2141,8 @@ define amdgpu_kernel void @store_load_vidx_sidx_offset(i32 %sidx) {
 ; GFX11-NEXT:    s_load_b32 s0, s[2:3], 0x0
 ; GFX11-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
-; GFX11-NEXT:    v_add_nc_u32_e32 v0, 0, v0
 ; GFX11-NEXT:    scratch_store_b32 v0, v1, off offset:1024 dlc
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    scratch_load_b32 v0, v0, off offset:1024 glc dlc
@@ -2182,7 +2170,6 @@ define amdgpu_kernel void @store_load_vidx_sidx_offset(i32 %sidx) {
 ; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v1, 15
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; UNALIGNED_GFX9-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
-; UNALIGNED_GFX9-NEXT:    v_add_u32_e32 v0, 0, v0
 ; UNALIGNED_GFX9-NEXT:    scratch_store_dword v0, v1, off offset:1024
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX9-NEXT:    scratch_load_dword v0, v0, off offset:1024 glc
@@ -2199,7 +2186,6 @@ define amdgpu_kernel void @store_load_vidx_sidx_offset(i32 %sidx) {
 ; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v1, 15
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; UNALIGNED_GFX10-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
-; UNALIGNED_GFX10-NEXT:    v_add_nc_u32_e32 v0, 0, v0
 ; UNALIGNED_GFX10-NEXT:    scratch_store_dword v0, v1, off offset:1024
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; UNALIGNED_GFX10-NEXT:    scratch_load_dword v0, v0, off offset:1024 glc dlc
@@ -2213,7 +2199,6 @@ define amdgpu_kernel void @store_load_vidx_sidx_offset(i32 %sidx) {
 ; UNALIGNED_GFX940-NEXT:    v_mov_b32_e32 v1, 15
 ; UNALIGNED_GFX940-NEXT:    s_waitcnt lgkmcnt(0)
 ; UNALIGNED_GFX940-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
-; UNALIGNED_GFX940-NEXT:    v_add_u32_e32 v0, 0, v0
 ; UNALIGNED_GFX940-NEXT:    scratch_store_dword v0, v1, off offset:1024 sc0 sc1
 ; UNALIGNED_GFX940-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX940-NEXT:    scratch_load_dword v0, v0, off offset:1024 sc0 sc1
@@ -2225,9 +2210,8 @@ define amdgpu_kernel void @store_load_vidx_sidx_offset(i32 %sidx) {
 ; UNALIGNED_GFX11-NEXT:    s_load_b32 s0, s[2:3], 0x0
 ; UNALIGNED_GFX11-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; UNALIGNED_GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; UNALIGNED_GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; UNALIGNED_GFX11-NEXT:    v_add_lshl_u32 v0, s0, v0, 2
-; UNALIGNED_GFX11-NEXT:    v_add_nc_u32_e32 v0, 0, v0
 ; UNALIGNED_GFX11-NEXT:    scratch_store_b32 v0, v1, off offset:1024 dlc
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; UNALIGNED_GFX11-NEXT:    scratch_load_b32 v0, v0, off offset:1024 glc dlc
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-global-old-legalization.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-global-old-legalization.mir
new file mode 100644
index 0000000000000..0c42b70edda65
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-global-old-legalization.mir
@@ -0,0 +1,3279 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=instruction-select -verify-machineinstrs  -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX6 %s
+# RUN: llc -mtriple=amdgcn -mcpu=hawaii -run-pass=instruction-select -verify-machineinstrs  -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX7 %s
+# RUN: llc -mtriple=amdgcn -mcpu=hawaii -mattr=+flat-for-global -run-pass=instruction-select -verify-machineinstrs  -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX7-FLAT %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX8 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX10 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX11 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX12 %s
+
+---
+
+name: load_global_s32_from_4
+
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s32_from_4
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_DWORD_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_s32_from_4
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_DWORD_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s32_from_4
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s32), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
+    ;
+    ; GFX8-LABEL: name: load_global_s32_from_4
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s32), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
+    ;
+    ; GFX9-LABEL: name: load_global_s32_from_4
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
+    ;
+    ; GFX10-LABEL: name: load_global_s32_from_4
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
+    ;
+    ; GFX11-LABEL: name: load_global_s32_from_4
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
+    ;
+    ; GFX12-LABEL: name: load_global_s32_from_4
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s32) = G_LOAD %0 :: (load (s32), align 4, addrspace 1)
+    $vgpr0 = COPY %1
+
+...
+
+---
+
+name: load_global_s32_from_2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s32_from_2
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_USHORT_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (s16), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_s32_from_2
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_USHORT_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (s16), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s32_from_2
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_USHORT:%[0-9]+]]:vgpr_32 = FLAT_LOAD_USHORT [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s16), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_USHORT]]
+    ;
+    ; GFX8-LABEL: name: load_global_s32_from_2
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[FLAT_LOAD_USHORT:%[0-9]+]]:vgpr_32 = FLAT_LOAD_USHORT [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s16), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_USHORT]]
+    ;
+    ; GFX9-LABEL: name: load_global_s32_from_2
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_USHORT:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT [[COPY]], 0, 0, implicit $exec :: (load (s16), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_USHORT]]
+    ;
+    ; GFX10-LABEL: name: load_global_s32_from_2
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_USHORT:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT [[COPY]], 0, 0, implicit $exec :: (load (s16), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_USHORT]]
+    ;
+    ; GFX11-LABEL: name: load_global_s32_from_2
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_USHORT:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT [[COPY]], 0, 0, implicit $exec :: (load (s16), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_USHORT]]
+    ;
+    ; GFX12-LABEL: name: load_global_s32_from_2
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_USHORT:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT [[COPY]], 0, 0, implicit $exec :: (load (s16), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_USHORT]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s32) = G_LOAD %0 :: (load (s16), align 2, addrspace 1)
+    $vgpr0 = COPY %1
+
+...
+
+---
+
+name: load_global_s32_from_1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s32_from_1
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_s32_from_1
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s32_from_1
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX8-LABEL: name: load_global_s32_from_1
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX9-LABEL: name: load_global_s32_from_1
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX10-LABEL: name: load_global_s32_from_1
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX11-LABEL: name: load_global_s32_from_1
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX12-LABEL: name: load_global_s32_from_1
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s32) = G_LOAD %0 :: (load (s8), align 1, addrspace 1)
+    $vgpr0 = COPY %1
+
+...
+
+---
+
+name: load_global_v2s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_v2s32
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_DWORDX2_ADDR64_:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (<2 x s32>), addrspace 1)
+    ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[BUFFER_LOAD_DWORDX2_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_v2s32
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_DWORDX2_ADDR64_:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (<2 x s32>), addrspace 1)
+    ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[BUFFER_LOAD_DWORDX2_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_v2s32
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (<2 x s32>), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
+    ;
+    ; GFX8-LABEL: name: load_global_v2s32
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (<2 x s32>), addrspace 1)
+    ; GFX8-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
+    ;
+    ; GFX9-LABEL: name: load_global_v2s32
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (<2 x s32>), addrspace 1)
+    ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
+    ;
+    ; GFX10-LABEL: name: load_global_v2s32
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (<2 x s32>), addrspace 1)
+    ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
+    ;
+    ; GFX11-LABEL: name: load_global_v2s32
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (<2 x s32>), addrspace 1)
+    ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
+    ;
+    ; GFX12-LABEL: name: load_global_v2s32
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (<2 x s32>), addrspace 1)
+    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(<2 x s32>) = G_LOAD %0 :: (load (<2 x s32>), align 8, addrspace 1)
+    $vgpr0_vgpr1 = COPY %1
+
+...
+
+---
+
+name: load_global_v4s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_v4s32
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_DWORDX4_ADDR64_:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (<4 x s32>), align 4, addrspace 1)
+    ; GFX6-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUFFER_LOAD_DWORDX4_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_v4s32
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_DWORDX4_ADDR64_:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (<4 x s32>), align 4, addrspace 1)
+    ; GFX7-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUFFER_LOAD_DWORDX4_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_v4s32
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = FLAT_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (<4 x s32>), align 4, addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[FLAT_LOAD_DWORDX4_]]
+    ;
+    ; GFX8-LABEL: name: load_global_v4s32
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[FLAT_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = FLAT_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (<4 x s32>), align 4, addrspace 1)
+    ; GFX8-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[FLAT_LOAD_DWORDX4_]]
+    ;
+    ; GFX9-LABEL: name: load_global_v4s32
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec :: (load (<4 x s32>), align 4, addrspace 1)
+    ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[GLOBAL_LOAD_DWORDX4_]]
+    ;
+    ; GFX10-LABEL: name: load_global_v4s32
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec :: (load (<4 x s32>), align 4, addrspace 1)
+    ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[GLOBAL_LOAD_DWORDX4_]]
+    ;
+    ; GFX11-LABEL: name: load_global_v4s32
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec :: (load (<4 x s32>), align 4, addrspace 1)
+    ; GFX11-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[GLOBAL_LOAD_DWORDX4_]]
+    ;
+    ; GFX12-LABEL: name: load_global_v4s32
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec :: (load (<4 x s32>), align 4, addrspace 1)
+    ; GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[GLOBAL_LOAD_DWORDX4_]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(<4 x s32>) = G_LOAD %0 :: (load (<4 x s32>), align 4, addrspace 1)
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+
+...
+
+---
+
+name: load_global_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s64
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_DWORDX2_ADDR64_:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[BUFFER_LOAD_DWORDX2_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_s64
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_DWORDX2_ADDR64_:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[BUFFER_LOAD_DWORDX2_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s64
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s64), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
+    ;
+    ; GFX8-LABEL: name: load_global_s64
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s64), addrspace 1)
+    ; GFX8-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
+    ;
+    ; GFX9-LABEL: name: load_global_s64
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
+    ;
+    ; GFX10-LABEL: name: load_global_s64
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
+    ;
+    ; GFX11-LABEL: name: load_global_s64
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
+    ;
+    ; GFX12-LABEL: name: load_global_s64
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = G_LOAD %0 :: (load (s64), align 8, addrspace 1)
+    $vgpr0_vgpr1 = COPY %1
+
+...
+
+---
+
+name: load_global_v2s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_v2s64
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_DWORDX4_ADDR64_:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (<2 x s64>), align 4, addrspace 1)
+    ; GFX6-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUFFER_LOAD_DWORDX4_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_v2s64
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_DWORDX4_ADDR64_:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (<2 x s64>), align 4, addrspace 1)
+    ; GFX7-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUFFER_LOAD_DWORDX4_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_v2s64
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = FLAT_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (<2 x s64>), align 4, addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[FLAT_LOAD_DWORDX4_]]
+    ;
+    ; GFX8-LABEL: name: load_global_v2s64
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[FLAT_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = FLAT_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (<2 x s64>), align 4, addrspace 1)
+    ; GFX8-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[FLAT_LOAD_DWORDX4_]]
+    ;
+    ; GFX9-LABEL: name: load_global_v2s64
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec :: (load (<2 x s64>), align 4, addrspace 1)
+    ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[GLOBAL_LOAD_DWORDX4_]]
+    ;
+    ; GFX10-LABEL: name: load_global_v2s64
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec :: (load (<2 x s64>), align 4, addrspace 1)
+    ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[GLOBAL_LOAD_DWORDX4_]]
+    ;
+    ; GFX11-LABEL: name: load_global_v2s64
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec :: (load (<2 x s64>), align 4, addrspace 1)
+    ; GFX11-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[GLOBAL_LOAD_DWORDX4_]]
+    ;
+    ; GFX12-LABEL: name: load_global_v2s64
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec :: (load (<2 x s64>), align 4, addrspace 1)
+    ; GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[GLOBAL_LOAD_DWORDX4_]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(<2 x s64>) = G_LOAD %0 :: (load (<2 x s64>), align 4, addrspace 1)
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+
+...
+
+---
+
+name: load_global_v2p1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_v2p1
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x p1>) = G_LOAD [[COPY]](p1) :: (load (<2 x p1>), align 4, addrspace 1)
+    ; GFX6-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x p1>)
+    ;
+    ; GFX7-LABEL: name: load_global_v2p1
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x p1>) = G_LOAD [[COPY]](p1) :: (load (<2 x p1>), align 4, addrspace 1)
+    ; GFX7-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x p1>)
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_v2p1
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x p1>) = G_LOAD [[COPY]](p1) :: (load (<2 x p1>), align 4, addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x p1>)
+    ;
+    ; GFX8-LABEL: name: load_global_v2p1
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x p1>) = G_LOAD [[COPY]](p1) :: (load (<2 x p1>), align 4, addrspace 1)
+    ; GFX8-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x p1>)
+    ;
+    ; GFX9-LABEL: name: load_global_v2p1
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x p1>) = G_LOAD [[COPY]](p1) :: (load (<2 x p1>), align 4, addrspace 1)
+    ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x p1>)
+    ;
+    ; GFX10-LABEL: name: load_global_v2p1
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x p1>) = G_LOAD [[COPY]](p1) :: (load (<2 x p1>), align 4, addrspace 1)
+    ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x p1>)
+    ;
+    ; GFX11-LABEL: name: load_global_v2p1
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x p1>) = G_LOAD [[COPY]](p1) :: (load (<2 x p1>), align 4, addrspace 1)
+    ; GFX11-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x p1>)
+    ;
+    ; GFX12-LABEL: name: load_global_v2p1
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x p1>) = G_LOAD [[COPY]](p1) :: (load (<2 x p1>), align 4, addrspace 1)
+    ; GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x p1>)
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(<2 x p1>) = G_LOAD %0 :: (load (<2 x p1>), align 4, addrspace 1)
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+
+...
+
+---
+
+name: load_global_s128
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s128
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[LOAD:%[0-9]+]]:vgpr(s128) = G_LOAD [[COPY]](p1) :: (load (s128), align 4, addrspace 1)
+    ; GFX6-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](s128)
+    ;
+    ; GFX7-LABEL: name: load_global_s128
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vgpr(s128) = G_LOAD [[COPY]](p1) :: (load (s128), align 4, addrspace 1)
+    ; GFX7-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](s128)
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s128
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[LOAD:%[0-9]+]]:vgpr(s128) = G_LOAD [[COPY]](p1) :: (load (s128), align 4, addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](s128)
+    ;
+    ; GFX8-LABEL: name: load_global_s128
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[LOAD:%[0-9]+]]:vgpr(s128) = G_LOAD [[COPY]](p1) :: (load (s128), align 4, addrspace 1)
+    ; GFX8-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](s128)
+    ;
+    ; GFX9-LABEL: name: load_global_s128
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vgpr(s128) = G_LOAD [[COPY]](p1) :: (load (s128), align 4, addrspace 1)
+    ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](s128)
+    ;
+    ; GFX10-LABEL: name: load_global_s128
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vgpr(s128) = G_LOAD [[COPY]](p1) :: (load (s128), align 4, addrspace 1)
+    ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](s128)
+    ;
+    ; GFX11-LABEL: name: load_global_s128
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[LOAD:%[0-9]+]]:vgpr(s128) = G_LOAD [[COPY]](p1) :: (load (s128), align 4, addrspace 1)
+    ; GFX11-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](s128)
+    ;
+    ; GFX12-LABEL: name: load_global_s128
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:vgpr(s128) = G_LOAD [[COPY]](p1) :: (load (s128), align 4, addrspace 1)
+    ; GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](s128)
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s128) = G_LOAD %0 :: (load (s128), align 4, addrspace 1)
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+
+...
+
+---
+
+name: load_global_p3_from_4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_p3_from_4
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_DWORD_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (p3), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_p3_from_4
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_DWORD_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (p3), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_p3_from_4
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (p3), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
+    ;
+    ; GFX8-LABEL: name: load_global_p3_from_4
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (p3), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
+    ;
+    ; GFX9-LABEL: name: load_global_p3_from_4
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (p3), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
+    ;
+    ; GFX10-LABEL: name: load_global_p3_from_4
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (p3), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
+    ;
+    ; GFX11-LABEL: name: load_global_p3_from_4
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (p3), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
+    ;
+    ; GFX12-LABEL: name: load_global_p3_from_4
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (p3), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(p3) = G_LOAD %0 :: (load (p3), align 4, addrspace 1)
+    $vgpr0 = COPY %1
+
+...
+
+---
+
+name: load_global_p1_from_8
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_p1_from_8
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_DWORDX2_ADDR64_:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (p1), addrspace 1)
+    ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[BUFFER_LOAD_DWORDX2_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_p1_from_8
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_DWORDX2_ADDR64_:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (p1), addrspace 1)
+    ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[BUFFER_LOAD_DWORDX2_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_p1_from_8
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (p1), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
+    ;
+    ; GFX8-LABEL: name: load_global_p1_from_8
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (p1), addrspace 1)
+    ; GFX8-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
+    ;
+    ; GFX9-LABEL: name: load_global_p1_from_8
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (p1), addrspace 1)
+    ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
+    ;
+    ; GFX10-LABEL: name: load_global_p1_from_8
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (p1), addrspace 1)
+    ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
+    ;
+    ; GFX11-LABEL: name: load_global_p1_from_8
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (p1), addrspace 1)
+    ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
+    ;
+    ; GFX12-LABEL: name: load_global_p1_from_8
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (p1), addrspace 1)
+    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(p1) = G_LOAD %0 :: (load (p1), align 8, addrspace 1)
+    $vgpr0_vgpr1 = COPY %1
+
+...
+
+---
+
+name: load_global_p999_from_8
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_p999_from_8
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[LOAD:%[0-9]+]]:vreg_64(p999) = G_LOAD [[COPY]](p1) :: (load (p999), addrspace 1)
+    ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p999)
+    ;
+    ; GFX7-LABEL: name: load_global_p999_from_8
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vreg_64(p999) = G_LOAD [[COPY]](p1) :: (load (p999), addrspace 1)
+    ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p999)
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_p999_from_8
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[LOAD:%[0-9]+]]:vreg_64(p999) = G_LOAD [[COPY]](p1) :: (load (p999), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p999)
+    ;
+    ; GFX8-LABEL: name: load_global_p999_from_8
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[LOAD:%[0-9]+]]:vreg_64(p999) = G_LOAD [[COPY]](p1) :: (load (p999), addrspace 1)
+    ; GFX8-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p999)
+    ;
+    ; GFX9-LABEL: name: load_global_p999_from_8
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vreg_64(p999) = G_LOAD [[COPY]](p1) :: (load (p999), addrspace 1)
+    ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p999)
+    ;
+    ; GFX10-LABEL: name: load_global_p999_from_8
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vreg_64(p999) = G_LOAD [[COPY]](p1) :: (load (p999), addrspace 1)
+    ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p999)
+    ;
+    ; GFX11-LABEL: name: load_global_p999_from_8
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[LOAD:%[0-9]+]]:vreg_64(p999) = G_LOAD [[COPY]](p1) :: (load (p999), addrspace 1)
+    ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p999)
+    ;
+    ; GFX12-LABEL: name: load_global_p999_from_8
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:vreg_64(p999) = G_LOAD [[COPY]](p1) :: (load (p999), addrspace 1)
+    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p999)
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(p999) = G_LOAD %0 :: (load (p999), align 8, addrspace 1)
+    $vgpr0_vgpr1 = COPY %1
+
+...
+
+---
+
+name: load_global_v2p3
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_v2p3
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), addrspace 1)
+    ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ;
+    ; GFX7-LABEL: name: load_global_v2p3
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), addrspace 1)
+    ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_v2p3
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ;
+    ; GFX8-LABEL: name: load_global_v2p3
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), addrspace 1)
+    ; GFX8-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ;
+    ; GFX9-LABEL: name: load_global_v2p3
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), addrspace 1)
+    ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ;
+    ; GFX10-LABEL: name: load_global_v2p3
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), addrspace 1)
+    ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ;
+    ; GFX11-LABEL: name: load_global_v2p3
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), addrspace 1)
+    ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ;
+    ; GFX12-LABEL: name: load_global_v2p3
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), addrspace 1)
+    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(<2 x p3>) = G_LOAD %0 :: (load (<2 x p3>), align 8, addrspace 1)
+    $vgpr0_vgpr1 = COPY %1
+
+...
+
+---
+
+name: load_global_v2s16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_v2s16
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_DWORD_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (<2 x s16>), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_v2s16
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_DWORD_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (<2 x s16>), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_v2s16
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (<2 x s16>), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
+    ;
+    ; GFX8-LABEL: name: load_global_v2s16
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (<2 x s16>), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
+    ;
+    ; GFX9-LABEL: name: load_global_v2s16
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (<2 x s16>), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
+    ;
+    ; GFX10-LABEL: name: load_global_v2s16
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (<2 x s16>), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
+    ;
+    ; GFX11-LABEL: name: load_global_v2s16
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (<2 x s16>), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
+    ;
+    ; GFX12-LABEL: name: load_global_v2s16
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (<2 x s16>), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(<2 x s16>) = G_LOAD %0 :: (load (<2 x s16>), align 4, addrspace 1)
+    $vgpr0 = COPY %1
+
+...
+
+---
+
+name: load_global_v4s16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_v4s16
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_DWORDX2_ADDR64_:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (<4 x s16>), addrspace 1)
+    ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[BUFFER_LOAD_DWORDX2_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_v4s16
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_DWORDX2_ADDR64_:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load (<4 x s16>), addrspace 1)
+    ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[BUFFER_LOAD_DWORDX2_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_v4s16
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (<4 x s16>), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
+    ;
+    ; GFX8-LABEL: name: load_global_v4s16
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (<4 x s16>), addrspace 1)
+    ; GFX8-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
+    ;
+    ; GFX9-LABEL: name: load_global_v4s16
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (<4 x s16>), addrspace 1)
+    ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
+    ;
+    ; GFX10-LABEL: name: load_global_v4s16
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (<4 x s16>), addrspace 1)
+    ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
+    ;
+    ; GFX11-LABEL: name: load_global_v4s16
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (<4 x s16>), addrspace 1)
+    ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
+    ;
+    ; GFX12-LABEL: name: load_global_v4s16
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (<4 x s16>), addrspace 1)
+    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(<4 x s16>) = G_LOAD %0 :: (load (<4 x s16>), align 8, addrspace 1)
+    $vgpr0_vgpr1 = COPY %1
+
+...
+
+---
+
+name: load_global_v8s16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_v8s16
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[LOAD:%[0-9]+]]:vgpr(<8 x s16>) = G_LOAD [[COPY]](p1) :: (load (<8 x s16>), align 4, addrspace 1)
+    ; GFX6-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<8 x s16>)
+    ;
+    ; GFX7-LABEL: name: load_global_v8s16
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vgpr(<8 x s16>) = G_LOAD [[COPY]](p1) :: (load (<8 x s16>), align 4, addrspace 1)
+    ; GFX7-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<8 x s16>)
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_v8s16
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[LOAD:%[0-9]+]]:vgpr(<8 x s16>) = G_LOAD [[COPY]](p1) :: (load (<8 x s16>), align 4, addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<8 x s16>)
+    ;
+    ; GFX8-LABEL: name: load_global_v8s16
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[LOAD:%[0-9]+]]:vgpr(<8 x s16>) = G_LOAD [[COPY]](p1) :: (load (<8 x s16>), align 4, addrspace 1)
+    ; GFX8-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<8 x s16>)
+    ;
+    ; GFX9-LABEL: name: load_global_v8s16
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vgpr(<8 x s16>) = G_LOAD [[COPY]](p1) :: (load (<8 x s16>), align 4, addrspace 1)
+    ; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<8 x s16>)
+    ;
+    ; GFX10-LABEL: name: load_global_v8s16
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vgpr(<8 x s16>) = G_LOAD [[COPY]](p1) :: (load (<8 x s16>), align 4, addrspace 1)
+    ; GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<8 x s16>)
+    ;
+    ; GFX11-LABEL: name: load_global_v8s16
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[LOAD:%[0-9]+]]:vgpr(<8 x s16>) = G_LOAD [[COPY]](p1) :: (load (<8 x s16>), align 4, addrspace 1)
+    ; GFX11-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<8 x s16>)
+    ;
+    ; GFX12-LABEL: name: load_global_v8s16
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:vgpr(<8 x s16>) = G_LOAD [[COPY]](p1) :: (load (<8 x s16>), align 4, addrspace 1)
+    ; GFX12-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<8 x s16>)
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(<8 x s16>) = G_LOAD %0 :: (load (<8 x s16>), align 4, addrspace 1)
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+
+...
+
+################################################################################
+### Stress addressing modes
+################################################################################
+
+---
+
+name: load_global_s32_from_1_gep_2047
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s32_from_1_gep_2047
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 2047, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_s32_from_1_gep_2047
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 2047, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s32_from_1_gep_2047
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 2047, implicit $exec
+    ; GFX7-FLAT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-FLAT-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-FLAT-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX8-LABEL: name: load_global_s32_from_1_gep_2047
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 2047, implicit $exec
+    ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX8-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX8-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX8-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX9-LABEL: name: load_global_s32_from_1_gep_2047
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], 2047, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX10-LABEL: name: load_global_s32_from_1_gep_2047
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], 2047, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX11-LABEL: name: load_global_s32_from_1_gep_2047
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], 2047, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX12-LABEL: name: load_global_s32_from_1_gep_2047
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], 2047, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = G_CONSTANT i64 2047
+    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 1)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_global_s32_from_1_gep_2048
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s32_from_1_gep_2048
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 2048, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_s32_from_1_gep_2048
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 2048, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s32_from_1_gep_2048
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 2048, implicit $exec
+    ; GFX7-FLAT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-FLAT-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-FLAT-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX8-LABEL: name: load_global_s32_from_1_gep_2048
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 2048, implicit $exec
+    ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX8-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX8-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX8-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX9-LABEL: name: load_global_s32_from_1_gep_2048
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], 2048, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX10-LABEL: name: load_global_s32_from_1_gep_2048
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 2048, implicit $exec
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX10-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX10-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX11-LABEL: name: load_global_s32_from_1_gep_2048
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], 2048, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX12-LABEL: name: load_global_s32_from_1_gep_2048
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], 2048, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = G_CONSTANT i64 2048
+    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 1)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_global_s32_from_1_gep_m2047
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s32_from_1_gep_m2047
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -2047, implicit $exec
+    ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX6-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX6-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX6-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX6-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX6-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], 0, 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_s32_from_1_gep_m2047
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -2047, implicit $exec
+    ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], 0, 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s32_from_1_gep_m2047
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -2047, implicit $exec
+    ; GFX7-FLAT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-FLAT-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-FLAT-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX8-LABEL: name: load_global_s32_from_1_gep_m2047
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -2047, implicit $exec
+    ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX8-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX8-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX8-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX9-LABEL: name: load_global_s32_from_1_gep_m2047
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], -2047, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX10-LABEL: name: load_global_s32_from_1_gep_m2047
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], -2047, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX11-LABEL: name: load_global_s32_from_1_gep_m2047
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], -2047, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX12-LABEL: name: load_global_s32_from_1_gep_m2047
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], -2047, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = G_CONSTANT i64 -2047
+    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 1)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_global_s32_from_1_gep_m2048
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s32_from_1_gep_m2048
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -2048, implicit $exec
+    ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX6-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX6-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX6-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX6-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX6-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], 0, 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_s32_from_1_gep_m2048
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -2048, implicit $exec
+    ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], 0, 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s32_from_1_gep_m2048
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -2048, implicit $exec
+    ; GFX7-FLAT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-FLAT-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-FLAT-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX8-LABEL: name: load_global_s32_from_1_gep_m2048
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -2048, implicit $exec
+    ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX8-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX8-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX8-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX9-LABEL: name: load_global_s32_from_1_gep_m2048
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], -2048, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX10-LABEL: name: load_global_s32_from_1_gep_m2048
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], -2048, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX11-LABEL: name: load_global_s32_from_1_gep_m2048
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], -2048, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX12-LABEL: name: load_global_s32_from_1_gep_m2048
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], -2048, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = G_CONSTANT i64 -2048
+    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 1)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_global_s32_from_1_gep_4095
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s32_from_1_gep_4095
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 4095, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_s32_from_1_gep_4095
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 4095, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s32_from_1_gep_4095
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 4095, implicit $exec
+    ; GFX7-FLAT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-FLAT-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-FLAT-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX8-LABEL: name: load_global_s32_from_1_gep_4095
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 4095, implicit $exec
+    ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX8-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX8-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX8-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX9-LABEL: name: load_global_s32_from_1_gep_4095
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], 4095, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX10-LABEL: name: load_global_s32_from_1_gep_4095
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 4095, implicit $exec
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX10-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX10-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX11-LABEL: name: load_global_s32_from_1_gep_4095
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], 4095, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX12-LABEL: name: load_global_s32_from_1_gep_4095
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], 4095, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = G_CONSTANT i64 4095
+    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 1)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_global_s32_from_1_gep_4096
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s32_from_1_gep_4096
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+    ; GFX6-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_s32_from_1_gep_4096
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+    ; GFX7-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s32_from_1_gep_4096
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 4096, implicit $exec
+    ; GFX7-FLAT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-FLAT-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-FLAT-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX8-LABEL: name: load_global_s32_from_1_gep_4096
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 4096, implicit $exec
+    ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX8-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX8-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX8-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX9-LABEL: name: load_global_s32_from_1_gep_4096
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 4096, implicit $exec
+    ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX9-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX9-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX9-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX9-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX9-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX9-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX10-LABEL: name: load_global_s32_from_1_gep_4096
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 4096, implicit $exec
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX10-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX10-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX11-LABEL: name: load_global_s32_from_1_gep_4096
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 4096, implicit $exec
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX11-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX11-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX12-LABEL: name: load_global_s32_from_1_gep_4096
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], 4096, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = G_CONSTANT i64 4096
+    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 1)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_global_s32_from_1_gep_m4095
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s32_from_1_gep_m4095
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -4095, implicit $exec
+    ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX6-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX6-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX6-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX6-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX6-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], 0, 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_s32_from_1_gep_m4095
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -4095, implicit $exec
+    ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], 0, 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s32_from_1_gep_m4095
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -4095, implicit $exec
+    ; GFX7-FLAT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-FLAT-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-FLAT-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX8-LABEL: name: load_global_s32_from_1_gep_m4095
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -4095, implicit $exec
+    ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX8-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX8-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX8-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX9-LABEL: name: load_global_s32_from_1_gep_m4095
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], -4095, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX10-LABEL: name: load_global_s32_from_1_gep_m4095
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -4095, implicit $exec
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX10-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX10-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX11-LABEL: name: load_global_s32_from_1_gep_m4095
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], -4095, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX12-LABEL: name: load_global_s32_from_1_gep_m4095
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], -4095, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = G_CONSTANT i64 -4095
+    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 1)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_global_s32_from_1_gep_m4096
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s32_from_1_gep_m4096
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -4096, implicit $exec
+    ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX6-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX6-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX6-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX6-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX6-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], 0, 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_s32_from_1_gep_m4096
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -4096, implicit $exec
+    ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], 0, 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s32_from_1_gep_m4096
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -4096, implicit $exec
+    ; GFX7-FLAT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-FLAT-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-FLAT-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX8-LABEL: name: load_global_s32_from_1_gep_m4096
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -4096, implicit $exec
+    ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX8-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX8-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX8-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX9-LABEL: name: load_global_s32_from_1_gep_m4096
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], -4096, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX10-LABEL: name: load_global_s32_from_1_gep_m4096
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -4096, implicit $exec
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX10-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX10-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX11-LABEL: name: load_global_s32_from_1_gep_m4096
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], -4096, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX12-LABEL: name: load_global_s32_from_1_gep_m4096
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], -4096, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = G_CONSTANT i64 -4096
+    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 1)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_global_s32_from_1_gep_8191
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s32_from_1_gep_8191
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 8191
+    ; GFX6-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_s32_from_1_gep_8191
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 8191
+    ; GFX7-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s32_from_1_gep_8191
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 8191, implicit $exec
+    ; GFX7-FLAT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-FLAT-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-FLAT-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX8-LABEL: name: load_global_s32_from_1_gep_8191
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 8191, implicit $exec
+    ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX8-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX8-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX8-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX9-LABEL: name: load_global_s32_from_1_gep_8191
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 8191, implicit $exec
+    ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX9-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX9-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX9-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX9-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX9-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX9-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX10-LABEL: name: load_global_s32_from_1_gep_8191
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 8191, implicit $exec
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX10-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX10-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX11-LABEL: name: load_global_s32_from_1_gep_8191
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 8191, implicit $exec
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX11-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX11-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX12-LABEL: name: load_global_s32_from_1_gep_8191
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], 8191, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = G_CONSTANT i64 8191
+    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 1)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_global_s32_from_1_gep_8192
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s32_from_1_gep_8192
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 8192
+    ; GFX6-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_s32_from_1_gep_8192
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 8192
+    ; GFX7-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s32_from_1_gep_8192
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 8192, implicit $exec
+    ; GFX7-FLAT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-FLAT-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-FLAT-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX8-LABEL: name: load_global_s32_from_1_gep_8192
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 8192, implicit $exec
+    ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX8-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX8-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX8-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX9-LABEL: name: load_global_s32_from_1_gep_8192
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 8192, implicit $exec
+    ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX9-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX9-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX9-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX9-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX9-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX9-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX10-LABEL: name: load_global_s32_from_1_gep_8192
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 8192, implicit $exec
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX10-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX10-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX11-LABEL: name: load_global_s32_from_1_gep_8192
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 8192, implicit $exec
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX11-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX11-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX12-LABEL: name: load_global_s32_from_1_gep_8192
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], 8192, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = G_CONSTANT i64 8192
+    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 1)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_global_s32_from_1_gep_m8191
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s32_from_1_gep_m8191
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8191, implicit $exec
+    ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX6-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX6-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX6-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX6-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX6-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], 0, 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_s32_from_1_gep_m8191
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8191, implicit $exec
+    ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], 0, 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s32_from_1_gep_m8191
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8191, implicit $exec
+    ; GFX7-FLAT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-FLAT-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-FLAT-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX8-LABEL: name: load_global_s32_from_1_gep_m8191
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8191, implicit $exec
+    ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX8-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX8-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX8-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX9-LABEL: name: load_global_s32_from_1_gep_m8191
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8191, implicit $exec
+    ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX9-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX9-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX9-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX9-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX9-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX9-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX10-LABEL: name: load_global_s32_from_1_gep_m8191
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8191, implicit $exec
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX10-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX10-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX11-LABEL: name: load_global_s32_from_1_gep_m8191
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8191, implicit $exec
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX11-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX11-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX12-LABEL: name: load_global_s32_from_1_gep_m8191
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], -8191, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = G_CONSTANT i64 -8191
+    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 1)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_global_s32_from_1_gep_m8192
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s32_from_1_gep_m8192
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8192, implicit $exec
+    ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX6-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX6-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX6-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX6-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX6-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], 0, 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_s32_from_1_gep_m8192
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8192, implicit $exec
+    ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], 0, 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s32_from_1_gep_m8192
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8192, implicit $exec
+    ; GFX7-FLAT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-FLAT-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-FLAT-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX8-LABEL: name: load_global_s32_from_1_gep_m8192
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8192, implicit $exec
+    ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX8-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX8-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX8-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX9-LABEL: name: load_global_s32_from_1_gep_m8192
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8192, implicit $exec
+    ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX9-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX9-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX9-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX9-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX9-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX9-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX10-LABEL: name: load_global_s32_from_1_gep_m8192
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8192, implicit $exec
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX10-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX10-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX11-LABEL: name: load_global_s32_from_1_gep_m8192
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8192, implicit $exec
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX11-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX11-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX12-LABEL: name: load_global_s32_from_1_gep_m8192
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], -8192, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = G_CONSTANT i64 -8192
+    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 1)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_global_s32_from_1_gep_24bit_max
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s32_from_1_gep_24bit_max
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 8388607
+    ; GFX6-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_s32_from_1_gep_24bit_max
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 8388607
+    ; GFX7-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s32_from_1_gep_24bit_max
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 8388607, implicit $exec
+    ; GFX7-FLAT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-FLAT-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-FLAT-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX8-LABEL: name: load_global_s32_from_1_gep_24bit_max
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 8388607, implicit $exec
+    ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX8-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX8-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX8-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX9-LABEL: name: load_global_s32_from_1_gep_24bit_max
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 8388607, implicit $exec
+    ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX9-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX9-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX9-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX9-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX9-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX9-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX10-LABEL: name: load_global_s32_from_1_gep_24bit_max
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 8388607, implicit $exec
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX10-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX10-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX11-LABEL: name: load_global_s32_from_1_gep_24bit_max
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 8388607, implicit $exec
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX11-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX11-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX12-LABEL: name: load_global_s32_from_1_gep_24bit_max
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], 8388607, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = G_CONSTANT i64 8388607
+    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 1)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_global_s32_from_1_gep_2x_24bit_max
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s32_from_1_gep_2x_24bit_max
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 16777214
+    ; GFX6-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_s32_from_1_gep_2x_24bit_max
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 16777214
+    ; GFX7-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s32_from_1_gep_2x_24bit_max
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 16777214, implicit $exec
+    ; GFX7-FLAT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-FLAT-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-FLAT-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX8-LABEL: name: load_global_s32_from_1_gep_2x_24bit_max
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 16777214, implicit $exec
+    ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX8-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX8-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX8-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX9-LABEL: name: load_global_s32_from_1_gep_2x_24bit_max
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 16777214, implicit $exec
+    ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX9-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX9-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX9-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX9-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX9-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX9-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX10-LABEL: name: load_global_s32_from_1_gep_2x_24bit_max
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 16777214, implicit $exec
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX10-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX10-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX11-LABEL: name: load_global_s32_from_1_gep_2x_24bit_max
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 16777214, implicit $exec
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX11-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX11-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX12-LABEL: name: load_global_s32_from_1_gep_2x_24bit_max
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 16777214, implicit $exec
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX12-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX12-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = G_CONSTANT i64 16777214
+    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 1)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_global_s32_from_1_gep_24bit_min
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s32_from_1_gep_24bit_min
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8388608, implicit $exec
+    ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX6-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX6-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX6-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX6-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX6-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], 0, 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_s32_from_1_gep_24bit_min
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8388608, implicit $exec
+    ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], 0, 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s32_from_1_gep_24bit_min
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8388608, implicit $exec
+    ; GFX7-FLAT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-FLAT-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-FLAT-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX8-LABEL: name: load_global_s32_from_1_gep_24bit_min
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8388608, implicit $exec
+    ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX8-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX8-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX8-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX9-LABEL: name: load_global_s32_from_1_gep_24bit_min
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8388608, implicit $exec
+    ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX9-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX9-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX9-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX9-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX9-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX9-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX10-LABEL: name: load_global_s32_from_1_gep_24bit_min
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8388608, implicit $exec
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX10-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX10-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX11-LABEL: name: load_global_s32_from_1_gep_24bit_min
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -8388608, implicit $exec
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX11-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX11-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX12-LABEL: name: load_global_s32_from_1_gep_24bit_min
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], -8388608, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = G_CONSTANT i64 -8388608
+    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 1)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_global_s32_from_1_gep_2x_24bit_min
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1
+
+    ; GFX6-LABEL: name: load_global_s32_from_1_gep_2x_24bit_min
+    ; GFX6: liveins: $vgpr0_vgpr1
+    ; GFX6-NEXT: {{  $}}
+    ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX6-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -16777215, implicit $exec
+    ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX6-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX6-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX6-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX6-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX6-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX6-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+    ; GFX6-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], 0, 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-LABEL: name: load_global_s32_from_1_gep_2x_24bit_min
+    ; GFX7: liveins: $vgpr0_vgpr1
+    ; GFX7-NEXT: {{  $}}
+    ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -16777215, implicit $exec
+    ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
+    ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+    ; GFX7-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+    ; GFX7-NEXT: [[BUFFER_LOAD_UBYTE_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_ADDR64 [[REG_SEQUENCE]], [[REG_SEQUENCE2]], 0, 0, 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_ADDR64_]]
+    ;
+    ; GFX7-FLAT-LABEL: name: load_global_s32_from_1_gep_2x_24bit_min
+    ; GFX7-FLAT: liveins: $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: {{  $}}
+    ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX7-FLAT-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -16777215, implicit $exec
+    ; GFX7-FLAT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX7-FLAT-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX7-FLAT-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX7-FLAT-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX7-FLAT-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX7-FLAT-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX8-LABEL: name: load_global_s32_from_1_gep_2x_24bit_min
+    ; GFX8: liveins: $vgpr0_vgpr1
+    ; GFX8-NEXT: {{  $}}
+    ; GFX8-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX8-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -16777215, implicit $exec
+    ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX8-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX8-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX8-NEXT: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s8), addrspace 1)
+    ; GFX8-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
+    ;
+    ; GFX9-LABEL: name: load_global_s32_from_1_gep_2x_24bit_min
+    ; GFX9: liveins: $vgpr0_vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -16777215, implicit $exec
+    ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX9-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX9-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX9-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX9-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX9-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX9-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX9-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX10-LABEL: name: load_global_s32_from_1_gep_2x_24bit_min
+    ; GFX10: liveins: $vgpr0_vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX10-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -16777215, implicit $exec
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX10-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX10-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX10-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX11-LABEL: name: load_global_s32_from_1_gep_2x_24bit_min
+    ; GFX11: liveins: $vgpr0_vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX11-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -16777215, implicit $exec
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX11-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX11-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX11-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX11-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    ;
+    ; GFX12-LABEL: name: load_global_s32_from_1_gep_2x_24bit_min
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; GFX12-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO -16777215, implicit $exec
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub0
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_MOV_B]].sub1
+    ; GFX12-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY1]], [[COPY2]], 0, implicit $exec
+    ; GFX12-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 [[COPY3]], [[COPY4]], killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+    ; GFX12-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8), addrspace 1)
+    ; GFX12-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(s64) = G_CONSTANT i64 -16777215
+    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 1)
+    $vgpr0 = COPY %3
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local.mir
index 280c7a5a492da..87018100abf68 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local.mir
@@ -24,6 +24,7 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (s32), addrspace 3)
     ; GFX6-NEXT: $vgpr0 = COPY [[DS_READ_B32_]]
+    ;
     ; GFX7-LABEL: name: load_local_s32_from_4
     ; GFX7: liveins: $vgpr0
     ; GFX7-NEXT: {{  $}}
@@ -31,12 +32,14 @@ body: |
     ; GFX7-NEXT: $m0 = S_MOV_B32 -1
     ; GFX7-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (s32), addrspace 3)
     ; GFX7-NEXT: $vgpr0 = COPY [[DS_READ_B32_]]
+    ;
     ; GFX9-LABEL: name: load_local_s32_from_4
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9-NEXT: [[DS_READ_B32_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_B32_gfx9 [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 3)
     ; GFX9-NEXT: $vgpr0 = COPY [[DS_READ_B32_gfx9_]]
+    ;
     ; GFX10-LABEL: name: load_local_s32_from_4
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
@@ -67,6 +70,7 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[DS_READ_U16_:%[0-9]+]]:vgpr_32 = DS_READ_U16 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (s16), addrspace 3)
     ; GFX6-NEXT: $vgpr0 = COPY [[DS_READ_U16_]]
+    ;
     ; GFX7-LABEL: name: load_local_s32_from_2
     ; GFX7: liveins: $vgpr0
     ; GFX7-NEXT: {{  $}}
@@ -74,12 +78,14 @@ body: |
     ; GFX7-NEXT: $m0 = S_MOV_B32 -1
     ; GFX7-NEXT: [[DS_READ_U16_:%[0-9]+]]:vgpr_32 = DS_READ_U16 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (s16), addrspace 3)
     ; GFX7-NEXT: $vgpr0 = COPY [[DS_READ_U16_]]
+    ;
     ; GFX9-LABEL: name: load_local_s32_from_2
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9-NEXT: [[DS_READ_U16_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_U16_gfx9 [[COPY]], 0, 0, implicit $exec :: (load (s16), addrspace 3)
     ; GFX9-NEXT: $vgpr0 = COPY [[DS_READ_U16_gfx9_]]
+    ;
     ; GFX10-LABEL: name: load_local_s32_from_2
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
@@ -113,6 +119,7 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[DS_READ_U8_:%[0-9]+]]:vgpr_32 = DS_READ_U8 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (s8), addrspace 3)
     ; GFX6-NEXT: $vgpr0 = COPY [[DS_READ_U8_]]
+    ;
     ; GFX7-LABEL: name: load_local_s32_from_1
     ; GFX7: liveins: $vgpr0
     ; GFX7-NEXT: {{  $}}
@@ -120,12 +127,14 @@ body: |
     ; GFX7-NEXT: $m0 = S_MOV_B32 -1
     ; GFX7-NEXT: [[DS_READ_U8_:%[0-9]+]]:vgpr_32 = DS_READ_U8 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (s8), addrspace 3)
     ; GFX7-NEXT: $vgpr0 = COPY [[DS_READ_U8_]]
+    ;
     ; GFX9-LABEL: name: load_local_s32_from_1
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9-NEXT: [[DS_READ_U8_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_U8_gfx9 [[COPY]], 0, 0, implicit $exec :: (load (s8), addrspace 3)
     ; GFX9-NEXT: $vgpr0 = COPY [[DS_READ_U8_gfx9_]]
+    ;
     ; GFX10-LABEL: name: load_local_s32_from_1
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
@@ -156,6 +165,7 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[DS_READ_B64_:%[0-9]+]]:vreg_64 = DS_READ_B64 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (<2 x s32>), addrspace 3)
     ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ_B64_]]
+    ;
     ; GFX7-LABEL: name: load_local_v2s32
     ; GFX7: liveins: $vgpr0
     ; GFX7-NEXT: {{  $}}
@@ -163,12 +173,14 @@ body: |
     ; GFX7-NEXT: $m0 = S_MOV_B32 -1
     ; GFX7-NEXT: [[DS_READ_B64_:%[0-9]+]]:vreg_64 = DS_READ_B64 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (<2 x s32>), addrspace 3)
     ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ_B64_]]
+    ;
     ; GFX9-LABEL: name: load_local_v2s32
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9-NEXT: [[DS_READ_B64_gfx9_:%[0-9]+]]:vreg_64 = DS_READ_B64_gfx9 [[COPY]], 0, 0, implicit $exec :: (load (<2 x s32>), addrspace 3)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ_B64_gfx9_]]
+    ;
     ; GFX10-LABEL: name: load_local_v2s32
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
@@ -199,6 +211,7 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load (<2 x s32>), align 4, addrspace 3)
     ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>)
+    ;
     ; GFX7-LABEL: name: load_local_v2s32_align4
     ; GFX7: liveins: $vgpr0
     ; GFX7-NEXT: {{  $}}
@@ -206,12 +219,14 @@ body: |
     ; GFX7-NEXT: $m0 = S_MOV_B32 -1
     ; GFX7-NEXT: [[DS_READ2_B32_:%[0-9]+]]:vreg_64 = DS_READ2_B32 [[COPY]], 0, 1, 0, implicit $m0, implicit $exec :: (load (<2 x s32>), align 4, addrspace 3)
     ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ2_B32_]]
+    ;
     ; GFX9-LABEL: name: load_local_v2s32_align4
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9-NEXT: [[DS_READ2_B32_gfx9_:%[0-9]+]]:vreg_64 = DS_READ2_B32_gfx9 [[COPY]], 0, 1, 0, implicit $exec :: (load (<2 x s32>), align 4, addrspace 3)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ2_B32_gfx9_]]
+    ;
     ; GFX10-LABEL: name: load_local_v2s32_align4
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
@@ -242,6 +257,7 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[DS_READ_B64_:%[0-9]+]]:vreg_64 = DS_READ_B64 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (s64), addrspace 3)
     ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ_B64_]]
+    ;
     ; GFX7-LABEL: name: load_local_s64
     ; GFX7: liveins: $vgpr0
     ; GFX7-NEXT: {{  $}}
@@ -249,12 +265,14 @@ body: |
     ; GFX7-NEXT: $m0 = S_MOV_B32 -1
     ; GFX7-NEXT: [[DS_READ_B64_:%[0-9]+]]:vreg_64 = DS_READ_B64 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (s64), addrspace 3)
     ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ_B64_]]
+    ;
     ; GFX9-LABEL: name: load_local_s64
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9-NEXT: [[DS_READ_B64_gfx9_:%[0-9]+]]:vreg_64 = DS_READ_B64_gfx9 [[COPY]], 0, 0, implicit $exec :: (load (s64), addrspace 3)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ_B64_gfx9_]]
+    ;
     ; GFX10-LABEL: name: load_local_s64
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
@@ -285,6 +303,7 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[LOAD:%[0-9]+]]:vreg_64(s64) = G_LOAD [[COPY]](p3) :: (load (s64), align 4, addrspace 3)
     ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](s64)
+    ;
     ; GFX7-LABEL: name: load_local_s64_align4
     ; GFX7: liveins: $vgpr0
     ; GFX7-NEXT: {{  $}}
@@ -292,12 +311,14 @@ body: |
     ; GFX7-NEXT: $m0 = S_MOV_B32 -1
     ; GFX7-NEXT: [[DS_READ2_B32_:%[0-9]+]]:vreg_64 = DS_READ2_B32 [[COPY]], 0, 1, 0, implicit $m0, implicit $exec :: (load (s64), align 4, addrspace 3)
     ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ2_B32_]]
+    ;
     ; GFX9-LABEL: name: load_local_s64_align4
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9-NEXT: [[DS_READ2_B32_gfx9_:%[0-9]+]]:vreg_64 = DS_READ2_B32_gfx9 [[COPY]], 0, 1, 0, implicit $exec :: (load (s64), align 4, addrspace 3)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ2_B32_gfx9_]]
+    ;
     ; GFX10-LABEL: name: load_local_s64_align4
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
@@ -328,6 +349,7 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (p3), addrspace 3)
     ; GFX6-NEXT: $vgpr0 = COPY [[DS_READ_B32_]]
+    ;
     ; GFX7-LABEL: name: load_local_p3_from_4
     ; GFX7: liveins: $vgpr0
     ; GFX7-NEXT: {{  $}}
@@ -335,12 +357,14 @@ body: |
     ; GFX7-NEXT: $m0 = S_MOV_B32 -1
     ; GFX7-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (p3), addrspace 3)
     ; GFX7-NEXT: $vgpr0 = COPY [[DS_READ_B32_]]
+    ;
     ; GFX9-LABEL: name: load_local_p3_from_4
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9-NEXT: [[DS_READ_B32_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_B32_gfx9 [[COPY]], 0, 0, implicit $exec :: (load (p3), addrspace 3)
     ; GFX9-NEXT: $vgpr0 = COPY [[DS_READ_B32_gfx9_]]
+    ;
     ; GFX10-LABEL: name: load_local_p3_from_4
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
@@ -371,6 +395,7 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (p5), addrspace 3)
     ; GFX6-NEXT: $vgpr0 = COPY [[DS_READ_B32_]]
+    ;
     ; GFX7-LABEL: name: load_local_p5_from_4
     ; GFX7: liveins: $vgpr0
     ; GFX7-NEXT: {{  $}}
@@ -378,12 +403,14 @@ body: |
     ; GFX7-NEXT: $m0 = S_MOV_B32 -1
     ; GFX7-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (p5), addrspace 3)
     ; GFX7-NEXT: $vgpr0 = COPY [[DS_READ_B32_]]
+    ;
     ; GFX9-LABEL: name: load_local_p5_from_4
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9-NEXT: [[DS_READ_B32_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_B32_gfx9 [[COPY]], 0, 0, implicit $exec :: (load (p5), addrspace 3)
     ; GFX9-NEXT: $vgpr0 = COPY [[DS_READ_B32_gfx9_]]
+    ;
     ; GFX10-LABEL: name: load_local_p5_from_4
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
@@ -414,6 +441,7 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[DS_READ_B64_:%[0-9]+]]:vreg_64 = DS_READ_B64 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (p1), addrspace 3)
     ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ_B64_]]
+    ;
     ; GFX7-LABEL: name: load_local_p1_align8
     ; GFX7: liveins: $vgpr0
     ; GFX7-NEXT: {{  $}}
@@ -421,12 +449,14 @@ body: |
     ; GFX7-NEXT: $m0 = S_MOV_B32 -1
     ; GFX7-NEXT: [[DS_READ_B64_:%[0-9]+]]:vreg_64 = DS_READ_B64 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (p1), addrspace 3)
     ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ_B64_]]
+    ;
     ; GFX9-LABEL: name: load_local_p1_align8
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9-NEXT: [[DS_READ_B64_gfx9_:%[0-9]+]]:vreg_64 = DS_READ_B64_gfx9 [[COPY]], 0, 0, implicit $exec :: (load (p1), addrspace 3)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ_B64_gfx9_]]
+    ;
     ; GFX10-LABEL: name: load_local_p1_align8
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
@@ -457,6 +487,7 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[LOAD:%[0-9]+]]:vreg_64(p1) = G_LOAD [[COPY]](p3) :: (load (p1), align 4, addrspace 3)
     ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p1)
+    ;
     ; GFX7-LABEL: name: load_local_p1_align4
     ; GFX7: liveins: $vgpr0
     ; GFX7-NEXT: {{  $}}
@@ -464,12 +495,14 @@ body: |
     ; GFX7-NEXT: $m0 = S_MOV_B32 -1
     ; GFX7-NEXT: [[DS_READ2_B32_:%[0-9]+]]:vreg_64 = DS_READ2_B32 [[COPY]], 0, 1, 0, implicit $m0, implicit $exec :: (load (p1), align 4, addrspace 3)
     ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ2_B32_]]
+    ;
     ; GFX9-LABEL: name: load_local_p1_align4
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9-NEXT: [[DS_READ2_B32_gfx9_:%[0-9]+]]:vreg_64 = DS_READ2_B32_gfx9 [[COPY]], 0, 1, 0, implicit $exec :: (load (p1), align 4, addrspace 3)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ2_B32_gfx9_]]
+    ;
     ; GFX10-LABEL: name: load_local_p1_align4
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
@@ -500,6 +533,7 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[LOAD:%[0-9]+]]:vreg_64(p999) = G_LOAD [[COPY]](p3) :: (load (p999), addrspace 3)
     ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p999)
+    ;
     ; GFX7-LABEL: name: load_local_p999_from_8
     ; GFX7: liveins: $vgpr0
     ; GFX7-NEXT: {{  $}}
@@ -507,12 +541,14 @@ body: |
     ; GFX7-NEXT: $m0 = S_MOV_B32 -1
     ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vreg_64(p999) = G_LOAD [[COPY]](p3) :: (load (p999), addrspace 3)
     ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p999)
+    ;
     ; GFX9-LABEL: name: load_local_p999_from_8
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
     ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vreg_64(p999) = G_LOAD [[COPY]](p3) :: (load (p999), addrspace 3)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](p999)
+    ;
     ; GFX10-LABEL: name: load_local_p999_from_8
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
@@ -543,24 +579,26 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<2 x p3>) = G_LOAD [[COPY]](p3) :: (load (<2 x p3>), addrspace 3)
     ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ;
     ; GFX7-LABEL: name: load_local_v2p3
     ; GFX7: liveins: $vgpr0
     ; GFX7-NEXT: {{  $}}
     ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
-    ; GFX7-NEXT: $m0 = S_MOV_B32 -1
-    ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<2 x p3>) = G_LOAD [[COPY]](p3) :: (load (<2 x p3>), addrspace 3)
+    ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x p3>) = G_LOAD [[COPY]](p3) :: (load (<2 x p3>), addrspace 3)
     ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ;
     ; GFX9-LABEL: name: load_local_v2p3
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<2 x p3>) = G_LOAD [[COPY]](p3) :: (load (<2 x p3>), addrspace 3)
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x p3>) = G_LOAD [[COPY]](p3) :: (load (<2 x p3>), addrspace 3)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ;
     ; GFX10-LABEL: name: load_local_v2p3
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<2 x p3>) = G_LOAD [[COPY]](p3) :: (load (<2 x p3>), addrspace 3)
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vgpr(<2 x p3>) = G_LOAD [[COPY]](p3) :: (load (<2 x p3>), addrspace 3)
     ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
     %0:vgpr(p3) = COPY $vgpr0
     %1:vgpr(<2 x p3>) = G_LOAD %0 :: (load (<2 x p3>), align 8, addrspace 3)
@@ -586,6 +624,7 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (<2 x s16>), addrspace 3)
     ; GFX6-NEXT: $vgpr0 = COPY [[DS_READ_B32_]]
+    ;
     ; GFX7-LABEL: name: load_local_v2s16
     ; GFX7: liveins: $vgpr0
     ; GFX7-NEXT: {{  $}}
@@ -593,12 +632,14 @@ body: |
     ; GFX7-NEXT: $m0 = S_MOV_B32 -1
     ; GFX7-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (<2 x s16>), addrspace 3)
     ; GFX7-NEXT: $vgpr0 = COPY [[DS_READ_B32_]]
+    ;
     ; GFX9-LABEL: name: load_local_v2s16
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9-NEXT: [[DS_READ_B32_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_B32_gfx9 [[COPY]], 0, 0, implicit $exec :: (load (<2 x s16>), addrspace 3)
     ; GFX9-NEXT: $vgpr0 = COPY [[DS_READ_B32_gfx9_]]
+    ;
     ; GFX10-LABEL: name: load_local_v2s16
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
@@ -629,6 +670,7 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[DS_READ_B64_:%[0-9]+]]:vreg_64 = DS_READ_B64 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (<4 x s16>), addrspace 3)
     ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ_B64_]]
+    ;
     ; GFX7-LABEL: name: load_local_v4s16
     ; GFX7: liveins: $vgpr0
     ; GFX7-NEXT: {{  $}}
@@ -636,12 +678,14 @@ body: |
     ; GFX7-NEXT: $m0 = S_MOV_B32 -1
     ; GFX7-NEXT: [[DS_READ_B64_:%[0-9]+]]:vreg_64 = DS_READ_B64 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load (<4 x s16>), addrspace 3)
     ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ_B64_]]
+    ;
     ; GFX9-LABEL: name: load_local_v4s16
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9-NEXT: [[DS_READ_B64_gfx9_:%[0-9]+]]:vreg_64 = DS_READ_B64_gfx9 [[COPY]], 0, 0, implicit $exec :: (load (<4 x s16>), addrspace 3)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ_B64_gfx9_]]
+    ;
     ; GFX10-LABEL: name: load_local_v4s16
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
@@ -698,6 +742,7 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[DS_READ_U8_:%[0-9]+]]:vgpr_32 = DS_READ_U8 [[V_ADD_CO_U32_e64_]], 0, 0, implicit $m0, implicit $exec :: (load (s8), addrspace 3)
     ; GFX6-NEXT: $vgpr0 = COPY [[DS_READ_U8_]]
+    ;
     ; GFX7-LABEL: name: load_local_s32_from_1_gep_65535
     ; GFX7: liveins: $vgpr0
     ; GFX7-NEXT: {{  $}}
@@ -705,12 +750,14 @@ body: |
     ; GFX7-NEXT: $m0 = S_MOV_B32 -1
     ; GFX7-NEXT: [[DS_READ_U8_:%[0-9]+]]:vgpr_32 = DS_READ_U8 [[COPY]], 65535, 0, implicit $m0, implicit $exec :: (load (s8), addrspace 3)
     ; GFX7-NEXT: $vgpr0 = COPY [[DS_READ_U8_]]
+    ;
     ; GFX9-LABEL: name: load_local_s32_from_1_gep_65535
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9-NEXT: [[DS_READ_U8_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_U8_gfx9 [[COPY]], 65535, 0, implicit $exec :: (load (s8), addrspace 3)
     ; GFX9-NEXT: $vgpr0 = COPY [[DS_READ_U8_gfx9_]]
+    ;
     ; GFX10-LABEL: name: load_local_s32_from_1_gep_65535
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
@@ -745,6 +792,7 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[DS_READ_U8_:%[0-9]+]]:vgpr_32 = DS_READ_U8 [[V_AND_B32_e64_]], 65535, 0, implicit $m0, implicit $exec :: (load (s8), addrspace 3)
     ; GFX6-NEXT: $vgpr0 = COPY [[DS_READ_U8_]]
+    ;
     ; GFX7-LABEL: name: load_local_s32_from_1_gep_65535_known_bits_base_address
     ; GFX7: liveins: $vgpr0
     ; GFX7-NEXT: {{  $}}
@@ -754,6 +802,7 @@ body: |
     ; GFX7-NEXT: $m0 = S_MOV_B32 -1
     ; GFX7-NEXT: [[DS_READ_U8_:%[0-9]+]]:vgpr_32 = DS_READ_U8 [[V_AND_B32_e64_]], 65535, 0, implicit $m0, implicit $exec :: (load (s8), addrspace 3)
     ; GFX7-NEXT: $vgpr0 = COPY [[DS_READ_U8_]]
+    ;
     ; GFX9-LABEL: name: load_local_s32_from_1_gep_65535_known_bits_base_address
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
@@ -762,6 +811,7 @@ body: |
     ; GFX9-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
     ; GFX9-NEXT: [[DS_READ_U8_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_U8_gfx9 [[V_AND_B32_e64_]], 65535, 0, implicit $exec :: (load (s8), addrspace 3)
     ; GFX9-NEXT: $vgpr0 = COPY [[DS_READ_U8_gfx9_]]
+    ;
     ; GFX10-LABEL: name: load_local_s32_from_1_gep_65535_known_bits_base_address
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
@@ -801,6 +851,7 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[DS_READ_U8_:%[0-9]+]]:vgpr_32 = DS_READ_U8 [[V_ADD_CO_U32_e64_]], 0, 0, implicit $m0, implicit $exec :: (load (s8), addrspace 3)
     ; GFX6-NEXT: $vgpr0 = COPY [[DS_READ_U8_]]
+    ;
     ; GFX7-LABEL: name: load_local_s32_from_1_gep_65536
     ; GFX7: liveins: $vgpr0
     ; GFX7-NEXT: {{  $}}
@@ -810,6 +861,7 @@ body: |
     ; GFX7-NEXT: $m0 = S_MOV_B32 -1
     ; GFX7-NEXT: [[DS_READ_U8_:%[0-9]+]]:vgpr_32 = DS_READ_U8 [[V_ADD_CO_U32_e64_]], 0, 0, implicit $m0, implicit $exec :: (load (s8), addrspace 3)
     ; GFX7-NEXT: $vgpr0 = COPY [[DS_READ_U8_]]
+    ;
     ; GFX9-LABEL: name: load_local_s32_from_1_gep_65536
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
@@ -818,6 +870,7 @@ body: |
     ; GFX9-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
     ; GFX9-NEXT: [[DS_READ_U8_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_U8_gfx9 [[V_ADD_U32_e64_]], 0, 0, implicit $exec :: (load (s8), addrspace 3)
     ; GFX9-NEXT: $vgpr0 = COPY [[DS_READ_U8_gfx9_]]
+    ;
     ; GFX10-LABEL: name: load_local_s32_from_1_gep_65536
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
@@ -854,6 +907,7 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[DS_READ_U8_:%[0-9]+]]:vgpr_32 = DS_READ_U8 [[V_ADD_CO_U32_e64_]], 0, 0, implicit $m0, implicit $exec :: (load (s8), addrspace 3)
     ; GFX6-NEXT: $vgpr0 = COPY [[DS_READ_U8_]]
+    ;
     ; GFX7-LABEL: name: load_local_s32_from_1_gep_m1
     ; GFX7: liveins: $vgpr0
     ; GFX7-NEXT: {{  $}}
@@ -863,6 +917,7 @@ body: |
     ; GFX7-NEXT: $m0 = S_MOV_B32 -1
     ; GFX7-NEXT: [[DS_READ_U8_:%[0-9]+]]:vgpr_32 = DS_READ_U8 [[V_ADD_CO_U32_e64_]], 0, 0, implicit $m0, implicit $exec :: (load (s8), addrspace 3)
     ; GFX7-NEXT: $vgpr0 = COPY [[DS_READ_U8_]]
+    ;
     ; GFX9-LABEL: name: load_local_s32_from_1_gep_m1
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
@@ -871,6 +926,7 @@ body: |
     ; GFX9-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
     ; GFX9-NEXT: [[DS_READ_U8_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_U8_gfx9 [[V_ADD_U32_e64_]], 0, 0, implicit $exec :: (load (s8), addrspace 3)
     ; GFX9-NEXT: $vgpr0 = COPY [[DS_READ_U8_gfx9_]]
+    ;
     ; GFX10-LABEL: name: load_local_s32_from_1_gep_m1
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
@@ -907,6 +963,7 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[LOAD:%[0-9]+]]:vreg_64(s64) = G_LOAD [[PTR_ADD]](p3) :: (load (s64), align 4, addrspace 3)
     ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](s64)
+    ;
     ; GFX7-LABEL: name: load_local_s64_align4_from_1_gep_1016
     ; GFX7: liveins: $vgpr0_vgpr1
     ; GFX7-NEXT: {{  $}}
@@ -914,12 +971,14 @@ body: |
     ; GFX7-NEXT: $m0 = S_MOV_B32 -1
     ; GFX7-NEXT: [[DS_READ2_B32_:%[0-9]+]]:vreg_64 = DS_READ2_B32 [[COPY]], 254, 255, 0, implicit $m0, implicit $exec :: (load (s64), align 4, addrspace 3)
     ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ2_B32_]]
+    ;
     ; GFX9-LABEL: name: load_local_s64_align4_from_1_gep_1016
     ; GFX9: liveins: $vgpr0_vgpr1
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9-NEXT: [[DS_READ2_B32_gfx9_:%[0-9]+]]:vreg_64 = DS_READ2_B32_gfx9 [[COPY]], 254, 255, 0, implicit $exec :: (load (s64), align 4, addrspace 3)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ2_B32_gfx9_]]
+    ;
     ; GFX10-LABEL: name: load_local_s64_align4_from_1_gep_1016
     ; GFX10: liveins: $vgpr0_vgpr1
     ; GFX10-NEXT: {{  $}}
@@ -954,6 +1013,7 @@ body: |
     ; GFX6-NEXT: $m0 = S_MOV_B32 -1
     ; GFX6-NEXT: [[LOAD:%[0-9]+]]:vreg_64(s64) = G_LOAD [[PTR_ADD]](p3) :: (load (s64), align 4, addrspace 3)
     ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](s64)
+    ;
     ; GFX7-LABEL: name: load_local_s64_align4_from_1_gep_1020
     ; GFX7: liveins: $vgpr0_vgpr1
     ; GFX7-NEXT: {{  $}}
@@ -963,6 +1023,7 @@ body: |
     ; GFX7-NEXT: $m0 = S_MOV_B32 -1
     ; GFX7-NEXT: [[DS_READ2_B32_:%[0-9]+]]:vreg_64 = DS_READ2_B32 [[V_ADD_CO_U32_e64_]], 0, 1, 0, implicit $m0, implicit $exec :: (load (s64), align 4, addrspace 3)
     ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ2_B32_]]
+    ;
     ; GFX9-LABEL: name: load_local_s64_align4_from_1_gep_1020
     ; GFX9: liveins: $vgpr0_vgpr1
     ; GFX9-NEXT: {{  $}}
@@ -971,6 +1032,7 @@ body: |
     ; GFX9-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
     ; GFX9-NEXT: [[DS_READ2_B32_gfx9_:%[0-9]+]]:vreg_64 = DS_READ2_B32_gfx9 [[V_ADD_U32_e64_]], 0, 1, 0, implicit $exec :: (load (s64), align 4, addrspace 3)
     ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[DS_READ2_B32_gfx9_]]
+    ;
     ; GFX10-LABEL: name: load_local_s64_align4_from_1_gep_1020
     ; GFX10: liveins: $vgpr0_vgpr1
     ; GFX10-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir
index a63df136e003c..ff1d3fe379673 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir
@@ -6604,22 +6604,25 @@ body: |
     ; CI: liveins: $vgpr0_vgpr1
     ; CI-NEXT: {{  $}}
     ; CI-NEXT: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
-    ; CI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p4) :: (load (<2 x p3>), addrspace 4)
-    ; CI-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; CI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p4) :: (load (<2 x s32>), addrspace 4)
+    ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; CI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; VI-LABEL: name: test_load_constant_v2p3_align8
     ; VI: liveins: $vgpr0_vgpr1
     ; VI-NEXT: {{  $}}
     ; VI-NEXT: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
-    ; VI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p4) :: (load (<2 x p3>), addrspace 4)
-    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; VI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p4) :: (load (<2 x s32>), addrspace 4)
+    ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX9-LABEL: name: test_load_constant_v2p3_align8
     ; GFX9: liveins: $vgpr0_vgpr1
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p4) :: (load (<2 x p3>), addrspace 4)
-    ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p4) :: (load (<2 x s32>), addrspace 4)
+    ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     %0:_(p4) = COPY $vgpr0_vgpr1
     %1:_(<2 x p3>) = G_LOAD %0 :: (load (<2 x p3>), align 8, addrspace 4)
     $vgpr0_vgpr1 = COPY %1
@@ -6635,22 +6638,25 @@ body: |
     ; CI: liveins: $vgpr0_vgpr1
     ; CI-NEXT: {{  $}}
     ; CI-NEXT: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
-    ; CI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p4) :: (load (<2 x p3>), align 4, addrspace 4)
-    ; CI-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; CI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p4) :: (load (<2 x s32>), align 4, addrspace 4)
+    ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; CI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; VI-LABEL: name: test_load_constant_v2p3_align4
     ; VI: liveins: $vgpr0_vgpr1
     ; VI-NEXT: {{  $}}
     ; VI-NEXT: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
-    ; VI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p4) :: (load (<2 x p3>), align 4, addrspace 4)
-    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; VI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p4) :: (load (<2 x s32>), align 4, addrspace 4)
+    ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX9-LABEL: name: test_load_constant_v2p3_align4
     ; GFX9: liveins: $vgpr0_vgpr1
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p4) :: (load (<2 x p3>), align 4, addrspace 4)
-    ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p4) :: (load (<2 x s32>), align 4, addrspace 4)
+    ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     %0:_(p4) = COPY $vgpr0_vgpr1
     %1:_(<2 x p3>) = G_LOAD %0 :: (load (<2 x p3>), align 4, addrspace 4)
     $vgpr0_vgpr1 = COPY %1
@@ -6683,7 +6689,6 @@ body: |
     ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
     ; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; CI-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
     ; CI-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; CI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C4]](s64)
     ; CI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p4) :: (load (s8) from unknown-address + 4, addrspace 4)
@@ -6699,9 +6704,9 @@ body: |
     ; CI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
     ; CI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
     ; CI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; CI-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR5]](s32)
-    ; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[INTTOPTR]](p3), [[INTTOPTR1]](p3)
-    ; CI-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
+    ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; CI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; VI-LABEL: name: test_load_constant_v2p3_align1
     ; VI: liveins: $vgpr0_vgpr1
@@ -6724,7 +6729,6 @@ body: |
     ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
     ; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; VI-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
     ; VI-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C4]](s64)
     ; VI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p4) :: (load (s8) from unknown-address + 4, addrspace 4)
@@ -6740,9 +6744,9 @@ body: |
     ; VI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
     ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
     ; VI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; VI-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR5]](s32)
-    ; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[INTTOPTR]](p3), [[INTTOPTR1]](p3)
-    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
+    ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX9-LABEL: name: test_load_constant_v2p3_align1
     ; GFX9: liveins: $vgpr0_vgpr1
@@ -6765,7 +6769,6 @@ body: |
     ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
     ; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
     ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C4]](s64)
     ; GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p4) :: (load (s8) from unknown-address + 4, addrspace 4)
@@ -6781,9 +6784,9 @@ body: |
     ; GFX9-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
     ; GFX9-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
     ; GFX9-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR5]](s32)
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[INTTOPTR]](p3), [[INTTOPTR1]](p3)
-    ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
+    ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     %0:_(p4) = COPY $vgpr0_vgpr1
     %1:_(<2 x p3>) = G_LOAD %0 :: (load (<2 x p3>), align 1, addrspace 4)
     $vgpr0_vgpr1 = COPY %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
index b1d7d36f9912e..3b166660a84b7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
@@ -15192,65 +15192,73 @@ body: |
     ; CI: liveins: $vgpr0_vgpr1
     ; CI-NEXT: {{  $}}
     ; CI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; CI-NEXT: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p0) :: (load (p3), align 8)
+    ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 8)
     ; CI-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(p3) = G_LOAD [[PTR_ADD]](p0) :: (load (p3) from unknown-address + 4)
-    ; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[LOAD]](p3), [[LOAD1]](p3)
-    ; CI-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
+    ; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32)
+    ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; CI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; VI-LABEL: name: test_load_flat_v2p3_align8
     ; VI: liveins: $vgpr0_vgpr1
     ; VI-NEXT: {{  $}}
     ; VI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; VI-NEXT: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p0) :: (load (p3), align 8)
+    ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 8)
     ; VI-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(p3) = G_LOAD [[PTR_ADD]](p0) :: (load (p3) from unknown-address + 4)
-    ; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[LOAD]](p3), [[LOAD1]](p3)
-    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
+    ; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32)
+    ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX9PLUS-LABEL: name: test_load_flat_v2p3_align8
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p0) :: (load (<2 x p3>))
-    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p0) :: (load (<2 x s32>))
+    ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_v2p3_align8
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p0) :: (load (<2 x p3>))
-    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p0) :: (load (<2 x s32>))
+    ; GFX11PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX12-LABEL: name: test_load_flat_v2p3_align8
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p0) :: (load (<2 x p3>))
-    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p0) :: (load (<2 x s32>))
+    ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_v2p3_align8
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
     ; UNALIGNED_GFX9PLUS-NEXT: {{  $}}
     ; UNALIGNED_GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; UNALIGNED_GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p0) :: (load (<2 x p3>))
-    ; UNALIGNED_GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; UNALIGNED_GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p0) :: (load (<2 x s32>))
+    ; UNALIGNED_GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; UNALIGNED_GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; UNALIGNED_GFX11PLUS-LABEL: name: test_load_flat_v2p3_align8
     ; UNALIGNED_GFX11PLUS: liveins: $vgpr0_vgpr1
     ; UNALIGNED_GFX11PLUS-NEXT: {{  $}}
     ; UNALIGNED_GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; UNALIGNED_GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p0) :: (load (<2 x p3>))
-    ; UNALIGNED_GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; UNALIGNED_GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p0) :: (load (<2 x s32>))
+    ; UNALIGNED_GFX11PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; UNALIGNED_GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_flat_v2p3_align8
     ; UNALIGNED_GFX12: liveins: $vgpr0_vgpr1
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p0) :: (load (<2 x p3>))
-    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p0) :: (load (<2 x s32>))
+    ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     %0:_(p0) = COPY $vgpr0_vgpr1
     %1:_(<2 x p3>) = G_LOAD %0 :: (load (<2 x p3>), align 8, addrspace 0)
     $vgpr0_vgpr1 = COPY %1
@@ -15266,65 +15274,73 @@ body: |
     ; CI: liveins: $vgpr0_vgpr1
     ; CI-NEXT: {{  $}}
     ; CI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; CI-NEXT: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p0) :: (load (p3))
+    ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32))
     ; CI-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(p3) = G_LOAD [[PTR_ADD]](p0) :: (load (p3) from unknown-address + 4)
-    ; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[LOAD]](p3), [[LOAD1]](p3)
-    ; CI-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
+    ; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32)
+    ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; CI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; VI-LABEL: name: test_load_flat_v2p3_align4
     ; VI: liveins: $vgpr0_vgpr1
     ; VI-NEXT: {{  $}}
     ; VI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; VI-NEXT: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p0) :: (load (p3))
+    ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32))
     ; VI-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
-    ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(p3) = G_LOAD [[PTR_ADD]](p0) :: (load (p3) from unknown-address + 4)
-    ; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[LOAD]](p3), [[LOAD1]](p3)
-    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
+    ; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32)
+    ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX9PLUS-LABEL: name: test_load_flat_v2p3_align4
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
     ; GFX9PLUS-NEXT: {{  $}}
     ; GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p0) :: (load (<2 x p3>), align 4)
-    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p0) :: (load (<2 x s32>), align 4)
+    ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_v2p3_align4
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
     ; GFX11PLUS-NEXT: {{  $}}
     ; GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p0) :: (load (<2 x p3>), align 4)
-    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p0) :: (load (<2 x s32>), align 4)
+    ; GFX11PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX12-LABEL: name: test_load_flat_v2p3_align4
     ; GFX12: liveins: $vgpr0_vgpr1
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p0) :: (load (<2 x p3>), align 4)
-    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p0) :: (load (<2 x s32>), align 4)
+    ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_v2p3_align4
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
     ; UNALIGNED_GFX9PLUS-NEXT: {{  $}}
     ; UNALIGNED_GFX9PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; UNALIGNED_GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p0) :: (load (<2 x p3>), align 4)
-    ; UNALIGNED_GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; UNALIGNED_GFX9PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p0) :: (load (<2 x s32>), align 4)
+    ; UNALIGNED_GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; UNALIGNED_GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; UNALIGNED_GFX11PLUS-LABEL: name: test_load_flat_v2p3_align4
     ; UNALIGNED_GFX11PLUS: liveins: $vgpr0_vgpr1
     ; UNALIGNED_GFX11PLUS-NEXT: {{  $}}
     ; UNALIGNED_GFX11PLUS-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; UNALIGNED_GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p0) :: (load (<2 x p3>), align 4)
-    ; UNALIGNED_GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; UNALIGNED_GFX11PLUS-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p0) :: (load (<2 x s32>), align 4)
+    ; UNALIGNED_GFX11PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; UNALIGNED_GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_flat_v2p3_align4
     ; UNALIGNED_GFX12: liveins: $vgpr0_vgpr1
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p0) :: (load (<2 x p3>), align 4)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p0) :: (load (<2 x s32>), align 4)
+    ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     %0:_(p0) = COPY $vgpr0_vgpr1
     %1:_(<2 x p3>) = G_LOAD %0 :: (load (<2 x p3>), align 4, addrspace 0)
     $vgpr0_vgpr1 = COPY %1
@@ -15357,7 +15373,6 @@ body: |
     ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
     ; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; CI-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
     ; CI-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; CI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
     ; CI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
@@ -15373,9 +15388,9 @@ body: |
     ; CI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
     ; CI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
     ; CI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; CI-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR5]](s32)
-    ; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[INTTOPTR]](p3), [[INTTOPTR1]](p3)
-    ; CI-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
+    ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; CI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; VI-LABEL: name: test_load_flat_v2p3_align1
     ; VI: liveins: $vgpr0_vgpr1
@@ -15398,7 +15413,6 @@ body: |
     ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
     ; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; VI-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
     ; VI-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
     ; VI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
@@ -15414,9 +15428,9 @@ body: |
     ; VI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
     ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
     ; VI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; VI-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR5]](s32)
-    ; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[INTTOPTR]](p3), [[INTTOPTR1]](p3)
-    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
+    ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX9PLUS-LABEL: name: test_load_flat_v2p3_align1
     ; GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -15439,7 +15453,6 @@ body: |
     ; GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
     ; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9PLUS-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
     ; GFX9PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; GFX9PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
     ; GFX9PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
@@ -15455,9 +15468,9 @@ body: |
     ; GFX9PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
     ; GFX9PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
     ; GFX9PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9PLUS-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR5]](s32)
-    ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[INTTOPTR]](p3), [[INTTOPTR1]](p3)
-    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
+    ; GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX11PLUS-LABEL: name: test_load_flat_v2p3_align1
     ; GFX11PLUS: liveins: $vgpr0_vgpr1
@@ -15480,7 +15493,6 @@ body: |
     ; GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
     ; GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX11PLUS-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
     ; GFX11PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; GFX11PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
     ; GFX11PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
@@ -15496,9 +15508,9 @@ body: |
     ; GFX11PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
     ; GFX11PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
     ; GFX11PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX11PLUS-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR5]](s32)
-    ; GFX11PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[INTTOPTR]](p3), [[INTTOPTR1]](p3)
-    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; GFX11PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
+    ; GFX11PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX12-LABEL: name: test_load_flat_v2p3_align1
     ; GFX12: liveins: $vgpr0_vgpr1
@@ -15521,7 +15533,6 @@ body: |
     ; GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
     ; GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX12-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
     ; GFX12-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
     ; GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
@@ -15537,9 +15548,9 @@ body: |
     ; GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
     ; GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
     ; GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX12-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR5]](s32)
-    ; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[INTTOPTR]](p3), [[INTTOPTR1]](p3)
-    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
+    ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; UNALIGNED_GFX9PLUS-LABEL: name: test_load_flat_v2p3_align1
     ; UNALIGNED_GFX9PLUS: liveins: $vgpr0_vgpr1
@@ -15562,7 +15573,6 @@ body: |
     ; UNALIGNED_GFX9PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; UNALIGNED_GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
     ; UNALIGNED_GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; UNALIGNED_GFX9PLUS-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
     ; UNALIGNED_GFX9PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; UNALIGNED_GFX9PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
     ; UNALIGNED_GFX9PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
@@ -15578,9 +15588,9 @@ body: |
     ; UNALIGNED_GFX9PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
     ; UNALIGNED_GFX9PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
     ; UNALIGNED_GFX9PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; UNALIGNED_GFX9PLUS-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR5]](s32)
-    ; UNALIGNED_GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[INTTOPTR]](p3), [[INTTOPTR1]](p3)
-    ; UNALIGNED_GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; UNALIGNED_GFX9PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
+    ; UNALIGNED_GFX9PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; UNALIGNED_GFX9PLUS-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; UNALIGNED_GFX11PLUS-LABEL: name: test_load_flat_v2p3_align1
     ; UNALIGNED_GFX11PLUS: liveins: $vgpr0_vgpr1
@@ -15603,7 +15613,6 @@ body: |
     ; UNALIGNED_GFX11PLUS-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; UNALIGNED_GFX11PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
     ; UNALIGNED_GFX11PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; UNALIGNED_GFX11PLUS-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
     ; UNALIGNED_GFX11PLUS-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; UNALIGNED_GFX11PLUS-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
     ; UNALIGNED_GFX11PLUS-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
@@ -15619,9 +15628,9 @@ body: |
     ; UNALIGNED_GFX11PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
     ; UNALIGNED_GFX11PLUS-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
     ; UNALIGNED_GFX11PLUS-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; UNALIGNED_GFX11PLUS-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR5]](s32)
-    ; UNALIGNED_GFX11PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[INTTOPTR]](p3), [[INTTOPTR1]](p3)
-    ; UNALIGNED_GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; UNALIGNED_GFX11PLUS-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
+    ; UNALIGNED_GFX11PLUS-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; UNALIGNED_GFX11PLUS-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_flat_v2p3_align1
     ; UNALIGNED_GFX12: liveins: $vgpr0_vgpr1
@@ -15644,7 +15653,6 @@ body: |
     ; UNALIGNED_GFX12-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; UNALIGNED_GFX12-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
     ; UNALIGNED_GFX12-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; UNALIGNED_GFX12-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
     ; UNALIGNED_GFX12-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; UNALIGNED_GFX12-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
     ; UNALIGNED_GFX12-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
@@ -15660,9 +15668,9 @@ body: |
     ; UNALIGNED_GFX12-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
     ; UNALIGNED_GFX12-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
     ; UNALIGNED_GFX12-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; UNALIGNED_GFX12-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR5]](s32)
-    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[INTTOPTR]](p3), [[INTTOPTR1]](p3)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; UNALIGNED_GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
+    ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     %0:_(p0) = COPY $vgpr0_vgpr1
     %1:_(<2 x p3>) = G_LOAD %0 :: (load (<2 x p3>), align 1, addrspace 0)
     $vgpr0_vgpr1 = COPY %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir
index d6acc6ecdfc66..f384114ee4cde 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir
@@ -13448,43 +13448,49 @@ body: |
     ; SI: liveins: $vgpr0_vgpr1
     ; SI-NEXT: {{  $}}
     ; SI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; SI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), addrspace 1)
-    ; SI-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; SI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load (<2 x s32>), addrspace 1)
+    ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; SI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; CI-HSA-LABEL: name: test_load_global_v2p3_align8
     ; CI-HSA: liveins: $vgpr0_vgpr1
     ; CI-HSA-NEXT: {{  $}}
     ; CI-HSA-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CI-HSA-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), addrspace 1)
-    ; CI-HSA-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; CI-HSA-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load (<2 x s32>), addrspace 1)
+    ; CI-HSA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; CI-HSA-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; CI-MESA-LABEL: name: test_load_global_v2p3_align8
     ; CI-MESA: liveins: $vgpr0_vgpr1
     ; CI-MESA-NEXT: {{  $}}
     ; CI-MESA-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CI-MESA-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), addrspace 1)
-    ; CI-MESA-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; CI-MESA-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load (<2 x s32>), addrspace 1)
+    ; CI-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; CI-MESA-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; VI-LABEL: name: test_load_global_v2p3_align8
     ; VI: liveins: $vgpr0_vgpr1
     ; VI-NEXT: {{  $}}
     ; VI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; VI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), addrspace 1)
-    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; VI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load (<2 x s32>), addrspace 1)
+    ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX9-HSA-LABEL: name: test_load_global_v2p3_align8
     ; GFX9-HSA: liveins: $vgpr0_vgpr1
     ; GFX9-HSA-NEXT: {{  $}}
     ; GFX9-HSA-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX9-HSA-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), addrspace 1)
-    ; GFX9-HSA-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX9-HSA-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load (<2 x s32>), addrspace 1)
+    ; GFX9-HSA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX9-HSA-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX9-MESA-LABEL: name: test_load_global_v2p3_align8
     ; GFX9-MESA: liveins: $vgpr0_vgpr1
     ; GFX9-MESA-NEXT: {{  $}}
     ; GFX9-MESA-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX9-MESA-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), addrspace 1)
-    ; GFX9-MESA-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX9-MESA-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load (<2 x s32>), addrspace 1)
+    ; GFX9-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX9-MESA-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<2 x p3>) = G_LOAD %0 :: (load (<2 x p3>), align 8, addrspace 1)
     $vgpr0_vgpr1 = COPY %1
@@ -13500,43 +13506,49 @@ body: |
     ; SI: liveins: $vgpr0_vgpr1
     ; SI-NEXT: {{  $}}
     ; SI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; SI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), align 4, addrspace 1)
-    ; SI-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; SI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load (<2 x s32>), align 4, addrspace 1)
+    ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; SI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; CI-HSA-LABEL: name: test_load_global_v2p3_align4
     ; CI-HSA: liveins: $vgpr0_vgpr1
     ; CI-HSA-NEXT: {{  $}}
     ; CI-HSA-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CI-HSA-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), align 4, addrspace 1)
-    ; CI-HSA-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; CI-HSA-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load (<2 x s32>), align 4, addrspace 1)
+    ; CI-HSA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; CI-HSA-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; CI-MESA-LABEL: name: test_load_global_v2p3_align4
     ; CI-MESA: liveins: $vgpr0_vgpr1
     ; CI-MESA-NEXT: {{  $}}
     ; CI-MESA-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CI-MESA-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), align 4, addrspace 1)
-    ; CI-MESA-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; CI-MESA-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load (<2 x s32>), align 4, addrspace 1)
+    ; CI-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; CI-MESA-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; VI-LABEL: name: test_load_global_v2p3_align4
     ; VI: liveins: $vgpr0_vgpr1
     ; VI-NEXT: {{  $}}
     ; VI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; VI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), align 4, addrspace 1)
-    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; VI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load (<2 x s32>), align 4, addrspace 1)
+    ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX9-HSA-LABEL: name: test_load_global_v2p3_align4
     ; GFX9-HSA: liveins: $vgpr0_vgpr1
     ; GFX9-HSA-NEXT: {{  $}}
     ; GFX9-HSA-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX9-HSA-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), align 4, addrspace 1)
-    ; GFX9-HSA-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX9-HSA-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load (<2 x s32>), align 4, addrspace 1)
+    ; GFX9-HSA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX9-HSA-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX9-MESA-LABEL: name: test_load_global_v2p3_align4
     ; GFX9-MESA: liveins: $vgpr0_vgpr1
     ; GFX9-MESA-NEXT: {{  $}}
     ; GFX9-MESA-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX9-MESA-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), align 4, addrspace 1)
-    ; GFX9-MESA-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX9-MESA-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load (<2 x s32>), align 4, addrspace 1)
+    ; GFX9-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX9-MESA-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<2 x p3>) = G_LOAD %0 :: (load (<2 x p3>), align 4, addrspace 1)
     $vgpr0_vgpr1 = COPY %1
@@ -13569,7 +13581,6 @@ body: |
     ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
     ; SI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; SI-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
     ; SI-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; SI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C4]](s64)
     ; SI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p1) :: (load (s8) from unknown-address + 4, addrspace 1)
@@ -13585,16 +13596,17 @@ body: |
     ; SI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
     ; SI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
     ; SI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; SI-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR5]](s32)
-    ; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[INTTOPTR]](p3), [[INTTOPTR1]](p3)
-    ; SI-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
+    ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; SI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; CI-HSA-LABEL: name: test_load_global_v2p3_align1
     ; CI-HSA: liveins: $vgpr0_vgpr1
     ; CI-HSA-NEXT: {{  $}}
     ; CI-HSA-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CI-HSA-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), align 1, addrspace 1)
-    ; CI-HSA-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; CI-HSA-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load (<2 x s32>), align 1, addrspace 1)
+    ; CI-HSA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; CI-HSA-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; CI-MESA-LABEL: name: test_load_global_v2p3_align1
     ; CI-MESA: liveins: $vgpr0_vgpr1
@@ -13617,7 +13629,6 @@ body: |
     ; CI-MESA-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
     ; CI-MESA-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; CI-MESA-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
     ; CI-MESA-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; CI-MESA-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C4]](s64)
     ; CI-MESA-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p1) :: (load (s8) from unknown-address + 4, addrspace 1)
@@ -13633,9 +13644,9 @@ body: |
     ; CI-MESA-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
     ; CI-MESA-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
     ; CI-MESA-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; CI-MESA-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR5]](s32)
-    ; CI-MESA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[INTTOPTR]](p3), [[INTTOPTR1]](p3)
-    ; CI-MESA-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; CI-MESA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
+    ; CI-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; CI-MESA-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; VI-LABEL: name: test_load_global_v2p3_align1
     ; VI: liveins: $vgpr0_vgpr1
@@ -13658,7 +13669,6 @@ body: |
     ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
     ; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; VI-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
     ; VI-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C4]](s64)
     ; VI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p1) :: (load (s8) from unknown-address + 4, addrspace 1)
@@ -13674,16 +13684,17 @@ body: |
     ; VI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
     ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
     ; VI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; VI-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR5]](s32)
-    ; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[INTTOPTR]](p3), [[INTTOPTR1]](p3)
-    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
+    ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX9-HSA-LABEL: name: test_load_global_v2p3_align1
     ; GFX9-HSA: liveins: $vgpr0_vgpr1
     ; GFX9-HSA-NEXT: {{  $}}
     ; GFX9-HSA-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; GFX9-HSA-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p1) :: (load (<2 x p3>), align 1, addrspace 1)
-    ; GFX9-HSA-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX9-HSA-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load (<2 x s32>), align 1, addrspace 1)
+    ; GFX9-HSA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX9-HSA-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX9-MESA-LABEL: name: test_load_global_v2p3_align1
     ; GFX9-MESA: liveins: $vgpr0_vgpr1
@@ -13706,7 +13717,6 @@ body: |
     ; GFX9-MESA-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX9-MESA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
     ; GFX9-MESA-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; GFX9-MESA-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR2]](s32)
     ; GFX9-MESA-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; GFX9-MESA-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C4]](s64)
     ; GFX9-MESA-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p1) :: (load (s8) from unknown-address + 4, addrspace 1)
@@ -13722,9 +13732,9 @@ body: |
     ; GFX9-MESA-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
     ; GFX9-MESA-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
     ; GFX9-MESA-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
-    ; GFX9-MESA-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR5]](s32)
-    ; GFX9-MESA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[INTTOPTR]](p3), [[INTTOPTR1]](p3)
-    ; GFX9-MESA-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; GFX9-MESA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
+    ; GFX9-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-MESA-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<2 x p3>) = G_LOAD %0 :: (load (<2 x p3>), align 1, addrspace 1)
     $vgpr0_vgpr1 = COPY %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
index 1249de647bb75..1608234d6b2bc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
@@ -14595,71 +14595,81 @@ body: |
     ; SI: liveins: $vgpr0
     ; SI-NEXT: {{  $}}
     ; SI-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; SI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p3) :: (load (<2 x p3>), addrspace 3)
-    ; SI-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; SI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load (<2 x s32>), addrspace 3)
+    ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; SI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; CI-LABEL: name: test_load_local_v2p3_align8
     ; CI: liveins: $vgpr0
     ; CI-NEXT: {{  $}}
     ; CI-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; CI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p3) :: (load (<2 x p3>), addrspace 3)
-    ; CI-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; CI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load (<2 x s32>), addrspace 3)
+    ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; CI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; CI-DS128-LABEL: name: test_load_local_v2p3_align8
     ; CI-DS128: liveins: $vgpr0
     ; CI-DS128-NEXT: {{  $}}
     ; CI-DS128-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; CI-DS128-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p3) :: (load (<2 x p3>), addrspace 3)
-    ; CI-DS128-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; CI-DS128-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load (<2 x s32>), addrspace 3)
+    ; CI-DS128-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; CI-DS128-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; VI-LABEL: name: test_load_local_v2p3_align8
     ; VI: liveins: $vgpr0
     ; VI-NEXT: {{  $}}
     ; VI-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; VI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p3) :: (load (<2 x p3>), addrspace 3)
-    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; VI-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load (<2 x s32>), addrspace 3)
+    ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX9-LABEL: name: test_load_local_v2p3_align8
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p3) :: (load (<2 x p3>), addrspace 3)
-    ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load (<2 x s32>), addrspace 3)
+    ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX9-UNALIGNED-LABEL: name: test_load_local_v2p3_align8
     ; GFX9-UNALIGNED: liveins: $vgpr0
     ; GFX9-UNALIGNED-NEXT: {{  $}}
     ; GFX9-UNALIGNED-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX9-UNALIGNED-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p3) :: (load (<2 x p3>), addrspace 3)
-    ; GFX9-UNALIGNED-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX9-UNALIGNED-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load (<2 x s32>), addrspace 3)
+    ; GFX9-UNALIGNED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX9-UNALIGNED-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX10-LABEL: name: test_load_local_v2p3_align8
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p3) :: (load (<2 x p3>), addrspace 3)
-    ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load (<2 x s32>), addrspace 3)
+    ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX10-UNALIGNED-LABEL: name: test_load_local_v2p3_align8
     ; GFX10-UNALIGNED: liveins: $vgpr0
     ; GFX10-UNALIGNED-NEXT: {{  $}}
     ; GFX10-UNALIGNED-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX10-UNALIGNED-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p3) :: (load (<2 x p3>), addrspace 3)
-    ; GFX10-UNALIGNED-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX10-UNALIGNED-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load (<2 x s32>), addrspace 3)
+    ; GFX10-UNALIGNED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX10-UNALIGNED-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX11-LABEL: name: test_load_local_v2p3_align8
     ; GFX11: liveins: $vgpr0
     ; GFX11-NEXT: {{  $}}
     ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p3) :: (load (<2 x p3>), addrspace 3)
-    ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load (<2 x s32>), addrspace 3)
+    ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX11-UNALIGNED-LABEL: name: test_load_local_v2p3_align8
     ; GFX11-UNALIGNED: liveins: $vgpr0
     ; GFX11-UNALIGNED-NEXT: {{  $}}
     ; GFX11-UNALIGNED-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX11-UNALIGNED-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p3) :: (load (<2 x p3>), addrspace 3)
-    ; GFX11-UNALIGNED-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX11-UNALIGNED-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load (<2 x s32>), addrspace 3)
+    ; GFX11-UNALIGNED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX11-UNALIGNED-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     %0:_(p3) = COPY $vgpr0
     %1:_(<2 x p3>) = G_LOAD %0 :: (load (<2 x p3>), align 8, addrspace 3)
     $vgpr0_vgpr1 = COPY %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
index 741f878c86f8b..472cbe559e56f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
@@ -16526,106 +16526,117 @@ body: |
     ; SI: liveins: $vgpr0
     ; SI-NEXT: {{  $}}
     ; SI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; SI-NEXT: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p5) :: (load (p3), align 8, addrspace 5)
+    ; SI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
     ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(p3) = G_LOAD [[PTR_ADD]](p5) :: (load (p3) from unknown-address + 4, addrspace 5)
-    ; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[LOAD]](p3), [[LOAD1]](p3)
-    ; SI-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
+    ; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32)
+    ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; SI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; CI-LABEL: name: test_load_private_v2p3_align8
     ; CI: liveins: $vgpr0
     ; CI-NEXT: {{  $}}
     ; CI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; CI-NEXT: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p5) :: (load (p3), align 8, addrspace 5)
+    ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
     ; CI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(p3) = G_LOAD [[PTR_ADD]](p5) :: (load (p3) from unknown-address + 4, addrspace 5)
-    ; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[LOAD]](p3), [[LOAD1]](p3)
-    ; CI-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
+    ; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32)
+    ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; CI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; VI-LABEL: name: test_load_private_v2p3_align8
     ; VI: liveins: $vgpr0
     ; VI-NEXT: {{  $}}
     ; VI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; VI-NEXT: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p5) :: (load (p3), align 8, addrspace 5)
+    ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
     ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(p3) = G_LOAD [[PTR_ADD]](p5) :: (load (p3) from unknown-address + 4, addrspace 5)
-    ; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[LOAD]](p3), [[LOAD1]](p3)
-    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
+    ; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32)
+    ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX9-LABEL: name: test_load_private_v2p3_align8
     ; GFX9: liveins: $vgpr0
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p5) :: (load (p3), align 8, addrspace 5)
+    ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
     ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(p3) = G_LOAD [[PTR_ADD]](p5) :: (load (p3) from unknown-address + 4, addrspace 5)
-    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[LOAD]](p3), [[LOAD1]](p3)
-    ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
+    ; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32)
+    ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX10-LABEL: name: test_load_private_v2p3_align8
     ; GFX10: liveins: $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p5) :: (load (p3), align 8, addrspace 5)
+    ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
     ; GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(p3) = G_LOAD [[PTR_ADD]](p5) :: (load (p3) from unknown-address + 4, addrspace 5)
-    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[LOAD]](p3), [[LOAD1]](p3)
-    ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
+    ; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32)
+    ; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX11-LABEL: name: test_load_private_v2p3_align8
     ; GFX11: liveins: $vgpr0
     ; GFX11-NEXT: {{  $}}
     ; GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p5) :: (load (<2 x p3>), addrspace 5)
-    ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p5) :: (load (<2 x s32>), addrspace 5)
+    ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; GFX12-LABEL: name: test_load_private_v2p3_align8
     ; GFX12: liveins: $vgpr0
     ; GFX12-NEXT: {{  $}}
     ; GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p5) :: (load (<2 x p3>), addrspace 5)
-    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p5) :: (load (<2 x s32>), addrspace 5)
+    ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; UNALIGNED_GFX9-LABEL: name: test_load_private_v2p3_align8
     ; UNALIGNED_GFX9: liveins: $vgpr0
     ; UNALIGNED_GFX9-NEXT: {{  $}}
     ; UNALIGNED_GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX9-NEXT: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p5) :: (load (p3), align 8, addrspace 5)
+    ; UNALIGNED_GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
     ; UNALIGNED_GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; UNALIGNED_GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(p3) = G_LOAD [[PTR_ADD]](p5) :: (load (p3) from unknown-address + 4, addrspace 5)
-    ; UNALIGNED_GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[LOAD]](p3), [[LOAD1]](p3)
-    ; UNALIGNED_GFX9-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32)
+    ; UNALIGNED_GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; UNALIGNED_GFX9-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; UNALIGNED_GFX10-LABEL: name: test_load_private_v2p3_align8
     ; UNALIGNED_GFX10: liveins: $vgpr0
     ; UNALIGNED_GFX10-NEXT: {{  $}}
     ; UNALIGNED_GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX10-NEXT: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p5) :: (load (p3), align 8, addrspace 5)
+    ; UNALIGNED_GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
     ; UNALIGNED_GFX10-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; UNALIGNED_GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(p3) = G_LOAD [[PTR_ADD]](p5) :: (load (p3) from unknown-address + 4, addrspace 5)
-    ; UNALIGNED_GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[LOAD]](p3), [[LOAD1]](p3)
-    ; UNALIGNED_GFX10-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
+    ; UNALIGNED_GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LOAD]](s32), [[LOAD1]](s32)
+    ; UNALIGNED_GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[BUILD_VECTOR]](<2 x s32>)
+    ; UNALIGNED_GFX10-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; UNALIGNED_GFX11-LABEL: name: test_load_private_v2p3_align8
     ; UNALIGNED_GFX11: liveins: $vgpr0
     ; UNALIGNED_GFX11-NEXT: {{  $}}
     ; UNALIGNED_GFX11-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p5) :: (load (<2 x p3>), addrspace 5)
-    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; UNALIGNED_GFX11-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p5) :: (load (<2 x s32>), addrspace 5)
+    ; UNALIGNED_GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; UNALIGNED_GFX11-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     ;
     ; UNALIGNED_GFX12-LABEL: name: test_load_private_v2p3_align8
     ; UNALIGNED_GFX12: liveins: $vgpr0
     ; UNALIGNED_GFX12-NEXT: {{  $}}
     ; UNALIGNED_GFX12-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
-    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x p3>) = G_LOAD [[COPY]](p5) :: (load (<2 x p3>), addrspace 5)
-    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; UNALIGNED_GFX12-NEXT: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p5) :: (load (<2 x s32>), addrspace 5)
+    ; UNALIGNED_GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x p3>) = G_BITCAST [[LOAD]](<2 x s32>)
+    ; UNALIGNED_GFX12-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x p3>)
     %0:_(p5) = COPY $vgpr0
     %1:_(<2 x p3>) = G_LOAD %0 :: (load (<2 x p3>), align 8, addrspace 5)
     $vgpr0_vgpr1 = COPY %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store-global.mir
index f2a88a21a286e..b9c72d39ed45b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store-global.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store-global.mir
@@ -2310,9 +2310,9 @@ body: |
     ; SI-NEXT: {{  $}}
     ; SI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr2_vgpr3
-    ; SI-NEXT: [[UV:%[0-9]+]]:_(p3), [[UV1:%[0-9]+]]:_(p3) = G_UNMERGE_VALUES [[COPY1]](<2 x p3>)
-    ; SI-NEXT: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[UV]](p3)
-    ; SI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[PTRTOINT]](s32)
+    ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY1]](<2 x p3>)
+    ; SI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST]](<2 x s32>)
+    ; SI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
     ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C]](s32)
     ; SI-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
@@ -2332,8 +2332,7 @@ body: |
     ; SI-NEXT: G_STORE [[LSHR2]](s32), [[PTR_ADD2]](p1) :: (store (s8) into unknown-address + 3, addrspace 1)
     ; SI-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; SI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C5]](s64)
-    ; SI-NEXT: [[PTRTOINT1:%[0-9]+]]:_(s32) = G_PTRTOINT [[UV1]](p3)
-    ; SI-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[PTRTOINT1]](s32)
+    ; SI-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
     ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY4]], [[C]](s32)
     ; SI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p1) = G_PTR_ADD [[PTR_ADD3]], [[C1]](s64)
     ; SI-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
@@ -2353,16 +2352,17 @@ body: |
     ; CI-NEXT: {{  $}}
     ; CI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr2_vgpr3
-    ; CI-NEXT: G_STORE [[COPY1]](<2 x p3>), [[COPY]](p1) :: (store (<2 x p3>), align 1, addrspace 1)
+    ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY1]](<2 x p3>)
+    ; CI-NEXT: G_STORE [[BITCAST]](<2 x s32>), [[COPY]](p1) :: (store (<2 x s32>), align 1, addrspace 1)
     ;
     ; VI-LABEL: name: test_store_global_v2p3_align1
     ; VI: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
     ; VI-NEXT: {{  $}}
     ; VI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr2_vgpr3
-    ; VI-NEXT: [[UV:%[0-9]+]]:_(p3), [[UV1:%[0-9]+]]:_(p3) = G_UNMERGE_VALUES [[COPY1]](<2 x p3>)
-    ; VI-NEXT: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[UV]](p3)
-    ; VI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[PTRTOINT]](s32)
+    ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY1]](<2 x p3>)
+    ; VI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST]](<2 x s32>)
+    ; VI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
     ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C]](s32)
     ; VI-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
@@ -2383,8 +2383,7 @@ body: |
     ; VI-NEXT: G_STORE [[ANYEXT1]](s32), [[PTR_ADD2]](p1) :: (store (s8) into unknown-address + 3, addrspace 1)
     ; VI-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; VI-NEXT: [[PTRTOINT1:%[0-9]+]]:_(s32) = G_PTRTOINT [[UV1]](p3)
-    ; VI-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[PTRTOINT1]](s32)
+    ; VI-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
     ; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C]](s32)
     ; VI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p1) = G_PTR_ADD [[PTR_ADD3]], [[C1]](s64)
     ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY3]](s32)
@@ -2405,7 +2404,8 @@ body: |
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr2_vgpr3
-    ; GFX9-NEXT: G_STORE [[COPY1]](<2 x p3>), [[COPY]](p1) :: (store (<2 x p3>), align 1, addrspace 1)
+    ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY1]](<2 x p3>)
+    ; GFX9-NEXT: G_STORE [[BITCAST]](<2 x s32>), [[COPY]](p1) :: (store (<2 x s32>), align 1, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<2 x p3>) = COPY $vgpr2_vgpr3
     G_STORE %1, %0 :: (store (<2 x p3>), align 1, addrspace 1)
@@ -2422,9 +2422,9 @@ body: |
     ; SI-NEXT: {{  $}}
     ; SI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr2_vgpr3
-    ; SI-NEXT: [[UV:%[0-9]+]]:_(p3), [[UV1:%[0-9]+]]:_(p3) = G_UNMERGE_VALUES [[COPY1]](<2 x p3>)
-    ; SI-NEXT: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[UV]](p3)
-    ; SI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[PTRTOINT]](s32)
+    ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY1]](<2 x p3>)
+    ; SI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST]](<2 x s32>)
+    ; SI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
     ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C]](s32)
     ; SI-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
@@ -2433,8 +2433,7 @@ body: |
     ; SI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p1) :: (store (s16) into unknown-address + 2, addrspace 1)
     ; SI-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; SI-NEXT: [[PTRTOINT1:%[0-9]+]]:_(s32) = G_PTRTOINT [[UV1]](p3)
-    ; SI-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[PTRTOINT1]](s32)
+    ; SI-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
     ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C]](s32)
     ; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[PTR_ADD1]], [[C1]](s64)
     ; SI-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p1) :: (store (s16) into unknown-address + 4, addrspace 1)
@@ -2445,16 +2444,17 @@ body: |
     ; CI-NEXT: {{  $}}
     ; CI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr2_vgpr3
-    ; CI-NEXT: G_STORE [[COPY1]](<2 x p3>), [[COPY]](p1) :: (store (<2 x p3>), align 2, addrspace 1)
+    ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY1]](<2 x p3>)
+    ; CI-NEXT: G_STORE [[BITCAST]](<2 x s32>), [[COPY]](p1) :: (store (<2 x s32>), align 2, addrspace 1)
     ;
     ; VI-LABEL: name: test_store_global_v2p3_align2
     ; VI: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
     ; VI-NEXT: {{  $}}
     ; VI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr2_vgpr3
-    ; VI-NEXT: [[UV:%[0-9]+]]:_(p3), [[UV1:%[0-9]+]]:_(p3) = G_UNMERGE_VALUES [[COPY1]](<2 x p3>)
-    ; VI-NEXT: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[UV]](p3)
-    ; VI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[PTRTOINT]](s32)
+    ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY1]](<2 x p3>)
+    ; VI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST]](<2 x s32>)
+    ; VI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
     ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C]](s32)
     ; VI-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
@@ -2463,8 +2463,7 @@ body: |
     ; VI-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p1) :: (store (s16) into unknown-address + 2, addrspace 1)
     ; VI-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; VI-NEXT: [[PTRTOINT1:%[0-9]+]]:_(s32) = G_PTRTOINT [[UV1]](p3)
-    ; VI-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[PTRTOINT1]](s32)
+    ; VI-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
     ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C]](s32)
     ; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[PTR_ADD1]], [[C1]](s64)
     ; VI-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p1) :: (store (s16) into unknown-address + 4, addrspace 1)
@@ -2475,7 +2474,8 @@ body: |
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr2_vgpr3
-    ; GFX9-NEXT: G_STORE [[COPY1]](<2 x p3>), [[COPY]](p1) :: (store (<2 x p3>), align 2, addrspace 1)
+    ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY1]](<2 x p3>)
+    ; GFX9-NEXT: G_STORE [[BITCAST]](<2 x s32>), [[COPY]](p1) :: (store (<2 x s32>), align 2, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<2 x p3>) = COPY $vgpr2_vgpr3
     G_STORE %1, %0 :: (store (<2 x p3>), align 2, addrspace 1)
@@ -2492,28 +2492,32 @@ body: |
     ; SI-NEXT: {{  $}}
     ; SI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr2_vgpr3
-    ; SI-NEXT: G_STORE [[COPY1]](<2 x p3>), [[COPY]](p1) :: (store (<2 x p3>), align 4, addrspace 1)
+    ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY1]](<2 x p3>)
+    ; SI-NEXT: G_STORE [[BITCAST]](<2 x s32>), [[COPY]](p1) :: (store (<2 x s32>), align 4, addrspace 1)
     ;
     ; CI-LABEL: name: test_store_global_v2p3_align4
     ; CI: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
     ; CI-NEXT: {{  $}}
     ; CI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr2_vgpr3
-    ; CI-NEXT: G_STORE [[COPY1]](<2 x p3>), [[COPY]](p1) :: (store (<2 x p3>), align 4, addrspace 1)
+    ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY1]](<2 x p3>)
+    ; CI-NEXT: G_STORE [[BITCAST]](<2 x s32>), [[COPY]](p1) :: (store (<2 x s32>), align 4, addrspace 1)
     ;
     ; VI-LABEL: name: test_store_global_v2p3_align4
     ; VI: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
     ; VI-NEXT: {{  $}}
     ; VI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr2_vgpr3
-    ; VI-NEXT: G_STORE [[COPY1]](<2 x p3>), [[COPY]](p1) :: (store (<2 x p3>), align 4, addrspace 1)
+    ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY1]](<2 x p3>)
+    ; VI-NEXT: G_STORE [[BITCAST]](<2 x s32>), [[COPY]](p1) :: (store (<2 x s32>), align 4, addrspace 1)
     ;
     ; GFX9-LABEL: name: test_store_global_v2p3_align4
     ; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr2_vgpr3
-    ; GFX9-NEXT: G_STORE [[COPY1]](<2 x p3>), [[COPY]](p1) :: (store (<2 x p3>), align 4, addrspace 1)
+    ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY1]](<2 x p3>)
+    ; GFX9-NEXT: G_STORE [[BITCAST]](<2 x s32>), [[COPY]](p1) :: (store (<2 x s32>), align 4, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<2 x p3>) = COPY $vgpr2_vgpr3
     G_STORE %1, %0 :: (store (<2 x p3>), align 4, addrspace 1)
@@ -2530,28 +2534,32 @@ body: |
     ; SI-NEXT: {{  $}}
     ; SI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr2_vgpr3
-    ; SI-NEXT: G_STORE [[COPY1]](<2 x p3>), [[COPY]](p1) :: (store (<2 x p3>), addrspace 1)
+    ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY1]](<2 x p3>)
+    ; SI-NEXT: G_STORE [[BITCAST]](<2 x s32>), [[COPY]](p1) :: (store (<2 x s32>), addrspace 1)
     ;
     ; CI-LABEL: name: test_store_global_v2p3_align8
     ; CI: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
     ; CI-NEXT: {{  $}}
     ; CI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr2_vgpr3
-    ; CI-NEXT: G_STORE [[COPY1]](<2 x p3>), [[COPY]](p1) :: (store (<2 x p3>), addrspace 1)
+    ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY1]](<2 x p3>)
+    ; CI-NEXT: G_STORE [[BITCAST]](<2 x s32>), [[COPY]](p1) :: (store (<2 x s32>), addrspace 1)
     ;
     ; VI-LABEL: name: test_store_global_v2p3_align8
     ; VI: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
     ; VI-NEXT: {{  $}}
     ; VI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr2_vgpr3
-    ; VI-NEXT: G_STORE [[COPY1]](<2 x p3>), [[COPY]](p1) :: (store (<2 x p3>), addrspace 1)
+    ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY1]](<2 x p3>)
+    ; VI-NEXT: G_STORE [[BITCAST]](<2 x s32>), [[COPY]](p1) :: (store (<2 x s32>), addrspace 1)
     ;
     ; GFX9-LABEL: name: test_store_global_v2p3_align8
     ; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr2_vgpr3
-    ; GFX9-NEXT: G_STORE [[COPY1]](<2 x p3>), [[COPY]](p1) :: (store (<2 x p3>), addrspace 1)
+    ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY1]](<2 x p3>)
+    ; GFX9-NEXT: G_STORE [[BITCAST]](<2 x s32>), [[COPY]](p1) :: (store (<2 x s32>), addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<2 x p3>) = COPY $vgpr2_vgpr3
     G_STORE %1, %0 :: (store (<2 x p3>), align 8, addrspace 1)
@@ -2568,28 +2576,32 @@ body: |
     ; SI-NEXT: {{  $}}
     ; SI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr2_vgpr3
-    ; SI-NEXT: G_STORE [[COPY1]](<2 x p3>), [[COPY]](p1) :: (store (<2 x p3>), align 16, addrspace 1)
+    ; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY1]](<2 x p3>)
+    ; SI-NEXT: G_STORE [[BITCAST]](<2 x s32>), [[COPY]](p1) :: (store (<2 x s32>), align 16, addrspace 1)
     ;
     ; CI-LABEL: name: test_store_global_v2p3_align16
     ; CI: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
     ; CI-NEXT: {{  $}}
     ; CI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr2_vgpr3
-    ; CI-NEXT: G_STORE [[COPY1]](<2 x p3>), [[COPY]](p1) :: (store (<2 x p3>), align 16, addrspace 1)
+    ; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY1]](<2 x p3>)
+    ; CI-NEXT: G_STORE [[BITCAST]](<2 x s32>), [[COPY]](p1) :: (store (<2 x s32>), align 16, addrspace 1)
     ;
     ; VI-LABEL: name: test_store_global_v2p3_align16
     ; VI: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
     ; VI-NEXT: {{  $}}
     ; VI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr2_vgpr3
-    ; VI-NEXT: G_STORE [[COPY1]](<2 x p3>), [[COPY]](p1) :: (store (<2 x p3>), align 16, addrspace 1)
+    ; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY1]](<2 x p3>)
+    ; VI-NEXT: G_STORE [[BITCAST]](<2 x s32>), [[COPY]](p1) :: (store (<2 x s32>), align 16, addrspace 1)
     ;
     ; GFX9-LABEL: name: test_store_global_v2p3_align16
     ; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr2_vgpr3
-    ; GFX9-NEXT: G_STORE [[COPY1]](<2 x p3>), [[COPY]](p1) :: (store (<2 x p3>), align 16, addrspace 1)
+    ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY1]](<2 x p3>)
+    ; GFX9-NEXT: G_STORE [[BITCAST]](<2 x s32>), [[COPY]](p1) :: (store (<2 x s32>), align 16, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<2 x p3>) = COPY $vgpr2_vgpr3
     G_STORE %1, %0 :: (store (<2 x p3>), align 16, addrspace 1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-or.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-or.mir
index 38a1cab549f20..e92c11144cd45 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-or.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-or.mir
@@ -327,6 +327,28 @@ body: |
     %2:_(s64) = G_OR %0, %1
 ...
 
+---
+name: or_s64_vv_flags
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    ; CHECK-LABEL: name: or_s64_vv_flags
+    ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; CHECK-NEXT: %3:vgpr(s32) = disjoint G_OR [[UV]], [[UV2]]
+    ; CHECK-NEXT: %4:vgpr(s32) = disjoint G_OR [[UV1]], [[UV3]]
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES %3(s32), %4(s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(s64) = disjoint G_OR %0, %1
+...
+
 ---
 name: or_s64_vv_user
 legalized: true
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-select.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-select.mir
index c45acadd2ad45..c728fdd193393 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-select.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-select.mir
@@ -19,6 +19,7 @@ body: |
     ; FAST-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP]](s32)
     ; FAST-NEXT: [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC]](s1)
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[ZEXT]](s32), [[COPY2]], [[COPY3]]
+    ;
     ; GREEDY-LABEL: name: select_s32_scc_ss
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3
     ; GREEDY-NEXT: {{  $}}
@@ -56,6 +57,7 @@ body: |
     ; FAST-NEXT: [[COPY4:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
     ; FAST-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[COPY5]], [[COPY3]]
+    ;
     ; GREEDY-LABEL: name: select_s32_scc_sv
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0
     ; GREEDY-NEXT: {{  $}}
@@ -95,6 +97,7 @@ body: |
     ; FAST-NEXT: [[COPY4:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
     ; FAST-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[COPY3]], [[COPY5]]
+    ;
     ; GREEDY-LABEL: name: select_s32_scc_vs
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0
     ; GREEDY-NEXT: {{  $}}
@@ -132,6 +135,7 @@ body: |
     ; FAST-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP]](s32)
     ; FAST-NEXT: [[COPY4:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[COPY2]], [[COPY3]]
+    ;
     ; GREEDY-LABEL: name: select_s32_scc_vv
     ; GREEDY: liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
     ; GREEDY-NEXT: {{  $}}
@@ -168,6 +172,7 @@ body: |
     ; FAST-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
     ; FAST-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[COPY4]], [[COPY5]]
+    ;
     ; GREEDY-LABEL: name: select_s32_vcc_ss
     ; GREEDY: liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
     ; GREEDY-NEXT: {{  $}}
@@ -203,6 +208,7 @@ body: |
     ; FAST-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY1]](s32), [[COPY2]]
     ; FAST-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[COPY4]], [[COPY3]]
+    ;
     ; GREEDY-LABEL: name: select_s32_vcc_sv
     ; GREEDY: liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2
     ; GREEDY-NEXT: {{  $}}
@@ -237,6 +243,7 @@ body: |
     ; FAST-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY1]](s32), [[COPY2]]
     ; FAST-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[COPY3]], [[COPY4]]
+    ;
     ; GREEDY-LABEL: name: select_s32_vcc_vs
     ; GREEDY: liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2
     ; GREEDY-NEXT: {{  $}}
@@ -270,6 +277,7 @@ body: |
     ; FAST-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
     ; FAST-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[COPY2]], [[COPY3]]
+    ;
     ; GREEDY-LABEL: name: select_s32_vcc_vv
     ; GREEDY: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
     ; GREEDY-NEXT: {{  $}}
@@ -304,6 +312,7 @@ body: |
     ; FAST-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP]](s32)
     ; FAST-NEXT: [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC]](s1)
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:sgpr(s64) = G_SELECT [[ZEXT]](s32), [[COPY2]], [[COPY3]]
+    ;
     ; GREEDY-LABEL: name: select_s64_sss
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $sgpr4_sgpr5
     ; GREEDY-NEXT: {{  $}}
@@ -344,6 +353,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_s64_ssv
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $vgpr0_vgpr1
     ; GREEDY-NEXT: {{  $}}
@@ -389,6 +399,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_s64_svs
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $vgpr0_vgpr1
     ; GREEDY-NEXT: {{  $}}
@@ -433,6 +444,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_s64_svv
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $vgpr0_vgpr1, $vgpr2_vgpr3
     ; GREEDY-NEXT: {{  $}}
@@ -475,6 +487,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_s64_vss
     ; GREEDY: liveins: $sgpr0_sgpr1, $sgpr2_sgpr3, $vgpr0, $vgpr1
     ; GREEDY-NEXT: {{  $}}
@@ -515,6 +528,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_s64_vsv
     ; GREEDY: liveins: $sgpr0_sgpr1, $vgpr0, $vgpr1, $vgpr2_vgpr3
     ; GREEDY-NEXT: {{  $}}
@@ -555,6 +569,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_s64_vvs
     ; GREEDY: liveins: $sgpr0_sgpr1, $vgpr0, $vgpr1, $vgpr2_vgpr3
     ; GREEDY-NEXT: {{  $}}
@@ -595,6 +610,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_s64_vvv
     ; GREEDY: liveins: $vgpr0, $vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
     ; GREEDY-NEXT: {{  $}}
@@ -616,6 +632,47 @@ body: |
     %5:_(s64) = G_SELECT %4, %2, %3
 ...
 
+---
+name: select_s64_vvv_flags
+legalized: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
+    ; FAST-LABEL: name: select_s64_vvv_flags
+    ; FAST: liveins: $vgpr0, $vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
+    ; FAST-NEXT: {{  $}}
+    ; FAST-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; FAST-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; FAST-NEXT: [[COPY2:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+    ; FAST-NEXT: [[COPY3:%[0-9]+]]:vgpr(s64) = COPY $vgpr4_vgpr5
+    ; FAST-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+    ; FAST-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY2]](s64)
+    ; FAST-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY3]](s64)
+    ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = nnan G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
+    ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = nnan G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
+    ; FAST-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    ;
+    ; GREEDY-LABEL: name: select_s64_vvv_flags
+    ; GREEDY: liveins: $vgpr0, $vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
+    ; GREEDY-NEXT: {{  $}}
+    ; GREEDY-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GREEDY-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; GREEDY-NEXT: [[COPY2:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+    ; GREEDY-NEXT: [[COPY3:%[0-9]+]]:vgpr(s64) = COPY $vgpr4_vgpr5
+    ; GREEDY-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+    ; GREEDY-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY2]](s64)
+    ; GREEDY-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY3]](s64)
+    ; GREEDY-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = nnan G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
+    ; GREEDY-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = nnan G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
+    ; GREEDY-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s64) = COPY $vgpr2_vgpr3
+    %3:_(s64) = COPY $vgpr4_vgpr5
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(s64) = nnan G_SELECT %4, %2, %3
+...
+
 ---
 name: select_v2s32_scc_ss
 legalized: true
@@ -633,6 +690,7 @@ body: |
     ; FAST-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP]](s32)
     ; FAST-NEXT: [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC]](s1)
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:sgpr(<2 x s32>) = G_SELECT [[ZEXT]](s32), [[COPY2]], [[COPY3]]
+    ;
     ; GREEDY-LABEL: name: select_v2s32_scc_ss
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $sgpr4_sgpr5
     ; GREEDY-NEXT: {{  $}}
@@ -673,6 +731,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_v2s32_scc_sv
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $vgpr0_vgpr1
     ; GREEDY-NEXT: {{  $}}
@@ -718,6 +777,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_v2s32_scc_vs
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $vgpr0_vgpr1
     ; GREEDY-NEXT: {{  $}}
@@ -762,6 +822,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_v2s32_scc_vv
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $vgpr0_vgpr1, $vgpr2_vgpr3
     ; GREEDY-NEXT: {{  $}}
@@ -804,6 +865,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_v2s32_vcc_ss
     ; GREEDY: liveins: $sgpr0_sgpr1, $sgpr2_sgpr3, $vgpr0, $vgpr1
     ; GREEDY-NEXT: {{  $}}
@@ -844,6 +906,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_v2s32_vcc_sv
     ; GREEDY: liveins: $sgpr0_sgpr1, $vgpr0, $vgpr1, $vgpr2_vgpr3
     ; GREEDY-NEXT: {{  $}}
@@ -884,6 +947,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_v2s32_vcc_vs
     ; GREEDY: liveins: $sgpr0_sgpr1, $vgpr0, $vgpr1, $vgpr2_vgpr3
     ; GREEDY-NEXT: {{  $}}
@@ -924,6 +988,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_v2s32_vcc_vv
     ; GREEDY: liveins: $vgpr0, $vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
     ; GREEDY-NEXT: {{  $}}
@@ -962,6 +1027,7 @@ body: |
     ; FAST-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP]](s32)
     ; FAST-NEXT: [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC]](s1)
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:sgpr(<4 x s16>) = G_SELECT [[ZEXT]](s32), [[COPY2]], [[COPY3]]
+    ;
     ; GREEDY-LABEL: name: select_v4s16_scc_ss
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $sgpr4_sgpr5
     ; GREEDY-NEXT: {{  $}}
@@ -1002,6 +1068,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(<2 x s16>) = G_SELECT [[COPY4]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(<2 x s16>) = G_SELECT [[COPY4]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<4 x s16>) = G_CONCAT_VECTORS [[SELECT]](<2 x s16>), [[SELECT1]](<2 x s16>)
+    ;
     ; GREEDY-LABEL: name: select_v4s16_scc_sv
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $vgpr0_vgpr1
     ; GREEDY-NEXT: {{  $}}
@@ -1047,6 +1114,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(<2 x s16>) = G_SELECT [[COPY4]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(<2 x s16>) = G_SELECT [[COPY4]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<4 x s16>) = G_CONCAT_VECTORS [[SELECT]](<2 x s16>), [[SELECT1]](<2 x s16>)
+    ;
     ; GREEDY-LABEL: name: select_v4s16_scc_vs
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $vgpr0_vgpr1
     ; GREEDY-NEXT: {{  $}}
@@ -1091,6 +1159,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(<2 x s16>) = G_SELECT [[COPY4]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(<2 x s16>) = G_SELECT [[COPY4]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<4 x s16>) = G_CONCAT_VECTORS [[SELECT]](<2 x s16>), [[SELECT1]](<2 x s16>)
+    ;
     ; GREEDY-LABEL: name: select_v4s16_scc_vv
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $vgpr0_vgpr1, $vgpr2_vgpr3
     ; GREEDY-NEXT: {{  $}}
@@ -1133,6 +1202,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(<2 x s16>) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(<2 x s16>) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<4 x s16>) = G_CONCAT_VECTORS [[SELECT]](<2 x s16>), [[SELECT1]](<2 x s16>)
+    ;
     ; GREEDY-LABEL: name: select_v4s16_vcc_ss
     ; GREEDY: liveins: $sgpr0_sgpr1, $sgpr2_sgpr3, $vgpr0, $vgpr1
     ; GREEDY-NEXT: {{  $}}
@@ -1173,6 +1243,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(<2 x s16>) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(<2 x s16>) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<4 x s16>) = G_CONCAT_VECTORS [[SELECT]](<2 x s16>), [[SELECT1]](<2 x s16>)
+    ;
     ; GREEDY-LABEL: name: select_v4s16_vcc_sv
     ; GREEDY: liveins: $sgpr0_sgpr1, $vgpr0, $vgpr1, $vgpr2_vgpr3
     ; GREEDY-NEXT: {{  $}}
@@ -1213,6 +1284,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(<2 x s16>) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(<2 x s16>) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<4 x s16>) = G_CONCAT_VECTORS [[SELECT]](<2 x s16>), [[SELECT1]](<2 x s16>)
+    ;
     ; GREEDY-LABEL: name: select_v4s16_vcc_vs
     ; GREEDY: liveins: $sgpr0_sgpr1, $vgpr0, $vgpr1, $vgpr2_vgpr3
     ; GREEDY-NEXT: {{  $}}
@@ -1253,6 +1325,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(<2 x s16>) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(<2 x s16>) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:vgpr(<4 x s16>) = G_CONCAT_VECTORS [[SELECT]](<2 x s16>), [[SELECT1]](<2 x s16>)
+    ;
     ; GREEDY-LABEL: name: select_v4s16_vcc_vv
     ; GREEDY: liveins: $vgpr0, $vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
     ; GREEDY-NEXT: {{  $}}
@@ -1291,6 +1364,7 @@ body: |
     ; FAST-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP]](s32)
     ; FAST-NEXT: [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC]](s1)
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:sgpr(p1) = G_SELECT [[ZEXT]](s32), [[COPY2]], [[COPY3]]
+    ;
     ; GREEDY-LABEL: name: select_p1_scc_ss
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $sgpr4_sgpr5
     ; GREEDY-NEXT: {{  $}}
@@ -1327,6 +1401,7 @@ body: |
     ; FAST-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP]](s32)
     ; FAST-NEXT: [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC]](s1)
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:sgpr(p999) = G_SELECT [[ZEXT]](s32), [[COPY2]], [[COPY3]]
+    ;
     ; GREEDY-LABEL: name: select_p999_scc_ss
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $sgpr4_sgpr5
     ; GREEDY-NEXT: {{  $}}
@@ -1367,6 +1442,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_p1_scc_sv
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $vgpr0_vgpr1
     ; GREEDY-NEXT: {{  $}}
@@ -1412,6 +1488,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_p1_scc_vs
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $vgpr0_vgpr1
     ; GREEDY-NEXT: {{  $}}
@@ -1456,6 +1533,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_p1_scc_vv
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $vgpr0_vgpr1, $vgpr2_vgpr3
     ; GREEDY-NEXT: {{  $}}
@@ -1498,6 +1576,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_p1_vcc_ss
     ; GREEDY: liveins: $sgpr0_sgpr1, $sgpr2_sgpr3, $vgpr0, $vgpr1
     ; GREEDY-NEXT: {{  $}}
@@ -1538,6 +1617,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_p1_vcc_sv
     ; GREEDY: liveins: $sgpr0_sgpr1, $vgpr0, $vgpr1, $vgpr2_vgpr3
     ; GREEDY-NEXT: {{  $}}
@@ -1578,6 +1658,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_p1_vcc_vs
     ; GREEDY: liveins: $sgpr0_sgpr1, $vgpr0, $vgpr1, $vgpr2_vgpr3
     ; GREEDY-NEXT: {{  $}}
@@ -1618,6 +1699,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[MV:%[0-9]+]]:vgpr(p1) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_p1_vcc_vv
     ; GREEDY: liveins: $vgpr0, $vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
     ; GREEDY-NEXT: {{  $}}
@@ -1658,6 +1740,7 @@ body: |
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
     ; FAST-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
     ; FAST-NEXT: [[MV:%[0-9]+]]:vgpr(p999) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    ;
     ; GREEDY-LABEL: name: select_p999_vcc_vv
     ; GREEDY: liveins: $vgpr0, $vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
     ; GREEDY-NEXT: {{  $}}
@@ -1694,6 +1777,7 @@ body: |
     ; FAST-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY]](s32)
     ; FAST-NEXT: [[COPY3:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY3]](s1), [[COPY1]], [[COPY2]]
+    ;
     ; GREEDY-LABEL: name: select_s32_vgpr_vv
     ; GREEDY: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
     ; GREEDY-NEXT: {{  $}}
@@ -1727,6 +1811,7 @@ body: |
     ; FAST-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
     ; FAST-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY3]](s1), [[COPY4]], [[COPY5]]
+    ;
     ; GREEDY-LABEL: name: select_s32_vgpr_ss
     ; GREEDY: liveins: $vgpr0, $sgpr0, $sgpr1
     ; GREEDY-NEXT: {{  $}}
@@ -1760,6 +1845,7 @@ body: |
     ; FAST-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY]](s32)
     ; FAST-NEXT: [[COPY3:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY3]](s1), [[COPY1]], [[COPY2]]
+    ;
     ; GREEDY-LABEL: name: select_s32_sgpr_vv
     ; GREEDY: liveins: $sgpr0, $vgpr0, $vgpr1
     ; GREEDY-NEXT: {{  $}}
@@ -1792,6 +1878,7 @@ body: |
     ; FAST-NEXT: [[COPY3:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
     ; FAST-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY3]](s1), [[COPY1]], [[COPY4]]
+    ;
     ; GREEDY-LABEL: name: select_s32_sgpr_vs
     ; GREEDY: liveins: $sgpr0, $vgpr0, $sgpr1
     ; GREEDY-NEXT: {{  $}}
@@ -1825,6 +1912,7 @@ body: |
     ; FAST-NEXT: [[COPY3:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
     ; FAST-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY3]](s1), [[COPY4]], [[COPY2]]
+    ;
     ; GREEDY-LABEL: name: select_s32_sgpr_sv
     ; GREEDY: liveins: $sgpr0, $sgpr0, $vgpr0
     ; GREEDY-NEXT: {{  $}}
@@ -1858,6 +1946,7 @@ body: |
     ; FAST-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY]](s32)
     ; FAST-NEXT: [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC]](s1)
     ; FAST-NEXT: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[ZEXT]](s32), [[COPY1]], [[COPY2]]
+    ;
     ; GREEDY-LABEL: name: select_s32_sgpr_ss
     ; GREEDY: liveins: $sgpr0, $sgpr1, $sgpr2
     ; GREEDY-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-cc.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-cc.ll
index e34ae52fc673a..06f66e05d6747 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-cc.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-cc.ll
@@ -863,7 +863,7 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_dont_realign_stack(i32 %idx) {
 ; GISEL-GFX11-NEXT:    s_mov_b32 s0, 1
 ; GISEL-GFX11-NEXT:    v_lshlrev_b32_e32 v0, 4, v8
 ; GISEL-GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX11-NEXT:    v_add_nc_u32_e32 v4, 0, v0
+; GISEL-GFX11-NEXT:    v_mov_b32_e32 v4, v0
 ; GISEL-GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
 ; GISEL-GFX11-NEXT:    v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
 ; GISEL-GFX11-NEXT:    scratch_store_b128 v4, v[0:3], off dlc
@@ -878,7 +878,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_dont_realign_stack(i32 %idx) {
 ; GISEL-GFX10-NEXT:    v_mov_b32_e32 v2, 2
 ; GISEL-GFX10-NEXT:    v_mov_b32_e32 v3, 3
 ; GISEL-GFX10-NEXT:    v_mov_b32_e32 v4, 4
-; GISEL-GFX10-NEXT:    v_add_nc_u32_e32 v0, 0, v0
 ; GISEL-GFX10-NEXT:    buffer_store_dword v1, v0, s[48:51], 0 offen
 ; GISEL-GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GISEL-GFX10-NEXT:    buffer_store_dword v2, v0, s[48:51], 0 offen offset:4
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-preserve-cc.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-preserve-cc.ll
index 320268564f4db..4ba9f0729ea1f 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-preserve-cc.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-preserve-cc.ll
@@ -589,7 +589,7 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_dont_realign_stac
 ; GISEL-GFX11-NEXT:    s_mov_b32 s0, 1
 ; GISEL-GFX11-NEXT:    v_lshlrev_b32_e32 v0, 4, v8
 ; GISEL-GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX11-NEXT:    v_add_nc_u32_e32 v4, 0, v0
+; GISEL-GFX11-NEXT:    v_mov_b32_e32 v4, v0
 ; GISEL-GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
 ; GISEL-GFX11-NEXT:    v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
 ; GISEL-GFX11-NEXT:    scratch_store_b128 v4, v[0:3], off dlc
@@ -604,7 +604,6 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_dont_realign_stac
 ; GISEL-GFX10-NEXT:    v_mov_b32_e32 v2, 2
 ; GISEL-GFX10-NEXT:    v_mov_b32_e32 v3, 3
 ; GISEL-GFX10-NEXT:    v_mov_b32_e32 v4, 4
-; GISEL-GFX10-NEXT:    v_add_nc_u32_e32 v0, 0, v0
 ; GISEL-GFX10-NEXT:    buffer_store_dword v1, v0, s[48:51], 0 offen
 ; GISEL-GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GISEL-GFX10-NEXT:    buffer_store_dword v2, v0, s[48:51], 0 offen offset:4
diff --git a/llvm/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll b/llvm/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll
index d33196ba33109..ac8782e1e542f 100644
--- a/llvm/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll
@@ -12,7 +12,12 @@ declare void @llvm.amdgcn.s.barrier() #2
 
 ; SI-LABEL: {{^}}test_private_array_ptr_calc:
 
-; SI-ALLOCA: v_add_i32_e32 [[PTRREG:v[0-9]+]], vcc, 0, v{{[0-9]+}}
+; SI-ALLOCA: buffer_load_dword [[LOAD_A:v[0-9]+]]
+; SI-ALLOCA: buffer_load_dword [[LOAD_B:v[0-9]+]]
+
+; SI-ALLOCA: v_lshlrev_b32_e32 [[SIZE_SCALE:v[0-9]+]], 2, [[LOAD_A]]
+
+; SI-ALLOCA: v_mov_b32_e32 [[PTRREG:v[0-9]+]], [[SIZE_SCALE]]
 ; SI-ALLOCA: buffer_store_dword {{v[0-9]+}}, [[PTRREG]], s[{{[0-9]+:[0-9]+}}], 0 offen offset:64
 ; SI-ALLOCA: s_barrier
 ; SI-ALLOCA: buffer_load_dword {{v[0-9]+}}, [[PTRREG]], s[{{[0-9]+:[0-9]+}}], 0 offen offset:64
diff --git a/llvm/test/CodeGen/AMDGPU/branch-condition-and.ll b/llvm/test/CodeGen/AMDGPU/branch-condition-and.ll
index cc05129b1b2af..6efc9f2ae77b8 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-condition-and.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-condition-and.ll
@@ -1,5 +1,6 @@
-; RUN: llc -mtriple=amdgcn -verify-machineinstrs -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -check-prefix=GCN %s
-; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -check-prefix=GCN %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=tahiti -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn -mcpu=tonga -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -check-prefix=GCN %s
 
 ; This used to crash because during intermediate control flow lowering, there
 ; was a sequence
@@ -9,20 +10,21 @@
 ;       s_mov_b64_term exec, s[2:3]
 ; that was not treated correctly.
 ;
-; GCN-LABEL: {{^}}ham:
-; GCN-DAG: v_cmp_lt_f32_e64 [[OTHERCC:s\[[0-9]+:[0-9]+\]]],
-; GCN-DAG: v_cmp_lt_f32_e32 vcc,
-; GCN: s_and_b64 [[AND:s\[[0-9]+:[0-9]+\]]], vcc, [[OTHERCC]]
-; GCN: s_and_saveexec_b64 [[SAVED:s\[[0-9]+:[0-9]+\]]], [[AND]]
-; GCN-NEXT: s_cbranch_execz .LBB0_{{[0-9]+}}
-
-; GCN-NEXT: ; %bb.{{[0-9]+}}: ; %bb4
-; GCN: ds_write_b32
-
-; GCN: .LBB0_{{[0-9]+}}: ; %UnifiedReturnBlock
-; GCN-NEXT: s_endpgm
-; GCN-NEXT: .Lfunc_end
 define amdgpu_ps void @ham(float %arg, float %arg1) #0 {
+; GCN-LABEL: ham:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cmp_lt_f32_e64 s[0:1], 0, v1
+; GCN-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
+; GCN-NEXT:    s_and_saveexec_b64 s[2:3], s[0:1]
+; GCN-NEXT:    s_cbranch_execz .LBB0_2
+; GCN-NEXT:  ; %bb.1: ; %bb4
+; GCN-NEXT:    v_mov_b32_e32 v0, 4
+; GCN-NEXT:    s_mov_b32 m0, -1
+; GCN-NEXT:    ds_write_b32 v0, v0
+; GCN-NEXT:    ; divergent unreachable
+; GCN-NEXT:  .LBB0_2: ; %UnifiedReturnBlock
+; GCN-NEXT:    s_endpgm
 bb:
   %tmp = fcmp ogt float %arg, 0.000000e+00
   %tmp2 = fcmp ogt float %arg1, 0.000000e+00
diff --git a/llvm/test/CodeGen/AMDGPU/captured-frame-index.ll b/llvm/test/CodeGen/AMDGPU/captured-frame-index.ll
index 927e45f029488..ca0c669056ee3 100644
--- a/llvm/test/CodeGen/AMDGPU/captured-frame-index.ll
+++ b/llvm/test/CodeGen/AMDGPU/captured-frame-index.ll
@@ -1,4 +1,6 @@
-; RUN: llc -mtriple=amdgcn-- -mcpu=tahiti -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-- -mcpu=tahiti -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+
+; TODO: Test with flat scratch
 
 ; GCN-LABEL: {{^}}store_fi_lifetime:
 ; GCN: v_mov_b32_e32 [[FI:v[0-9]+]], 0{{$}}
@@ -143,7 +145,7 @@ define amdgpu_kernel void @stored_fi_to_global_2_small_objects(ptr addrspace(1)
   ret void
 }
 
-; GCN-LABEL: {{^}}stored_fi_to_global_huge_frame_offset:
+; GCN-LABEL: {{^}}kernel_stored_fi_to_global_huge_frame_offset:
 ; GCN: v_mov_b32_e32 [[BASE_0:v[0-9]+]], 0{{$}}
 ; GCN: buffer_store_dword [[BASE_0]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:4{{$}}
 
@@ -158,7 +160,32 @@ define amdgpu_kernel void @stored_fi_to_global_2_small_objects(ptr addrspace(1)
 ; GCN: buffer_store_dword [[K]], [[BASE_1_OFF_1]], s{{\[[0-9]+:[0-9]+\]}}, 0 offen{{$}}
 
 ; GCN: buffer_store_dword [[BASE_1_OFF_2]], off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
-define amdgpu_kernel void @stored_fi_to_global_huge_frame_offset(ptr addrspace(1) %ptr) #0 {
+define amdgpu_kernel void @kernel_stored_fi_to_global_huge_frame_offset(ptr addrspace(1) %ptr) #0 {
+  %tmp0 = alloca [4096 x i32], addrspace(5)
+  %tmp1 = alloca [4096 x i32], addrspace(5)
+  store volatile i32 0, ptr addrspace(5) %tmp0
+  %gep1.tmp0 = getelementptr [4096 x i32], ptr addrspace(5) %tmp0, i32 0, i32 4095
+  store volatile i32 999, ptr addrspace(5) %gep1.tmp0
+  %gep0.tmp1 = getelementptr [4096 x i32], ptr addrspace(5) %tmp0, i32 0, i32 14
+  store ptr addrspace(5) %gep0.tmp1, ptr addrspace(1) %ptr
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_stored_fi_to_global_huge_frame_offset:
+; GCN: v_mov_b32_e32 [[BASE_0:v[0-9]+]], 0{{$}}
+; GCN: buffer_store_dword [[BASE_0]], off, s{{\[[0-9]+:[0-9]+\]}}, s32 offset:4{{$}}
+
+; GCN: v_lshr_b32_e64 [[FI_TMP:v[0-9]+]], s32, 6
+; GCN: v_add_i32_e32 [[BASE_0_1:v[0-9]+]], vcc, 4, [[FI_TMP]]{{$}}
+
+; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7{{$}}
+; GCN-DAG: v_add_i32_e32 [[BASE_1_OFF_1:v[0-9]+]], vcc, 0x3ffc, [[BASE_0_1]]
+
+; GCN: v_add_i32_e32 [[BASE_1_OFF_2:v[0-9]+]], vcc, 56, [[BASE_0_1]]
+; GCN: buffer_store_dword [[K]], [[BASE_1_OFF_1]], s{{\[[0-9]+:[0-9]+\]}}, 0 offen{{$}}
+
+; GCN: buffer_store_dword [[BASE_1_OFF_2]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64
+define void @func_stored_fi_to_global_huge_frame_offset(ptr addrspace(1) %ptr) #0 {
   %tmp0 = alloca [4096 x i32], addrspace(5)
   %tmp1 = alloca [4096 x i32], addrspace(5)
   store volatile i32 0, ptr addrspace(5) %tmp0
@@ -190,6 +217,124 @@ entry:
   ret void
 }
 
+; FIXME: This is broken, and the sgpr input just gets replaced with a VGPR
+; GCN-LABEL: {{^}}func_alloca_offset0__use_asm_sgpr:
+; GCN: v_lshr_b32_e64 [[FI:v[0-9]+]], s32, 6
+; GCN: ; use [[FI]]
+define void @func_alloca_offset0__use_asm_sgpr() {
+  %alloca = alloca i32, addrspace(5)
+  call void asm sideeffect "; use $0", "s"(ptr addrspace(5) %alloca)
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_alloca_offset0__use_asm_vgpr:
+; GCN: v_lshr_b32_e64 [[FI:v[0-9]+]], s32, 6
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: ; use [[FI]]
+define void @func_alloca_offset0__use_asm_vgpr() {
+  %alloca = alloca i32, addrspace(5)
+  call void asm sideeffect "; use $0", "v"(ptr addrspace(5) %alloca)
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_alloca_offset0__use_asm_phys_sgpr:
+; GCN: s_lshr_b32 s8, s32, 6
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: ; use s8
+define void @func_alloca_offset0__use_asm_phys_sgpr() {
+  %alloca = alloca i32, addrspace(5)
+  call void asm sideeffect "; use $0", "{s8}"(ptr addrspace(5) %alloca)
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_alloca_offset0__use_asm_phys_vgpr:
+; GCN: v_lshr_b32_e64 v8, s32, 6
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: ; use v8
+define void @func_alloca_offset0__use_asm_phys_vgpr() {
+  %alloca = alloca i32, addrspace(5)
+  call void asm sideeffect "; use $0", "{v8}"(ptr addrspace(5) %alloca)
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_alloca_offset_use_asm_sgpr:
+; GCN: v_lshr_b32_e64 [[FI0_TMP0:v[0-9]+]], s32, 6
+; GCN-NEXT: v_add_i32_e32 [[FI0:v[0-9]+]], vcc, 16, [[FI0_TMP0]]
+
+; GCN: v_lshr_b32_e64 [[TMP:v[0-9]+]], s32, 6
+; GCN-NEXT: s_movk_i32 vcc_lo, 0x4010
+; GCN-NEXT: v_add_i32_e32 [[TMP]], vcc, vcc_lo, [[TMP]]
+; GCN-NEXT: ;;#ASMSTART
+; GCN: ; use [[TMP]]
+define void @func_alloca_offset_use_asm_sgpr() {
+  %alloca0 = alloca [4096 x i32], align 16, addrspace(5)
+  %alloca1 = alloca i32, addrspace(5)
+  call void asm sideeffect "; use $0", "s"(ptr addrspace(5) %alloca0)
+  call void asm sideeffect "; use $0", "s"(ptr addrspace(5) %alloca1)
+  ret void
+}
+
+; FIXME: Shouldn't need to materialize constant
+; GCN-LABEL: {{^}}func_alloca_offset_use_asm_vgpr:
+; GCN: v_lshr_b32_e64 [[FI0_TMP:v[0-9]+]], s32, 6
+; GCN-NEXT: v_add_i32_e32 [[FI0:v[0-9]+]], vcc, 16, [[FI0_TMP]]
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: ; use [[FI0]]
+; GCN-NEXT: ;;#ASMEND
+
+; GCN: v_lshr_b32_e64 [[FI1_TMP:v[0-9]+]], s32, 6
+; GCN-NEXT: s_movk_i32 vcc_lo, 0x4010
+; GCN-NEXT: v_add_i32_e32 [[FI1:v[0-9]+]], vcc, vcc_lo, [[FI1_TMP]]
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: ; use [[FI1]]
+; GCN-NEXT: ;;#ASMEND
+define void @func_alloca_offset_use_asm_vgpr() {
+  %alloca0 = alloca [4096 x i32], align 16, addrspace(5)
+  %alloca1 = alloca i32, addrspace(5)
+  call void asm sideeffect "; use $0", "s"(ptr addrspace(5) %alloca0)
+  call void asm sideeffect "; use $0", "v"(ptr addrspace(5) %alloca1)
+  ret void
+}
+
+; FIXME: Using VGPR for SGPR input
+; GCN-LABEL: {{^}}kernel_alloca_offset_use_asm_sgpr:
+; GCN: v_mov_b32_e32 v0, 16
+; GCN-NOT: v0
+; GCN: ;;#ASMSTART
+; GCN-NEXT: ; use v0
+; GCN-NEXT: ;;#ASMEND
+
+; GCN: v_mov_b32_e32 v0, 0x4010
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: ; use v0
+; GCN-NEXT: ;;#ASMEND
+define amdgpu_kernel void @kernel_alloca_offset_use_asm_sgpr() {
+  %alloca0 = alloca [4096 x i32], align 16, addrspace(5)
+  %alloca1 = alloca i32, addrspace(5)
+  call void asm sideeffect "; use $0", "s"(ptr addrspace(5) %alloca0)
+  call void asm sideeffect "; use $0", "s"(ptr addrspace(5) %alloca1)
+  ret void
+}
+
+; GCN-LABEL: {{^}}kernel_alloca_offset_use_asm_vgpr:
+; GCN: v_mov_b32_e32 v0, 16
+; GCN-NOT: v0
+; GCN: ;;#ASMSTART
+; GCN-NEXT: ; use v0
+; GCN-NEXT: ;;#ASMEND
+
+; GCN: v_mov_b32_e32 v0, 0x4010
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: ; use v0
+; GCN-NEXT: ;;#ASMEND
+define amdgpu_kernel void @kernel_alloca_offset_use_asm_vgpr() {
+  %alloca0 = alloca [4096 x i32], align 16, addrspace(5)
+  %alloca1 = alloca i32, addrspace(5)
+  call void asm sideeffect "; use $0", "v"(ptr addrspace(5) %alloca0)
+  call void asm sideeffect "; use $0", "v"(ptr addrspace(5) %alloca1)
+  ret void
+}
+
 declare void @llvm.lifetime.start.p5(i64, ptr addrspace(5) nocapture) #1
 declare void @llvm.lifetime.end.p5(i64, ptr addrspace(5) nocapture) #1
 
diff --git a/llvm/test/CodeGen/AMDGPU/dag-preserve-disjoint-flag.ll b/llvm/test/CodeGen/AMDGPU/dag-preserve-disjoint-flag.ll
new file mode 100644
index 0000000000000..89f9583234291
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/dag-preserve-disjoint-flag.ll
@@ -0,0 +1,120 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -stop-after=finalize-isel -simplify-mir -o - %s | FileCheck %s
+
+; Make sure disjoint flag is preserved on or instructions through selection
+
+define amdgpu_ps i32 @s_or_i32_disjoint(i32 inreg %a, i32 inreg %b) {
+  ; CHECK-LABEL: name: s_or_i32_disjoint
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $sgpr0, $sgpr1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; CHECK-NEXT:   %3:sreg_32 = disjoint S_OR_B32 [[COPY1]], [[COPY]], implicit-def dead $scc
+  ; CHECK-NEXT:   $sgpr0 = COPY %3
+  ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $sgpr0
+  %result = or disjoint i32 %a, %b
+  ret i32 %result
+}
+
+define amdgpu_ps <2 x i32> @s_or_v2i32_disjoint(<2 x i32> inreg %a, <2 x i32> inreg %b) {
+  ; CHECK-LABEL: name: s_or_v2i32_disjoint
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; CHECK-NEXT:   %5:sreg_32 = disjoint S_OR_B32 [[COPY3]], [[COPY1]], implicit-def dead $scc
+  ; CHECK-NEXT:   %6:sreg_32 = disjoint S_OR_B32 [[COPY2]], [[COPY]], implicit-def dead $scc
+  ; CHECK-NEXT:   $sgpr0 = COPY %5
+  ; CHECK-NEXT:   $sgpr1 = COPY %6
+  ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
+  %result = or disjoint <2 x i32> %a, %b
+  ret <2 x i32> %result
+}
+
+define i32 @v_or_i32_disjoint(i32 %a, i32 %b) {
+  ; CHECK-LABEL: name: v_or_i32_disjoint
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $vgpr0, $vgpr1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK-NEXT:   %9:vgpr_32 = disjoint V_OR_B32_e64 [[COPY1]], [[COPY]], implicit $exec
+  ; CHECK-NEXT:   $vgpr0 = COPY %9
+  ; CHECK-NEXT:   SI_RETURN implicit $vgpr0
+  %result = or disjoint i32 %a, %b
+  ret i32 %result
+}
+
+define <2 x i32> @v_or_v2i32_disjoint(<2 x i32> %a, <2 x i32> %b) {
+  ; CHECK-LABEL: name: v_or_v2i32_disjoint
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK-NEXT:   %11:vgpr_32 = disjoint V_OR_B32_e64 [[COPY3]], [[COPY1]], implicit $exec
+  ; CHECK-NEXT:   %12:vgpr_32 = disjoint V_OR_B32_e64 [[COPY2]], [[COPY]], implicit $exec
+  ; CHECK-NEXT:   $vgpr0 = COPY %11
+  ; CHECK-NEXT:   $vgpr1 = COPY %12
+  ; CHECK-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1
+  %result = or disjoint <2 x i32> %a, %b
+  ret <2 x i32> %result
+}
+
+define amdgpu_ps i64 @s_or_i64_disjoint(i64 inreg %a, i64 inreg %b) {
+  ; CHECK-LABEL: name: s_or_i64_disjoint
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
+  ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+  ; CHECK-NEXT:   %7:sreg_64 = disjoint S_OR_B64 killed [[REG_SEQUENCE1]], killed [[REG_SEQUENCE]], implicit-def dead $scc
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY %7.sub1
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY %7.sub0
+  ; CHECK-NEXT:   $sgpr0 = COPY [[COPY5]]
+  ; CHECK-NEXT:   $sgpr1 = COPY [[COPY4]]
+  ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $sgpr0, $sgpr1
+  %result = or disjoint i64 %a, %b
+  ret i64 %result
+}
+
+define i64 @v_or_i64_disjoint(i64 %a, i64 %b) {
+  ; CHECK-LABEL: name: v_or_i64_disjoint
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+  ; CHECK-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+  ; CHECK-NEXT:   [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 killed [[COPY5]], killed [[COPY4]], implicit $exec
+  ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+  ; CHECK-NEXT:   [[V_OR_B32_e64_1:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 killed [[COPY7]], killed [[COPY6]], implicit $exec
+  ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE killed [[V_OR_B32_e64_1]], %subreg.sub0, killed [[V_OR_B32_e64_]], %subreg.sub1
+  ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE2]].sub1
+  ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE2]].sub0
+  ; CHECK-NEXT:   $vgpr0 = COPY [[COPY9]]
+  ; CHECK-NEXT:   $vgpr1 = COPY [[COPY8]]
+  ; CHECK-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1
+  %result = or disjoint i64 %a, %b
+  ret i64 %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/eliminate-frame-index-v-add-co-u32-wave32.mir b/llvm/test/CodeGen/AMDGPU/eliminate-frame-index-v-add-co-u32-wave32.mir
index a680b63a34b9a..348743644ce4f 100644
--- a/llvm/test/CodeGen/AMDGPU/eliminate-frame-index-v-add-co-u32-wave32.mir
+++ b/llvm/test/CodeGen/AMDGPU/eliminate-frame-index-v-add-co-u32-wave32.mir
@@ -16,11 +16,11 @@ machineFunctionInfo:
 body:             |
   bb.0:
     ; MUBUFW32-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset0
-    ; MUBUFW32: renamable $vgpr0, dead $vcc_lo = V_ADD_CO_U32_e64 12, $sgpr32, 0, implicit $exec
+    ; MUBUFW32: renamable $vgpr0, dead $vcc_lo = V_ADD_CO_U32_e64 $sgpr32, 12, 0, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0, implicit $sgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset0
-    ; FLATSCRW32: renamable $vgpr0, dead $vcc_lo = V_ADD_CO_U32_e64 12, $sgpr32, 0, implicit $exec
+    ; FLATSCRW32: renamable $vgpr0, dead $vcc_lo = V_ADD_CO_U32_e64 $sgpr32, 12, 0, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0, implicit $sgpr0
     renamable $vgpr0, dead $vcc_lo = V_ADD_CO_U32_e64 12, %stack.0, 0, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $sgpr0
@@ -39,11 +39,13 @@ machineFunctionInfo:
 body:             |
   bb.0:
     ; MUBUFW32-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset0__live_vcc
-    ; MUBUFW32: renamable $vgpr0, $vcc_lo = V_ADD_CO_U32_e64 12, $sgpr32, 0, implicit $exec
+    ; MUBUFW32: renamable $vgpr0, $vcc_lo = V_ADD_CO_U32_e64 $sgpr32, 12, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0, $vcc_lo = V_ADD_CO_U32_e64 killed $vgpr0, 0, 0, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0, implicit $sgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset0__live_vcc
-    ; FLATSCRW32: renamable $vgpr0, $vcc_lo = V_ADD_CO_U32_e64 12, $sgpr32, 0, implicit $exec
+    ; FLATSCRW32: renamable $vgpr0, $vcc_lo = V_ADD_CO_U32_e64 $sgpr32, 12, 0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0, $vcc_lo = V_ADD_CO_U32_e64 killed $vgpr0, 0, 0, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0, implicit $sgpr0
     renamable $vgpr0, $vcc_lo = V_ADD_CO_U32_e64 12, %stack.0, 0, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $sgpr0
@@ -62,11 +64,13 @@ machineFunctionInfo:
 body:             |
   bb.0:
     ; MUBUFW32-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset0__clamp
-    ; MUBUFW32: renamable $vgpr0, dead $vcc_lo = V_ADD_CO_U32_e64 12, $sgpr32, 1, implicit $exec
+    ; MUBUFW32: renamable $vgpr0, dead $vcc_lo = V_ADD_CO_U32_e64 $sgpr32, 12, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0, dead $vcc_lo = V_ADD_CO_U32_e64 killed $vgpr0, 0, 1, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset0__clamp
-    ; FLATSCRW32: renamable $vgpr0, dead $vcc_lo = V_ADD_CO_U32_e64 12, $sgpr32, 1, implicit $exec
+    ; FLATSCRW32: renamable $vgpr0, dead $vcc_lo = V_ADD_CO_U32_e64 $sgpr32, 12, 0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0, dead $vcc_lo = V_ADD_CO_U32_e64 killed $vgpr0, 0, 1, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0, dead $vcc_lo = V_ADD_CO_U32_e64 12, %stack.0, 1, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -85,11 +89,13 @@ machineFunctionInfo:
 body:             |
   bb.0:
     ; MUBUFW32-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset0__live_vcc_clamp
-    ; MUBUFW32: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 12, $sgpr32, 1, implicit $exec
+    ; MUBUFW32: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 $sgpr32, 12, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 killed $vgpr0, 0, 1, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc_lo
     ;
     ; FLATSCRW32-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset0__live_vcc_clamp
-    ; FLATSCRW32: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 12, $sgpr32, 1, implicit $exec
+    ; FLATSCRW32: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 $sgpr32, 12, 0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 killed $vgpr0, 0, 1, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc_lo
     renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 12, %stack.0, 1, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $vcc_lo
@@ -112,15 +118,15 @@ body:             |
     ; MUBUFW32-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr
     ; MUBUFW32: liveins: $sgpr8
     ; MUBUFW32-NEXT: {{  $}}
-    ; MUBUFW32-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; MUBUFW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 killed $sgpr4, $sgpr8, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 $sgpr32, $sgpr8, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 128, killed $vgpr0, 0, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc_lo
     ;
     ; FLATSCRW32-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr
     ; FLATSCRW32: liveins: $sgpr8
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 killed $sgpr4, $sgpr8, 0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 $sgpr32, $sgpr8, 0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 128, killed $vgpr0, 0, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc_lo
     renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 %stack.1, $sgpr8, 0, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $vcc_lo
@@ -143,15 +149,15 @@ body:             |
     ; MUBUFW32-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr_clamp
     ; MUBUFW32: liveins: $sgpr8
     ; MUBUFW32-NEXT: {{  $}}
-    ; MUBUFW32-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; MUBUFW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 killed $sgpr4, $sgpr8, 1, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 $sgpr32, $sgpr8, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 128, killed $vgpr0, 1, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc_lo
     ;
     ; FLATSCRW32-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr_clamp
     ; FLATSCRW32: liveins: $sgpr8
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 killed $sgpr4, $sgpr8, 1, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 $sgpr32, $sgpr8, 0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 128, killed $vgpr0, 1, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc_lo
     renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 %stack.1, $sgpr8, 1, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $vcc_lo
@@ -175,15 +181,15 @@ body:             |
     ; MUBUFW32-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr
     ; MUBUFW32: liveins: $vgpr8
     ; MUBUFW32-NEXT: {{  $}}
-    ; MUBUFW32-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; MUBUFW32-NEXT: renamable $vgpr0, dead renamable $vcc_lo = V_ADD_CO_U32_e64 killed $sgpr4, $vgpr8, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0, dead renamable $vcc_lo = V_ADD_CO_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0, dead renamable $vcc_lo = V_ADD_CO_U32_e64 128, killed $vgpr0, 0, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr
     ; FLATSCRW32: liveins: $vgpr8
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW32-NEXT: renamable $vgpr0, dead renamable $vcc_lo = V_ADD_CO_U32_e64 killed $sgpr4, $vgpr8, 0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0, dead renamable $vcc_lo = V_ADD_CO_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0, dead renamable $vcc_lo = V_ADD_CO_U32_e64 128, killed $vgpr0, 0, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0, renamable dead $vcc_lo = V_ADD_CO_U32_e64 %stack.1, $vgpr8, 0, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -207,15 +213,15 @@ body:             |
     ; MUBUFW32-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__clamp
     ; MUBUFW32: liveins: $vgpr8
     ; MUBUFW32-NEXT: {{  $}}
-    ; MUBUFW32-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; MUBUFW32-NEXT: renamable $vgpr0, dead renamable $vcc_lo = V_ADD_CO_U32_e64 killed $sgpr4, $vgpr8, 1, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0, dead renamable $vcc_lo = V_ADD_CO_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0, dead renamable $vcc_lo = V_ADD_CO_U32_e64 128, killed $vgpr0, 1, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__clamp
     ; FLATSCRW32: liveins: $vgpr8
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW32-NEXT: renamable $vgpr0, dead renamable $vcc_lo = V_ADD_CO_U32_e64 killed $sgpr4, $vgpr8, 1, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0, dead renamable $vcc_lo = V_ADD_CO_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0, dead renamable $vcc_lo = V_ADD_CO_U32_e64 128, killed $vgpr0, 1, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0, renamable dead $vcc_lo = V_ADD_CO_U32_e64 %stack.1, $vgpr8, 1, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -239,15 +245,15 @@ body:             |
     ; MUBUFW32-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__live_vcc
     ; MUBUFW32: liveins: $vgpr8
     ; MUBUFW32-NEXT: {{  $}}
-    ; MUBUFW32-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; MUBUFW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 killed $sgpr4, $vgpr8, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 128, killed $vgpr0, 0, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc_lo
     ;
     ; FLATSCRW32-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__live_vcc
     ; FLATSCRW32: liveins: $vgpr8
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 killed $sgpr4, $vgpr8, 0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 128, killed $vgpr0, 0, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc_lo
     renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 %stack.1, $vgpr8, 0, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $vcc_lo
@@ -268,11 +274,11 @@ machineFunctionInfo:
 body:             |
   bb.0:
     ; MUBUFW32-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset_literal__kernel__clamp
-    ; MUBUFW32: renamable $vgpr0, dead $vcc_lo = V_ADD_CO_U32_e64 12, 72, 1, implicit $exec
+    ; MUBUFW32: renamable $vgpr0, dead $vcc_lo = V_ADD_CO_U32_e64 84, 0, 1, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset_literal__kernel__clamp
-    ; FLATSCRW32: renamable $vgpr0, dead $vcc_lo = V_ADD_CO_U32_e64 12, 72, 1, implicit $exec
+    ; FLATSCRW32: renamable $vgpr0, dead $vcc_lo = V_ADD_CO_U32_e64 84, 0, 1, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0, dead $vcc_lo = V_ADD_CO_U32_e64 12, %stack.1, 1, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -296,16 +302,16 @@ body:             |
     ; MUBUFW32-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr__scavenge_spill_required
     ; MUBUFW32: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
     ; MUBUFW32-NEXT: {{  $}}
-    ; MUBUFW32-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; MUBUFW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 killed $sgpr4, $sgpr8, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 $sgpr32, $sgpr8, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 128, killed $vgpr0, 0, implicit $exec
     ; MUBUFW32-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr__scavenge_spill_required
     ; FLATSCRW32: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 killed $sgpr4, $sgpr8, 0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 $sgpr32, $sgpr8, 0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 128, killed $vgpr0, 0, implicit $exec
     ; FLATSCRW32-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0, renamable $vcc_lo = V_ADD_CO_U32_e64 %stack.1, $sgpr8, 0, implicit $exec
@@ -390,13 +396,11 @@ body:             |
     ; MUBUFW32-LABEL: name: v_add_co_u32_e64__identity_vgpr__fi_offset0__kernel
     ; MUBUFW32: liveins: $vgpr0
     ; MUBUFW32-NEXT: {{  $}}
-    ; MUBUFW32-NEXT: renamable $vgpr0, dead renamable $vcc_lo = V_ADD_CO_U32_e64 $vgpr0, 0, 0, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_co_u32_e64__identity_vgpr__fi_offset0__kernel
     ; FLATSCRW32: liveins: $vgpr0
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: renamable $vgpr0, dead renamable $vcc_lo = V_ADD_CO_U32_e64 $vgpr0, 0, 0, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0, renamable dead $vcc_lo = V_ADD_CO_U32_e64 $vgpr0, %stack.0, 0, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -420,13 +424,11 @@ body:             |
     ; MUBUFW32-LABEL: name: v_add_co_u32_e64__fi_offset0__identity_vgpr__kernel
     ; MUBUFW32: liveins: $vgpr0
     ; MUBUFW32-NEXT: {{  $}}
-    ; MUBUFW32-NEXT: renamable $vgpr0, dead renamable $vcc_lo = V_ADD_CO_U32_e64 0, $vgpr0, 0, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_co_u32_e64__fi_offset0__identity_vgpr__kernel
     ; FLATSCRW32: liveins: $vgpr0
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: renamable $vgpr0, dead renamable $vcc_lo = V_ADD_CO_U32_e64 0, $vgpr0, 0, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0, renamable dead $vcc_lo = V_ADD_CO_U32_e64 %stack.0, $vgpr0, 0, implicit $exec
     SI_RETURN implicit $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/eliminate-frame-index-v-add-co-u32.mir b/llvm/test/CodeGen/AMDGPU/eliminate-frame-index-v-add-co-u32.mir
index fa442aa849d17..e7e56c1a3fdaa 100644
--- a/llvm/test/CodeGen/AMDGPU/eliminate-frame-index-v-add-co-u32.mir
+++ b/llvm/test/CodeGen/AMDGPU/eliminate-frame-index-v-add-co-u32.mir
@@ -22,7 +22,7 @@ machineFunctionInfo:
 body:             |
   bb.0:
     ; MUBUFW64-LABEL: name: v_add_co_u32_e32__inline_imm__fi_offset0
-    ; MUBUFW64: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUFW64: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
     ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def dead $vcc, implicit $exec
     ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
@@ -47,13 +47,15 @@ machineFunctionInfo:
 body:             |
   bb.0:
     ; MUBUFW64-LABEL: name: v_add_co_u32_e32__inline_imm__fi_offset0_live_vcc
-    ; MUBUFW64: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def $vcc, implicit $exec
+    ; MUBUFW64: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
     ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__inline_imm__fi_offset0_live_vcc
     ; FLATSCRW64: $vgpr1 = V_MOV_B32_e32 $sgpr32, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def $vcc, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     renamable $vgpr0 = V_ADD_CO_U32_e32 12, %stack.0, implicit-def $vcc, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $vcc
@@ -72,40 +74,14 @@ machineFunctionInfo:
   stackPtrOffsetReg: '$sgpr32'
 body:             |
   bb.0:
-    ; GFX7-LABEL: name: v_add_co_u32_e32__inline_imm___fi_offset_inline_imm
-    ; GFX7: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX7-NEXT: $vgpr1, dead $vcc = V_ADD_CO_U32_e64 16, killed $vgpr1, 0, implicit $exec
-    ; GFX7-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX7-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX8-LABEL: name: v_add_co_u32_e32__inline_imm___fi_offset_inline_imm
-    ; GFX8: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX8-NEXT: $vgpr1, dead $vcc = V_ADD_CO_U32_e64 16, killed $vgpr1, 0, implicit $exec
-    ; GFX8-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX8-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX900-LABEL: name: v_add_co_u32_e32__inline_imm___fi_offset_inline_imm
-    ; GFX900: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX900-NEXT: $vgpr1 = V_ADD_U32_e32 16, killed $vgpr1, implicit $exec
-    ; GFX900-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX900-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX90A-LABEL: name: v_add_co_u32_e32__inline_imm___fi_offset_inline_imm
-    ; GFX90A: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX90A-NEXT: $vgpr1 = V_ADD_U32_e32 16, killed $vgpr1, implicit $exec
-    ; GFX90A-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX90A-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX10-LABEL: name: v_add_co_u32_e32__inline_imm___fi_offset_inline_imm
-    ; GFX10: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX10-NEXT: $vgpr1 = V_ADD_U32_e32 16, killed $vgpr1, implicit $exec
-    ; GFX10-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX10-NEXT: SI_RETURN implicit $vgpr0
+    ; MUBUFW64-LABEL: name: v_add_co_u32_e32__inline_imm___fi_offset_inline_imm
+    ; MUBUFW64: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 28, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__inline_imm___fi_offset_inline_imm
-    ; FLATSCRW64: $sgpr4 = S_ADD_I32 $sgpr32, 16, implicit-def $scc
-    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; FLATSCRW64: $vgpr1 = V_MOV_B32_e32 $sgpr32, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 28, killed $vgpr1, implicit-def dead $vcc, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_CO_U32_e32 12, %stack.1, implicit-def dead $vcc, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -124,40 +100,16 @@ machineFunctionInfo:
   stackPtrOffsetReg: '$sgpr32'
 body:             |
   bb.0:
-    ; GFX7-LABEL: name: v_add_co_u32_e32__inline_imm___fi_offset_inline_imm_live_vcc
-    ; GFX7: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX7-NEXT: $vgpr1, dead $sgpr4_sgpr5 = V_ADD_CO_U32_e64 16, killed $vgpr1, 0, implicit $exec
-    ; GFX7-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def $vcc, implicit $exec
-    ; GFX7-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
-    ;
-    ; GFX8-LABEL: name: v_add_co_u32_e32__inline_imm___fi_offset_inline_imm_live_vcc
-    ; GFX8: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX8-NEXT: $vgpr1, dead $sgpr4_sgpr5 = V_ADD_CO_U32_e64 16, killed $vgpr1, 0, implicit $exec
-    ; GFX8-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def $vcc, implicit $exec
-    ; GFX8-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
-    ;
-    ; GFX900-LABEL: name: v_add_co_u32_e32__inline_imm___fi_offset_inline_imm_live_vcc
-    ; GFX900: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX900-NEXT: $vgpr1 = V_ADD_U32_e32 16, killed $vgpr1, implicit $exec
-    ; GFX900-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def $vcc, implicit $exec
-    ; GFX900-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
-    ;
-    ; GFX90A-LABEL: name: v_add_co_u32_e32__inline_imm___fi_offset_inline_imm_live_vcc
-    ; GFX90A: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX90A-NEXT: $vgpr1 = V_ADD_U32_e32 16, killed $vgpr1, implicit $exec
-    ; GFX90A-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def $vcc, implicit $exec
-    ; GFX90A-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
-    ;
-    ; GFX10-LABEL: name: v_add_co_u32_e32__inline_imm___fi_offset_inline_imm_live_vcc
-    ; GFX10: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX10-NEXT: $vgpr1 = V_ADD_U32_e32 16, killed $vgpr1, implicit $exec
-    ; GFX10-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def $vcc, implicit $exec
-    ; GFX10-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
+    ; MUBUFW64-LABEL: name: v_add_co_u32_e32__inline_imm___fi_offset_inline_imm_live_vcc
+    ; MUBUFW64: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 28, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
+    ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__inline_imm___fi_offset_inline_imm_live_vcc
-    ; FLATSCRW64: $sgpr4 = S_ADD_I32 $sgpr32, 16, implicit-def $scc
-    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def $vcc, implicit $exec
+    ; FLATSCRW64: $vgpr1 = V_MOV_B32_e32 $sgpr32, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 28, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     renamable $vgpr0 = V_ADD_CO_U32_e32 12, %stack.1, implicit-def $vcc, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $vcc
@@ -176,7 +128,7 @@ machineFunctionInfo:
 body:             |
   bb.0:
     ; MUBUFW64-LABEL: name: v_add_co_u32_e32__literal__fi_offset0
-    ; MUBUFW64: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUFW64: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
     ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 68, killed $vgpr1, implicit-def dead $vcc, implicit $exec
     ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
@@ -201,13 +153,15 @@ machineFunctionInfo:
 body:             |
   bb.0:
     ; MUBUFW64-LABEL: name: v_add_co_u32_e32__literal__fi_offset0_live_vcc
-    ; MUBUFW64: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 68, killed $vgpr1, implicit-def $vcc, implicit $exec
+    ; MUBUFW64: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 68, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
     ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__literal__fi_offset0_live_vcc
     ; FLATSCRW64: $vgpr1 = V_MOV_B32_e32 $sgpr32, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 68, killed $vgpr1, implicit-def $vcc, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 68, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     renamable $vgpr0 = V_ADD_CO_U32_e32 68, %stack.0, implicit-def $vcc, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $vcc
@@ -226,40 +180,14 @@ machineFunctionInfo:
   stackPtrOffsetReg: '$sgpr32'
 body:             |
   bb.0:
-    ; GFX7-LABEL: name: v_add_co_u32_e32__literal__fi_offset0__offset_inlineimm
-    ; GFX7: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX7-NEXT: $vgpr1, dead $vcc = V_ADD_CO_U32_e64 32, killed $vgpr1, 0, implicit $exec
-    ; GFX7-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 68, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX7-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX8-LABEL: name: v_add_co_u32_e32__literal__fi_offset0__offset_inlineimm
-    ; GFX8: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX8-NEXT: $vgpr1, dead $vcc = V_ADD_CO_U32_e64 32, killed $vgpr1, 0, implicit $exec
-    ; GFX8-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 68, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX8-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX900-LABEL: name: v_add_co_u32_e32__literal__fi_offset0__offset_inlineimm
-    ; GFX900: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX900-NEXT: $vgpr1 = V_ADD_U32_e32 32, killed $vgpr1, implicit $exec
-    ; GFX900-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 68, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX900-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX90A-LABEL: name: v_add_co_u32_e32__literal__fi_offset0__offset_inlineimm
-    ; GFX90A: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX90A-NEXT: $vgpr1 = V_ADD_U32_e32 32, killed $vgpr1, implicit $exec
-    ; GFX90A-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 68, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX90A-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX10-LABEL: name: v_add_co_u32_e32__literal__fi_offset0__offset_inlineimm
-    ; GFX10: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX10-NEXT: $vgpr1 = V_ADD_U32_e32 32, killed $vgpr1, implicit $exec
-    ; GFX10-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 68, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX10-NEXT: SI_RETURN implicit $vgpr0
+    ; MUBUFW64-LABEL: name: v_add_co_u32_e32__literal__fi_offset0__offset_inlineimm
+    ; MUBUFW64: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 100, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__literal__fi_offset0__offset_inlineimm
-    ; FLATSCRW64: $sgpr4 = S_ADD_I32 $sgpr32, 32, implicit-def $scc
-    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 68, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; FLATSCRW64: $vgpr1 = V_MOV_B32_e32 $sgpr32, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 100, killed $vgpr1, implicit-def dead $vcc, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_CO_U32_e32 68, %stack.1, implicit-def dead $vcc, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -278,40 +206,16 @@ machineFunctionInfo:
   stackPtrOffsetReg: '$sgpr32'
 body:             |
   bb.0:
-    ; GFX7-LABEL: name: v_add_co_u32_e32__literal__fi_offset0__offset_inlineimm_live_vcc
-    ; GFX7: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX7-NEXT: $vgpr1, dead $sgpr4_sgpr5 = V_ADD_CO_U32_e64 32, killed $vgpr1, 0, implicit $exec
-    ; GFX7-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 68, killed $vgpr1, implicit-def $vcc, implicit $exec
-    ; GFX7-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
-    ;
-    ; GFX8-LABEL: name: v_add_co_u32_e32__literal__fi_offset0__offset_inlineimm_live_vcc
-    ; GFX8: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX8-NEXT: $vgpr1, dead $sgpr4_sgpr5 = V_ADD_CO_U32_e64 32, killed $vgpr1, 0, implicit $exec
-    ; GFX8-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 68, killed $vgpr1, implicit-def $vcc, implicit $exec
-    ; GFX8-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
-    ;
-    ; GFX900-LABEL: name: v_add_co_u32_e32__literal__fi_offset0__offset_inlineimm_live_vcc
-    ; GFX900: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX900-NEXT: $vgpr1 = V_ADD_U32_e32 32, killed $vgpr1, implicit $exec
-    ; GFX900-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 68, killed $vgpr1, implicit-def $vcc, implicit $exec
-    ; GFX900-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
-    ;
-    ; GFX90A-LABEL: name: v_add_co_u32_e32__literal__fi_offset0__offset_inlineimm_live_vcc
-    ; GFX90A: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX90A-NEXT: $vgpr1 = V_ADD_U32_e32 32, killed $vgpr1, implicit $exec
-    ; GFX90A-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 68, killed $vgpr1, implicit-def $vcc, implicit $exec
-    ; GFX90A-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
-    ;
-    ; GFX10-LABEL: name: v_add_co_u32_e32__literal__fi_offset0__offset_inlineimm_live_vcc
-    ; GFX10: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX10-NEXT: $vgpr1 = V_ADD_U32_e32 32, killed $vgpr1, implicit $exec
-    ; GFX10-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 68, killed $vgpr1, implicit-def $vcc, implicit $exec
-    ; GFX10-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
+    ; MUBUFW64-LABEL: name: v_add_co_u32_e32__literal__fi_offset0__offset_inlineimm_live_vcc
+    ; MUBUFW64: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 100, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
+    ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__literal__fi_offset0__offset_inlineimm_live_vcc
-    ; FLATSCRW64: $sgpr4 = S_ADD_I32 $sgpr32, 32, implicit-def $scc
-    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 68, killed $vgpr1, implicit-def $vcc, implicit $exec
+    ; FLATSCRW64: $vgpr1 = V_MOV_B32_e32 $sgpr32, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 100, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     renamable $vgpr0 = V_ADD_CO_U32_e32 68, %stack.1, implicit-def $vcc, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $vcc
@@ -333,15 +237,14 @@ body:             |
     ; MUBUFW64-LABEL: name: v_add_co_u32_e32__vgpr__fi_offset0
     ; MUBUFW64: liveins: $vgpr1
     ; MUBUFW64-NEXT: {{  $}}
-    ; MUBUFW64-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
     ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr1, killed $vgpr2, implicit-def dead $vcc, implicit $exec
     ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__vgpr__fi_offset0
     ; FLATSCRW64: liveins: $vgpr1
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $vgpr2 = V_MOV_B32_e32 $sgpr32, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr1, killed $vgpr2, implicit-def dead $vcc, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $sgpr32, $vgpr1, implicit-def dead $vcc, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr1, %stack.0, implicit-def dead $vcc, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -363,8 +266,8 @@ body:             |
     ; MUBUFW64-LABEL: name: v_add_co_u32_e32__fi_offset0__vgpr
     ; MUBUFW64: liveins: $vgpr1
     ; MUBUFW64-NEXT: {{  $}}
-    ; MUBUFW64-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 killed $vgpr2, $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr1, killed $vgpr2, implicit-def dead $vcc, implicit $exec
     ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__fi_offset0__vgpr
@@ -390,54 +293,19 @@ machineFunctionInfo:
 body:             |
   bb.0:
     liveins: $vgpr1
-    ; GFX7-LABEL: name: v_add_co_u32_e32__vgpr__fi_literal_offset
-    ; GFX7: liveins: $vgpr1
-    ; GFX7-NEXT: {{  $}}
-    ; GFX7-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX7-NEXT: $vcc_lo = S_MOV_B32 128
-    ; GFX7-NEXT: $vgpr2, dead $vcc = V_ADD_CO_U32_e64 killed $vcc_lo, killed $vgpr2, 0, implicit $exec
-    ; GFX7-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr1, killed $vgpr2, implicit-def dead $vcc, implicit $exec
-    ; GFX7-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX8-LABEL: name: v_add_co_u32_e32__vgpr__fi_literal_offset
-    ; GFX8: liveins: $vgpr1
-    ; GFX8-NEXT: {{  $}}
-    ; GFX8-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX8-NEXT: $vcc_lo = S_MOV_B32 128
-    ; GFX8-NEXT: $vgpr2, dead $vcc = V_ADD_CO_U32_e64 killed $vcc_lo, killed $vgpr2, 0, implicit $exec
-    ; GFX8-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr1, killed $vgpr2, implicit-def dead $vcc, implicit $exec
-    ; GFX8-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX900-LABEL: name: v_add_co_u32_e32__vgpr__fi_literal_offset
-    ; GFX900: liveins: $vgpr1
-    ; GFX900-NEXT: {{  $}}
-    ; GFX900-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX900-NEXT: $vgpr2 = V_ADD_U32_e32 128, killed $vgpr2, implicit $exec
-    ; GFX900-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr1, killed $vgpr2, implicit-def dead $vcc, implicit $exec
-    ; GFX900-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX90A-LABEL: name: v_add_co_u32_e32__vgpr__fi_literal_offset
-    ; GFX90A: liveins: $vgpr1
-    ; GFX90A-NEXT: {{  $}}
-    ; GFX90A-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX90A-NEXT: $vgpr2 = V_ADD_U32_e32 128, killed $vgpr2, implicit $exec
-    ; GFX90A-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr1, killed $vgpr2, implicit-def dead $vcc, implicit $exec
-    ; GFX90A-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX10-LABEL: name: v_add_co_u32_e32__vgpr__fi_literal_offset
-    ; GFX10: liveins: $vgpr1
-    ; GFX10-NEXT: {{  $}}
-    ; GFX10-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX10-NEXT: $vgpr2 = V_ADD_U32_e32 128, killed $vgpr2, implicit $exec
-    ; GFX10-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr1, killed $vgpr2, implicit-def dead $vcc, implicit $exec
-    ; GFX10-NEXT: SI_RETURN implicit $vgpr0
+    ; MUBUFW64-LABEL: name: v_add_co_u32_e32__vgpr__fi_literal_offset
+    ; MUBUFW64: liveins: $vgpr1
+    ; MUBUFW64-NEXT: {{  $}}
+    ; MUBUFW64-NEXT: renamable $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr1, killed $vgpr2, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 128, killed $vgpr0, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__vgpr__fi_literal_offset
     ; FLATSCRW64: liveins: $vgpr1
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW64-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr1, killed $vgpr2, implicit-def dead $vcc, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $sgpr32, $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 128, killed $vgpr0, implicit-def dead $vcc, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr1, %stack.1, implicit-def dead $vcc, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -457,53 +325,19 @@ machineFunctionInfo:
 body:             |
   bb.0:
     liveins: $vgpr1
-    ; GFX7-LABEL: name: v_add_co_u32_e32__fi_literal_offset__vgpr
-    ; GFX7: liveins: $vgpr1
-    ; GFX7-NEXT: {{  $}}
-    ; GFX7-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX7-NEXT: $vcc_lo = S_MOV_B32 128
-    ; GFX7-NEXT: $vgpr2, dead $vcc = V_ADD_CO_U32_e64 killed $vcc_lo, killed $vgpr2, 0, implicit $exec
-    ; GFX7-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 killed $vgpr2, $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX7-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX8-LABEL: name: v_add_co_u32_e32__fi_literal_offset__vgpr
-    ; GFX8: liveins: $vgpr1
-    ; GFX8-NEXT: {{  $}}
-    ; GFX8-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX8-NEXT: $vcc_lo = S_MOV_B32 128
-    ; GFX8-NEXT: $vgpr2, dead $vcc = V_ADD_CO_U32_e64 killed $vcc_lo, killed $vgpr2, 0, implicit $exec
-    ; GFX8-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 killed $vgpr2, $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX8-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX900-LABEL: name: v_add_co_u32_e32__fi_literal_offset__vgpr
-    ; GFX900: liveins: $vgpr1
-    ; GFX900-NEXT: {{  $}}
-    ; GFX900-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX900-NEXT: $vgpr2 = V_ADD_U32_e32 128, killed $vgpr2, implicit $exec
-    ; GFX900-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 killed $vgpr2, $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX900-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX90A-LABEL: name: v_add_co_u32_e32__fi_literal_offset__vgpr
-    ; GFX90A: liveins: $vgpr1
-    ; GFX90A-NEXT: {{  $}}
-    ; GFX90A-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX90A-NEXT: $vgpr2 = V_ADD_U32_e32 128, killed $vgpr2, implicit $exec
-    ; GFX90A-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 killed $vgpr2, $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX90A-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX10-LABEL: name: v_add_co_u32_e32__fi_literal_offset__vgpr
-    ; GFX10: liveins: $vgpr1
-    ; GFX10-NEXT: {{  $}}
-    ; GFX10-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX10-NEXT: $vgpr2 = V_ADD_U32_e32 128, killed $vgpr2, implicit $exec
-    ; GFX10-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 killed $vgpr2, $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX10-NEXT: SI_RETURN implicit $vgpr0
+    ; MUBUFW64-LABEL: name: v_add_co_u32_e32__fi_literal_offset__vgpr
+    ; MUBUFW64: liveins: $vgpr1
+    ; MUBUFW64-NEXT: {{  $}}
+    ; MUBUFW64-NEXT: renamable $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr1, killed $vgpr2, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 128, killed $vgpr0, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__fi_literal_offset__vgpr
     ; FLATSCRW64: liveins: $vgpr1
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 killed $sgpr4, $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $sgpr32, $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 128, killed $vgpr0, implicit-def dead $vcc, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_CO_U32_e32 %stack.1, $vgpr1, implicit-def dead $vcc, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -523,54 +357,20 @@ machineFunctionInfo:
 body:             |
   bb.0:
     liveins: $sgpr8
-    ; GFX7-LABEL: name: v_add_co_u32_e32__sgpr__fi_literal_offset
-    ; GFX7: liveins: $sgpr8
-    ; GFX7-NEXT: {{  $}}
-    ; GFX7-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX7-NEXT: $vcc_lo = S_MOV_B32 128
-    ; GFX7-NEXT: $vgpr1, dead $vcc = V_ADD_CO_U32_e64 killed $vcc_lo, killed $vgpr1, 0, implicit $exec
-    ; GFX7-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $sgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX7-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX8-LABEL: name: v_add_co_u32_e32__sgpr__fi_literal_offset
-    ; GFX8: liveins: $sgpr8
-    ; GFX8-NEXT: {{  $}}
-    ; GFX8-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX8-NEXT: $vcc_lo = S_MOV_B32 128
-    ; GFX8-NEXT: $vgpr1, dead $vcc = V_ADD_CO_U32_e64 killed $vcc_lo, killed $vgpr1, 0, implicit $exec
-    ; GFX8-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $sgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX8-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX900-LABEL: name: v_add_co_u32_e32__sgpr__fi_literal_offset
-    ; GFX900: liveins: $sgpr8
-    ; GFX900-NEXT: {{  $}}
-    ; GFX900-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX900-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX900-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $sgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX900-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX90A-LABEL: name: v_add_co_u32_e32__sgpr__fi_literal_offset
-    ; GFX90A: liveins: $sgpr8
-    ; GFX90A-NEXT: {{  $}}
-    ; GFX90A-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX90A-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX90A-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $sgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX90A-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX10-LABEL: name: v_add_co_u32_e32__sgpr__fi_literal_offset
-    ; GFX10: liveins: $sgpr8
-    ; GFX10-NEXT: {{  $}}
-    ; GFX10-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX10-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX10-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $sgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX10-NEXT: SI_RETURN implicit $vgpr0
+    ; MUBUFW64-LABEL: name: v_add_co_u32_e32__sgpr__fi_literal_offset
+    ; MUBUFW64: liveins: $sgpr8
+    ; MUBUFW64-NEXT: {{  $}}
+    ; MUBUFW64-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $sgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 128, killed $vgpr0, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__sgpr__fi_literal_offset
     ; FLATSCRW64: liveins: $sgpr8
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
+    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 $sgpr32, implicit $exec
     ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $sgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 128, killed $vgpr0, implicit-def dead $vcc, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_CO_U32_e32 $sgpr8, %stack.1, implicit-def dead $vcc, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -589,12 +389,14 @@ machineFunctionInfo:
 body:             |
   bb.0:
     ; MUBUFW64-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset0
-    ; MUBUFW64: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUFW64: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
     ; MUBUFW64-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 12, killed $vgpr1, 0, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr0, 0, 0, implicit $exec
     ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset0
-    ; FLATSCRW64: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 12, $sgpr32, 0, implicit $exec
+    ; FLATSCRW64: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr32, 12, 0, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr0, 0, 0, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 12, %stack.0, 0, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $vcc
@@ -613,12 +415,14 @@ machineFunctionInfo:
 body:             |
   bb.0:
     ; MUBUFW64-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset0__clamp
-    ; MUBUFW64: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; MUBUFW64-NEXT: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 12, killed $vgpr1, 1, implicit $exec
+    ; MUBUFW64: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 12, killed $vgpr1, 0, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 killed $vgpr0, 0, 1, implicit $exec
     ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset0__clamp
-    ; FLATSCRW64: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 12, $sgpr32, 1, implicit $exec
+    ; FLATSCRW64: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 $sgpr32, 12, 0, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 killed $vgpr0, 0, 1, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 12, %stack.0, 1, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -637,12 +441,14 @@ machineFunctionInfo:
 body:             |
   bb.0:
     ; MUBUFW64-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset0__live_vcc_clamp
-    ; MUBUFW64: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; MUBUFW64-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 12, killed $vgpr1, 1, implicit $exec
+    ; MUBUFW64: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 12, killed $vgpr1, 0, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr0, 0, 1, implicit $exec
     ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset0__live_vcc_clamp
-    ; FLATSCRW64: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 12, $sgpr32, 1, implicit $exec
+    ; FLATSCRW64: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr32, 12, 0, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr0, 0, 1, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 12, %stack.0, 1, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $vcc
@@ -665,65 +471,68 @@ body:             |
     ; GFX7-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr
     ; GFX7: liveins: $sgpr8
     ; GFX7-NEXT: {{  $}}
-    ; GFX7-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX7-NEXT: $sgpr4 = S_MOV_B32 128
-    ; GFX7-NEXT: $vgpr1, dead $sgpr4_sgpr5 = V_ADD_CO_U32_e64 killed $sgpr4, killed $vgpr1, 0, implicit $exec
-    ; GFX7-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $sgpr8, 0, implicit $exec
+    ; GFX7-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX7-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX7-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX7-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; GFX7-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; GFX8-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr
     ; GFX8: liveins: $sgpr8
     ; GFX8-NEXT: {{  $}}
-    ; GFX8-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX8-NEXT: $sgpr4 = S_MOV_B32 128
-    ; GFX8-NEXT: $vgpr1, dead $sgpr4_sgpr5 = V_ADD_CO_U32_e64 killed $sgpr4, killed $vgpr1, 0, implicit $exec
-    ; GFX8-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $sgpr8, 0, implicit $exec
+    ; GFX8-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX8-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX8-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX8-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; GFX8-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; GFX900-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr
     ; GFX900: liveins: $sgpr8
     ; GFX900-NEXT: {{  $}}
-    ; GFX900-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX900-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX900-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $sgpr8, 0, implicit $exec
+    ; GFX900-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX900-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX900-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX900-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; GFX900-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; GFX90A-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr
     ; GFX90A: liveins: $sgpr8
     ; GFX90A-NEXT: {{  $}}
-    ; GFX90A-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX90A-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX90A-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $sgpr8, 0, implicit $exec
+    ; GFX90A-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX90A-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX90A-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX90A-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; GFX90A-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; GFX10-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr
     ; GFX10: liveins: $sgpr8
     ; GFX10-NEXT: {{  $}}
-    ; GFX10-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX10-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX10-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $sgpr8, 0, implicit $exec
+    ; GFX10-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX10-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX10-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 128, killed $vgpr0, 0, implicit $exec
     ; GFX10-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; GFX940-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr
     ; GFX940: liveins: $sgpr8
     ; GFX940-NEXT: {{  $}}
-    ; GFX940-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; GFX940-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
-    ; GFX940-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $sgpr8, 0, implicit $exec
+    ; GFX940-NEXT: $vgpr1 = V_MOV_B32_e32 $sgpr32, implicit $exec
+    ; GFX940-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX940-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX940-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; GFX940-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; GFX11-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr
     ; GFX11: liveins: $sgpr8
     ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; GFX11-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $sgpr4, $sgpr8, 0, implicit $exec
+    ; GFX11-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr32, $sgpr8, 0, implicit $exec
+    ; GFX11-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 128, killed $vgpr0, 0, implicit $exec
     ; GFX11-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; GFX12-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr
     ; GFX12: liveins: $sgpr8
     ; GFX12-NEXT: {{  $}}
-    ; GFX12-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; GFX12-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $sgpr4, $sgpr8, 0, implicit $exec
+    ; GFX12-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr32, $sgpr8, 0, implicit $exec
+    ; GFX12-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 128, killed $vgpr0, 0, implicit $exec
     ; GFX12-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 %stack.1, $sgpr8, 0, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $vcc
@@ -746,65 +555,68 @@ body:             |
     ; GFX7-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr_clamp
     ; GFX7: liveins: $sgpr8
     ; GFX7-NEXT: {{  $}}
-    ; GFX7-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX7-NEXT: $sgpr4 = S_MOV_B32 128
-    ; GFX7-NEXT: $vgpr1, dead $sgpr4_sgpr5 = V_ADD_CO_U32_e64 killed $sgpr4, killed $vgpr1, 0, implicit $exec
-    ; GFX7-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $sgpr8, 1, implicit $exec
+    ; GFX7-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX7-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX7-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX7-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 1, implicit $exec
     ; GFX7-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; GFX8-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr_clamp
     ; GFX8: liveins: $sgpr8
     ; GFX8-NEXT: {{  $}}
-    ; GFX8-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX8-NEXT: $sgpr4 = S_MOV_B32 128
-    ; GFX8-NEXT: $vgpr1, dead $sgpr4_sgpr5 = V_ADD_CO_U32_e64 killed $sgpr4, killed $vgpr1, 0, implicit $exec
-    ; GFX8-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $sgpr8, 1, implicit $exec
+    ; GFX8-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX8-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX8-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX8-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 1, implicit $exec
     ; GFX8-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; GFX900-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr_clamp
     ; GFX900: liveins: $sgpr8
     ; GFX900-NEXT: {{  $}}
-    ; GFX900-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX900-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX900-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $sgpr8, 1, implicit $exec
+    ; GFX900-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX900-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX900-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX900-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 1, implicit $exec
     ; GFX900-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; GFX90A-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr_clamp
     ; GFX90A: liveins: $sgpr8
     ; GFX90A-NEXT: {{  $}}
-    ; GFX90A-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX90A-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX90A-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $sgpr8, 1, implicit $exec
+    ; GFX90A-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX90A-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX90A-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX90A-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 1, implicit $exec
     ; GFX90A-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; GFX10-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr_clamp
     ; GFX10: liveins: $sgpr8
     ; GFX10-NEXT: {{  $}}
-    ; GFX10-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX10-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX10-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $sgpr8, 1, implicit $exec
+    ; GFX10-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX10-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX10-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 128, killed $vgpr0, 1, implicit $exec
     ; GFX10-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; GFX940-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr_clamp
     ; GFX940: liveins: $sgpr8
     ; GFX940-NEXT: {{  $}}
-    ; GFX940-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; GFX940-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
-    ; GFX940-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $sgpr8, 1, implicit $exec
+    ; GFX940-NEXT: $vgpr1 = V_MOV_B32_e32 $sgpr32, implicit $exec
+    ; GFX940-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX940-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX940-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 1, implicit $exec
     ; GFX940-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; GFX11-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr_clamp
     ; GFX11: liveins: $sgpr8
     ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; GFX11-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $sgpr4, $sgpr8, 1, implicit $exec
+    ; GFX11-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr32, $sgpr8, 0, implicit $exec
+    ; GFX11-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 128, killed $vgpr0, 1, implicit $exec
     ; GFX11-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; GFX12-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr_clamp
     ; GFX12: liveins: $sgpr8
     ; GFX12-NEXT: {{  $}}
-    ; GFX12-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; GFX12-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $sgpr4, $sgpr8, 1, implicit $exec
+    ; GFX12-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr32, $sgpr8, 0, implicit $exec
+    ; GFX12-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 128, killed $vgpr0, 1, implicit $exec
     ; GFX12-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 %stack.1, $sgpr8, 1, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $vcc
@@ -828,51 +640,68 @@ body:             |
     ; GFX7-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr
     ; GFX7: liveins: $vgpr8
     ; GFX7-NEXT: {{  $}}
-    ; GFX7-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX7-NEXT: $vcc_lo = S_MOV_B32 128
-    ; GFX7-NEXT: $vgpr1, dead $vcc = V_ADD_CO_U32_e64 killed $vcc_lo, killed $vgpr1, 0, implicit $exec
-    ; GFX7-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $vgpr8, 0, implicit $exec
+    ; GFX7-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX7-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX7-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX7-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; GFX7-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; GFX8-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr
     ; GFX8: liveins: $vgpr8
     ; GFX8-NEXT: {{  $}}
-    ; GFX8-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX8-NEXT: $vcc_lo = S_MOV_B32 128
-    ; GFX8-NEXT: $vgpr1, dead $vcc = V_ADD_CO_U32_e64 killed $vcc_lo, killed $vgpr1, 0, implicit $exec
-    ; GFX8-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $vgpr8, 0, implicit $exec
+    ; GFX8-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX8-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX8-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX8-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; GFX8-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; GFX900-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr
     ; GFX900: liveins: $vgpr8
     ; GFX900-NEXT: {{  $}}
-    ; GFX900-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX900-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX900-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $vgpr8, 0, implicit $exec
+    ; GFX900-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX900-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX900-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX900-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; GFX900-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; GFX90A-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr
     ; GFX90A: liveins: $vgpr8
     ; GFX90A-NEXT: {{  $}}
-    ; GFX90A-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX90A-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX90A-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $vgpr8, 0, implicit $exec
+    ; GFX90A-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX90A-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX90A-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX90A-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; GFX90A-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; GFX10-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr
     ; GFX10: liveins: $vgpr8
     ; GFX10-NEXT: {{  $}}
-    ; GFX10-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX10-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX10-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $vgpr8, 0, implicit $exec
+    ; GFX10-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX10-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX10-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 128, killed $vgpr0, 0, implicit $exec
     ; GFX10-NEXT: SI_RETURN implicit $vgpr0
     ;
-    ; FLATSCRW64-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr
-    ; FLATSCRW64: liveins: $vgpr8
-    ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW64-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $sgpr4, $vgpr8, 0, implicit $exec
-    ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
+    ; GFX940-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr
+    ; GFX940: liveins: $vgpr8
+    ; GFX940-NEXT: {{  $}}
+    ; GFX940-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; GFX940-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX940-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
+    ; GFX940-NEXT: SI_RETURN implicit $vgpr0
+    ;
+    ; GFX11-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr
+    ; GFX11: liveins: $vgpr8
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; GFX11-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 128, killed $vgpr0, 0, implicit $exec
+    ; GFX11-NEXT: SI_RETURN implicit $vgpr0
+    ;
+    ; GFX12-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr
+    ; GFX12: liveins: $vgpr8
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; GFX12-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 128, killed $vgpr0, 0, implicit $exec
+    ; GFX12-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0, renamable dead $vcc = V_ADD_CO_U32_e64 %stack.1, $vgpr8, 0, implicit $exec
     SI_RETURN implicit $vgpr0
 
@@ -895,51 +724,68 @@ body:             |
     ; GFX7-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__clamp
     ; GFX7: liveins: $vgpr8
     ; GFX7-NEXT: {{  $}}
-    ; GFX7-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX7-NEXT: $vcc_lo = S_MOV_B32 128
-    ; GFX7-NEXT: $vgpr1, dead $vcc = V_ADD_CO_U32_e64 killed $vcc_lo, killed $vgpr1, 0, implicit $exec
-    ; GFX7-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $vgpr8, 1, implicit $exec
+    ; GFX7-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX7-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX7-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX7-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 1, implicit $exec
     ; GFX7-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; GFX8-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__clamp
     ; GFX8: liveins: $vgpr8
     ; GFX8-NEXT: {{  $}}
-    ; GFX8-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX8-NEXT: $vcc_lo = S_MOV_B32 128
-    ; GFX8-NEXT: $vgpr1, dead $vcc = V_ADD_CO_U32_e64 killed $vcc_lo, killed $vgpr1, 0, implicit $exec
-    ; GFX8-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $vgpr8, 1, implicit $exec
+    ; GFX8-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX8-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX8-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX8-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 1, implicit $exec
     ; GFX8-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; GFX900-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__clamp
     ; GFX900: liveins: $vgpr8
     ; GFX900-NEXT: {{  $}}
-    ; GFX900-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX900-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX900-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $vgpr8, 1, implicit $exec
+    ; GFX900-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX900-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX900-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX900-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 1, implicit $exec
     ; GFX900-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; GFX90A-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__clamp
     ; GFX90A: liveins: $vgpr8
     ; GFX90A-NEXT: {{  $}}
-    ; GFX90A-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX90A-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX90A-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $vgpr8, 1, implicit $exec
+    ; GFX90A-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX90A-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX90A-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX90A-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 1, implicit $exec
     ; GFX90A-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; GFX10-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__clamp
     ; GFX10: liveins: $vgpr8
     ; GFX10-NEXT: {{  $}}
-    ; GFX10-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX10-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX10-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $vgpr8, 1, implicit $exec
+    ; GFX10-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX10-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX10-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 128, killed $vgpr0, 1, implicit $exec
     ; GFX10-NEXT: SI_RETURN implicit $vgpr0
     ;
-    ; FLATSCRW64-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__clamp
-    ; FLATSCRW64: liveins: $vgpr8
-    ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW64-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $sgpr4, $vgpr8, 1, implicit $exec
-    ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
+    ; GFX940-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__clamp
+    ; GFX940: liveins: $vgpr8
+    ; GFX940-NEXT: {{  $}}
+    ; GFX940-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; GFX940-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX940-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 1, implicit $exec
+    ; GFX940-NEXT: SI_RETURN implicit $vgpr0
+    ;
+    ; GFX11-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__clamp
+    ; GFX11: liveins: $vgpr8
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; GFX11-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 128, killed $vgpr0, 1, implicit $exec
+    ; GFX11-NEXT: SI_RETURN implicit $vgpr0
+    ;
+    ; GFX12-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__clamp
+    ; GFX12: liveins: $vgpr8
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; GFX12-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 128, killed $vgpr0, 1, implicit $exec
+    ; GFX12-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0, renamable dead $vcc = V_ADD_CO_U32_e64 %stack.1, $vgpr8, 1, implicit $exec
     SI_RETURN implicit $vgpr0
 
@@ -962,51 +808,68 @@ body:             |
     ; GFX7-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__live_vcc
     ; GFX7: liveins: $vgpr8
     ; GFX7-NEXT: {{  $}}
-    ; GFX7-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX7-NEXT: $sgpr4 = S_MOV_B32 128
-    ; GFX7-NEXT: $vgpr1, dead $sgpr4_sgpr5 = V_ADD_CO_U32_e64 killed $sgpr4, killed $vgpr1, 0, implicit $exec
-    ; GFX7-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $vgpr8, 0, implicit $exec
+    ; GFX7-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX7-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX7-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX7-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; GFX7-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; GFX8-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__live_vcc
     ; GFX8: liveins: $vgpr8
     ; GFX8-NEXT: {{  $}}
-    ; GFX8-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX8-NEXT: $sgpr4 = S_MOV_B32 128
-    ; GFX8-NEXT: $vgpr1, dead $sgpr4_sgpr5 = V_ADD_CO_U32_e64 killed $sgpr4, killed $vgpr1, 0, implicit $exec
-    ; GFX8-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $vgpr8, 0, implicit $exec
+    ; GFX8-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX8-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX8-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX8-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; GFX8-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; GFX900-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__live_vcc
     ; GFX900: liveins: $vgpr8
     ; GFX900-NEXT: {{  $}}
-    ; GFX900-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX900-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX900-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $vgpr8, 0, implicit $exec
+    ; GFX900-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX900-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX900-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX900-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; GFX900-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; GFX90A-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__live_vcc
     ; GFX90A: liveins: $vgpr8
     ; GFX90A-NEXT: {{  $}}
-    ; GFX90A-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX90A-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX90A-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $vgpr8, 0, implicit $exec
+    ; GFX90A-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX90A-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX90A-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX90A-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; GFX90A-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
     ; GFX10-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__live_vcc
     ; GFX10: liveins: $vgpr8
     ; GFX10-NEXT: {{  $}}
-    ; GFX10-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX10-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX10-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $vgpr8, 0, implicit $exec
+    ; GFX10-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX10-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX10-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 128, killed $vgpr0, 0, implicit $exec
     ; GFX10-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     ;
-    ; FLATSCRW64-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__live_vcc
-    ; FLATSCRW64: liveins: $vgpr8
-    ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW64-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $sgpr4, $vgpr8, 0, implicit $exec
-    ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
+    ; GFX940-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__live_vcc
+    ; GFX940: liveins: $vgpr8
+    ; GFX940-NEXT: {{  $}}
+    ; GFX940-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; GFX940-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX940-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
+    ; GFX940-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
+    ;
+    ; GFX11-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__live_vcc
+    ; GFX11: liveins: $vgpr8
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; GFX11-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 128, killed $vgpr0, 0, implicit $exec
+    ; GFX11-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
+    ;
+    ; GFX12-LABEL: name: v_add_co_u32_e64__fi_literal_offset__vgpr__live_vcc
+    ; GFX12: liveins: $vgpr8
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; GFX12-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 128, killed $vgpr0, 0, implicit $exec
+    ; GFX12-NEXT: SI_RETURN implicit $vgpr0, implicit $vcc
     renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 %stack.1, $vgpr8, 0, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $vcc
 
@@ -1029,13 +892,11 @@ body:             |
     ; MUBUFW64-NEXT: {{  $}}
     ; MUBUFW64-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW64-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW64-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
-    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 12, implicit $exec
     ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__inline_imm__fi_offset0__kernel
-    ; FLATSCRW64: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; FLATSCRW64: renamable $vgpr0 = V_MOV_B32_e32 12, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_CO_U32_e32 12, %stack.0, implicit-def dead $vcc, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -1090,13 +951,11 @@ body:             |
     ; MUBUFW64-NEXT: {{  $}}
     ; MUBUFW64-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW64-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW64-NEXT: $vgpr1 = V_MOV_B32_e32 72, implicit $exec
-    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 84, implicit $exec
     ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__inline_imm__fi_offset_literal__kernel
-    ; FLATSCRW64: $vgpr1 = V_MOV_B32_e32 72, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 12, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; FLATSCRW64: renamable $vgpr0 = V_MOV_B32_e32 84, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_CO_U32_e32 12, %stack.1, implicit-def dead $vcc, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -1121,8 +980,8 @@ body:             |
     ; GFX7-NEXT: {{  $}}
     ; GFX7-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; GFX7-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; GFX7-NEXT: $vgpr1 = V_MOV_B32_e32 72, implicit $exec
-    ; GFX7-NEXT: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 12, killed $vgpr1, 1, implicit $exec
+    ; GFX7-NEXT: $vgpr1 = V_MOV_B32_e32 84, implicit $exec
+    ; GFX7-NEXT: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 killed $vgpr1, 0, 1, implicit $exec
     ; GFX7-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; GFX8-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset_literal__kernel__clamp
@@ -1130,8 +989,8 @@ body:             |
     ; GFX8-NEXT: {{  $}}
     ; GFX8-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; GFX8-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; GFX8-NEXT: $vgpr1 = V_MOV_B32_e32 72, implicit $exec
-    ; GFX8-NEXT: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 12, killed $vgpr1, 1, implicit $exec
+    ; GFX8-NEXT: $vgpr1 = V_MOV_B32_e32 84, implicit $exec
+    ; GFX8-NEXT: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 killed $vgpr1, 0, 1, implicit $exec
     ; GFX8-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; GFX900-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset_literal__kernel__clamp
@@ -1139,8 +998,8 @@ body:             |
     ; GFX900-NEXT: {{  $}}
     ; GFX900-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; GFX900-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; GFX900-NEXT: $vgpr1 = V_MOV_B32_e32 72, implicit $exec
-    ; GFX900-NEXT: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 12, killed $vgpr1, 1, implicit $exec
+    ; GFX900-NEXT: $vgpr1 = V_MOV_B32_e32 84, implicit $exec
+    ; GFX900-NEXT: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 killed $vgpr1, 0, 1, implicit $exec
     ; GFX900-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; GFX90A-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset_literal__kernel__clamp
@@ -1148,8 +1007,8 @@ body:             |
     ; GFX90A-NEXT: {{  $}}
     ; GFX90A-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; GFX90A-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; GFX90A-NEXT: $vgpr1 = V_MOV_B32_e32 72, implicit $exec
-    ; GFX90A-NEXT: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 12, killed $vgpr1, 1, implicit $exec
+    ; GFX90A-NEXT: $vgpr1 = V_MOV_B32_e32 84, implicit $exec
+    ; GFX90A-NEXT: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 killed $vgpr1, 0, 1, implicit $exec
     ; GFX90A-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; GFX10-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset_literal__kernel__clamp
@@ -1157,20 +1016,20 @@ body:             |
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; GFX10-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; GFX10-NEXT: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 12, 72, 1, implicit $exec
+    ; GFX10-NEXT: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 84, 0, 1, implicit $exec
     ; GFX10-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; GFX940-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset_literal__kernel__clamp
-    ; GFX940: $sgpr4 = S_MOV_B32 72
-    ; GFX940-NEXT: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 12, killed $sgpr4, 1, implicit $exec
+    ; GFX940: $vgpr1 = V_MOV_B32_e32 84, implicit $exec
+    ; GFX940-NEXT: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 killed $vgpr1, 0, 1, implicit $exec
     ; GFX940-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; GFX11-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset_literal__kernel__clamp
-    ; GFX11: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 12, 72, 1, implicit $exec
+    ; GFX11: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 84, 0, 1, implicit $exec
     ; GFX11-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; GFX12-LABEL: name: v_add_co_u32_e64__inline_imm__fi_offset_literal__kernel__clamp
-    ; GFX12: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 12, 72, 1, implicit $exec
+    ; GFX12: renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 84, 0, 1, implicit $exec
     ; GFX12-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0, dead $vcc = V_ADD_CO_U32_e64 12, %stack.1, 1, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -1195,10 +1054,10 @@ body:             |
     ; GFX7: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
     ; GFX7-NEXT: {{  $}}
     ; GFX7-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (store (s32) into %stack.2, addrspace 5)
-    ; GFX7-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX7-NEXT: $vcc_lo = S_MOV_B32 128
-    ; GFX7-NEXT: $vgpr1, dead $vcc = V_ADD_CO_U32_e64 killed $vcc_lo, killed $vgpr1, 0, implicit $exec
-    ; GFX7-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $sgpr8, 0, implicit $exec
+    ; GFX7-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX7-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX7-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX7-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; GFX7-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
     ; GFX7-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
     ; GFX7-NEXT: SI_RETURN implicit $vgpr0
@@ -1207,10 +1066,10 @@ body:             |
     ; GFX8: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
     ; GFX8-NEXT: {{  $}}
     ; GFX8-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (store (s32) into %stack.2, addrspace 5)
-    ; GFX8-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX8-NEXT: $vcc_lo = S_MOV_B32 128
-    ; GFX8-NEXT: $vgpr1, dead $vcc = V_ADD_CO_U32_e64 killed $vcc_lo, killed $vgpr1, 0, implicit $exec
-    ; GFX8-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $sgpr8, 0, implicit $exec
+    ; GFX8-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX8-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX8-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX8-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; GFX8-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
     ; GFX8-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
     ; GFX8-NEXT: SI_RETURN implicit $vgpr0
@@ -1219,9 +1078,10 @@ body:             |
     ; GFX900: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
     ; GFX900-NEXT: {{  $}}
     ; GFX900-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (store (s32) into %stack.2, addrspace 5)
-    ; GFX900-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX900-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX900-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $sgpr8, 0, implicit $exec
+    ; GFX900-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX900-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX900-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX900-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; GFX900-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
     ; GFX900-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
     ; GFX900-NEXT: SI_RETURN implicit $vgpr0
@@ -1230,9 +1090,10 @@ body:             |
     ; GFX90A: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
     ; GFX90A-NEXT: {{  $}}
     ; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (store (s32) into %stack.2, addrspace 5)
-    ; GFX90A-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX90A-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX90A-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $sgpr8, 0, implicit $exec
+    ; GFX90A-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX90A-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX90A-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX90A-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; GFX90A-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
     ; GFX90A-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
     ; GFX90A-NEXT: SI_RETURN implicit $vgpr0
@@ -1241,9 +1102,9 @@ body:             |
     ; GFX10: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (store (s32) into %stack.2, addrspace 5)
-    ; GFX10-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX10-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX10-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $sgpr8, 0, implicit $exec
+    ; GFX10-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX10-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX10-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 128, killed $vgpr0, 0, implicit $exec
     ; GFX10-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
     ; GFX10-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
     ; GFX10-NEXT: SI_RETURN implicit $vgpr0
@@ -1252,9 +1113,10 @@ body:             |
     ; GFX940: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
     ; GFX940-NEXT: {{  $}}
     ; GFX940-NEXT: SCRATCH_STORE_DWORD_SADDR killed $vgpr1, $sgpr32, 132, 0, implicit $exec, implicit $flat_scr :: (store (s32) into %stack.2, addrspace 5)
-    ; GFX940-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; GFX940-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
-    ; GFX940-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, $sgpr8, 0, implicit $exec
+    ; GFX940-NEXT: $vgpr1 = V_MOV_B32_e32 $sgpr32, implicit $exec
+    ; GFX940-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; GFX940-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; GFX940-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; GFX940-NEXT: $vgpr1 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 132, 0, implicit $exec, implicit $flat_scr :: (load (s32) from %stack.2, addrspace 5)
     ; GFX940-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
     ; GFX940-NEXT: SI_RETURN implicit $vgpr0
@@ -1262,16 +1124,16 @@ body:             |
     ; GFX11-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr__scavenge_spill_required
     ; GFX11: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
     ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; GFX11-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $sgpr4, $sgpr8, 0, implicit $exec
+    ; GFX11-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr32, $sgpr8, 0, implicit $exec
+    ; GFX11-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 128, killed $vgpr0, 0, implicit $exec
     ; GFX11-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
     ; GFX11-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; GFX12-LABEL: name: v_add_co_u32_e64__fi_literal_offset__sgpr__scavenge_spill_required
     ; GFX12: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
     ; GFX12-NEXT: {{  $}}
-    ; GFX12-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; GFX12-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 killed $sgpr4, $sgpr8, 0, implicit $exec
+    ; GFX12-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 $sgpr32, $sgpr8, 0, implicit $exec
+    ; GFX12-NEXT: renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 128, killed $vgpr0, 0, implicit $exec
     ; GFX12-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
     ; GFX12-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0, renamable $vcc = V_ADD_CO_U32_e64 %stack.1, $sgpr8, 0, implicit $exec
@@ -1295,70 +1157,24 @@ body:             |
   bb.0:
     liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
 
-    ; GFX7-LABEL: name: v_add_co_u32_e32__fi_literal_offset__sgpr__scavenge_spill_required
-    ; GFX7: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
-    ; GFX7-NEXT: {{  $}}
-    ; GFX7-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (store (s32) into %stack.2, addrspace 5)
-    ; GFX7-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX7-NEXT: $vcc_lo = S_MOV_B32 128
-    ; GFX7-NEXT: $vgpr1, dead $vcc = V_ADD_CO_U32_e64 killed $vcc_lo, killed $vgpr1, 0, implicit $exec
-    ; GFX7-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $sgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX7-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
-    ; GFX7-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
-    ; GFX7-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX8-LABEL: name: v_add_co_u32_e32__fi_literal_offset__sgpr__scavenge_spill_required
-    ; GFX8: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
-    ; GFX8-NEXT: {{  $}}
-    ; GFX8-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (store (s32) into %stack.2, addrspace 5)
-    ; GFX8-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX8-NEXT: $vcc_lo = S_MOV_B32 128
-    ; GFX8-NEXT: $vgpr1, dead $vcc = V_ADD_CO_U32_e64 killed $vcc_lo, killed $vgpr1, 0, implicit $exec
-    ; GFX8-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $sgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX8-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
-    ; GFX8-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
-    ; GFX8-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX900-LABEL: name: v_add_co_u32_e32__fi_literal_offset__sgpr__scavenge_spill_required
-    ; GFX900: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
-    ; GFX900-NEXT: {{  $}}
-    ; GFX900-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (store (s32) into %stack.2, addrspace 5)
-    ; GFX900-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX900-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX900-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $sgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX900-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
-    ; GFX900-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
-    ; GFX900-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX90A-LABEL: name: v_add_co_u32_e32__fi_literal_offset__sgpr__scavenge_spill_required
-    ; GFX90A: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
-    ; GFX90A-NEXT: {{  $}}
-    ; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (store (s32) into %stack.2, addrspace 5)
-    ; GFX90A-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX90A-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX90A-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $sgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX90A-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
-    ; GFX90A-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
-    ; GFX90A-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX10-LABEL: name: v_add_co_u32_e32__fi_literal_offset__sgpr__scavenge_spill_required
-    ; GFX10: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
-    ; GFX10-NEXT: {{  $}}
-    ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (store (s32) into %stack.2, addrspace 5)
-    ; GFX10-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX10-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX10-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $sgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX10-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
-    ; GFX10-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
-    ; GFX10-NEXT: SI_RETURN implicit $vgpr0
+    ; MUBUFW64-LABEL: name: v_add_co_u32_e32__fi_literal_offset__sgpr__scavenge_spill_required
+    ; MUBUFW64: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
+    ; MUBUFW64-NEXT: {{  $}}
+    ; MUBUFW64-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (store (s32) into %stack.2, addrspace 5)
+    ; MUBUFW64-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $sgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 128, killed $vgpr0, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
+    ; MUBUFW64-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
+    ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__fi_literal_offset__sgpr__scavenge_spill_required
     ; FLATSCRW64: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
     ; FLATSCRW64-NEXT: {{  $}}
     ; FLATSCRW64-NEXT: SCRATCH_STORE_DWORD_SADDR killed $vgpr1, $sgpr32, 132, 0, implicit $exec, implicit $flat_scr :: (store (s32) into %stack.2, addrspace 5)
-    ; FLATSCRW64-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
+    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 $sgpr32, implicit $exec
     ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $sgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 128, killed $vgpr0, implicit-def dead $vcc, implicit $exec
     ; FLATSCRW64-NEXT: $vgpr1 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 132, 0, implicit $exec, implicit $flat_scr :: (load (s32) from %stack.2, addrspace 5)
     ; FLATSCRW64-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
@@ -1383,71 +1199,22 @@ body:             |
   bb.0:
     liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
 
-    ; GFX7-LABEL: name: v_add_co_u32_e32__fi_literal_offset__vgpr__scavenge_spill_required
-    ; GFX7: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
-    ; GFX7-NEXT: {{  $}}
-    ; GFX7-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (store (s32) into %stack.2, addrspace 5)
-    ; GFX7-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX7-NEXT: $vcc_lo = S_MOV_B32 128
-    ; GFX7-NEXT: $vgpr1, dead $vcc = V_ADD_CO_U32_e64 killed $vcc_lo, killed $vgpr1, 0, implicit $exec
-    ; GFX7-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX7-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
-    ; GFX7-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
-    ; GFX7-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX8-LABEL: name: v_add_co_u32_e32__fi_literal_offset__vgpr__scavenge_spill_required
-    ; GFX8: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
-    ; GFX8-NEXT: {{  $}}
-    ; GFX8-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (store (s32) into %stack.2, addrspace 5)
-    ; GFX8-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX8-NEXT: $vcc_lo = S_MOV_B32 128
-    ; GFX8-NEXT: $vgpr1, dead $vcc = V_ADD_CO_U32_e64 killed $vcc_lo, killed $vgpr1, 0, implicit $exec
-    ; GFX8-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX8-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
-    ; GFX8-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
-    ; GFX8-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX900-LABEL: name: v_add_co_u32_e32__fi_literal_offset__vgpr__scavenge_spill_required
-    ; GFX900: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
-    ; GFX900-NEXT: {{  $}}
-    ; GFX900-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (store (s32) into %stack.2, addrspace 5)
-    ; GFX900-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX900-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX900-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX900-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
-    ; GFX900-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
-    ; GFX900-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX90A-LABEL: name: v_add_co_u32_e32__fi_literal_offset__vgpr__scavenge_spill_required
-    ; GFX90A: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
-    ; GFX90A-NEXT: {{  $}}
-    ; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (store (s32) into %stack.2, addrspace 5)
-    ; GFX90A-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX90A-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX90A-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX90A-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
-    ; GFX90A-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
-    ; GFX90A-NEXT: SI_RETURN implicit $vgpr0
-    ;
-    ; GFX10-LABEL: name: v_add_co_u32_e32__fi_literal_offset__vgpr__scavenge_spill_required
-    ; GFX10: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
-    ; GFX10-NEXT: {{  $}}
-    ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (store (s32) into %stack.2, addrspace 5)
-    ; GFX10-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GFX10-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; GFX10-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; GFX10-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
-    ; GFX10-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
-    ; GFX10-NEXT: SI_RETURN implicit $vgpr0
+    ; MUBUFW64-LABEL: name: v_add_co_u32_e32__fi_literal_offset__vgpr__scavenge_spill_required
+    ; MUBUFW64: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
+    ; MUBUFW64-NEXT: {{  $}}
+    ; MUBUFW64-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (store (s32) into %stack.2, addrspace 5)
+    ; MUBUFW64-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 128, killed $vgpr0, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 132, 0, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
+    ; MUBUFW64-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
+    ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__fi_literal_offset__vgpr__scavenge_spill_required
     ; FLATSCRW64: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251, $vgpr252_vgpr253, $vgpr254, $vgpr255, $sgpr8
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: SCRATCH_STORE_DWORD_SADDR killed $vgpr1, $sgpr32, 132, 0, implicit $exec, implicit $flat_scr :: (store (s32) into %stack.2, addrspace 5)
-    ; FLATSCRW64-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr8, killed $vgpr1, implicit-def dead $vcc, implicit $exec
-    ; FLATSCRW64-NEXT: $vgpr1 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 132, 0, implicit $exec, implicit $flat_scr :: (load (s32) from %stack.2, addrspace 5)
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $sgpr32, $vgpr8, implicit-def dead $vcc, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 128, killed $vgpr0, implicit-def dead $vcc, implicit $exec
     ; FLATSCRW64-NEXT: S_NOP 0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, implicit $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, implicit $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, implicit $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, implicit $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, implicit $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, implicit $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, implicit $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, implicit $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, implicit $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, implicit $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, implicit $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, implicit $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, implicit $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, implicit $vgpr248_vgpr249_vgpr250_vgpr251, implicit $vgpr252_vgpr253, implicit $vgpr254, implicit $vgpr255
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr8, %stack.1, implicit-def dead $vcc, implicit $exec
@@ -1475,15 +1242,13 @@ body:             |
     ; MUBUFW64-NEXT: {{  $}}
     ; MUBUFW64-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW64-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW64-NEXT: $vgpr2 = V_MOV_B32_e32 0, implicit $exec
-    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr1, killed $vgpr2, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
     ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__kernel_fi_offset0__other_vgpr_live_after
     ; FLATSCRW64: liveins: $vgpr1
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $vgpr2 = V_MOV_B32_e32 0, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr1, killed $vgpr2, implicit-def dead $vcc, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
     renamable $vgpr0 = V_ADD_CO_U32_e32 renamable $vgpr1, %stack.0, implicit-def dead $vcc, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $vgpr1
@@ -1571,15 +1336,11 @@ body:             |
     ; MUBUFW64-NEXT: {{  $}}
     ; MUBUFW64-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW64-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW64-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
-    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr0, killed $vgpr1, implicit-def dead $vcc, implicit $exec
     ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__identity_vgpr__fi_offset0__kernel
     ; FLATSCRW64: liveins: $vgpr0
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr0, killed $vgpr1, implicit-def dead $vcc, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr0, %stack.0, implicit-def dead $vcc, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -1604,13 +1365,11 @@ body:             |
     ; MUBUFW64-NEXT: {{  $}}
     ; MUBUFW64-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW64-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 0, $vgpr0, implicit-def dead $vcc, implicit $exec
     ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__fi_offset0__identity_vgpr__kernel
     ; FLATSCRW64: liveins: $vgpr0
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 0, $vgpr0, implicit-def dead $vcc, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_CO_U32_e32 %stack.0, $vgpr0, implicit-def dead $vcc, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -1636,13 +1395,11 @@ body:             |
     ; MUBUFW64-NEXT: {{  $}}
     ; MUBUFW64-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW64-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW64-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 $vgpr0, 0, 0, implicit $exec
     ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e64__identity_vgpr__fi_offset0__kernel
     ; FLATSCRW64: liveins: $vgpr0
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 $vgpr0, 0, 0, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0, renamable dead $vcc = V_ADD_CO_U32_e64 $vgpr0, %stack.0, 0, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -1668,13 +1425,11 @@ body:             |
     ; MUBUFW64-NEXT: {{  $}}
     ; MUBUFW64-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW64-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW64-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 0, $vgpr0, 0, implicit $exec
     ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e64__fi_offset0__identity_vgpr__kernel
     ; FLATSCRW64: liveins: $vgpr0
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: renamable $vgpr0, dead renamable $vcc = V_ADD_CO_U32_e64 0, $vgpr0, 0, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0, renamable dead $vcc = V_ADD_CO_U32_e64 %stack.0, $vgpr0, 0, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -1699,13 +1454,11 @@ body:             |
     ; MUBUFW64-NEXT: {{  $}}
     ; MUBUFW64-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW64-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 0, killed $vgpr0, implicit-def dead $vcc, implicit $exec
     ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__fi_offset0__identity_vgpr__kernel_kill
     ; FLATSCRW64: liveins: $vgpr0
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 0, killed $vgpr0, implicit-def dead $vcc, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_CO_U32_e32 %stack.0, killed $vgpr0, implicit-def dead $vcc, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -1763,15 +1516,13 @@ body:             |
     ; MUBUFW64-NEXT: {{  $}}
     ; MUBUFW64-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW64-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW64-NEXT: $vgpr1 = V_MOV_B32_e32 32, implicit $exec
-    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr0, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 32, $vgpr0, implicit-def dead $vcc, implicit $exec
     ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__identity_vgpr__fi_offset32__kernel
     ; FLATSCRW64: liveins: $vgpr0
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 32, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr0, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 32, $vgpr0, implicit-def dead $vcc, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr0, %stack.1, implicit-def dead $vcc, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -1799,15 +1550,13 @@ body:             |
     ; MUBUFW64-NEXT: {{  $}}
     ; MUBUFW64-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW64-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW64-NEXT: $vgpr1 = V_MOV_B32_e32 72, implicit $exec
-    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr0, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; MUBUFW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 72, $vgpr0, implicit-def dead $vcc, implicit $exec
     ; MUBUFW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_co_u32_e32__identity_vgpr__fi_offset72__kernel
     ; FLATSCRW64: liveins: $vgpr0
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 72, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr0, killed $vgpr1, implicit-def dead $vcc, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 72, $vgpr0, implicit-def dead $vcc, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_CO_U32_e32 $vgpr0, %stack.1, implicit-def dead $vcc, implicit $exec
     SI_RETURN implicit $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/eliminate-frame-index-v-add-u32.mir b/llvm/test/CodeGen/AMDGPU/eliminate-frame-index-v-add-u32.mir
index af6823c1ab64a..388bbc26bbea7 100644
--- a/llvm/test/CodeGen/AMDGPU/eliminate-frame-index-v-add-u32.mir
+++ b/llvm/test/CodeGen/AMDGPU/eliminate-frame-index-v-add-u32.mir
@@ -18,12 +18,12 @@ machineFunctionInfo:
 body:             |
   bb.0:
     ; MUBUF-LABEL: name: v_add_u32_e32__inline_imm__fi_offset0
-    ; MUBUF: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUF: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
     ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 12, killed $vgpr1, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e32__inline_imm__fi_offset0
-    ; MUBUFW32: $vgpr1 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
+    ; MUBUFW32: renamable $vgpr1 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
     ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 12, killed $vgpr1, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
@@ -54,27 +54,23 @@ machineFunctionInfo:
 body:             |
   bb.0:
     ; MUBUF-LABEL: name: v_add_u32_e32__inline_imm___fi_offset_inline_imm
-    ; MUBUF: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; MUBUF-NEXT: $vgpr1 = V_ADD_U32_e32 16, killed $vgpr1, implicit $exec
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 12, killed $vgpr1, implicit $exec
+    ; MUBUF: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 28, killed $vgpr1, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e32__inline_imm___fi_offset_inline_imm
-    ; MUBUFW32: $vgpr1 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
-    ; MUBUFW32-NEXT: $vgpr1 = V_ADD_U32_e32 16, killed $vgpr1, implicit $exec
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 12, killed $vgpr1, implicit $exec
+    ; MUBUFW32: renamable $vgpr1 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 28, killed $vgpr1, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e32__inline_imm___fi_offset_inline_imm
-    ; FLATSCRW64: $sgpr4 = S_ADD_I32 $sgpr32, 16, implicit-def $scc
-    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 12, killed $vgpr1, implicit $exec
+    ; FLATSCRW64: $vgpr1 = V_MOV_B32_e32 $sgpr32, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 28, killed $vgpr1, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e32__inline_imm___fi_offset_inline_imm
-    ; FLATSCRW32: $sgpr4 = S_ADD_I32 $sgpr32, 16, implicit-def $scc
-    ; FLATSCRW32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 12, killed $vgpr1, implicit $exec
+    ; FLATSCRW32: $vgpr1 = V_MOV_B32_e32 $sgpr32, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 28, killed $vgpr1, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e32 12, %stack.1, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -93,12 +89,12 @@ machineFunctionInfo:
 body:             |
   bb.0:
     ; MUBUF-LABEL: name: v_add_u32_e32__literal__fi_offset0
-    ; MUBUF: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUF: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
     ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 68, killed $vgpr1, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e32__literal__fi_offset0
-    ; MUBUFW32: $vgpr1 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
+    ; MUBUFW32: renamable $vgpr1 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
     ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 68, killed $vgpr1, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
@@ -129,27 +125,23 @@ machineFunctionInfo:
 body:             |
   bb.0:
     ; MUBUF-LABEL: name: v_add_u32_e32__literal__fi_offset0__offset_inlineimm
-    ; MUBUF: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; MUBUF-NEXT: $vgpr1 = V_ADD_U32_e32 32, killed $vgpr1, implicit $exec
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 68, killed $vgpr1, implicit $exec
+    ; MUBUF: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 100, killed $vgpr1, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e32__literal__fi_offset0__offset_inlineimm
-    ; MUBUFW32: $vgpr1 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
-    ; MUBUFW32-NEXT: $vgpr1 = V_ADD_U32_e32 32, killed $vgpr1, implicit $exec
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 68, killed $vgpr1, implicit $exec
+    ; MUBUFW32: renamable $vgpr1 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 100, killed $vgpr1, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e32__literal__fi_offset0__offset_inlineimm
-    ; FLATSCRW64: $sgpr4 = S_ADD_I32 $sgpr32, 32, implicit-def $scc
-    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 68, killed $vgpr1, implicit $exec
+    ; FLATSCRW64: $vgpr1 = V_MOV_B32_e32 $sgpr32, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 100, killed $vgpr1, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e32__literal__fi_offset0__offset_inlineimm
-    ; FLATSCRW32: $sgpr4 = S_ADD_I32 $sgpr32, 32, implicit-def $scc
-    ; FLATSCRW32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 68, killed $vgpr1, implicit $exec
+    ; FLATSCRW32: $vgpr1 = V_MOV_B32_e32 $sgpr32, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 100, killed $vgpr1, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e32 68, %stack.1, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -171,29 +163,27 @@ body:             |
     ; MUBUF-LABEL: name: v_add_u32_e32__vgpr__fi_offset0
     ; MUBUF: liveins: $vgpr1
     ; MUBUF-NEXT: {{  $}}
-    ; MUBUF-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
     ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr1, killed $vgpr2, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e32__vgpr__fi_offset0
     ; MUBUFW32: liveins: $vgpr1
     ; MUBUFW32-NEXT: {{  $}}
-    ; MUBUFW32-NEXT: $vgpr2 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr2 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
     ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr1, killed $vgpr2, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e32__vgpr__fi_offset0
     ; FLATSCRW64: liveins: $vgpr1
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $vgpr2 = V_MOV_B32_e32 $sgpr32, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr1, killed $vgpr2, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $sgpr32, $vgpr1, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e32__vgpr__fi_offset0
     ; FLATSCRW32: liveins: $vgpr1
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: $vgpr2 = V_MOV_B32_e32 $sgpr32, implicit $exec
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr1, killed $vgpr2, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $sgpr32, $vgpr1, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e32 $vgpr1, %stack.0, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -215,15 +205,15 @@ body:             |
     ; MUBUF-LABEL: name: v_add_u32_e32__fi_offset0__vgpr
     ; MUBUF: liveins: $vgpr1
     ; MUBUF-NEXT: {{  $}}
-    ; MUBUF-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr2, $vgpr1, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr1, killed $vgpr2, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e32__fi_offset0__vgpr
     ; MUBUFW32: liveins: $vgpr1
     ; MUBUFW32-NEXT: {{  $}}
-    ; MUBUFW32-NEXT: $vgpr2 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr2, $vgpr1, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr2 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr1, killed $vgpr2, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e32__fi_offset0__vgpr
@@ -258,33 +248,31 @@ body:             |
     ; MUBUF-LABEL: name: v_add_u32_e32__vgpr__fi_literal_offset
     ; MUBUF: liveins: $vgpr1
     ; MUBUF-NEXT: {{  $}}
-    ; MUBUF-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; MUBUF-NEXT: $vgpr2 = V_ADD_U32_e32 128, killed $vgpr2, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
     ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr1, killed $vgpr2, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 128, killed $vgpr0, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e32__vgpr__fi_literal_offset
     ; MUBUFW32: liveins: $vgpr1
     ; MUBUFW32-NEXT: {{  $}}
-    ; MUBUFW32-NEXT: $vgpr2 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
-    ; MUBUFW32-NEXT: $vgpr2 = V_ADD_U32_e32 128, killed $vgpr2, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr2 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
     ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr1, killed $vgpr2, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 128, killed $vgpr0, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e32__vgpr__fi_literal_offset
     ; FLATSCRW64: liveins: $vgpr1
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW64-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr1, killed $vgpr2, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $sgpr32, $vgpr1, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 128, killed $vgpr0, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e32__vgpr__fi_literal_offset
     ; FLATSCRW32: liveins: $vgpr1
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW32-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr1, killed $vgpr2, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $sgpr32, $vgpr1, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 128, killed $vgpr0, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e32 $vgpr1, %stack.1, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -307,31 +295,31 @@ body:             |
     ; MUBUF-LABEL: name: v_add_u32_e32__fi_literal_offset__vgpr
     ; MUBUF: liveins: $vgpr1
     ; MUBUF-NEXT: {{  $}}
-    ; MUBUF-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; MUBUF-NEXT: $vgpr2 = V_ADD_U32_e32 128, killed $vgpr2, implicit $exec
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr2, $vgpr1, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr1, killed $vgpr2, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 128, killed $vgpr0, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e32__fi_literal_offset__vgpr
     ; MUBUFW32: liveins: $vgpr1
     ; MUBUFW32-NEXT: {{  $}}
-    ; MUBUFW32-NEXT: $vgpr2 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
-    ; MUBUFW32-NEXT: $vgpr2 = V_ADD_U32_e32 128, killed $vgpr2, implicit $exec
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $vgpr2, $vgpr1, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr2 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr1, killed $vgpr2, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 128, killed $vgpr0, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e32__fi_literal_offset__vgpr
     ; FLATSCRW64: liveins: $vgpr1
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $sgpr4, $vgpr1, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $sgpr32, $vgpr1, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 128, killed $vgpr0, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e32__fi_literal_offset__vgpr
     ; FLATSCRW32: liveins: $vgpr1
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 killed $sgpr4, $vgpr1, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $sgpr32, $vgpr1, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 128, killed $vgpr0, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e32 %stack.1, $vgpr1, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -354,33 +342,33 @@ body:             |
     ; MUBUF-LABEL: name: v_add_u32_e32__sgpr__fi_literal_offset
     ; MUBUF: liveins: $sgpr8
     ; MUBUF-NEXT: {{  $}}
-    ; MUBUF-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; MUBUF-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
     ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $sgpr8, killed $vgpr1, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 128, killed $vgpr0, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e32__sgpr__fi_literal_offset
     ; MUBUFW32: liveins: $sgpr8
     ; MUBUFW32-NEXT: {{  $}}
-    ; MUBUFW32-NEXT: $vgpr1 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
-    ; MUBUFW32-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
     ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $sgpr8, killed $vgpr1, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 128, killed $vgpr0, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e32__sgpr__fi_literal_offset
     ; FLATSCRW64: liveins: $sgpr8
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
+    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 $sgpr32, implicit $exec
     ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $sgpr8, killed $vgpr1, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 128, killed $vgpr0, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e32__sgpr__fi_literal_offset
     ; FLATSCRW32: liveins: $sgpr8
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
+    ; FLATSCRW32-NEXT: $vgpr1 = V_MOV_B32_e32 $sgpr32, implicit $exec
     ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $sgpr8, killed $vgpr1, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 128, killed $vgpr0, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e32 $sgpr8, %stack.1, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -399,21 +387,21 @@ machineFunctionInfo:
 body:             |
   bb.0:
     ; MUBUF-LABEL: name: v_add_u32_e64__inline_imm__fi_offset0
-    ; MUBUF: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUF: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
     ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 12, killed $vgpr1, 0, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e64__inline_imm__fi_offset0
-    ; MUBUFW32: $vgpr1 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
+    ; MUBUFW32: renamable $vgpr1 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
     ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 12, killed $vgpr1, 0, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e64__inline_imm__fi_offset0
-    ; FLATSCRW64: renamable $vgpr0 = V_ADD_U32_e64 12, $sgpr32, 0, implicit $exec
+    ; FLATSCRW64: renamable $vgpr0 = V_ADD_U32_e64 $sgpr32, 12, 0, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e64__inline_imm__fi_offset0
-    ; FLATSCRW32: renamable $vgpr0 = V_ADD_U32_e64 12, $sgpr32, 0, implicit $exec
+    ; FLATSCRW32: renamable $vgpr0 = V_ADD_U32_e64 $sgpr32, 12, 0, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e64 12, %stack.0, 0, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -436,32 +424,34 @@ body:             |
     ; MUBUF-LABEL: name: v_add_u32_e64__fi_literal_offset__sgpr
     ; MUBUF: liveins: $sgpr8
     ; MUBUF-NEXT: {{  $}}
-    ; MUBUF-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; MUBUF-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $vgpr1, $sgpr8, 0, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; MUBUF-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e64__fi_literal_offset__sgpr
     ; MUBUFW32: liveins: $sgpr8
     ; MUBUFW32-NEXT: {{  $}}
-    ; MUBUFW32-NEXT: $vgpr1 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
-    ; MUBUFW32-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $vgpr1, $sgpr8, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 128, killed $vgpr0, 0, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e64__fi_literal_offset__sgpr
     ; FLATSCRW64: liveins: $sgpr8
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr4, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $vgpr1, $sgpr8, 0, implicit $exec
+    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 $sgpr32, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e64__fi_literal_offset__sgpr
     ; FLATSCRW32: liveins: $sgpr8
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $sgpr4, $sgpr8, 0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $sgpr32, $sgpr8, 0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 128, killed $vgpr0, 0, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e64 %stack.1, $sgpr8, 0, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -484,31 +474,33 @@ body:             |
     ; MUBUF-LABEL: name: v_add_u32_e64__vgpr__fi_literal_offset
     ; MUBUF: liveins: $vgpr8
     ; MUBUF-NEXT: {{  $}}
-    ; MUBUF-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; MUBUF-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
     ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; MUBUF-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e64__vgpr__fi_literal_offset
     ; MUBUFW32: liveins: $vgpr8
     ; MUBUFW32-NEXT: {{  $}}
-    ; MUBUFW32-NEXT: $vgpr1 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
-    ; MUBUFW32-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
     ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $vgpr0, 128, 0, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e64__vgpr__fi_literal_offset
     ; FLATSCRW64: liveins: $vgpr8
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr8, killed $sgpr4, 0, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $vgpr1, killed $vgpr0, 0, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e64__vgpr__fi_literal_offset
     ; FLATSCRW32: liveins: $vgpr8
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr8, killed $sgpr4, 0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $vgpr0, 128, 0, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e64 $vgpr8, %stack.1, 0, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -531,31 +523,33 @@ body:             |
     ; MUBUF-LABEL: name: v_add_u32_e64__vgpr__fi_literal_offset__clamp
     ; MUBUF: liveins: $vgpr8
     ; MUBUF-NEXT: {{  $}}
-    ; MUBUF-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; MUBUF-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr8, killed $vgpr1, 1, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; MUBUF-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $vgpr1, killed $vgpr0, 1, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e64__vgpr__fi_literal_offset__clamp
     ; MUBUFW32: liveins: $vgpr8
     ; MUBUFW32-NEXT: {{  $}}
-    ; MUBUFW32-NEXT: $vgpr1 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
-    ; MUBUFW32-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr8, killed $vgpr1, 1, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $vgpr0, 128, 1, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e64__vgpr__fi_literal_offset__clamp
     ; FLATSCRW64: liveins: $vgpr8
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr8, killed $sgpr4, 1, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $vgpr1, killed $vgpr0, 1, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e64__vgpr__fi_literal_offset__clamp
     ; FLATSCRW32: liveins: $vgpr8
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr8, killed $sgpr4, 1, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $vgpr0, 128, 1, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e64 $vgpr8, %stack.1, 1, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -578,31 +572,33 @@ body:             |
     ; MUBUF-LABEL: name: v_add_u32_e64__fi_literal_offset__vgpr__clamp
     ; MUBUF: liveins: $vgpr8
     ; MUBUF-NEXT: {{  $}}
-    ; MUBUF-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; MUBUF-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $vgpr1, $vgpr8, 1, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; MUBUF-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $vgpr1, killed $vgpr0, 1, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e64__fi_literal_offset__vgpr__clamp
     ; MUBUFW32: liveins: $vgpr8
     ; MUBUFW32-NEXT: {{  $}}
-    ; MUBUFW32-NEXT: $vgpr1 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
-    ; MUBUFW32-NEXT: $vgpr1 = V_ADD_U32_e32 128, killed $vgpr1, implicit $exec
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $vgpr1, $vgpr8, 1, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 5, $sgpr32, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr8, killed $vgpr1, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 128, killed $vgpr0, 1, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e64__fi_literal_offset__vgpr__clamp
     ; FLATSCRW64: liveins: $vgpr8
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $sgpr4, $vgpr8, 1, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $vgpr1, killed $vgpr0, 1, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e64__fi_literal_offset__vgpr__clamp
     ; FLATSCRW32: liveins: $vgpr8
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: $sgpr4 = S_ADD_I32 $sgpr32, 128, implicit-def $scc
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $sgpr4, $vgpr8, 1, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $sgpr32, $vgpr8, 0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 128, killed $vgpr0, 1, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e64 %stack.1, $vgpr8, 1, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -643,8 +639,8 @@ body:             |
     ; FLATSCRW64-LABEL: name: v_add_u32_e64__fi_literal_offset__vgpr__clamp__kernel
     ; FLATSCRW64: liveins: $vgpr8
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $sgpr4 = S_MOV_B32 128
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $sgpr4, $vgpr8, 1, implicit $exec
+    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 128, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $vgpr1, $vgpr8, 1, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e64__fi_literal_offset__vgpr__clamp__kernel
@@ -674,8 +670,7 @@ body:             |
     ; MUBUF-NEXT: {{  $}}
     ; MUBUF-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUF-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 12, killed $vgpr1, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_MOV_B32_e32 12, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e32__inline_imm__fi_offset0__kernel
@@ -683,18 +678,15 @@ body:             |
     ; MUBUFW32-NEXT: {{  $}}
     ; MUBUFW32-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW32-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW32-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 12, killed $vgpr1, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 12, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e32__inline_imm__fi_offset0__kernel
-    ; FLATSCRW64: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 12, killed $vgpr1, implicit $exec
+    ; FLATSCRW64: renamable $vgpr0 = V_MOV_B32_e32 12, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e32__inline_imm__fi_offset0__kernel
-    ; FLATSCRW32: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 12, killed $vgpr1, implicit $exec
+    ; FLATSCRW32: renamable $vgpr0 = V_MOV_B32_e32 12, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e32 12, %stack.0, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -718,7 +710,7 @@ body:             |
     ; MUBUF-NEXT: {{  $}}
     ; MUBUF-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 12, 0, 0, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_MOV_B32_e32 12, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e64__inline_imm__fi_offset0__kernel
@@ -726,15 +718,15 @@ body:             |
     ; MUBUFW32-NEXT: {{  $}}
     ; MUBUFW32-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW32-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 12, 0, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 12, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e64__inline_imm__fi_offset0__kernel
-    ; FLATSCRW64: renamable $vgpr0 = V_ADD_U32_e64 12, 0, 0, implicit $exec
+    ; FLATSCRW64: renamable $vgpr0 = V_MOV_B32_e32 12, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e64__inline_imm__fi_offset0__kernel
-    ; FLATSCRW32: renamable $vgpr0 = V_ADD_U32_e64 12, 0, 0, implicit $exec
+    ; FLATSCRW32: renamable $vgpr0 = V_MOV_B32_e32 12, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e64 12, %stack.0, 0, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -761,8 +753,7 @@ body:             |
     ; MUBUF-NEXT: {{  $}}
     ; MUBUF-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUF-NEXT: $vgpr1 = V_MOV_B32_e32 32, implicit $exec
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 12, killed $vgpr1, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_MOV_B32_e32 44, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e32__inline_imm__fi_literal__kernel
@@ -770,18 +761,15 @@ body:             |
     ; MUBUFW32-NEXT: {{  $}}
     ; MUBUFW32-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW32-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW32-NEXT: $vgpr1 = V_MOV_B32_e32 32, implicit $exec
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 12, killed $vgpr1, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 44, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e32__inline_imm__fi_literal__kernel
-    ; FLATSCRW64: $vgpr1 = V_MOV_B32_e32 32, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 12, killed $vgpr1, implicit $exec
+    ; FLATSCRW64: renamable $vgpr0 = V_MOV_B32_e32 44, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e32__inline_imm__fi_literal__kernel
-    ; FLATSCRW32: $vgpr1 = V_MOV_B32_e32 32, implicit $exec
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 12, killed $vgpr1, implicit $exec
+    ; FLATSCRW32: renamable $vgpr0 = V_MOV_B32_e32 44, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e32 12, %stack.1, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -806,7 +794,7 @@ body:             |
     ; MUBUF-NEXT: {{  $}}
     ; MUBUF-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 12, 32, 0, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_MOV_B32_e32 44, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e64__inline_imm__fi_literal__kernel
@@ -814,15 +802,15 @@ body:             |
     ; MUBUFW32-NEXT: {{  $}}
     ; MUBUFW32-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW32-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 12, 32, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 44, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e64__inline_imm__fi_literal__kernel
-    ; FLATSCRW64: renamable $vgpr0 = V_ADD_U32_e64 12, 32, 0, implicit $exec
+    ; FLATSCRW64: renamable $vgpr0 = V_MOV_B32_e32 44, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e64__inline_imm__fi_literal__kernel
-    ; FLATSCRW32: renamable $vgpr0 = V_ADD_U32_e64 12, 32, 0, implicit $exec
+    ; FLATSCRW32: renamable $vgpr0 = V_MOV_B32_e32 44, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e64 12, %stack.1, 0, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -847,7 +835,7 @@ body:             |
     ; MUBUF-NEXT: {{  $}}
     ; MUBUF-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 32, 12, 0, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_MOV_B32_e32 44, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e64__fi_literal__inline_imm__kernel
@@ -855,15 +843,15 @@ body:             |
     ; MUBUFW32-NEXT: {{  $}}
     ; MUBUFW32-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW32-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 32, 12, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 44, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e64__fi_literal__inline_imm__kernel
-    ; FLATSCRW64: renamable $vgpr0 = V_ADD_U32_e64 32, 12, 0, implicit $exec
+    ; FLATSCRW64: renamable $vgpr0 = V_MOV_B32_e32 44, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e64__fi_literal__inline_imm__kernel
-    ; FLATSCRW32: renamable $vgpr0 = V_ADD_U32_e64 32, 12, 0, implicit $exec
+    ; FLATSCRW32: renamable $vgpr0 = V_MOV_B32_e32 44, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e64 %stack.1, 12, 0, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -888,7 +876,7 @@ body:             |
     ; MUBUF-NEXT: {{  $}}
     ; MUBUF-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 12, 32, 1, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 44, 0, 1, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e64__inline_imm__fi_literal__kernel__clamp
@@ -896,15 +884,15 @@ body:             |
     ; MUBUFW32-NEXT: {{  $}}
     ; MUBUFW32-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW32-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 12, 32, 1, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 44, 0, 1, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e64__inline_imm__fi_literal__kernel__clamp
-    ; FLATSCRW64: renamable $vgpr0 = V_ADD_U32_e64 12, 32, 1, implicit $exec
+    ; FLATSCRW64: renamable $vgpr0 = V_ADD_U32_e64 44, 0, 1, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e64__inline_imm__fi_literal__kernel__clamp
-    ; FLATSCRW32: renamable $vgpr0 = V_ADD_U32_e64 12, 32, 1, implicit $exec
+    ; FLATSCRW32: renamable $vgpr0 = V_ADD_U32_e64 44, 0, 1, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e64 12, %stack.1, 1, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -932,7 +920,7 @@ body:             |
     ; MUBUF-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: renamable $vgpr0 = V_LSHLREV_B32_e32 2, killed $vgpr0, implicit $exec
-    ; MUBUF-NEXT: renamable $vgpr1 = V_ADD_U32_e32 0, $vgpr0, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr1 = V_MOV_B32_e32 $vgpr0, implicit $exec
     ; MUBUF-NEXT: renamable $vgpr2 = V_MOV_B32_e32 15, implicit $exec
     ; MUBUF-NEXT: SCRATCH_STORE_DWORD killed renamable $vgpr2, killed renamable $vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (volatile store (s32), addrspace 5)
     ; MUBUF-NEXT: renamable $vgpr0 = V_SUB_U32_e32 0, killed $vgpr0, implicit $exec
@@ -945,7 +933,7 @@ body:             |
     ; MUBUFW32-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW32-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW32-NEXT: renamable $vgpr0 = V_LSHLREV_B32_e32 2, killed $vgpr0, implicit $exec
-    ; MUBUFW32-NEXT: renamable $vgpr1 = V_ADD_U32_e32 0, $vgpr0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr1 = V_MOV_B32_e32 $vgpr0, implicit $exec
     ; MUBUFW32-NEXT: renamable $vgpr2 = V_MOV_B32_e32 15, implicit $exec
     ; MUBUFW32-NEXT: SCRATCH_STORE_DWORD killed renamable $vgpr2, killed renamable $vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (volatile store (s32), addrspace 5)
     ; MUBUFW32-NEXT: renamable $vgpr0 = V_SUB_U32_e32 0, killed $vgpr0, implicit $exec
@@ -956,7 +944,7 @@ body:             |
     ; FLATSCRW64: liveins: $vgpr0
     ; FLATSCRW64-NEXT: {{  $}}
     ; FLATSCRW64-NEXT: renamable $vgpr0 = V_LSHLREV_B32_e32 2, killed $vgpr0, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr1 = V_ADD_U32_e32 0, $vgpr0, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr1 = V_MOV_B32_e32 $vgpr0, implicit $exec
     ; FLATSCRW64-NEXT: renamable $vgpr2 = V_MOV_B32_e32 15, implicit $exec
     ; FLATSCRW64-NEXT: SCRATCH_STORE_DWORD killed renamable $vgpr2, killed renamable $vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (volatile store (s32), addrspace 5)
     ; FLATSCRW64-NEXT: renamable $vgpr0 = V_SUB_U32_e32 0, killed $vgpr0, implicit $exec
@@ -967,7 +955,7 @@ body:             |
     ; FLATSCRW32: liveins: $vgpr0
     ; FLATSCRW32-NEXT: {{  $}}
     ; FLATSCRW32-NEXT: renamable $vgpr0 = V_LSHLREV_B32_e32 2, killed $vgpr0, implicit $exec
-    ; FLATSCRW32-NEXT: renamable $vgpr1 = V_ADD_U32_e32 0, $vgpr0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr1 = V_MOV_B32_e32 $vgpr0, implicit $exec
     ; FLATSCRW32-NEXT: renamable $vgpr2 = V_MOV_B32_e32 15, implicit $exec
     ; FLATSCRW32-NEXT: SCRATCH_STORE_DWORD killed renamable $vgpr2, killed renamable $vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (volatile store (s32), addrspace 5)
     ; FLATSCRW32-NEXT: renamable $vgpr0 = V_SUB_U32_e32 0, killed $vgpr0, implicit $exec
@@ -1001,8 +989,7 @@ body:             |
     ; MUBUF-NEXT: {{  $}}
     ; MUBUF-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUF-NEXT: $vgpr2 = V_MOV_B32_e32 0, implicit $exec
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr1, killed $vgpr2, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e32__kernel_fi_offset0__other_vgpr_live_after
@@ -1010,22 +997,19 @@ body:             |
     ; MUBUFW32-NEXT: {{  $}}
     ; MUBUFW32-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW32-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW32-NEXT: $vgpr2 = V_MOV_B32_e32 0, implicit $exec
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr1, killed $vgpr2, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e32__kernel_fi_offset0__other_vgpr_live_after
     ; FLATSCRW64: liveins: $vgpr1
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $vgpr2 = V_MOV_B32_e32 0, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr1, killed $vgpr2, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e32__kernel_fi_offset0__other_vgpr_live_after
     ; FLATSCRW32: liveins: $vgpr1
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: $vgpr2 = V_MOV_B32_e32 0, implicit $exec
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr1, killed $vgpr2, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
     renamable $vgpr0 = V_ADD_U32_e32 renamable $vgpr1, %stack.0, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $vgpr1
@@ -1050,7 +1034,7 @@ body:             |
     ; MUBUF-NEXT: {{  $}}
     ; MUBUF-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 0, $vgpr1, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e32__kernel_other_vgpr_live_after__fi_offset0
@@ -1058,19 +1042,19 @@ body:             |
     ; MUBUFW32-NEXT: {{  $}}
     ; MUBUFW32-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW32-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 0, $vgpr1, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e32__kernel_other_vgpr_live_after__fi_offset0
     ; FLATSCRW64: liveins: $vgpr1
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 0, $vgpr1, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e32__kernel_other_vgpr_live_after__fi_offset0
     ; FLATSCRW32: liveins: $vgpr1
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 0, $vgpr1, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
     renamable $vgpr0 = V_ADD_U32_e32 %stack.0, renamable $vgpr1, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $vgpr1
@@ -1095,8 +1079,7 @@ body:             |
     ; MUBUF-NEXT: {{  $}}
     ; MUBUF-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUF-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $sgpr8, killed $vgpr1, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_MOV_B32_e32 $sgpr8, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0, implicit $sgpr8
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e32__kernel_fi_offset0__sgpr_live_after
@@ -1104,22 +1087,19 @@ body:             |
     ; MUBUFW32-NEXT: {{  $}}
     ; MUBUFW32-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW32-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW32-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $sgpr8, killed $vgpr1, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 $sgpr8, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0, implicit $sgpr8
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e32__kernel_fi_offset0__sgpr_live_after
     ; FLATSCRW64: liveins: $sgpr8
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $sgpr8, killed $vgpr1, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 $sgpr8, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0, implicit $sgpr8
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e32__kernel_fi_offset0__sgpr_live_after
     ; FLATSCRW32: liveins: $sgpr8
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $sgpr8, killed $vgpr1, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 $sgpr8, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0, implicit $sgpr8
     renamable $vgpr0 = V_ADD_U32_e32 renamable $sgpr8, %stack.0, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $sgpr8
@@ -1144,7 +1124,7 @@ body:             |
     ; MUBUF-NEXT: {{  $}}
     ; MUBUF-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr1, 0, 0, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e64__kernel_fi_offset0__other_vgpr_live_after
@@ -1152,19 +1132,19 @@ body:             |
     ; MUBUFW32-NEXT: {{  $}}
     ; MUBUFW32-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW32-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr1, 0, 0, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e64__kernel_fi_offset0__other_vgpr_live_after
     ; FLATSCRW64: liveins: $vgpr1
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr1, 0, 0, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e64__kernel_fi_offset0__other_vgpr_live_after
     ; FLATSCRW32: liveins: $vgpr1
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr1, 0, 0, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
     renamable $vgpr0 = V_ADD_U32_e64 renamable $vgpr1, %stack.0, 0, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $vgpr1
@@ -1190,8 +1170,8 @@ body:             |
     ; MUBUF-NEXT: {{  $}}
     ; MUBUF-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUF-NEXT: $vgpr1 = V_MOV_B32_e32 72, implicit $exec
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $sgpr8, killed $vgpr1, implicit $exec
+    ; MUBUF-NEXT: $vgpr1 = V_MOV_B32_e32 $sgpr8, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 72, killed $vgpr1, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0, implicit $sgpr8
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e32__kernel_fi_offset72__sgpr_live_after
@@ -1199,22 +1179,22 @@ body:             |
     ; MUBUFW32-NEXT: {{  $}}
     ; MUBUFW32-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW32-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW32-NEXT: $vgpr1 = V_MOV_B32_e32 72, implicit $exec
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $sgpr8, killed $vgpr1, implicit $exec
+    ; MUBUFW32-NEXT: $vgpr1 = V_MOV_B32_e32 $sgpr8, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 72, killed $vgpr1, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0, implicit $sgpr8
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e32__kernel_fi_offset72__sgpr_live_after
     ; FLATSCRW64: liveins: $sgpr8
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 72, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $sgpr8, killed $vgpr1, implicit $exec
+    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 $sgpr8, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 72, killed $vgpr1, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0, implicit $sgpr8
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e32__kernel_fi_offset72__sgpr_live_after
     ; FLATSCRW32: liveins: $sgpr8
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: $vgpr1 = V_MOV_B32_e32 72, implicit $exec
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $sgpr8, killed $vgpr1, implicit $exec
+    ; FLATSCRW32-NEXT: $vgpr1 = V_MOV_B32_e32 $sgpr8, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 72, killed $vgpr1, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0, implicit $sgpr8
     renamable $vgpr0 = V_ADD_U32_e32 renamable $sgpr8, %stack.1, implicit $exec
     SI_RETURN implicit $vgpr0, implicit $sgpr8
@@ -1240,8 +1220,9 @@ body:             |
     ; MUBUF-NEXT: {{  $}}
     ; MUBUF-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
+    ; MUBUF-NEXT: $vgpr1 = V_MOV_B32_e32 $sgpr8, implicit $exec
     ; MUBUF-NEXT: $vgpr1 = V_MOV_B32_e32 72, implicit $exec
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $vgpr1, killed $vgpr1, 0, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0, implicit $sgpr8
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e64__kernel_fi_offset72__sgpr_live_after
@@ -1255,8 +1236,9 @@ body:             |
     ; FLATSCRW64-LABEL: name: v_add_u32_e64__kernel_fi_offset72__sgpr_live_after
     ; FLATSCRW64: liveins: $sgpr8
     ; FLATSCRW64-NEXT: {{  $}}
+    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 $sgpr8, implicit $exec
     ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 72, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $sgpr8, killed $vgpr1, 0, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e64 killed $vgpr1, killed $vgpr1, 0, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0, implicit $sgpr8
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e64__kernel_fi_offset72__sgpr_live_after
@@ -1288,8 +1270,6 @@ body:             |
     ; MUBUF-NEXT: {{  $}}
     ; MUBUF-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUF-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr0, killed $vgpr1, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e32__identity_vgpr__fi_offset0__kernel
@@ -1297,22 +1277,16 @@ body:             |
     ; MUBUFW32-NEXT: {{  $}}
     ; MUBUFW32-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW32-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW32-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr0, killed $vgpr1, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e32__identity_vgpr__fi_offset0__kernel
     ; FLATSCRW64: liveins: $vgpr0
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr0, killed $vgpr1, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e32__identity_vgpr__fi_offset0__kernel
     ; FLATSCRW32: liveins: $vgpr0
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr0, killed $vgpr1, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e32 $vgpr0, %stack.0, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -1337,7 +1311,6 @@ body:             |
     ; MUBUF-NEXT: {{  $}}
     ; MUBUF-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 0, $vgpr0, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e32__fi_offset0__identity_vgpr__kernel
@@ -1345,19 +1318,16 @@ body:             |
     ; MUBUFW32-NEXT: {{  $}}
     ; MUBUFW32-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW32-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 0, $vgpr0, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e32__fi_offset0__identity_vgpr__kernel
     ; FLATSCRW64: liveins: $vgpr0
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 0, $vgpr0, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e32__fi_offset0__identity_vgpr__kernel
     ; FLATSCRW32: liveins: $vgpr0
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 0, $vgpr0, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e32 %stack.0, $vgpr0, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -1383,7 +1353,6 @@ body:             |
     ; MUBUF-NEXT: {{  $}}
     ; MUBUF-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr0, 0, 0, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e64__identity_vgpr__fi_offset0__kernel
@@ -1391,19 +1360,16 @@ body:             |
     ; MUBUFW32-NEXT: {{  $}}
     ; MUBUFW32-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW32-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr0, 0, 0, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e64__identity_vgpr__fi_offset0__kernel
     ; FLATSCRW64: liveins: $vgpr0
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr0, 0, 0, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e64__identity_vgpr__fi_offset0__kernel
     ; FLATSCRW32: liveins: $vgpr0
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr0, 0, 0, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e64 $vgpr0, %stack.0, 0, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -1429,7 +1395,6 @@ body:             |
     ; MUBUF-NEXT: {{  $}}
     ; MUBUF-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e64 0, $vgpr0, 0, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e64__fi_offset0__identity_vgpr__kernel
@@ -1437,19 +1402,16 @@ body:             |
     ; MUBUFW32-NEXT: {{  $}}
     ; MUBUFW32-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW32-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 0, $vgpr0, 0, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e64__fi_offset0__identity_vgpr__kernel
     ; FLATSCRW64: liveins: $vgpr0
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e64 0, $vgpr0, 0, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e64__fi_offset0__identity_vgpr__kernel
     ; FLATSCRW32: liveins: $vgpr0
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e64 0, $vgpr0, 0, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e64 %stack.0, $vgpr0, 0, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -1474,7 +1436,6 @@ body:             |
     ; MUBUF-NEXT: {{  $}}
     ; MUBUF-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 0, killed $vgpr0, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e32__fi_offset0__identity_vgpr__kernel_kill
@@ -1482,19 +1443,16 @@ body:             |
     ; MUBUFW32-NEXT: {{  $}}
     ; MUBUFW32-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW32-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 0, killed $vgpr0, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e32__fi_offset0__identity_vgpr__kernel_kill
     ; FLATSCRW64: liveins: $vgpr0
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 0, killed $vgpr0, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e32__fi_offset0__identity_vgpr__kernel_kill
     ; FLATSCRW32: liveins: $vgpr0
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 0, killed $vgpr0, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e32 %stack.0, killed $vgpr0, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -1521,8 +1479,7 @@ body:             |
     ; MUBUF-NEXT: {{  $}}
     ; MUBUF-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUF-NEXT: $vgpr1 = V_MOV_B32_e32 32, implicit $exec
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr0, killed $vgpr1, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 32, $vgpr0, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e32__identity_vgpr__fi_offset32__kernel
@@ -1530,22 +1487,19 @@ body:             |
     ; MUBUFW32-NEXT: {{  $}}
     ; MUBUFW32-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW32-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW32-NEXT: $vgpr1 = V_MOV_B32_e32 32, implicit $exec
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr0, killed $vgpr1, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 32, $vgpr0, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e32__identity_vgpr__fi_offset32__kernel
     ; FLATSCRW64: liveins: $vgpr0
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 32, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr0, killed $vgpr1, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 32, $vgpr0, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e32__identity_vgpr__fi_offset32__kernel
     ; FLATSCRW32: liveins: $vgpr0
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: $vgpr1 = V_MOV_B32_e32 32, implicit $exec
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr0, killed $vgpr1, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 32, $vgpr0, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e32 $vgpr0, %stack.1, implicit $exec
     SI_RETURN implicit $vgpr0
@@ -1573,8 +1527,7 @@ body:             |
     ; MUBUF-NEXT: {{  $}}
     ; MUBUF-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUF-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUF-NEXT: $vgpr1 = V_MOV_B32_e32 72, implicit $exec
-    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr0, killed $vgpr1, implicit $exec
+    ; MUBUF-NEXT: renamable $vgpr0 = V_ADD_U32_e32 72, $vgpr0, implicit $exec
     ; MUBUF-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; MUBUFW32-LABEL: name: v_add_u32_e32__identity_vgpr__fi_offset72__kernel
@@ -1582,22 +1535,19 @@ body:             |
     ; MUBUFW32-NEXT: {{  $}}
     ; MUBUFW32-NEXT: $sgpr0 = S_ADD_U32 $sgpr0, $noreg, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
     ; MUBUFW32-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
-    ; MUBUFW32-NEXT: $vgpr1 = V_MOV_B32_e32 72, implicit $exec
-    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr0, killed $vgpr1, implicit $exec
+    ; MUBUFW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 72, $vgpr0, implicit $exec
     ; MUBUFW32-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW64-LABEL: name: v_add_u32_e32__identity_vgpr__fi_offset72__kernel
     ; FLATSCRW64: liveins: $vgpr0
     ; FLATSCRW64-NEXT: {{  $}}
-    ; FLATSCRW64-NEXT: $vgpr1 = V_MOV_B32_e32 72, implicit $exec
-    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr0, killed $vgpr1, implicit $exec
+    ; FLATSCRW64-NEXT: renamable $vgpr0 = V_ADD_U32_e32 72, $vgpr0, implicit $exec
     ; FLATSCRW64-NEXT: SI_RETURN implicit $vgpr0
     ;
     ; FLATSCRW32-LABEL: name: v_add_u32_e32__identity_vgpr__fi_offset72__kernel
     ; FLATSCRW32: liveins: $vgpr0
     ; FLATSCRW32-NEXT: {{  $}}
-    ; FLATSCRW32-NEXT: $vgpr1 = V_MOV_B32_e32 72, implicit $exec
-    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 $vgpr0, killed $vgpr1, implicit $exec
+    ; FLATSCRW32-NEXT: renamable $vgpr0 = V_ADD_U32_e32 72, $vgpr0, implicit $exec
     ; FLATSCRW32-NEXT: SI_RETURN implicit $vgpr0
     renamable $vgpr0 = V_ADD_U32_e32 $vgpr0, %stack.1, implicit $exec
     SI_RETURN implicit $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/flat-scratch.ll b/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
index ef9590b3fd33f..3e4b43d9cfcd3 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
@@ -676,7 +676,7 @@ define amdgpu_kernel void @store_load_vindex_kernel() {
 ; GFX9-NEXT:    s_add_u32 flat_scratch_lo, s6, s11
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX9-NEXT:    s_addc_u32 flat_scratch_hi, s7, 0
-; GFX9-NEXT:    v_add_u32_e32 v1, 0, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 15
 ; GFX9-NEXT:    scratch_store_dword v1, v2, off
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -693,7 +693,7 @@ define amdgpu_kernel void @store_load_vindex_kernel() {
 ; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s7
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 15
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 0, v0
+; GFX10-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX10-NEXT:    v_sub_nc_u32_e32 v0, 0, v0
 ; GFX10-NEXT:    scratch_store_dword v1, v2, off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
@@ -731,7 +731,7 @@ define amdgpu_kernel void @store_load_vindex_kernel() {
 ; GFX9-PAL-NEXT:    s_mov_b32 s10, s0
 ; GFX9-PAL-NEXT:    s_load_dwordx2 s[10:11], s[10:11], 0x0
 ; GFX9-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-PAL-NEXT:    v_add_u32_e32 v1, 0, v0
+; GFX9-PAL-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v2, 15
 ; GFX9-PAL-NEXT:    v_sub_u32_e32 v0, 0, v0
 ; GFX9-PAL-NEXT:    s_waitcnt lgkmcnt(0)
@@ -769,7 +769,7 @@ define amdgpu_kernel void @store_load_vindex_kernel() {
 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s11
 ; GFX10-PAL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 15
-; GFX10-PAL-NEXT:    v_add_nc_u32_e32 v1, 0, v0
+; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX10-PAL-NEXT:    v_sub_nc_u32_e32 v0, 0, v0
 ; GFX10-PAL-NEXT:    scratch_store_dword v1, v2, off
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/fold-operands-frame-index.gfx10.mir b/llvm/test/CodeGen/AMDGPU/fold-operands-frame-index.gfx10.mir
new file mode 100644
index 0000000000000..0d6511cbfceb2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fold-operands-frame-index.gfx10.mir
@@ -0,0 +1,131 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs -run-pass=si-fold-operands %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass=si-fold-operands %s -o - | FileCheck %s
+
+---
+name:  fold_frame_index__v_add_u32_e32__const_v_fi
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fold_frame_index__v_add_u32_e32__const_v_fi
+    ; CHECK: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 128, [[V_MOV_B32_e32_]], implicit $exec
+    ; CHECK-NEXT: $vgpr0 = COPY [[V_ADD_U32_e32_]]
+    ; CHECK-NEXT: SI_RETURN implicit $vgpr0
+    %0:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    %1:vgpr_32 = V_ADD_U32_e32 128, %0, implicit $exec
+    $vgpr0 = COPY %1
+    SI_RETURN implicit $vgpr0
+...
+
+---
+name:  fold_frame_index__v_add_co_u32_e64__v_fi_const
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fold_frame_index__v_add_co_u32_e64__v_fi_const
+    ; CHECK: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    ; CHECK-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32 = V_ADD_CO_U32_e64 [[V_MOV_B32_e32_]], 128, 0, implicit $exec
+    ; CHECK-NEXT: $vgpr0 = COPY [[V_ADD_CO_U32_e64_]]
+    ; CHECK-NEXT: SI_RETURN implicit $vgpr0
+    %0:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    %1:vgpr_32, %2:sreg_32 = V_ADD_CO_U32_e64 %0, 128, 0, implicit $exec
+    $vgpr0 = COPY %1
+    SI_RETURN implicit $vgpr0
+...
+
+
+
+---
+name:  fold_frame_index__v_add_u32_e64__const_v_fi
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fold_frame_index__v_add_u32_e64__const_v_fi
+    ; CHECK: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    ; CHECK-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 128, [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; CHECK-NEXT: $sgpr4 = COPY [[V_ADD_U32_e64_]]
+    ; CHECK-NEXT: SI_RETURN implicit $sgpr4
+    %0:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    %1:vgpr_32 = V_ADD_U32_e64 128, %0, 0, implicit $exec
+    $sgpr4 = COPY %1
+    SI_RETURN implicit $sgpr4
+...
+
+---
+name:  fold_frame_index__v_add_u32_e64___v_fi_const
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fold_frame_index__v_add_u32_e64___v_fi_const
+    ; CHECK: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    ; CHECK-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[V_MOV_B32_e32_]], 128, 0, implicit $exec
+    ; CHECK-NEXT: $sgpr4 = COPY [[V_ADD_U32_e64_]]
+    ; CHECK-NEXT: SI_RETURN implicit $sgpr4
+    %0:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    %1:vgpr_32 = V_ADD_U32_e64 %0, 128, 0, implicit $exec
+    $sgpr4 = COPY %1
+    SI_RETURN implicit $sgpr4
+...
+
+---
+name:  fold_frame_index__v_add_co_u32_e64___fi_const_v
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fold_frame_index__v_add_co_u32_e64___fi_const_v
+    ; CHECK: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    ; CHECK-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32 = V_ADD_CO_U32_e64 128, [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; CHECK-NEXT: $vgpr0 = COPY [[V_ADD_CO_U32_e64_]]
+    ; CHECK-NEXT: SI_RETURN implicit $vgpr0
+    %0:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    %1:vgpr_32, %2:sreg_32 = V_ADD_CO_U32_e64 128, %0, 0, implicit $exec
+    $vgpr0 = COPY %1
+    SI_RETURN implicit $vgpr0
+...
+
+---
+name:  fold_frame_index__v_add_co_u32_e64__v_fi_imm
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fold_frame_index__v_add_co_u32_e64__v_fi_imm
+    ; CHECK: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32 = V_ADD_CO_U32_e64 %stack.0, 64, 0, implicit $exec
+    ; CHECK-NEXT: $vgpr0 = COPY [[V_ADD_CO_U32_e64_]]
+    ; CHECK-NEXT: SI_RETURN implicit $vgpr0
+    %0:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    %1:vgpr_32, %2:sreg_32 = V_ADD_CO_U32_e64 %0, 64, 0, implicit $exec
+    $vgpr0 = COPY %1
+    SI_RETURN implicit $vgpr0
+...
+
diff --git a/llvm/test/CodeGen/AMDGPU/fold-operands-frame-index.mir b/llvm/test/CodeGen/AMDGPU/fold-operands-frame-index.mir
new file mode 100644
index 0000000000000..aa91a4f9f988f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fold-operands-frame-index.mir
@@ -0,0 +1,446 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass=si-fold-operands %s -o - | FileCheck -check-prefixes=CHECK,GFX9 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -mattr=+wavefrontsize64 -verify-machineinstrs -run-pass=si-fold-operands %s -o - | FileCheck -check-prefixes=CHECK,GFX10 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -verify-machineinstrs -run-pass=si-fold-operands %s -o - | FileCheck -check-prefixes=CHECK,GFX12 %s
+
+---
+name:  fold_frame_index__s_add_i32__fi_const
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fold_frame_index__s_add_i32__fi_const
+    ; CHECK: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 %stack.0
+    ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_MOV_B32_]], 128, implicit-def $scc
+    ; CHECK-NEXT: $sgpr4 = COPY [[S_ADD_I32_]]
+    ; CHECK-NEXT: SI_RETURN implicit $sgpr4
+    %0:sreg_32 = S_MOV_B32 %stack.0
+    %1:sreg_32 = S_ADD_I32 %0, 128, implicit-def $scc
+    $sgpr4 = COPY %1
+    SI_RETURN implicit $sgpr4
+...
+
+---
+name:  fold_frame_index__s_add_i32__const_fi
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fold_frame_index__s_add_i32__const_fi
+    ; CHECK: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 %stack.0
+    ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 128, [[S_MOV_B32_]], implicit-def $scc
+    ; CHECK-NEXT: $sgpr4 = COPY [[S_ADD_I32_]]
+    ; CHECK-NEXT: SI_RETURN implicit $sgpr4
+    %0:sreg_32 = S_MOV_B32 %stack.0
+    %1:sreg_32 = S_ADD_I32 128, %0, implicit-def $scc
+    $sgpr4 = COPY %1
+    SI_RETURN implicit $sgpr4
+...
+
+---
+name:  fold_frame_index__s_add_i32__materializedconst_fi
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fold_frame_index__s_add_i32__materializedconst_fi
+    ; CHECK: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 %stack.0
+    ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 256, [[S_MOV_B32_]], implicit-def $scc
+    ; CHECK-NEXT: $sgpr4 = COPY [[S_ADD_I32_]]
+    ; CHECK-NEXT: SI_RETURN implicit $sgpr4
+    %0:sreg_32 = S_MOV_B32 256
+    %1:sreg_32 = S_MOV_B32 %stack.0
+    %2:sreg_32 = S_ADD_I32 %0, %1, implicit-def $scc
+    $sgpr4 = COPY %2
+    SI_RETURN implicit $sgpr4
+...
+
+---
+name:  fold_frame_index__s_add_i32__fi_materializedconst_0
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fold_frame_index__s_add_i32__fi_materializedconst_0
+    ; CHECK: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 256
+    ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 %stack.0, [[S_MOV_B32_]], implicit-def $scc
+    ; CHECK-NEXT: $sgpr4 = COPY [[S_ADD_I32_]]
+    ; CHECK-NEXT: SI_RETURN implicit $sgpr4
+    %0:sreg_32 = S_MOV_B32 %stack.0
+    %1:sreg_32 = S_MOV_B32 256
+    %2:sreg_32 = S_ADD_I32 %0, %1, implicit-def $scc
+    $sgpr4 = COPY %2
+    SI_RETURN implicit $sgpr4
+...
+
+
+---
+name:  fold_frame_index__s_add_i32__fi_materializedconst_1
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fold_frame_index__s_add_i32__fi_materializedconst_1
+    ; CHECK: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 %stack.0
+    ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 256, [[S_MOV_B32_]], implicit-def $scc
+    ; CHECK-NEXT: $sgpr4 = COPY [[S_ADD_I32_]]
+    ; CHECK-NEXT: SI_RETURN implicit $sgpr4
+    %0:sreg_32 = S_MOV_B32 256
+    %1:sreg_32 = S_MOV_B32 %stack.0
+    %2:sreg_32 = S_ADD_I32 %0, %1, implicit-def $scc
+    $sgpr4 = COPY %2
+    SI_RETURN implicit $sgpr4
+...
+
+---
+name:  fold_frame_index__s_add_i32__reg_fi
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    liveins: $sgpr4
+    ; CHECK-LABEL: name: fold_frame_index__s_add_i32__reg_fi
+    ; CHECK: liveins: $sgpr4
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr4
+    ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY]], %stack.0, implicit-def $scc
+    ; CHECK-NEXT: $sgpr4 = COPY [[S_ADD_I32_]]
+    ; CHECK-NEXT: SI_RETURN implicit $sgpr4
+    %0:sreg_32 = COPY $sgpr4
+    %1:sreg_32 = S_MOV_B32 %stack.0
+    %2:sreg_32 = S_ADD_I32 %0, %1, implicit-def $scc
+    $sgpr4 = COPY %2
+    SI_RETURN implicit $sgpr4
+...
+
+---
+name:  fold_frame_index__s_add_i32__fi_reg
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    liveins: $sgpr4
+    ; CHECK-LABEL: name: fold_frame_index__s_add_i32__fi_reg
+    ; CHECK: liveins: $sgpr4
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr4
+    ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 %stack.0, [[COPY]], implicit-def $scc
+    ; CHECK-NEXT: $sgpr4 = COPY [[S_ADD_I32_]]
+    ; CHECK-NEXT: SI_RETURN implicit $sgpr4
+    %0:sreg_32 = COPY $sgpr4
+    %1:sreg_32 = S_MOV_B32 %stack.0
+    %2:sreg_32 = S_ADD_I32 %1, %0, implicit-def $scc
+    $sgpr4 = COPY %2
+    SI_RETURN implicit $sgpr4
+...
+
+---
+name:  fold_frame_index__v_add_u32_e32__const_v_fi
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fold_frame_index__v_add_u32_e32__const_v_fi
+    ; CHECK: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 128, [[V_MOV_B32_e32_]], implicit $exec
+    ; CHECK-NEXT: $sgpr4 = COPY [[V_ADD_U32_e32_]]
+    ; CHECK-NEXT: SI_RETURN implicit $sgpr4
+    %0:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    %1:vgpr_32 = V_ADD_U32_e32 128, %0, implicit $exec
+    $sgpr4 = COPY %1
+    SI_RETURN implicit $sgpr4
+...
+
+---
+name:  fold_frame_index__v_add_u32_e32__materialized_v_const_v_fi
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fold_frame_index__v_add_u32_e32__materialized_v_const_v_fi
+    ; CHECK: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 128, implicit $exec
+    ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 %stack.0, [[V_MOV_B32_e32_]], implicit $exec
+    ; CHECK-NEXT: $sgpr4 = COPY [[V_ADD_U32_e32_]]
+    ; CHECK-NEXT: SI_RETURN implicit $sgpr4
+    %0:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    %1:vgpr_32 = V_MOV_B32_e32 128, implicit $exec
+    %2:vgpr_32 = V_ADD_U32_e32 %1, %0, implicit $exec
+    $sgpr4 = COPY %2
+    SI_RETURN implicit $sgpr4
+...
+
+---
+name:  fold_frame_index__v_add_u32_e64__imm_v_fi
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    ; GFX9-LABEL: name: fold_frame_index__v_add_u32_e64__imm_v_fi
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    ; GFX9-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 64, [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX9-NEXT: $sgpr4 = COPY [[V_ADD_U32_e64_]]
+    ; GFX9-NEXT: SI_RETURN implicit $sgpr4
+    ;
+    ; GFX10-LABEL: name: fold_frame_index__v_add_u32_e64__imm_v_fi
+    ; GFX10: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 64, %stack.0, 0, implicit $exec
+    ; GFX10-NEXT: $sgpr4 = COPY [[V_ADD_U32_e64_]]
+    ; GFX10-NEXT: SI_RETURN implicit $sgpr4
+    ;
+    ; GFX12-LABEL: name: fold_frame_index__v_add_u32_e64__imm_v_fi
+    ; GFX12: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 64, %stack.0, 0, implicit $exec
+    ; GFX12-NEXT: $sgpr4 = COPY [[V_ADD_U32_e64_]]
+    ; GFX12-NEXT: SI_RETURN implicit $sgpr4
+    %0:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    %1:vgpr_32 = V_ADD_U32_e64 64, %0, 0, implicit $exec
+    $sgpr4 = COPY %1
+    SI_RETURN implicit $sgpr4
+...
+
+---
+name:  fold_frame_index__v_add_u32_e64___v_fi_imm
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    ; GFX9-LABEL: name: fold_frame_index__v_add_u32_e64___v_fi_imm
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    ; GFX9-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[V_MOV_B32_e32_]], 64, 0, implicit $exec
+    ; GFX9-NEXT: $sgpr4 = COPY [[V_ADD_U32_e64_]]
+    ; GFX9-NEXT: SI_RETURN implicit $sgpr4
+    ;
+    ; GFX10-LABEL: name: fold_frame_index__v_add_u32_e64___v_fi_imm
+    ; GFX10: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 %stack.0, 64, 0, implicit $exec
+    ; GFX10-NEXT: $sgpr4 = COPY [[V_ADD_U32_e64_]]
+    ; GFX10-NEXT: SI_RETURN implicit $sgpr4
+    ;
+    ; GFX12-LABEL: name: fold_frame_index__v_add_u32_e64___v_fi_imm
+    ; GFX12: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 %stack.0, 64, 0, implicit $exec
+    ; GFX12-NEXT: $sgpr4 = COPY [[V_ADD_U32_e64_]]
+    ; GFX12-NEXT: SI_RETURN implicit $sgpr4
+    %0:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    %1:vgpr_32 = V_ADD_U32_e64 %0, 64, 0, implicit $exec
+    $sgpr4 = COPY %1
+    SI_RETURN implicit $sgpr4
+...
+
+---
+name:  fold_frame_index__v_add_co_u32_e32__const_v_fi
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fold_frame_index__v_add_co_u32_e32__const_v_fi
+    ; CHECK: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    ; CHECK-NEXT: [[V_ADD_CO_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_CO_U32_e32 128, [[V_MOV_B32_e32_]], implicit-def $vcc, implicit $exec
+    ; CHECK-NEXT: $vgpr0 = COPY [[V_ADD_CO_U32_e32_]]
+    ; CHECK-NEXT: SI_RETURN implicit $vgpr0
+    %0:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    %1:vgpr_32 = V_ADD_CO_U32_e32 128, %0, implicit-def $vcc, implicit $exec
+    $vgpr0 = COPY %1
+    SI_RETURN implicit $vgpr0
+...
+
+---
+name:  fold_frame_index__v_add_co_u32_e64__v_fi_imm
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    ; GFX9-LABEL: name: fold_frame_index__v_add_co_u32_e64__v_fi_imm
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    ; GFX9-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64 = V_ADD_CO_U32_e64 [[V_MOV_B32_e32_]], 64, 0, implicit $exec
+    ; GFX9-NEXT: $vgpr0 = COPY [[V_ADD_CO_U32_e64_]]
+    ; GFX9-NEXT: SI_RETURN implicit $vgpr0
+    ;
+    ; GFX10-LABEL: name: fold_frame_index__v_add_co_u32_e64__v_fi_imm
+    ; GFX10: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64 = V_ADD_CO_U32_e64 %stack.0, 64, 0, implicit $exec
+    ; GFX10-NEXT: $vgpr0 = COPY [[V_ADD_CO_U32_e64_]]
+    ; GFX10-NEXT: SI_RETURN implicit $vgpr0
+    ;
+    ; GFX12-LABEL: name: fold_frame_index__v_add_co_u32_e64__v_fi_imm
+    ; GFX12: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64 = V_ADD_CO_U32_e64 %stack.0, 64, 0, implicit $exec
+    ; GFX12-NEXT: $vgpr0 = COPY [[V_ADD_CO_U32_e64_]]
+    ; GFX12-NEXT: SI_RETURN implicit $vgpr0
+    %0:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    %1:vgpr_32, %2:sreg_64 = V_ADD_CO_U32_e64 %0, 64, 0, implicit $exec
+    $vgpr0 = COPY %1
+    SI_RETURN implicit $vgpr0
+...
+
+---
+name:  fold_frame_index__v_add_co_u32_e64__imm_v_fi
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    ; GFX9-LABEL: name: fold_frame_index__v_add_co_u32_e64__imm_v_fi
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    ; GFX9-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64 = V_ADD_CO_U32_e64 64, [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX9-NEXT: $vgpr0 = COPY [[V_ADD_CO_U32_e64_]]
+    ; GFX9-NEXT: SI_RETURN implicit $vgpr0
+    ;
+    ; GFX10-LABEL: name: fold_frame_index__v_add_co_u32_e64__imm_v_fi
+    ; GFX10: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64 = V_ADD_CO_U32_e64 64, %stack.0, 0, implicit $exec
+    ; GFX10-NEXT: $vgpr0 = COPY [[V_ADD_CO_U32_e64_]]
+    ; GFX10-NEXT: SI_RETURN implicit $vgpr0
+    ;
+    ; GFX12-LABEL: name: fold_frame_index__v_add_co_u32_e64__imm_v_fi
+    ; GFX12: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64 = V_ADD_CO_U32_e64 64, %stack.0, 0, implicit $exec
+    ; GFX12-NEXT: $vgpr0 = COPY [[V_ADD_CO_U32_e64_]]
+    ; GFX12-NEXT: SI_RETURN implicit $vgpr0
+    %0:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    %1:vgpr_32, %2:sreg_64 = V_ADD_CO_U32_e64 64, %0, 0, implicit $exec
+    $vgpr0 = COPY %1
+    SI_RETURN implicit $vgpr0
+...
+
+---
+name:  multi_use_scalar_fi__add_imm_add_inline_imm
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+
+    ; CHECK-LABEL: name: multi_use_scalar_fi__add_imm_add_inline_imm
+    ; CHECK: liveins: $sgpr0, $sgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+    ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 %stack.0
+    ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_MOV_B32_]], 16380, implicit-def dead $scc
+    ; CHECK-NEXT: [[S_ADD_I32_1:%[0-9]+]]:sreg_32 = S_ADD_I32 %stack.0, 56, implicit-def dead $scc
+    ; CHECK-NEXT: $sgpr4 = COPY [[S_ADD_I32_]]
+    ; CHECK-NEXT: $sgpr5 = COPY [[S_ADD_I32_1]]
+    ; CHECK-NEXT: SI_RETURN implicit $sgpr4, implicit $sgpr5
+    %0:sreg_32 = COPY $sgpr0
+    %1:sreg_32 = COPY $sgpr1
+    %2:sreg_32 = S_MOV_B32 16380
+    %3:sreg_32 = S_MOV_B32 56
+    %4:sreg_32 = S_MOV_B32 %stack.0
+    %5:sreg_32 = S_ADD_I32 %4, killed %2, implicit-def dead $scc
+    %6:sreg_32 = S_ADD_I32 %4, killed %3, implicit-def dead $scc
+    $sgpr4 = COPY %5
+    $sgpr5 = COPY %6
+    SI_RETURN implicit $sgpr4, implicit $sgpr5
+...
+
+---
+name:  multi_add_use_vector_fi__add_imm_add_inline_imm
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+  localFrameSize:  16384
+stack:
+  - { id: 0, size: 16384, alignment: 4, local-offset: 0 }
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; GFX9-LABEL: name: multi_add_use_vector_fi__add_imm_add_inline_imm
+    ; GFX9: liveins: $vgpr0, $vgpr1
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX9-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    ; GFX9-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[V_MOV_B32_e32_]], killed [[COPY1]], 0, implicit $exec
+    ; GFX9-NEXT: [[V_ADD_U32_e64_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[V_MOV_B32_e32_]], 56, 0, implicit $exec
+    ; GFX9-NEXT: $vgpr0 = COPY [[V_ADD_U32_e64_]]
+    ; GFX9-NEXT: $vgpr1 = COPY [[V_ADD_U32_e64_1]]
+    ; GFX9-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
+    ;
+    ; GFX10-LABEL: name: multi_add_use_vector_fi__add_imm_add_inline_imm
+    ; GFX10: liveins: $vgpr0, $vgpr1
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX10-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 %stack.0, killed [[COPY1]], 0, implicit $exec
+    ; GFX10-NEXT: [[V_ADD_U32_e64_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 %stack.0, 56, 0, implicit $exec
+    ; GFX10-NEXT: $vgpr0 = COPY [[V_ADD_U32_e64_]]
+    ; GFX10-NEXT: $vgpr1 = COPY [[V_ADD_U32_e64_1]]
+    ; GFX10-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
+    ;
+    ; GFX12-LABEL: name: multi_add_use_vector_fi__add_imm_add_inline_imm
+    ; GFX12: liveins: $vgpr0, $vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX12-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 %stack.0, killed [[COPY1]], 0, implicit $exec
+    ; GFX12-NEXT: [[V_ADD_U32_e64_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 %stack.0, 56, 0, implicit $exec
+    ; GFX12-NEXT: $vgpr0 = COPY [[V_ADD_U32_e64_]]
+    ; GFX12-NEXT: $vgpr1 = COPY [[V_ADD_U32_e64_1]]
+    ; GFX12-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+    %2:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    %3:vgpr_32 = V_ADD_U32_e64 %2, killed %1, 0, implicit $exec
+    %4:vgpr_32 = V_MOV_B32_e32 999, implicit $exec
+    %5:vgpr_32 = COPY %3
+    %6:sreg_32 = S_MOV_B32 56
+    %7:vgpr_32 = V_ADD_U32_e64 %2, killed %6, 0, implicit $exec
+    %8:vgpr_32 = COPY %7
+    $vgpr0 = COPY %3
+    $vgpr1 = COPY %7
+    SI_RETURN implicit $vgpr0, implicit $vgpr1
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/frame-index.mir b/llvm/test/CodeGen/AMDGPU/frame-index.mir
index 0309a156171d7..5815d2bfe7bf8 100644
--- a/llvm/test/CodeGen/AMDGPU/frame-index.mir
+++ b/llvm/test/CodeGen/AMDGPU/frame-index.mir
@@ -23,8 +23,8 @@ body:             |
     ; GCN-NEXT: {{  $}}
     ; GCN-NEXT: renamable $vgpr0 = V_AND_B32_e32 1023, killed $vgpr31, implicit $exec
     ; GCN-NEXT: renamable $vgpr0 = V_LSHLREV_B32_e32 2, killed $vgpr0, implicit $exec
-    ; GCN-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GCN-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 killed $vgpr1, killed $vgpr0, implicit-def dead $vcc, implicit $exec
+    ; GCN-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 killed $vgpr0, killed $vgpr1, implicit-def dead $vcc, implicit $exec
     ; GCN-NEXT: renamable $vgpr0, dead renamable $vcc = nuw V_ADD_CO_U32_e64 4, killed $vgpr0, 0, implicit $exec
     ; GCN-NEXT: $m0 = S_MOV_B32 -1
     ; GCN-NEXT: DS_WRITE_B32 undef renamable $vgpr0, killed renamable $vgpr0, 0, 0, implicit $m0, implicit $exec
@@ -206,8 +206,8 @@ body:             |
     ; GCN-NEXT: {{  $}}
     ; GCN-NEXT: renamable $vgpr0 = V_AND_B32_e32 1023, killed $vgpr31, implicit $exec
     ; GCN-NEXT: renamable $vgpr0 = V_LSHLREV_B32_e32 2, killed $vgpr0, implicit $exec
-    ; GCN-NEXT: $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
-    ; GCN-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 killed $vgpr1, killed $vgpr0, implicit-def dead $vcc, implicit $exec
+    ; GCN-NEXT: renamable $vgpr1 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GCN-NEXT: renamable $vgpr0 = V_ADD_CO_U32_e32 killed $vgpr0, killed $vgpr1, implicit-def dead $vcc, implicit $exec
     renamable $vgpr0 = V_AND_B32_e32 1023, killed $vgpr31, implicit $exec
     renamable $vgpr0 = V_LSHLREV_B32_e32 2, killed $vgpr0, implicit $exec
     renamable $vgpr0 = V_ADD_CO_U32_e32 %stack.0, killed $vgpr0, implicit-def dead $vcc, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll b/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll
index 9e3264eb9c07f..d3a6b4e01ebfb 100644
--- a/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll
@@ -358,7 +358,7 @@ declare void @external() #0
 ; GCN:	.set multi_call_with_external.num_vgpr, max(41, amdgpu.max_num_vgpr)
 ; GCN:	.set multi_call_with_external.num_agpr, max(0, amdgpu.max_num_agpr)
 ; GCN:	.set multi_call_with_external.numbered_sgpr, max(42, amdgpu.max_num_sgpr)
-; GCN:	.set multi_call_with_external.private_seg_size, 0
+; GCN:	.set multi_call_with_external.private_seg_size, 0+(max(use_stack0.private_seg_size, use_stack1.private_seg_size))
 ; GCN:	.set multi_call_with_external.uses_vcc, 1
 ; GCN:	.set multi_call_with_external.uses_flat_scratch, 1
 ; GCN:	.set multi_call_with_external.has_dyn_sized_stack, 1
@@ -366,7 +366,7 @@ declare void @external() #0
 ; GCN:	.set multi_call_with_external.has_indirect_call, 1
 ; GCN: TotalNumSgprs: multi_call_with_external.numbered_sgpr+6
 ; GCN: NumVgprs: multi_call_with_external.num_vgpr
-; GCN: ScratchSize: 0
+; GCN: ScratchSize: 2052
 define amdgpu_kernel void @multi_call_with_external() #0 {
   call void @use_stack0()
   call void @use_stack1()
@@ -374,6 +374,29 @@ define amdgpu_kernel void @multi_call_with_external() #0 {
   ret void
 }
 
+; GCN-LABEL: {{^}}multi_call_with_external_and_duplicates:
+; GCN:	.set multi_call_with_external_and_duplicates.num_vgpr, max(41, amdgpu.max_num_vgpr)
+; GCN:	.set multi_call_with_external_and_duplicates.num_agpr, max(0, amdgpu.max_num_agpr)
+; GCN:	.set multi_call_with_external_and_duplicates.numbered_sgpr, max(44, amdgpu.max_num_sgpr)
+; GCN:	.set multi_call_with_external_and_duplicates.private_seg_size, 0+(max(use_stack0.private_seg_size, use_stack1.private_seg_size))
+; GCN:	.set multi_call_with_external_and_duplicates.uses_vcc, 1
+; GCN:	.set multi_call_with_external_and_duplicates.uses_flat_scratch, 1
+; GCN:	.set multi_call_with_external_and_duplicates.has_dyn_sized_stack, 1
+; GCN:	.set multi_call_with_external_and_duplicates.has_recursion, 0
+; GCN:	.set multi_call_with_external_and_duplicates.has_indirect_call, 1
+; GCN: TotalNumSgprs: multi_call_with_external_and_duplicates.numbered_sgpr+6
+; GCN: NumVgprs: multi_call_with_external_and_duplicates.num_vgpr
+; GCN: ScratchSize: 2052
+define amdgpu_kernel void @multi_call_with_external_and_duplicates() #0 {
+  call void @use_stack0()
+  call void @use_stack0()
+  call void @use_stack1()
+  call void @use_stack1()
+  call void @external()
+  call void @external()
+  ret void
+}
+
 ; GCN-LABEL: {{^}}usage_external:
 ; GCN:	.set usage_external.num_vgpr, max(32, amdgpu.max_num_vgpr)
 ; GCN:	.set usage_external.num_agpr, max(0, amdgpu.max_num_agpr)
diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w32.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w32.ll
index 05fd883768480..469d0453b9dfb 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w32.ll
@@ -73,10 +73,16 @@ define amdgpu_cs_chain void @chain_to_chain(<3 x i32> inreg %sgpr, { i32, ptr ad
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
-  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY1]]
@@ -97,12 +103,18 @@ define amdgpu_cs_chain void @chain_to_chain(<3 x i32> inreg %sgpr, { i32, ptr ad
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
-  ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
-  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY7]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY10]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY1]]
@@ -177,10 +189,16 @@ define amdgpu_cs void @cs_to_chain(<3 x i32> inreg %sgpr, { i32, ptr addrspace(5
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
-  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY1]]
@@ -201,12 +219,18 @@ define amdgpu_cs void @cs_to_chain(<3 x i32> inreg %sgpr, { i32, ptr addrspace(5
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
-  ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_128 = COPY $sgpr100_sgpr101_sgpr102_sgpr103
+  ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:sgpr_128 = COPY $sgpr100_sgpr101_sgpr102_sgpr103
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
-  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY7]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY10]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY1]]
@@ -281,10 +305,16 @@ define amdgpu_cs_chain void @chain_to_chain_preserve(<3 x i32> inreg %sgpr, { i3
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee_preserve
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee_preserve
   ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
-  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY1]]
@@ -305,12 +335,18 @@ define amdgpu_cs_chain void @chain_to_chain_preserve(<3 x i32> inreg %sgpr, { i3
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee_preserve
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee_preserve
   ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
-  ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
-  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY7]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY10]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY1]]
@@ -385,10 +421,16 @@ define amdgpu_cs void @cs_to_chain_preserve(<3 x i32> inreg %sgpr, { i32, ptr ad
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee_preserve
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee_preserve
   ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
-  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY1]]
@@ -409,12 +451,18 @@ define amdgpu_cs void @cs_to_chain_preserve(<3 x i32> inreg %sgpr, { i32, ptr ad
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee_preserve
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee_preserve
   ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
-  ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_128 = COPY $sgpr100_sgpr101_sgpr102_sgpr103
+  ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:sgpr_128 = COPY $sgpr100_sgpr101_sgpr102_sgpr103
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
-  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY7]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY10]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY1]]
@@ -486,16 +534,29 @@ define amdgpu_cs_chain void @indirect(ptr inreg %callee, <3 x i32> inreg %sgpr,
   ; DAGISEL-GFX11-NEXT:   [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr2
   ; DAGISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr1
   ; DAGISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:sgpr_32 = COPY $sgpr0
-  ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY10]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY11:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
+  ; DAGISEL-GFX11-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY12]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_1]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY13]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY14]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY15]], implicit $exec
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
-  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_2]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_3]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_4]]
   ; DAGISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY1]]
   ; DAGISEL-GFX11-NEXT:   $vgpr11 = COPY [[COPY]]
-  ; DAGISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W32 killed [[REG_SEQUENCE]], 0, 0, killed [[S_MOV_B32_]], amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
+  ; DAGISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W32 killed [[REG_SEQUENCE1]], 0, 0, killed [[S_MOV_B32_]], amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
   ;
   ; DAGISEL-GFX10-LABEL: name: indirect
   ; DAGISEL-GFX10: bb.0 (%ir-block.0):
@@ -510,18 +571,31 @@ define amdgpu_cs_chain void @indirect(ptr inreg %callee, <3 x i32> inreg %sgpr,
   ; DAGISEL-GFX10-NEXT:   [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr2
   ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr1
   ; DAGISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:sgpr_32 = COPY $sgpr0
-  ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; DAGISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; DAGISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
+  ; DAGISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY10]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
+  ; DAGISEL-GFX10-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY12]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_1]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_]], %subreg.sub1
+  ; DAGISEL-GFX10-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY13]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY14]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY15]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY16:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
-  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY9]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY16]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_2]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_3]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_4]]
   ; DAGISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY1]]
   ; DAGISEL-GFX10-NEXT:   $vgpr11 = COPY [[COPY]]
-  ; DAGISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W32 killed [[REG_SEQUENCE]], 0, 0, killed [[S_MOV_B32_]], amdgpu_allvgprs, implicit $sgpr48_sgpr49_sgpr50_sgpr51, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
+  ; DAGISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W32 killed [[REG_SEQUENCE1]], 0, 0, killed [[S_MOV_B32_]], amdgpu_allvgprs, implicit $sgpr48_sgpr49_sgpr50_sgpr51, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
   call void(ptr, i32, <3 x i32>, { i32, ptr addrspace(5), i32, i32 }, i32, ...) @llvm.amdgcn.cs.chain(ptr %callee, i32 -1, <3 x i32> inreg %sgpr, { i32, ptr addrspace(5), i32, i32 } %vgpr, i32 0)
   unreachable
 }
@@ -594,9 +668,15 @@ define amdgpu_cs_chain void @non_imm_exec(i32 inreg %exec, <3 x i32> inreg %sgpr
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
-  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY1]]
@@ -618,11 +698,17 @@ define amdgpu_cs_chain void @non_imm_exec(i32 inreg %exec, <3 x i32> inreg %sgpr
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
-  ; DAGISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
-  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY8]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY11]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY1]]
@@ -697,15 +783,28 @@ define amdgpu_cs_chain void @indirect_with_non_imm_exec(ptr inreg %callee, i32 i
   ; DAGISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr2
   ; DAGISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:sgpr_32 = COPY $sgpr1
   ; DAGISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:sgpr_32 = COPY $sgpr0
-  ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY8]], %subreg.sub1
-  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY8]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY10:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY10]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY11]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY12:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
+  ; DAGISEL-GFX11-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY12]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY13]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_1]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY14]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY15]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY16]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_2]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_3]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_4]]
   ; DAGISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY1]]
   ; DAGISEL-GFX11-NEXT:   $vgpr11 = COPY [[COPY]]
-  ; DAGISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W32 killed [[REG_SEQUENCE]], 0, 0, [[COPY7]], amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
+  ; DAGISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W32 killed [[REG_SEQUENCE1]], 0, 0, [[COPY7]], amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
   ;
   ; DAGISEL-GFX10-LABEL: name: indirect_with_non_imm_exec
   ; DAGISEL-GFX10: bb.0 (%ir-block.0):
@@ -721,17 +820,30 @@ define amdgpu_cs_chain void @indirect_with_non_imm_exec(ptr inreg %callee, i32 i
   ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr2
   ; DAGISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:sgpr_32 = COPY $sgpr1
   ; DAGISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:sgpr_32 = COPY $sgpr0
-  ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY8]], %subreg.sub1
-  ; DAGISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
-  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY10]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY8]], %subreg.sub1
+  ; DAGISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
+  ; DAGISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY10]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY11]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY12:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
+  ; DAGISEL-GFX10-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY12]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY13]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_1]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_]], %subreg.sub1
+  ; DAGISEL-GFX10-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY14]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY15]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY16]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY17:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY17]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_2]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_3]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_4]]
   ; DAGISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY1]]
   ; DAGISEL-GFX10-NEXT:   $vgpr11 = COPY [[COPY]]
-  ; DAGISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W32 killed [[REG_SEQUENCE]], 0, 0, [[COPY7]], amdgpu_allvgprs, implicit $sgpr48_sgpr49_sgpr50_sgpr51, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
+  ; DAGISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W32 killed [[REG_SEQUENCE1]], 0, 0, [[COPY7]], amdgpu_allvgprs, implicit $sgpr48_sgpr49_sgpr50_sgpr51, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
   call void(ptr, i32, <3 x i32>, { i32, ptr addrspace(5), i32, i32 }, i32, ...) @llvm.amdgcn.cs.chain(ptr %callee, i32 %exec, <3 x i32> inreg %sgpr, { i32, ptr addrspace(5), i32, i32 } %vgpr, i32 0)
   unreachable
 }
diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w64.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w64.ll
index 28492f1426921..51c28a02b7f82 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-amdgcn-cs-chain-intrinsic-w64.ll
@@ -73,10 +73,16 @@ define amdgpu_cs_chain void @chain_to_chain(<3 x i32> inreg %sgpr, { i32, ptr ad
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
-  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY1]]
@@ -97,12 +103,18 @@ define amdgpu_cs_chain void @chain_to_chain(<3 x i32> inreg %sgpr, { i32, ptr ad
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
-  ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
-  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY7]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY10]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY1]]
@@ -177,10 +189,16 @@ define amdgpu_cs void @cs_to_chain(<3 x i32> inreg %sgpr, { i32, ptr addrspace(5
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
-  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY1]]
@@ -201,12 +219,18 @@ define amdgpu_cs void @cs_to_chain(<3 x i32> inreg %sgpr, { i32, ptr addrspace(5
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
-  ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_128 = COPY $sgpr100_sgpr101_sgpr102_sgpr103
+  ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:sgpr_128 = COPY $sgpr100_sgpr101_sgpr102_sgpr103
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
-  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY7]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY10]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY1]]
@@ -281,10 +305,16 @@ define amdgpu_cs_chain void @chain_to_chain_preserve(<3 x i32> inreg %sgpr, { i3
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee_preserve
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee_preserve
   ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
-  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY1]]
@@ -305,12 +335,18 @@ define amdgpu_cs_chain void @chain_to_chain_preserve(<3 x i32> inreg %sgpr, { i3
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee_preserve
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee_preserve
   ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
-  ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
-  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY7]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY10]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY1]]
@@ -385,10 +421,16 @@ define amdgpu_cs void @cs_to_chain_preserve(<3 x i32> inreg %sgpr, { i32, ptr ad
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee_preserve
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee_preserve
   ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
-  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY1]]
@@ -409,12 +451,18 @@ define amdgpu_cs void @cs_to_chain_preserve(<3 x i32> inreg %sgpr, { i32, ptr ad
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee_preserve
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee_preserve
   ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
-  ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_128 = COPY $sgpr100_sgpr101_sgpr102_sgpr103
+  ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:sgpr_128 = COPY $sgpr100_sgpr101_sgpr102_sgpr103
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
-  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY7]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY10]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY1]]
@@ -486,16 +534,29 @@ define amdgpu_cs_chain void @indirect(ptr inreg %callee, <3 x i32> inreg %sgpr,
   ; DAGISEL-GFX11-NEXT:   [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr2
   ; DAGISEL-GFX11-NEXT:   [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr1
   ; DAGISEL-GFX11-NEXT:   [[COPY8:%[0-9]+]]:sgpr_32 = COPY $sgpr0
-  ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY10]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY11:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
+  ; DAGISEL-GFX11-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY12]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_1]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY13]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY14]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY15]], implicit $exec
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
-  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_2]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_3]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_4]]
   ; DAGISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY1]]
   ; DAGISEL-GFX11-NEXT:   $vgpr11 = COPY [[COPY]]
-  ; DAGISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W64 killed [[REG_SEQUENCE]], 0, 0, killed [[S_MOV_B64_]], amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
+  ; DAGISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W64 killed [[REG_SEQUENCE1]], 0, 0, killed [[S_MOV_B64_]], amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
   ;
   ; DAGISEL-GFX10-LABEL: name: indirect
   ; DAGISEL-GFX10: bb.0 (%ir-block.0):
@@ -510,18 +571,31 @@ define amdgpu_cs_chain void @indirect(ptr inreg %callee, <3 x i32> inreg %sgpr,
   ; DAGISEL-GFX10-NEXT:   [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr2
   ; DAGISEL-GFX10-NEXT:   [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr1
   ; DAGISEL-GFX10-NEXT:   [[COPY8:%[0-9]+]]:sgpr_32 = COPY $sgpr0
-  ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; DAGISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; DAGISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
+  ; DAGISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY10]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
+  ; DAGISEL-GFX10-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY12]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_1]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_]], %subreg.sub1
+  ; DAGISEL-GFX10-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY13]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY14]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY15]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY16:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
-  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY9]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY16]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_2]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_3]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_4]]
   ; DAGISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY1]]
   ; DAGISEL-GFX10-NEXT:   $vgpr11 = COPY [[COPY]]
-  ; DAGISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W64 killed [[REG_SEQUENCE]], 0, 0, killed [[S_MOV_B64_]], amdgpu_allvgprs, implicit $sgpr48_sgpr49_sgpr50_sgpr51, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
+  ; DAGISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W64 killed [[REG_SEQUENCE1]], 0, 0, killed [[S_MOV_B64_]], amdgpu_allvgprs, implicit $sgpr48_sgpr49_sgpr50_sgpr51, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
   call void(ptr, i64, <3 x i32>, { i32, ptr addrspace(5), i32, i32 }, i32, ...) @llvm.amdgcn.cs.chain(ptr %callee, i64 -1, <3 x i32> inreg %sgpr, { i32, ptr addrspace(5), i32, i32 } %vgpr, i32 0)
   unreachable
 }
@@ -600,9 +674,15 @@ define amdgpu_cs_chain void @non_imm_exec(i64 inreg %exec, <3 x i32> inreg %sgpr
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; DAGISEL-GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
-  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY1]]
@@ -626,11 +706,17 @@ define amdgpu_cs_chain void @non_imm_exec(i64 inreg %exec, <3 x i32> inreg %sgpr
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
   ; DAGISEL-GFX10-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
   ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
-  ; DAGISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
-  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY9]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY12:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY12]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY1]]
@@ -711,15 +797,28 @@ define amdgpu_cs_chain void @indirect_with_non_imm_exec(ptr inreg %callee, i64 i
   ; DAGISEL-GFX11-NEXT:   [[COPY9:%[0-9]+]]:sgpr_32 = COPY $sgpr1
   ; DAGISEL-GFX11-NEXT:   [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE [[COPY10]], %subreg.sub0, [[COPY9]], %subreg.sub1
-  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY10]], %subreg.sub0, [[COPY9]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY11:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE1]].sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY12]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY13:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE1]].sub0
+  ; DAGISEL-GFX11-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY13]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY14]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_1]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_]], %subreg.sub1
+  ; DAGISEL-GFX11-NEXT:   [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY15]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY16]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX11-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY17]], implicit $exec
+  ; DAGISEL-GFX11-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_2]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_3]]
+  ; DAGISEL-GFX11-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_4]]
   ; DAGISEL-GFX11-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX11-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX11-NEXT:   $vgpr10 = COPY [[COPY1]]
   ; DAGISEL-GFX11-NEXT:   $vgpr11 = COPY [[COPY]]
-  ; DAGISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W64 killed [[REG_SEQUENCE1]], 0, 0, killed [[REG_SEQUENCE]], amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
+  ; DAGISEL-GFX11-NEXT:   SI_CS_CHAIN_TC_W64 killed [[REG_SEQUENCE2]], 0, 0, killed [[REG_SEQUENCE]], amdgpu_allvgprs, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
   ;
   ; DAGISEL-GFX10-LABEL: name: indirect_with_non_imm_exec
   ; DAGISEL-GFX10: bb.0 (%ir-block.0):
@@ -737,17 +836,30 @@ define amdgpu_cs_chain void @indirect_with_non_imm_exec(ptr inreg %callee, i64 i
   ; DAGISEL-GFX10-NEXT:   [[COPY9:%[0-9]+]]:sgpr_32 = COPY $sgpr1
   ; DAGISEL-GFX10-NEXT:   [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE [[COPY10]], %subreg.sub0, [[COPY9]], %subreg.sub1
-  ; DAGISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
-  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY11]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[COPY6]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[COPY5]]
-  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY10]], %subreg.sub0, [[COPY9]], %subreg.sub1
+  ; DAGISEL-GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE1]].sub1
+  ; DAGISEL-GFX10-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY12]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY13:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE1]].sub0
+  ; DAGISEL-GFX10-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY13]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY14]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_1]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_]], %subreg.sub1
+  ; DAGISEL-GFX10-NEXT:   [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY15]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY16]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; DAGISEL-GFX10-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY17]], implicit $exec
+  ; DAGISEL-GFX10-NEXT:   [[COPY18:%[0-9]+]]:sgpr_128 = COPY $sgpr48_sgpr49_sgpr50_sgpr51
+  ; DAGISEL-GFX10-NEXT:   $sgpr48_sgpr49_sgpr50_sgpr51 = COPY [[COPY18]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_2]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_3]]
+  ; DAGISEL-GFX10-NEXT:   $sgpr2 = COPY [[V_READFIRSTLANE_B32_4]]
   ; DAGISEL-GFX10-NEXT:   $vgpr8 = COPY [[COPY3]]
   ; DAGISEL-GFX10-NEXT:   $vgpr9 = COPY [[COPY2]]
   ; DAGISEL-GFX10-NEXT:   $vgpr10 = COPY [[COPY1]]
   ; DAGISEL-GFX10-NEXT:   $vgpr11 = COPY [[COPY]]
-  ; DAGISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W64 killed [[REG_SEQUENCE1]], 0, 0, killed [[REG_SEQUENCE]], amdgpu_allvgprs, implicit $sgpr48_sgpr49_sgpr50_sgpr51, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
+  ; DAGISEL-GFX10-NEXT:   SI_CS_CHAIN_TC_W64 killed [[REG_SEQUENCE2]], 0, 0, killed [[REG_SEQUENCE]], amdgpu_allvgprs, implicit $sgpr48_sgpr49_sgpr50_sgpr51, implicit $sgpr0, implicit $sgpr1, implicit $sgpr2, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11
   call void(ptr, i64, <3 x i32>, { i32, ptr addrspace(5), i32, i32 }, i32, ...) @llvm.amdgcn.cs.chain(ptr %callee, i64 %exec, <3 x i32> inreg %sgpr, { i32, ptr addrspace(5), i32, i32 } %vgpr, i32 0)
   unreachable
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index b192fdec15739..646b1264f5dea 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -366,10 +366,12 @@
 ; GCN-O1-NEXT:        Machine Optimization Remark Emitter
 ; GCN-O1-NEXT:        Greedy Register Allocator
 ; GCN-O1-NEXT:        Virtual Register Rewriter
+; GCN-O1-NEXT:        Stack Slot Coloring
 ; GCN-O1-NEXT:        SI lower SGPR spill instructions
 ; GCN-O1-NEXT:        Virtual Register Map
 ; GCN-O1-NEXT:        Live Register Matrix
 ; GCN-O1-NEXT:        SI Pre-allocate WWM Registers
+; GCN-O1-NEXT:        Live Stack Slot Analysis
 ; GCN-O1-NEXT:        Greedy Register Allocator
 ; GCN-O1-NEXT:        SI Lower WWM Copies
 ; GCN-O1-NEXT:        Virtual Register Rewriter
@@ -674,10 +676,12 @@
 ; GCN-O1-OPTS-NEXT:        Machine Optimization Remark Emitter
 ; GCN-O1-OPTS-NEXT:        Greedy Register Allocator
 ; GCN-O1-OPTS-NEXT:        Virtual Register Rewriter
+; GCN-O1-OPTS-NEXT:        Stack Slot Coloring
 ; GCN-O1-OPTS-NEXT:        SI lower SGPR spill instructions
 ; GCN-O1-OPTS-NEXT:        Virtual Register Map
 ; GCN-O1-OPTS-NEXT:        Live Register Matrix
 ; GCN-O1-OPTS-NEXT:        SI Pre-allocate WWM Registers
+; GCN-O1-OPTS-NEXT:        Live Stack Slot Analysis
 ; GCN-O1-OPTS-NEXT:        Greedy Register Allocator
 ; GCN-O1-OPTS-NEXT:        SI Lower WWM Copies
 ; GCN-O1-OPTS-NEXT:        Virtual Register Rewriter
@@ -988,10 +992,12 @@
 ; GCN-O2-NEXT:        Machine Optimization Remark Emitter
 ; GCN-O2-NEXT:        Greedy Register Allocator
 ; GCN-O2-NEXT:        Virtual Register Rewriter
+; GCN-O2-NEXT:        Stack Slot Coloring
 ; GCN-O2-NEXT:        SI lower SGPR spill instructions
 ; GCN-O2-NEXT:        Virtual Register Map
 ; GCN-O2-NEXT:        Live Register Matrix
 ; GCN-O2-NEXT:        SI Pre-allocate WWM Registers
+; GCN-O2-NEXT:        Live Stack Slot Analysis
 ; GCN-O2-NEXT:        Greedy Register Allocator
 ; GCN-O2-NEXT:        SI Lower WWM Copies
 ; GCN-O2-NEXT:        Virtual Register Rewriter
@@ -1314,10 +1320,12 @@
 ; GCN-O3-NEXT:        Machine Optimization Remark Emitter
 ; GCN-O3-NEXT:        Greedy Register Allocator
 ; GCN-O3-NEXT:        Virtual Register Rewriter
+; GCN-O3-NEXT:        Stack Slot Coloring
 ; GCN-O3-NEXT:        SI lower SGPR spill instructions
 ; GCN-O3-NEXT:        Virtual Register Map
 ; GCN-O3-NEXT:        Live Register Matrix
 ; GCN-O3-NEXT:        SI Pre-allocate WWM Registers
+; GCN-O3-NEXT:        Live Stack Slot Analysis
 ; GCN-O3-NEXT:        Greedy Register Allocator
 ; GCN-O3-NEXT:        SI Lower WWM Copies
 ; GCN-O3-NEXT:        Virtual Register Rewriter
diff --git a/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.gfx10.ll b/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.gfx10.ll
index 8a789a4c6cda9..e86ef52e413b6 100644
--- a/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.gfx10.ll
+++ b/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.gfx10.ll
@@ -1651,13 +1651,13 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offse
 ; GFX10_1-NEXT:    s_lshl_b32 s4, s6, 2
 ; GFX10_1-NEXT:    v_writelane_b32 v2, s59, 0
 ; GFX10_1-NEXT:    v_lshrrev_b32_e64 v0, 5, s32
-; GFX10_1-NEXT:    v_add_nc_u32_e32 v3, 0x4040, v3
+; GFX10_1-NEXT:    v_add_nc_u32_e32 v1, s4, v3
+; GFX10_1-NEXT:    s_and_b32 s4, 0, exec_lo
 ; GFX10_1-NEXT:    v_add_nc_u32_e32 v0, 64, v0
 ; GFX10_1-NEXT:    ;;#ASMSTART
 ; GFX10_1-NEXT:    ; use alloca0 v0
 ; GFX10_1-NEXT:    ;;#ASMEND
-; GFX10_1-NEXT:    v_add_nc_u32_e32 v1, s4, v3
-; GFX10_1-NEXT:    s_and_b32 s4, 0, exec_lo
+; GFX10_1-NEXT:    v_add_nc_u32_e32 v1, 0x4040, v1
 ; GFX10_1-NEXT:    v_readfirstlane_b32 s59, v1
 ; GFX10_1-NEXT:    ;;#ASMSTART
 ; GFX10_1-NEXT:    ; use s59, scc
@@ -1682,13 +1682,13 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offse
 ; GFX10_3-NEXT:    s_lshl_b32 s4, s6, 2
 ; GFX10_3-NEXT:    v_writelane_b32 v2, s59, 0
 ; GFX10_3-NEXT:    v_lshrrev_b32_e64 v0, 5, s32
-; GFX10_3-NEXT:    v_add_nc_u32_e32 v3, 0x4040, v3
+; GFX10_3-NEXT:    v_add_nc_u32_e32 v1, s4, v3
+; GFX10_3-NEXT:    s_and_b32 s4, 0, exec_lo
 ; GFX10_3-NEXT:    v_add_nc_u32_e32 v0, 64, v0
 ; GFX10_3-NEXT:    ;;#ASMSTART
 ; GFX10_3-NEXT:    ; use alloca0 v0
 ; GFX10_3-NEXT:    ;;#ASMEND
-; GFX10_3-NEXT:    v_add_nc_u32_e32 v1, s4, v3
-; GFX10_3-NEXT:    s_and_b32 s4, 0, exec_lo
+; GFX10_3-NEXT:    v_add_nc_u32_e32 v1, 0x4040, v1
 ; GFX10_3-NEXT:    v_readfirstlane_b32 s59, v1
 ; GFX10_3-NEXT:    ;;#ASMSTART
 ; GFX10_3-NEXT:    ; use s59, scc
@@ -1710,15 +1710,15 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offse
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
 ; GFX11-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX11-NEXT:    s_add_i32 s1, s32, 64
-; GFX11-NEXT:    v_writelane_b32 v2, s59, 0
+; GFX11-NEXT:    v_add_nc_u32_e64 v1, s0, s32
 ; GFX11-NEXT:    v_mov_b32_e32 v0, s1
-; GFX11-NEXT:    s_add_i32 s1, s32, 0x4040
+; GFX11-NEXT:    v_writelane_b32 v2, s59, 0
+; GFX11-NEXT:    s_and_b32 s0, 0, exec_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_nc_u32_e32 v1, 0x4040, v1
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use alloca0 v0
 ; GFX11-NEXT:    ;;#ASMEND
-; GFX11-NEXT:    v_add_nc_u32_e64 v1, s0, s1
-; GFX11-NEXT:    s_and_b32 s0, 0, exec_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readfirstlane_b32 s59, v1
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use s59, scc
@@ -1743,16 +1743,16 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offse
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_mov_b32 exec_lo, s1
 ; GFX12-NEXT:    s_lshl_b32 s0, s0, 2
-; GFX12-NEXT:    s_add_co_i32 s1, s32, 0x4000
 ; GFX12-NEXT:    v_writelane_b32 v2, s59, 0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    v_add_nc_u32_e64 v1, s0, s1
+; GFX12-NEXT:    v_add_nc_u32_e64 v1, s0, s32
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s32
 ; GFX12-NEXT:    s_and_b32 s0, 0, exec_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_add_nc_u32_e32 v1, 0x4000, v1
 ; GFX12-NEXT:    ;;#ASMSTART
 ; GFX12-NEXT:    ; use alloca0 v0
 ; GFX12-NEXT:    ;;#ASMEND
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_readfirstlane_b32 s59, v1
 ; GFX12-NEXT:    ;;#ASMSTART
 ; GFX12-NEXT:    ; use s59, scc
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-offset-private.ll b/llvm/test/CodeGen/AMDGPU/mubuf-offset-private.ll
index 4aed9dc2fca6c..268ac4db000bd 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-offset-private.ll
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-offset-private.ll
@@ -137,11 +137,15 @@ define amdgpu_kernel void @store_private_offset_i8_max_offset_plus2() #0 {
 ; so a possibly negative base index can't be used for the vgpr offset.
 
 ; GCN-LABEL: {{^}}store_private_unknown_bits_vaddr:
-; SICIVI: v_add_{{i|u}}32_e32 [[ADDR0:v[0-9]+]], vcc, 0
-; SICIVI: v_add_{{i|u}}32_e32 [[ADDR1:v[0-9]+]], vcc, 32, [[ADDR0]]
+; SICIVI: {{buffer|flat}}_load_dword [[VADDR:v[0-9]+]],
+; SICIVI: v_lshlrev_b32_e32 [[ADDR:v[0-9]+]], 2, [[VADDR]]
+; SICIVI-NOT [[ADDR]]
+; SICIVI: v_add_{{i|u}}32_e32 [[ADDR1:v[0-9]+]], vcc, 32, [[ADDR]]
 ; SICIVI: buffer_store_dword v{{[0-9]+}}, [[ADDR1]], s{{\[[0-9]+:[0-9]+\]}}, 0 offen{{$}}
 
-; GFX9: v_add_u32_e32 [[ADDR:v[0-9]+]], 0,
+; GFX9: global_load_dword [[VADDR:v[0-9]+]],
+; GFX9: v_lshlrev_b32_e32 [[ADDR:v[0-9]+]], 2, [[VADDR]]
+; GFX9-NOT [[ADDR]]
 ; GFX9: buffer_store_dword v{{[0-9]+}}, [[ADDR]], s{{\[[0-9]+:[0-9]+\]}}, 0 offen offset:32
 define amdgpu_kernel void @store_private_unknown_bits_vaddr() #0 {
   %alloca = alloca [16 x i32], align 4, addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll b/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll
index 924340ec8a2a6..5dd2dfe9a77d1 100644
--- a/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll
+++ b/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll
@@ -196,39 +196,39 @@ define void @preserve_wwm_copy_dstreg(ptr %parg0, ptr %parg1, ptr %parg2) #0 {
 ; GFX906-NEXT:    ;;#ASMSTART
 ; GFX906-NEXT:    ; def s21
 ; GFX906-NEXT:    ;;#ASMEND
-; GFX906-NEXT:    v_writelane_b32 v39, s21, 24
+; GFX906-NEXT:    v_writelane_b32 v39, s21, 12
 ; GFX906-NEXT:    ;;#ASMSTART
 ; GFX906-NEXT:    ; def s22
 ; GFX906-NEXT:    ;;#ASMEND
-; GFX906-NEXT:    v_writelane_b32 v39, s22, 25
+; GFX906-NEXT:    v_writelane_b32 v39, s22, 13
 ; GFX906-NEXT:    ;;#ASMSTART
 ; GFX906-NEXT:    ; def s23
 ; GFX906-NEXT:    ;;#ASMEND
-; GFX906-NEXT:    v_writelane_b32 v39, s23, 26
+; GFX906-NEXT:    v_writelane_b32 v39, s23, 14
 ; GFX906-NEXT:    ;;#ASMSTART
 ; GFX906-NEXT:    ; def s24
 ; GFX906-NEXT:    ;;#ASMEND
-; GFX906-NEXT:    v_writelane_b32 v39, s24, 27
+; GFX906-NEXT:    v_writelane_b32 v39, s24, 15
 ; GFX906-NEXT:    ;;#ASMSTART
 ; GFX906-NEXT:    ; def s25
 ; GFX906-NEXT:    ;;#ASMEND
-; GFX906-NEXT:    v_writelane_b32 v39, s25, 28
+; GFX906-NEXT:    v_writelane_b32 v39, s25, 16
 ; GFX906-NEXT:    ;;#ASMSTART
 ; GFX906-NEXT:    ; def s26
 ; GFX906-NEXT:    ;;#ASMEND
-; GFX906-NEXT:    v_writelane_b32 v39, s26, 29
+; GFX906-NEXT:    v_writelane_b32 v39, s26, 17
 ; GFX906-NEXT:    ;;#ASMSTART
 ; GFX906-NEXT:    ; def s27
 ; GFX906-NEXT:    ;;#ASMEND
-; GFX906-NEXT:    v_writelane_b32 v39, s27, 30
+; GFX906-NEXT:    v_writelane_b32 v39, s27, 18
 ; GFX906-NEXT:    ;;#ASMSTART
 ; GFX906-NEXT:    ; def s28
 ; GFX906-NEXT:    ;;#ASMEND
-; GFX906-NEXT:    v_writelane_b32 v39, s28, 31
+; GFX906-NEXT:    v_writelane_b32 v39, s28, 19
 ; GFX906-NEXT:    ;;#ASMSTART
 ; GFX906-NEXT:    ; def s29
 ; GFX906-NEXT:    ;;#ASMEND
-; GFX906-NEXT:    v_writelane_b32 v39, s29, 32
+; GFX906-NEXT:    v_writelane_b32 v39, s29, 20
 ; GFX906-NEXT:    s_or_saveexec_b64 s[34:35], -1
 ; GFX906-NEXT:    buffer_store_dword v39, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
 ; GFX906-NEXT:    s_mov_b64 exec, s[34:35]
@@ -267,39 +267,39 @@ define void @preserve_wwm_copy_dstreg(ptr %parg0, ptr %parg1, ptr %parg2) #0 {
 ; GFX906-NEXT:    v_readlane_b32 s15, v39, 2
 ; GFX906-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX906-NEXT:    v_readlane_b32 s17, v39, 23
-; GFX906-NEXT:    v_readlane_b32 s21, v39, 24
+; GFX906-NEXT:    v_readlane_b32 s21, v39, 12
 ; GFX906-NEXT:    ;;#ASMSTART
 ; GFX906-NEXT:    ; use s21
 ; GFX906-NEXT:    ;;#ASMEND
-; GFX906-NEXT:    v_readlane_b32 s22, v39, 25
+; GFX906-NEXT:    v_readlane_b32 s22, v39, 13
 ; GFX906-NEXT:    ;;#ASMSTART
 ; GFX906-NEXT:    ; use s22
 ; GFX906-NEXT:    ;;#ASMEND
-; GFX906-NEXT:    v_readlane_b32 s23, v39, 26
+; GFX906-NEXT:    v_readlane_b32 s23, v39, 14
 ; GFX906-NEXT:    ;;#ASMSTART
 ; GFX906-NEXT:    ; use s23
 ; GFX906-NEXT:    ;;#ASMEND
-; GFX906-NEXT:    v_readlane_b32 s24, v39, 27
+; GFX906-NEXT:    v_readlane_b32 s24, v39, 15
 ; GFX906-NEXT:    ;;#ASMSTART
 ; GFX906-NEXT:    ; use s24
 ; GFX906-NEXT:    ;;#ASMEND
-; GFX906-NEXT:    v_readlane_b32 s25, v39, 28
+; GFX906-NEXT:    v_readlane_b32 s25, v39, 16
 ; GFX906-NEXT:    ;;#ASMSTART
 ; GFX906-NEXT:    ; use s25
 ; GFX906-NEXT:    ;;#ASMEND
-; GFX906-NEXT:    v_readlane_b32 s26, v39, 29
+; GFX906-NEXT:    v_readlane_b32 s26, v39, 17
 ; GFX906-NEXT:    ;;#ASMSTART
 ; GFX906-NEXT:    ; use s26
 ; GFX906-NEXT:    ;;#ASMEND
-; GFX906-NEXT:    v_readlane_b32 s27, v39, 30
+; GFX906-NEXT:    v_readlane_b32 s27, v39, 18
 ; GFX906-NEXT:    ;;#ASMSTART
 ; GFX906-NEXT:    ; use s27
 ; GFX906-NEXT:    ;;#ASMEND
-; GFX906-NEXT:    v_readlane_b32 s28, v39, 31
+; GFX906-NEXT:    v_readlane_b32 s28, v39, 19
 ; GFX906-NEXT:    ;;#ASMSTART
 ; GFX906-NEXT:    ; use s28
 ; GFX906-NEXT:    ;;#ASMEND
-; GFX906-NEXT:    v_readlane_b32 s29, v39, 32
+; GFX906-NEXT:    v_readlane_b32 s29, v39, 20
 ; GFX906-NEXT:    ;;#ASMSTART
 ; GFX906-NEXT:    ; use s29
 ; GFX906-NEXT:    ;;#ASMEND
@@ -575,39 +575,39 @@ define void @preserve_wwm_copy_dstreg(ptr %parg0, ptr %parg1, ptr %parg2) #0 {
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ; def s21
 ; GFX908-NEXT:    ;;#ASMEND
-; GFX908-NEXT:    v_writelane_b32 v39, s21, 24
+; GFX908-NEXT:    v_writelane_b32 v39, s21, 12
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ; def s22
 ; GFX908-NEXT:    ;;#ASMEND
-; GFX908-NEXT:    v_writelane_b32 v39, s22, 25
+; GFX908-NEXT:    v_writelane_b32 v39, s22, 13
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ; def s23
 ; GFX908-NEXT:    ;;#ASMEND
-; GFX908-NEXT:    v_writelane_b32 v39, s23, 26
+; GFX908-NEXT:    v_writelane_b32 v39, s23, 14
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ; def s24
 ; GFX908-NEXT:    ;;#ASMEND
-; GFX908-NEXT:    v_writelane_b32 v39, s24, 27
+; GFX908-NEXT:    v_writelane_b32 v39, s24, 15
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ; def s25
 ; GFX908-NEXT:    ;;#ASMEND
-; GFX908-NEXT:    v_writelane_b32 v39, s25, 28
+; GFX908-NEXT:    v_writelane_b32 v39, s25, 16
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ; def s26
 ; GFX908-NEXT:    ;;#ASMEND
-; GFX908-NEXT:    v_writelane_b32 v39, s26, 29
+; GFX908-NEXT:    v_writelane_b32 v39, s26, 17
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ; def s27
 ; GFX908-NEXT:    ;;#ASMEND
-; GFX908-NEXT:    v_writelane_b32 v39, s27, 30
+; GFX908-NEXT:    v_writelane_b32 v39, s27, 18
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ; def s28
 ; GFX908-NEXT:    ;;#ASMEND
-; GFX908-NEXT:    v_writelane_b32 v39, s28, 31
+; GFX908-NEXT:    v_writelane_b32 v39, s28, 19
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ; def s29
 ; GFX908-NEXT:    ;;#ASMEND
-; GFX908-NEXT:    v_writelane_b32 v39, s29, 32
+; GFX908-NEXT:    v_writelane_b32 v39, s29, 20
 ; GFX908-NEXT:    s_or_saveexec_b64 s[34:35], -1
 ; GFX908-NEXT:    buffer_store_dword v39, off, s[0:3], s33 offset:12 ; 4-byte Folded Spill
 ; GFX908-NEXT:    s_mov_b64 exec, s[34:35]
@@ -646,39 +646,39 @@ define void @preserve_wwm_copy_dstreg(ptr %parg0, ptr %parg1, ptr %parg2) #0 {
 ; GFX908-NEXT:    v_readlane_b32 s15, v39, 2
 ; GFX908-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX908-NEXT:    v_readlane_b32 s17, v39, 23
-; GFX908-NEXT:    v_readlane_b32 s21, v39, 24
+; GFX908-NEXT:    v_readlane_b32 s21, v39, 12
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ; use s21
 ; GFX908-NEXT:    ;;#ASMEND
-; GFX908-NEXT:    v_readlane_b32 s22, v39, 25
+; GFX908-NEXT:    v_readlane_b32 s22, v39, 13
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ; use s22
 ; GFX908-NEXT:    ;;#ASMEND
-; GFX908-NEXT:    v_readlane_b32 s23, v39, 26
+; GFX908-NEXT:    v_readlane_b32 s23, v39, 14
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ; use s23
 ; GFX908-NEXT:    ;;#ASMEND
-; GFX908-NEXT:    v_readlane_b32 s24, v39, 27
+; GFX908-NEXT:    v_readlane_b32 s24, v39, 15
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ; use s24
 ; GFX908-NEXT:    ;;#ASMEND
-; GFX908-NEXT:    v_readlane_b32 s25, v39, 28
+; GFX908-NEXT:    v_readlane_b32 s25, v39, 16
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ; use s25
 ; GFX908-NEXT:    ;;#ASMEND
-; GFX908-NEXT:    v_readlane_b32 s26, v39, 29
+; GFX908-NEXT:    v_readlane_b32 s26, v39, 17
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ; use s26
 ; GFX908-NEXT:    ;;#ASMEND
-; GFX908-NEXT:    v_readlane_b32 s27, v39, 30
+; GFX908-NEXT:    v_readlane_b32 s27, v39, 18
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ; use s27
 ; GFX908-NEXT:    ;;#ASMEND
-; GFX908-NEXT:    v_readlane_b32 s28, v39, 31
+; GFX908-NEXT:    v_readlane_b32 s28, v39, 19
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ; use s28
 ; GFX908-NEXT:    ;;#ASMEND
-; GFX908-NEXT:    v_readlane_b32 s29, v39, 32
+; GFX908-NEXT:    v_readlane_b32 s29, v39, 20
 ; GFX908-NEXT:    ;;#ASMSTART
 ; GFX908-NEXT:    ; use s29
 ; GFX908-NEXT:    ;;#ASMEND
diff --git a/llvm/test/CodeGen/AMDGPU/scratch-simple.ll b/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
index 4cc469b9b49a0..da6f90561b517 100644
--- a/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
+++ b/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
@@ -74,10 +74,9 @@
 ; GFX11-FLATSCR: v_and_b32_e32 [[CLAMP_IDX:v[0-9]+]], 0x1fc, v0
 
 ; MUBUF-DAG: v_add{{_|_nc_}}{{i|u}}32_e32 [[HI_OFF:v[0-9]+]],{{.*}} 0x200, [[CLAMP_IDX]]
-; MUBUF-DAG: v_add{{_|_nc_}}{{i|u}}32_e32 [[LO_OFF:v[0-9]+]],{{.*}} {{v2|0}}, [[CLAMP_IDX]]
-; FLATSCR: v_add{{_|_nc_}}{{u32|b32}}_e32 [[LO_OFF:v[0-9]+]],{{.*}} {{v2|0}}, [[CLAMP_IDX]]
+; FLATSCR:  v_mov_b32_e32 [[LO_OFF:v[0-9]+]], [[CLAMP_IDX]]
 
-; MUBUF: buffer_load_dword {{v[0-9]+}}, [[LO_OFF]], {{s\[[0-9]+:[0-9]+\]}}, 0 offen
+; MUBUF: buffer_load_dword {{v[0-9]+}}, [[CLAMP_IDX]], {{s\[[0-9]+:[0-9]+\]}}, 0 offen
 ; MUBUF: buffer_load_dword {{v[0-9]+}}, [[HI_OFF]], {{s\[[0-9]+:[0-9]+\]}}, 0 offen
 ; FLATSCR: scratch_load_dword {{v[0-9]+}}, [[LO_OFF]], off
 ; GFX11-FLATSCR: scratch_load_b32 {{v[0-9]+}}, [[CLAMP_IDX]], off{{$}}
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll b/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
index 14a02d4d2dcec..52ad7e5355207 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
@@ -17,10 +17,12 @@
 
 ; DEFAULT: Greedy Register Allocator
 ; DEFAULT-NEXT: Virtual Register Rewriter
+; DEFAULT-NEXT: Stack Slot Coloring
 ; DEFAULT-NEXT: SI lower SGPR spill instructions
 ; DEFAULT-NEXT: Virtual Register Map
 ; DEFAULT-NEXT: Live Register Matrix
 ; DEFAULT-NEXT: SI Pre-allocate WWM Registers
+; DEFAULT-NEXT: Live Stack Slot Analysis
 ; DEFAULT-NEXT: Greedy Register Allocator
 ; DEFAULT-NEXT: SI Lower WWM Copies
 ; DEFAULT-NEXT: Virtual Register Rewriter
@@ -57,10 +59,12 @@
 ; BASIC-DEFAULT-NEXT: Live Register Matrix
 ; BASIC-DEFAULT-NEXT: Basic Register Allocator
 ; BASIC-DEFAULT-NEXT: Virtual Register Rewriter
+; BASIC-DEFAULT-NEXT: Stack Slot Coloring
 ; BASIC-DEFAULT-NEXT: SI lower SGPR spill instructions
 ; BASIC-DEFAULT-NEXT: Virtual Register Map
 ; BASIC-DEFAULT-NEXT: Live Register Matrix
 ; BASIC-DEFAULT-NEXT: SI Pre-allocate WWM Registers
+; BASIC-DEFAULT-NEXT: Live Stack Slot Analysis
 ; BASIC-DEFAULT-NEXT: Bundle Machine CFG Edges
 ; BASIC-DEFAULT-NEXT: Spill Code Placement Analysis
 ; BASIC-DEFAULT-NEXT: Lazy Machine Block Frequency Analysis
@@ -81,10 +85,12 @@
 
 ; DEFAULT-BASIC: Greedy Register Allocator
 ; DEFAULT-BASIC-NEXT: Virtual Register Rewriter
+; DEFAULT-BASIC-NEXT: Stack Slot Coloring
 ; DEFAULT-BASIC-NEXT: SI lower SGPR spill instructions
 ; DEFAULT-BASIC-NEXT: Virtual Register Map
 ; DEFAULT-BASIC-NEXT: Live Register Matrix
 ; DEFAULT-BASIC-NEXT: SI Pre-allocate WWM Registers
+; DEFAULT-BASIC-NEXT: Live Stack Slot Analysis
 ; DEFAULT-BASIC-NEXT: Basic Register Allocator
 ; DEFAULT-BASIC-NEXT: SI Lower WWM Copies
 ; DEFAULT-BASIC-NEXT: Virtual Register Rewriter
@@ -107,10 +113,12 @@
 ; BASIC-BASIC-NEXT: Live Register Matrix
 ; BASIC-BASIC-NEXT: Basic Register Allocator
 ; BASIC-BASIC-NEXT: Virtual Register Rewriter
+; BASIC-BASIC-NEXT: Stack Slot Coloring
 ; BASIC-BASIC-NEXT: SI lower SGPR spill instructions
 ; BASIC-BASIC-NEXT: Virtual Register Map
 ; BASIC-BASIC-NEXT: Live Register Matrix
 ; BASIC-BASIC-NEXT: SI Pre-allocate WWM Registers
+; BASIC-BASIC-NEXT: Live Stack Slot Analysis
 ; BASIC-BASIC-NEXT: Basic Register Allocator
 ; BASIC-BASIC-NEXT: SI Lower WWM Copies
 ; BASIC-BASIC-NEXT: Virtual Register Rewriter
diff --git a/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-vgpr-lanes-usage.mir b/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-vgpr-lanes-usage.mir
index 0e6d9ce4a7f31..1ffef8e60d90d 100644
--- a/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-vgpr-lanes-usage.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-vgpr-lanes-usage.mir
@@ -2,6 +2,7 @@
 # RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -verify-machineinstrs -stress-regalloc=3 -start-before=greedy,0 -stop-after=si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=SGPR_SPILLED %s
 
 # INFO : The test starts from the sgpr-regalloc pipeline.
+# INFO : Now, StackSlotColoring pass comes just after sgpr-regalloc pipeline.
 
 # This file contains 3 test cases to observe the optimal stack slot usage for SGPR spills utilizing Stack Slot Coloring pass.
 # @stack-slot-share-equal-sized-spills : In this, the stack slot indices is shared among the spill stack objects of equal size.
@@ -41,20 +42,19 @@ body:             |
     ; SGPR_SPILLED-NEXT: $sgpr5 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 4
     ; SGPR_SPILLED-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed renamable $sgpr4_sgpr5, 0, csr_amdgpu, implicit undef $vgpr0
     ; SGPR_SPILLED-NEXT: $sgpr32 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 0
-    ; SGPR_SPILLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = SI_SPILL_S32_TO_VGPR $sgpr32, 5, [[DEF]]
     ; SGPR_SPILLED-NEXT: $sgpr0 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 1, implicit-def $sgpr0_sgpr1
     ; SGPR_SPILLED-NEXT: $sgpr1 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 2
-    ; SGPR_SPILLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = SI_SPILL_S32_TO_VGPR $sgpr2, 6, [[DEF]], implicit-def $sgpr2_sgpr3, implicit $sgpr2_sgpr3
-    ; SGPR_SPILLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = SI_SPILL_S32_TO_VGPR $sgpr3, 7, [[DEF]], implicit $sgpr2_sgpr3
+    ; SGPR_SPILLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = SI_SPILL_S32_TO_VGPR $sgpr2, 1, [[DEF]], implicit-def $sgpr2_sgpr3, implicit $sgpr2_sgpr3
+    ; SGPR_SPILLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = SI_SPILL_S32_TO_VGPR $sgpr3, 2, [[DEF]], implicit $sgpr2_sgpr3
     ; SGPR_SPILLED-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32, implicit $sgpr32
     ; SGPR_SPILLED-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32, implicit $sgpr32
     ; SGPR_SPILLED-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
     ; SGPR_SPILLED-NEXT: $sgpr4 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 3, implicit-def $sgpr4_sgpr5
     ; SGPR_SPILLED-NEXT: $sgpr5 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 4
     ; SGPR_SPILLED-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed renamable $sgpr4_sgpr5, 0, csr_amdgpu, implicit killed $vgpr0
-    ; SGPR_SPILLED-NEXT: $sgpr32 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 5
-    ; SGPR_SPILLED-NEXT: $sgpr2 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 6, implicit-def $sgpr2_sgpr3
-    ; SGPR_SPILLED-NEXT: $sgpr3 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 7
+    ; SGPR_SPILLED-NEXT: $sgpr32 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 0
+    ; SGPR_SPILLED-NEXT: $sgpr2 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 1, implicit-def $sgpr2_sgpr3
+    ; SGPR_SPILLED-NEXT: $sgpr3 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 2
     ; SGPR_SPILLED-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32, implicit $sgpr32
     %0:sreg_32_xm0 = COPY $sgpr32
     %5:sreg_64 = COPY $sgpr0_sgpr1
@@ -107,18 +107,17 @@ body:             |
     ; SGPR_SPILLED-NEXT: $sgpr5 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 4
     ; SGPR_SPILLED-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed renamable $sgpr4_sgpr5, 0, csr_amdgpu, implicit undef $vgpr0
     ; SGPR_SPILLED-NEXT: $sgpr32 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 0
-    ; SGPR_SPILLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = SI_SPILL_S32_TO_VGPR $sgpr32, 5, [[DEF]]
     ; SGPR_SPILLED-NEXT: $sgpr2 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 1, implicit-def $sgpr2_sgpr3
     ; SGPR_SPILLED-NEXT: $sgpr3 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 2
-    ; SGPR_SPILLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = SI_SPILL_S32_TO_VGPR $sgpr2, 6, [[DEF]]
+    ; SGPR_SPILLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = SI_SPILL_S32_TO_VGPR $sgpr2, 1, [[DEF]]
     ; SGPR_SPILLED-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32, implicit $sgpr32
     ; SGPR_SPILLED-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32, implicit $sgpr32
     ; SGPR_SPILLED-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
     ; SGPR_SPILLED-NEXT: $sgpr4 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 3, implicit-def $sgpr4_sgpr5
     ; SGPR_SPILLED-NEXT: $sgpr5 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 4
     ; SGPR_SPILLED-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed renamable $sgpr4_sgpr5, 0, csr_amdgpu, implicit killed $vgpr0
-    ; SGPR_SPILLED-NEXT: $sgpr32 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 5
-    ; SGPR_SPILLED-NEXT: $sgpr2 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 6
+    ; SGPR_SPILLED-NEXT: $sgpr32 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 0
+    ; SGPR_SPILLED-NEXT: $sgpr2 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 1
     ; SGPR_SPILLED-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32, implicit $sgpr32
     %0:sreg_32_xm0 = COPY $sgpr32
     %5:sreg_64 = COPY $sgpr2_sgpr3
@@ -163,26 +162,25 @@ body:             |
     ; SGPR_SPILLED-NEXT: [[DEF1:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
     ; SGPR_SPILLED-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[DEF1]], 0, 0, implicit $exec, implicit $flat_scr
     ; SGPR_SPILLED-NEXT: renamable $sgpr4_sgpr5 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-rel32-lo) 4, target-flags(amdgpu-rel32-hi) 4, implicit-def dead $scc
-    ; SGPR_SPILLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = SI_SPILL_S32_TO_VGPR $sgpr4, 2, [[DEF]], implicit-def $sgpr4_sgpr5, implicit $sgpr4_sgpr5
-    ; SGPR_SPILLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = SI_SPILL_S32_TO_VGPR killed $sgpr5, 3, [[DEF]], implicit killed $sgpr4_sgpr5
+    ; SGPR_SPILLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = SI_SPILL_S32_TO_VGPR $sgpr4, 3, [[DEF]], implicit-def $sgpr4_sgpr5, implicit $sgpr4_sgpr5
+    ; SGPR_SPILLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = SI_SPILL_S32_TO_VGPR killed $sgpr5, 4, [[DEF]], implicit killed $sgpr4_sgpr5
     ; SGPR_SPILLED-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32, implicit $sgpr32
-    ; SGPR_SPILLED-NEXT: $sgpr4 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 2, implicit-def $sgpr4_sgpr5
-    ; SGPR_SPILLED-NEXT: $sgpr5 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 3
+    ; SGPR_SPILLED-NEXT: $sgpr4 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 3, implicit-def $sgpr4_sgpr5
+    ; SGPR_SPILLED-NEXT: $sgpr5 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 4
     ; SGPR_SPILLED-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed renamable $sgpr4_sgpr5, 0, csr_amdgpu, implicit undef $vgpr0
     ; SGPR_SPILLED-NEXT: $sgpr32 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 0
-    ; SGPR_SPILLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = SI_SPILL_S32_TO_VGPR $sgpr32, 4, [[DEF]]
     ; SGPR_SPILLED-NEXT: $sgpr0 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 1
-    ; SGPR_SPILLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = SI_SPILL_S32_TO_VGPR $sgpr2, 5, [[DEF]], implicit-def $sgpr2_sgpr3, implicit $sgpr2_sgpr3
-    ; SGPR_SPILLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = SI_SPILL_S32_TO_VGPR $sgpr3, 6, [[DEF]], implicit $sgpr2_sgpr3
+    ; SGPR_SPILLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = SI_SPILL_S32_TO_VGPR $sgpr2, 1, [[DEF]], implicit-def $sgpr2_sgpr3, implicit $sgpr2_sgpr3
+    ; SGPR_SPILLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = SI_SPILL_S32_TO_VGPR $sgpr3, 2, [[DEF]], implicit $sgpr2_sgpr3
     ; SGPR_SPILLED-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32, implicit $sgpr32
     ; SGPR_SPILLED-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32, implicit $sgpr32
     ; SGPR_SPILLED-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
-    ; SGPR_SPILLED-NEXT: $sgpr4 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 2, implicit-def $sgpr4_sgpr5
-    ; SGPR_SPILLED-NEXT: $sgpr5 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 3
+    ; SGPR_SPILLED-NEXT: $sgpr4 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 3, implicit-def $sgpr4_sgpr5
+    ; SGPR_SPILLED-NEXT: $sgpr5 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 4
     ; SGPR_SPILLED-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed renamable $sgpr4_sgpr5, 0, csr_amdgpu, implicit killed $vgpr0
-    ; SGPR_SPILLED-NEXT: $sgpr32 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 4
-    ; SGPR_SPILLED-NEXT: $sgpr2 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 5, implicit-def $sgpr2_sgpr3
-    ; SGPR_SPILLED-NEXT: $sgpr3 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 6
+    ; SGPR_SPILLED-NEXT: $sgpr32 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 0
+    ; SGPR_SPILLED-NEXT: $sgpr2 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 1, implicit-def $sgpr2_sgpr3
+    ; SGPR_SPILLED-NEXT: $sgpr3 = SI_RESTORE_S32_FROM_VGPR [[DEF]], 2
     ; SGPR_SPILLED-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32, implicit $sgpr32
     %0:sreg_32_xm0 = COPY $sgpr32
     %5:sreg_32 = COPY $sgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll b/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
index 0e68c306fb8b8..76ecbc0863650 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
@@ -9779,7 +9779,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    s_mov_b64 s[4:5], s[2:3]
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v5, 8, v0
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:240
-; GFX6-NEXT:    s_mov_b32 s2, 0x84400
+; GFX6-NEXT:    s_mov_b32 s2, 0x86a00
 ; GFX6-NEXT:    s_mov_b64 s[8:9], exec
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -9788,7 +9788,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:224
-; GFX6-NEXT:    s_mov_b32 s2, 0x84000
+; GFX6-NEXT:    s_mov_b32 s2, 0x86600
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9796,7 +9796,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:208
-; GFX6-NEXT:    s_mov_b32 s2, 0x83c00
+; GFX6-NEXT:    s_mov_b32 s2, 0x86200
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9804,7 +9804,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:192
-; GFX6-NEXT:    s_mov_b32 s2, 0x83800
+; GFX6-NEXT:    s_mov_b32 s2, 0x85e00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9812,7 +9812,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:176
-; GFX6-NEXT:    s_mov_b32 s2, 0x83400
+; GFX6-NEXT:    s_mov_b32 s2, 0x85a00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9820,7 +9820,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:160
-; GFX6-NEXT:    s_mov_b32 s2, 0x83000
+; GFX6-NEXT:    s_mov_b32 s2, 0x85600
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9828,7 +9828,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:144
-; GFX6-NEXT:    s_mov_b32 s2, 0x82c00
+; GFX6-NEXT:    s_mov_b32 s2, 0x85200
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9836,7 +9836,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:128
-; GFX6-NEXT:    s_mov_b32 s2, 0x82800
+; GFX6-NEXT:    s_mov_b32 s2, 0x84e00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9844,7 +9844,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:112
-; GFX6-NEXT:    s_mov_b32 s2, 0x82400
+; GFX6-NEXT:    s_mov_b32 s2, 0x84a00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9852,7 +9852,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:96
-; GFX6-NEXT:    s_mov_b32 s2, 0x82000
+; GFX6-NEXT:    s_mov_b32 s2, 0x84600
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9860,7 +9860,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:80
-; GFX6-NEXT:    s_mov_b32 s2, 0x81c00
+; GFX6-NEXT:    s_mov_b32 s2, 0x84200
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9868,7 +9868,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:64
-; GFX6-NEXT:    s_mov_b32 s2, 0x81400
+; GFX6-NEXT:    s_mov_b32 s2, 0x83a00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9877,7 +9877,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64
 ; GFX6-NEXT:    buffer_load_dwordx4 v[7:10], v[5:6], s[4:7], 0 addr64 offset:16
-; GFX6-NEXT:    s_mov_b32 s2, 0x80c00
+; GFX6-NEXT:    s_mov_b32 s2, 0x83200
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v7, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v8, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9885,7 +9885,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_store_dword v10, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dwordx4 v[7:10], v[5:6], s[4:7], 0 addr64 offset:32
-; GFX6-NEXT:    s_mov_b32 s2, 0x81000
+; GFX6-NEXT:    s_mov_b32 s2, 0x83600
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dword v7, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    buffer_store_dword v8, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9906,7 +9906,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    s_mov_b64 exec, s[8:9]
 ; GFX6-NEXT:    buffer_load_dwordx4 v[7:10], v[5:6], s[4:7], 0 addr64 offset:48
-; GFX6-NEXT:    s_mov_b32 s0, 0x81800
+; GFX6-NEXT:    s_mov_b32 s0, 0x83e00
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 13, v0
 ; GFX6-NEXT:    v_add_i32_e32 v4, vcc, 16, v4
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
@@ -9932,7 +9932,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    v_writelane_b32 v4, s9, 5
 ; GFX6-NEXT:    v_writelane_b32 v4, s10, 6
 ; GFX6-NEXT:    v_writelane_b32 v4, s11, 7
-; GFX6-NEXT:    s_mov_b32 s2, 0x84800
+; GFX6-NEXT:    s_mov_b32 s2, 0x80c00
 ; GFX6-NEXT:    buffer_store_dword v4, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dword v4, off, s[40:43], 0
@@ -9973,7 +9973,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    v_writelane_b32 v4, s13, 5
 ; GFX6-NEXT:    v_writelane_b32 v4, s14, 6
 ; GFX6-NEXT:    v_writelane_b32 v4, s15, 7
-; GFX6-NEXT:    s_mov_b32 s34, 0x85000
+; GFX6-NEXT:    s_mov_b32 s34, 0x81400
 ; GFX6-NEXT:    buffer_store_dword v4, off, s[40:43], s34 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dword v4, off, s[40:43], 0
@@ -9982,7 +9982,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX6-NEXT:    s_mov_b64 exec, 0xff
 ; GFX6-NEXT:    buffer_store_dword v4, off, s[40:43], 0
-; GFX6-NEXT:    s_mov_b32 s34, 0x84800
+; GFX6-NEXT:    s_mov_b32 s34, 0x80c00
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dword v4, off, s[40:43], s34 ; 4-byte Folded Reload
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
@@ -10009,7 +10009,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    v_writelane_b32 v4, s21, 5
 ; GFX6-NEXT:    v_writelane_b32 v4, s22, 6
 ; GFX6-NEXT:    v_writelane_b32 v4, s23, 7
-; GFX6-NEXT:    s_mov_b32 s34, 0x85800
+; GFX6-NEXT:    s_mov_b32 s34, 0x81c00
 ; GFX6-NEXT:    buffer_store_dword v4, off, s[40:43], s34 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dword v4, off, s[40:43], 0
@@ -10018,7 +10018,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX6-NEXT:    s_mov_b64 exec, 0xff
 ; GFX6-NEXT:    buffer_store_dword v4, off, s[40:43], 0
-; GFX6-NEXT:    s_mov_b32 s34, 0x85000
+; GFX6-NEXT:    s_mov_b32 s34, 0x81400
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dword v4, off, s[40:43], s34 ; 4-byte Folded Reload
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
@@ -10045,7 +10045,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    v_writelane_b32 v4, s29, 5
 ; GFX6-NEXT:    v_writelane_b32 v4, s30, 6
 ; GFX6-NEXT:    v_writelane_b32 v4, s31, 7
-; GFX6-NEXT:    s_mov_b32 s34, 0x86000
+; GFX6-NEXT:    s_mov_b32 s34, 0x82400
 ; GFX6-NEXT:    buffer_store_dword v4, off, s[40:43], s34 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dword v4, off, s[40:43], 0
@@ -10054,7 +10054,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX6-NEXT:    s_mov_b64 exec, 0xff
 ; GFX6-NEXT:    buffer_store_dword v4, off, s[40:43], 0
-; GFX6-NEXT:    s_mov_b32 s34, 0x85800
+; GFX6-NEXT:    s_mov_b32 s34, 0x81c00
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dword v4, off, s[40:43], s34 ; 4-byte Folded Reload
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
@@ -10077,7 +10077,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    v_writelane_b32 v4, s1, 1
 ; GFX6-NEXT:    v_writelane_b32 v4, s2, 2
 ; GFX6-NEXT:    v_writelane_b32 v4, s3, 3
-; GFX6-NEXT:    s_mov_b32 s34, 0x86800
+; GFX6-NEXT:    s_mov_b32 s34, 0x82c00
 ; GFX6-NEXT:    buffer_store_dword v4, off, s[40:43], s34 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dword v4, off, s[40:43], 0
@@ -10089,7 +10089,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    v_writelane_b32 v4, s4, 0
 ; GFX6-NEXT:    v_writelane_b32 v4, s5, 1
-; GFX6-NEXT:    s_mov_b32 s2, 0x86c00
+; GFX6-NEXT:    s_mov_b32 s2, 0x83000
 ; GFX6-NEXT:    buffer_store_dword v4, off, s[40:43], s2 ; 4-byte Folded Spill
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dword v4, off, s[40:43], 0
@@ -10098,7 +10098,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    s_mov_b64 s[34:35], exec
 ; GFX6-NEXT:    s_mov_b64 exec, 0xff
 ; GFX6-NEXT:    buffer_store_dword v4, off, s[40:43], 0
-; GFX6-NEXT:    s_mov_b32 s36, 0x86000
+; GFX6-NEXT:    s_mov_b32 s36, 0x82400
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dword v4, off, s[40:43], s36 ; 4-byte Folded Reload
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
@@ -10116,7 +10116,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    s_mov_b64 s[34:35], exec
 ; GFX6-NEXT:    s_mov_b64 exec, 15
 ; GFX6-NEXT:    buffer_store_dword v4, off, s[40:43], 0
-; GFX6-NEXT:    s_mov_b32 s44, 0x86800
+; GFX6-NEXT:    s_mov_b32 s44, 0x82c00
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dword v4, off, s[40:43], s44 ; 4-byte Folded Reload
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
@@ -10130,7 +10130,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    s_mov_b64 s[44:45], exec
 ; GFX6-NEXT:    s_mov_b64 exec, 3
 ; GFX6-NEXT:    buffer_store_dword v4, off, s[40:43], 0
-; GFX6-NEXT:    v_mov_b32_e32 v7, 0x21b0
+; GFX6-NEXT:    v_mov_b32_e32 v7, 0x20c0
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    buffer_load_dword v4, v7, s[40:43], 0 offen ; 4-byte Folded Reload
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
@@ -10187,13 +10187,13 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v4, off, s[40:43], 0
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX6-NEXT:    s_mov_b32 s0, 0x84400
+; GFX6-NEXT:    s_mov_b32 s0, 0x86a00
 ; GFX6-NEXT:    buffer_load_dword v7, off, s[40:43], s0 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
 ; GFX6-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX6-NEXT:    s_mov_b32 s0, 0x84000
+; GFX6-NEXT:    s_mov_b32 s0, 0x86600
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:240
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10201,7 +10201,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x83c00
+; GFX6-NEXT:    s_mov_b32 s0, 0x86200
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:224
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10209,7 +10209,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x83800
+; GFX6-NEXT:    s_mov_b32 s0, 0x85e00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:208
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10217,7 +10217,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x83400
+; GFX6-NEXT:    s_mov_b32 s0, 0x85a00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:192
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10225,7 +10225,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x83000
+; GFX6-NEXT:    s_mov_b32 s0, 0x85600
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:176
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10233,7 +10233,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x82c00
+; GFX6-NEXT:    s_mov_b32 s0, 0x85200
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:160
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10241,7 +10241,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x82800
+; GFX6-NEXT:    s_mov_b32 s0, 0x84e00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:144
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10249,7 +10249,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x82400
+; GFX6-NEXT:    s_mov_b32 s0, 0x84a00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:128
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10257,7 +10257,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x82000
+; GFX6-NEXT:    s_mov_b32 s0, 0x84600
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:112
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10265,7 +10265,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x81c00
+; GFX6-NEXT:    s_mov_b32 s0, 0x84200
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:96
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10273,7 +10273,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x81400
+; GFX6-NEXT:    s_mov_b32 s0, 0x83a00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:80
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10281,7 +10281,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x81800
+; GFX6-NEXT:    s_mov_b32 s0, 0x83e00
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:64
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10289,7 +10289,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x81000
+; GFX6-NEXT:    s_mov_b32 s0, 0x83600
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:48
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -10297,7 +10297,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
 ; GFX6-NEXT:    buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
 ; GFX6-NEXT:    buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT:    s_mov_b32 s0, 0x80c00
+; GFX6-NEXT:    s_mov_b32 s0, 0x83200
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:32
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll b/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll
index 132775d81ca1a..ebc82248cc80c 100644
--- a/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll
+++ b/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll
@@ -34,7 +34,7 @@ define amdgpu_kernel void @kernel_background_evaluate(ptr addrspace(5) %kg, ptr
 ; MUBUF-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; MUBUF-NEXT:    s_cbranch_execz .LBB0_2
 ; MUBUF-NEXT:  ; %bb.1: ; %if.then4.i
-; MUBUF-NEXT:    v_add_nc_u32_e64 v0, 4, 0x4000
+; MUBUF-NEXT:    v_mov_b32_e32 v0, 0x4004
 ; MUBUF-NEXT:    s_mov_b32 s0, 0x41c64e6d
 ; MUBUF-NEXT:    s_clause 0x1
 ; MUBUF-NEXT:    buffer_load_dword v1, v0, s[36:39], 0 offen
diff --git a/llvm/test/CodeGen/AMDGPU/stack-realign.ll b/llvm/test/CodeGen/AMDGPU/stack-realign.ll
index be3c0d741ac5f..70bd63d31d5d7 100644
--- a/llvm/test/CodeGen/AMDGPU/stack-realign.ll
+++ b/llvm/test/CodeGen/AMDGPU/stack-realign.ll
@@ -11,7 +11,7 @@
 ; GCN-LABEL: {{^}}needs_align16_default_stack_align:
 ; GCN-DAG: v_lshlrev_b32_e32 [[SCALED_IDX:v[0-9]+]], 4, v0
 ; GCN-DAG: v_lshrrev_b32_e64 [[FRAMEDIFF:v[0-9]+]], 6, s32
-; GCN: v_add_u32_e32 [[FI:v[0-9]+]], vcc, [[FRAMEDIFF]], [[SCALED_IDX]]
+; GCN: v_add_u32_e32 [[FI:v[0-9]+]], vcc, [[SCALED_IDX]], [[FRAMEDIFF]]
 
 ; GCN-NOT: s32
 
diff --git a/llvm/test/CodeGen/AMDGPU/tail-call-inreg-arguments.convergencetokens.ll b/llvm/test/CodeGen/AMDGPU/tail-call-inreg-arguments.convergencetokens.ll
new file mode 100644
index 0000000000000..42a1f746be3fa
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tail-call-inreg-arguments.convergencetokens.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -stop-after=si-fix-sgpr-copies < %s | FileCheck %s
+
+; FIXME: Merge with tail-call-inreg-arguments.ll
+
+; Currently all functions with convergence tokens will fail the
+; machine verifier after register allocation.
+declare void @void_func_i64_inreg(i64 inreg)
+
+define void @tail_call_i64_inreg_uniform_in_vgpr_convergence_tokens() #0 {
+  ; CHECK-LABEL: name: tail_call_i64_inreg_uniform_in_vgpr_convergence_tokens
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5, $sgpr8_sgpr9, $sgpr10_sgpr11, $sgpr12, $sgpr13, $sgpr14, $sgpr15, $sgpr6_sgpr7, $vgpr31
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr31
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr13
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr12
+  ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr8_sgpr9
+  ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK-NEXT:   [[CONVERGENCECTRL_ENTRY:%[0-9]+]]:sreg_64 = CONVERGENCECTRL_ENTRY
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   [[DS_READ_B64_gfx9_:%[0-9]+]]:vreg_64 = DS_READ_B64_gfx9 killed [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3)
+  ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[DS_READ_B64_gfx9_]].sub1
+  ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[DS_READ_B64_gfx9_]].sub0
+  ; CHECK-NEXT:   CONVERGENCECTRL_GLUE [[CONVERGENCECTRL_ENTRY]]
+  ; CHECK-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY10]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY11]], implicit $exec, implicit [[CONVERGENCECTRL_ENTRY]]
+  ; CHECK-NEXT:   CONVERGENCECTRL_GLUE [[CONVERGENCECTRL_ENTRY]]
+  ; CHECK-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY12]], implicit $exec, implicit [[CONVERGENCECTRL_ENTRY]]
+  ; CHECK-NEXT:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @void_func_i64_inreg, target-flags(amdgpu-gotprel32-hi) @void_func_i64_inreg, implicit-def dead $scc
+  ; CHECK-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:ccr_sgpr_64 = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4)
+  ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY [[COPY8]]
+  ; CHECK-NEXT:   $sgpr6_sgpr7 = COPY [[COPY1]]
+  ; CHECK-NEXT:   $sgpr8_sgpr9 = COPY [[COPY7]]
+  ; CHECK-NEXT:   $sgpr10_sgpr11 = COPY [[COPY6]]
+  ; CHECK-NEXT:   $sgpr12 = COPY [[COPY5]]
+  ; CHECK-NEXT:   $sgpr13 = COPY [[COPY4]]
+  ; CHECK-NEXT:   $sgpr14 = COPY [[COPY3]]
+  ; CHECK-NEXT:   $sgpr15 = COPY [[COPY2]]
+  ; CHECK-NEXT:   $vgpr31 = COPY [[COPY]]
+  ; CHECK-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; CHECK-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; CHECK-NEXT:   CONVERGENCECTRL_GLUE [[CONVERGENCECTRL_ENTRY]]
+  ; CHECK-NEXT:   SI_TCRETURN killed [[S_LOAD_DWORDX2_IMM]], @void_func_i64_inreg, 0, csr_amdgpu, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $vgpr31, implicit $sgpr0, implicit $sgpr1, implicit [[CONVERGENCECTRL_ENTRY]]
+  %t = call token @llvm.experimental.convergence.entry()
+  %uniform.vgpr = load i64, ptr addrspace(3) null, align 8
+  tail call void @void_func_i64_inreg(i64 inreg %uniform.vgpr) #0 [ "convergencectrl"(token %t) ]
+  ret void
+}
+
+attributes #0 = { convergent }
diff --git a/llvm/test/CodeGen/AMDGPU/tail-call-inreg-arguments.error.ll b/llvm/test/CodeGen/AMDGPU/tail-call-inreg-arguments.error.ll
new file mode 100644
index 0000000000000..cd6cb4d1e9fe4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tail-call-inreg-arguments.error.ll
@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs=0 2> %t.err < %s | FileCheck %s
+; RUN: FileCheck -check-prefix=ERR %s < %t.err
+; FIXME: These tests cannot be tail called, and should be executed in a waterfall loop.
+
+declare hidden void @void_func_i32_inreg(i32 inreg)
+
+; ERR: error: <unknown>:0:0: in function tail_call_i32_inreg_divergent void (i32): illegal VGPR to SGPR copy
+; ERR: error: <unknown>:0:0: in function indirect_tail_call_i32_inreg_divergent void (i32): illegal VGPR to SGPR copy
+
+define void @tail_call_i32_inreg_divergent(i32 %vgpr) {
+; CHECK-LABEL: tail_call_i32_inreg_divergent:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s16, s33
+; CHECK-NEXT:    s_mov_b32 s33, s32
+; CHECK-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; CHECK-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
+; CHECK-NEXT:    s_mov_b64 exec, s[18:19]
+; CHECK-NEXT:    v_writelane_b32 v40, s16, 2
+; CHECK-NEXT:    s_addk_i32 s32, 0x400
+; CHECK-NEXT:    v_writelane_b32 v40, s30, 0
+; CHECK-NEXT:    v_writelane_b32 v40, s31, 1
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, void_func_i32_inreg@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, void_func_i32_inreg@rel32@hi+12
+; CHECK-NEXT:     ; illegal copy v0 to s0
+; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; CHECK-NEXT:    v_readlane_b32 s31, v40, 1
+; CHECK-NEXT:    v_readlane_b32 s30, v40, 0
+; CHECK-NEXT:    v_readlane_b32 s4, v40, 2
+; CHECK-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; CHECK-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
+; CHECK-NEXT:    s_mov_b64 exec, s[6:7]
+; CHECK-NEXT:    s_addk_i32 s32, 0xfc00
+; CHECK-NEXT:    s_mov_b32 s33, s4
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  tail call void @void_func_i32_inreg(i32 inreg %vgpr)
+  ret void
+}
+
+@constant = external hidden addrspace(4) constant ptr
+
+define void @indirect_tail_call_i32_inreg_divergent(i32 %vgpr) {
+; CHECK-LABEL: indirect_tail_call_i32_inreg_divergent:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s16, s33
+; CHECK-NEXT:    s_mov_b32 s33, s32
+; CHECK-NEXT:    s_or_saveexec_b64 s[18:19], -1
+; CHECK-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
+; CHECK-NEXT:    s_mov_b64 exec, s[18:19]
+; CHECK-NEXT:    s_addk_i32 s32, 0x400
+; CHECK-NEXT:    v_writelane_b32 v40, s16, 2
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, constant@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, constant@rel32@hi+12
+; CHECK-NEXT:    s_load_dwordx2 s[16:17], s[16:17], 0x0
+; CHECK-NEXT:    v_writelane_b32 v40, s30, 0
+; CHECK-NEXT:    v_writelane_b32 v40, s31, 1
+; CHECK-NEXT:     ; illegal copy v0 to s0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; CHECK-NEXT:    v_readlane_b32 s31, v40, 1
+; CHECK-NEXT:    v_readlane_b32 s30, v40, 0
+; CHECK-NEXT:    v_readlane_b32 s4, v40, 2
+; CHECK-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; CHECK-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
+; CHECK-NEXT:    s_mov_b64 exec, s[6:7]
+; CHECK-NEXT:    s_addk_i32 s32, 0xfc00
+; CHECK-NEXT:    s_mov_b32 s33, s4
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %fptr = load ptr, ptr addrspace(4) @constant, align 8
+  tail call void %fptr(i32 inreg %vgpr)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/tail-call-inreg-arguments.ll b/llvm/test/CodeGen/AMDGPU/tail-call-inreg-arguments.ll
new file mode 100644
index 0000000000000..9b992c40c28ee
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tail-call-inreg-arguments.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck %s
+
+declare hidden void @void_func_i32_inreg(i32 inreg)
+
+define void @tail_call_i32_inreg_uniform(i32 inreg %sgpr) {
+; CHECK-LABEL: tail_call_i32_inreg_uniform:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ; kill: def $sgpr6_sgpr7 killed $sgpr6_sgpr7 killed $sgpr7
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s0, s6
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, void_func_i32_inreg@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, void_func_i32_inreg@rel32@hi+12
+; CHECK-NEXT:    s_setpc_b64 s[16:17]
+  tail call void @void_func_i32_inreg(i32 inreg %sgpr)
+  ret void
+}
+
+@constant = external hidden addrspace(4) constant ptr
+
+define void @indirect_tail_call_i32_inreg_uniform(i32 inreg %sgpr) {
+; CHECK-LABEL: indirect_tail_call_i32_inreg_uniform:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, constant@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, constant@rel32@hi+12
+; CHECK-NEXT:    s_load_dwordx2 s[16:17], s[16:17], 0x0
+; CHECK-NEXT:    ; kill: def $sgpr6_sgpr7 killed $sgpr6_sgpr7 killed $sgpr7
+; CHECK-NEXT:    s_mov_b32 s0, s6
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[16:17]
+  %fptr = load ptr, ptr addrspace(4) @constant, align 8
+  tail call void %fptr(i32 inreg %sgpr)
+  ret void
+}
+
+declare void @void_func_i64_inreg(i64 inreg)
+
+define void @tail_call_i64_inreg_uniform(i64 inreg %sgpr) {
+; CHECK-LABEL: tail_call_i64_inreg_uniform:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, void_func_i64_inreg@gotpcrel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, void_func_i64_inreg@gotpcrel32@hi+12
+; CHECK-NEXT:    s_load_dwordx2 s[16:17], s[16:17], 0x0
+; CHECK-NEXT:    s_mov_b32 s1, s7
+; CHECK-NEXT:    s_mov_b32 s0, s6
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[16:17]
+  tail call void @void_func_i64_inreg(i64 inreg %sgpr)
+  ret void
+}
+
+define void @tail_call_i64_inreg_uniform_in_vgpr() {
+; CHECK-LABEL: tail_call_i64_inreg_uniform_in_vgpr:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    ds_read_b64 v[0:1], v0
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, void_func_i64_inreg@gotpcrel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, void_func_i64_inreg@gotpcrel32@hi+12
+; CHECK-NEXT:    s_load_dwordx2 s[16:17], s[16:17], 0x0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_readfirstlane_b32 s0, v0
+; CHECK-NEXT:    v_readfirstlane_b32 s1, v1
+; CHECK-NEXT:    s_setpc_b64 s[16:17]
+  %uniform.vgpr = load i64, ptr addrspace(3) null, align 8
+  tail call void @void_func_i64_inreg(i64 inreg %uniform.vgpr)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/tail-call-uniform-target-in-vgprs-issue110930.convergencetokens.ll b/llvm/test/CodeGen/AMDGPU/tail-call-uniform-target-in-vgprs-issue110930.convergencetokens.ll
new file mode 100644
index 0000000000000..3ca2adec90be5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tail-call-uniform-target-in-vgprs-issue110930.convergencetokens.ll
@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -stop-after=si-fix-sgpr-copies < %s | FileCheck %s
+
+; FIXME: This should merge with
+; tail-call-uniform-target-in-vgprs-issue110930.ll.  This is split
+; because all functions involving convergence tokens will fail the
+; machine verifier after register allocation.
+
+target triple = "amdgcn-amd-amdhsa"
+
+define void @tail_call_uniform_vgpr_value_convergence_tokens() #0 {
+  ; CHECK-LABEL: name: tail_call_uniform_vgpr_value_convergence_tokens
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5, $sgpr8_sgpr9, $sgpr10_sgpr11, $sgpr12, $sgpr13, $sgpr14, $sgpr15, $sgpr6_sgpr7, $vgpr31
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr31
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr13
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr12
+  ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr8_sgpr9
+  ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK-NEXT:   [[CONVERGENCECTRL_ENTRY:%[0-9]+]]:sreg_64 = CONVERGENCECTRL_ENTRY
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   [[DS_READ_B64_gfx9_:%[0-9]+]]:vreg_64 = DS_READ_B64_gfx9 killed [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3)
+  ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[DS_READ_B64_gfx9_]].sub1
+  ; CHECK-NEXT:   CONVERGENCECTRL_GLUE [[CONVERGENCECTRL_ENTRY]]
+  ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY10]], implicit $exec, implicit [[CONVERGENCECTRL_ENTRY]]
+  ; CHECK-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[DS_READ_B64_gfx9_]].sub0
+  ; CHECK-NEXT:   CONVERGENCECTRL_GLUE [[CONVERGENCECTRL_ENTRY]]
+  ; CHECK-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
+  ; CHECK-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 killed [[COPY12]], implicit $exec, implicit [[CONVERGENCECTRL_ENTRY]]
+  ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:ccr_sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_1]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_]], %subreg.sub1
+  ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY [[COPY8]]
+  ; CHECK-NEXT:   $sgpr6_sgpr7 = COPY [[COPY1]]
+  ; CHECK-NEXT:   $sgpr8_sgpr9 = COPY [[COPY7]]
+  ; CHECK-NEXT:   $sgpr10_sgpr11 = COPY [[COPY6]]
+  ; CHECK-NEXT:   $sgpr12 = COPY [[COPY5]]
+  ; CHECK-NEXT:   $sgpr13 = COPY [[COPY4]]
+  ; CHECK-NEXT:   $sgpr14 = COPY [[COPY3]]
+  ; CHECK-NEXT:   $sgpr15 = COPY [[COPY2]]
+  ; CHECK-NEXT:   $vgpr31 = COPY [[COPY]]
+  ; CHECK-NEXT:   CONVERGENCECTRL_GLUE [[CONVERGENCECTRL_ENTRY]]
+  ; CHECK-NEXT:   SI_TCRETURN killed [[REG_SEQUENCE]], 0, 0, csr_amdgpu, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $vgpr31, implicit [[CONVERGENCECTRL_ENTRY]]
+  %t = call token @llvm.experimental.convergence.entry()
+  %fptr = load ptr, ptr addrspace(3) null, align 8
+  tail call void %fptr() #0 [ "convergencectrl"(token %t) ]
+  ret void
+}
+
+attributes #0 = { convergent }
diff --git a/llvm/test/CodeGen/AMDGPU/tail-call-uniform-target-in-vgprs-issue110930.ll b/llvm/test/CodeGen/AMDGPU/tail-call-uniform-target-in-vgprs-issue110930.ll
new file mode 100644
index 0000000000000..b5a68720dc19f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/tail-call-uniform-target-in-vgprs-issue110930.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck %s
+
+target triple = "amdgcn-amd-amdhsa"
+
+; The tail call target is known uniform, but will be in a VGPR, so we
+; need readfirstlane to legalize it.
+define void @tail_call_uniform_vgpr_value() {
+; CHECK-LABEL: tail_call_uniform_vgpr_value:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    ds_read_b64 v[0:1], v0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_readfirstlane_b32 s17, v1
+; CHECK-NEXT:    v_readfirstlane_b32 s16, v0
+; CHECK-NEXT:    s_setpc_b64 s[16:17]
+  %fptr = load ptr, ptr addrspace(3) null, align 8
+  tail call void %fptr()
+  ret void
+}
+
+@constant = external hidden addrspace(4) constant ptr
+
+; readfirstlanes should fold out.
+define void @tail_call_uniform_sgpr_value() {
+; CHECK-LABEL: tail_call_uniform_sgpr_value:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_getpc_b64 s[16:17]
+; CHECK-NEXT:    s_add_u32 s16, s16, constant@rel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s17, s17, constant@rel32@hi+12
+; CHECK-NEXT:    s_load_dwordx2 s[16:17], s[16:17], 0x0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[16:17]
+  %fptr = load ptr, ptr addrspace(4) @constant, align 8
+  tail call void %fptr()
+  ret void
+}
diff --git a/llvm/test/CodeGen/BPF/32-bit-subreg-alu.ll b/llvm/test/CodeGen/BPF/32-bit-subreg-alu.ll
index 42888ff58f5e2..abec34990044a 100644
--- a/llvm/test/CodeGen/BPF/32-bit-subreg-alu.ll
+++ b/llvm/test/CodeGen/BPF/32-bit-subreg-alu.ll
@@ -200,7 +200,7 @@ entry:
 define dso_local i32 @div_i(i32 %a) local_unnamed_addr #0 {
 entry:
   %div = udiv i32 %a, 15
-; CHECK: w{{[0-9]+}} /= 15
+; CHECK-NOT: w{{[0-9]+}} /= 15
   ret i32 %div
 }
 
@@ -216,7 +216,7 @@ entry:
 define dso_local i32 @rem_i(i32 %a) local_unnamed_addr #0 {
 entry:
   %rem = urem i32 %a, 15
-; CHECK: w{{[0-9]+}} %= 15
+; CHECK-NOT: w{{[0-9]+}} %= 15
   ret i32 %rem
 }
 
diff --git a/llvm/test/CodeGen/BPF/bpf-fastcall-4.ll b/llvm/test/CodeGen/BPF/bpf-fastcall-4.ll
new file mode 100644
index 0000000000000..4f0c9fc94aa9d
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/bpf-fastcall-4.ll
@@ -0,0 +1,35 @@
+; RUN: llc -O2 --march=bpfel %s -o - | FileCheck %s
+
+; Generated from the following C code:
+;
+;   extern int foo(void) __attribute__((bpf_fastcall));
+;
+;   int bar(int a) {
+;     foo();
+;     return a;
+;   }
+;
+; Using the following command:
+;
+;   clang --target=bpf -emit-llvm -O2 -S -o - t.c
+;
+; (unnecessary attrs removed maually)
+
+; Check that function marked with bpf_fastcall does not clobber W1-W5.
+
+define dso_local i32 @bar(i32 %a) {
+entry:
+  %call = tail call i32 @foo() #0
+  ret i32 %a
+}
+
+; CHECK:      # %bb.0:
+; CHECK-NEXT:   *(u64 *)(r10 - 8) = r1
+; CHECK-NEXT:   call foo
+; CHECK-NEXT:   r1 = *(u64 *)(r10 - 8)
+; CHECK-NEXT:   w0 = w1
+; CHECK-NEXT:   exit
+
+declare dso_local i32 @foo() #0
+
+attributes #0 = { "bpf_fastcall" }
diff --git a/llvm/test/CodeGen/BPF/bpf-fastcall-5.ll b/llvm/test/CodeGen/BPF/bpf-fastcall-5.ll
new file mode 100644
index 0000000000000..608ab4a0bb0cf
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/bpf-fastcall-5.ll
@@ -0,0 +1,35 @@
+; RUN: llc -O2 --march=bpfel %s -o - | FileCheck %s
+
+; Generated from the following C code:
+;
+;   extern int foo(void) __attribute__((bpf_fastcall));
+;
+;   int bar(int a, int b, int c, int d, int e) {
+;     foo();
+;     return e;
+;   }
+;
+; Using the following command:
+;
+;   clang --target=bpf -emit-llvm -O2 -S -o - t.c
+;
+; (unnecessary attrs removed maually)
+
+; Check that function marked with bpf_fastcall does not clobber W1-W5.
+
+define dso_local i32 @bar(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) {
+entry:
+  %call = tail call i32 @foo() #0
+  ret i32 %e
+}
+
+; CHECK:      # %bb.0:
+; CHECK-NEXT:   *(u64 *)(r10 - 8) = r5
+; CHECK-NEXT:   call foo
+; CHECK-NEXT:   r5 = *(u64 *)(r10 - 8)
+; CHECK-NEXT:   w0 = w5
+; CHECK-NEXT:   exit
+
+declare dso_local i32 @foo() #0
+
+attributes #0 = { "bpf_fastcall" }
diff --git a/llvm/test/CodeGen/BPF/sdiv_error.ll b/llvm/test/CodeGen/BPF/sdiv_error.ll
index a5dcc6271224b..05394f3e05de8 100644
--- a/llvm/test/CodeGen/BPF/sdiv_error.ll
+++ b/llvm/test/CodeGen/BPF/sdiv_error.ll
@@ -2,7 +2,7 @@
 ; RUN: FileCheck %s < %t1
 ; CHECK: unsupported signed division
 
-define i32 @test(i32 %len) {
-  %1 = sdiv i32 %len, 15
-  ret i32 %1
+define i64 @test(i64 %len) {
+  %1 = sdiv i64 %len, 15
+  ret i64 %1
 }
diff --git a/llvm/test/CodeGen/BPF/sdiv_to_mul.ll b/llvm/test/CodeGen/BPF/sdiv_to_mul.ll
new file mode 100644
index 0000000000000..7bb1b7badac1f
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/sdiv_to_mul.ll
@@ -0,0 +1,57 @@
+; RUN: llc -march=bpfel -mcpu=v1 < %s | FileCheck --check-prefix=CHECK-V1 %s
+; RUN: llc -march=bpfel -mcpu=v4 < %s | FileCheck --check-prefix=CHECK-V4 %s
+
+target triple = "bpf"
+
+;   struct S {
+;     int var[3];
+;   };
+;   int foo1 (struct S *a, struct S *b)
+;   {
+;     return a - b;
+;   }
+define dso_local i32 @foo1(ptr noundef %a, ptr noundef %b) local_unnamed_addr {
+entry:
+  %sub.ptr.lhs.cast = ptrtoint ptr %a to i64
+  %sub.ptr.rhs.cast = ptrtoint ptr %b to i64
+  %sub.ptr.sub = sub i64 %sub.ptr.lhs.cast, %sub.ptr.rhs.cast
+  %sub.ptr.div = sdiv exact i64 %sub.ptr.sub, 12
+  %conv = trunc i64 %sub.ptr.div to i32
+  ret i32 %conv
+}
+; CHECK-V1:        r0 = r1
+; CHECK-V1:        r0 -= r2
+; CHECK-V1:        r0 s>>= 2
+; CHECK-V1:        r1 = -6148914691236517205 ll
+; CHECK-V1:        r0 *= r1
+; CHECK-V1:        exit
+
+; CHECK-V4:        r0 = r1
+; CHECK-V4:        r0 -= r2
+; CHECK-V4:        r0 >>= 2
+; CHECK-V4:        w0 *= -1431655765
+; CHECK-V4:        exit
+
+define dso_local noundef range(i32 -143165576, 143165577) i32 @foo2(i32 noundef %a) local_unnamed_addr {
+entry:
+  %div = sdiv i32 %a, 15
+  ret i32 %div
+}
+; CHECK-V1-NOT:   r[[#]] s/= 15
+; CHECK-V4-NOT:   w[[#]] s/= 15
+
+define dso_local noundef range(i32 -14, 15) i32 @foo3(i32 noundef %a) local_unnamed_addr {
+entry:
+  %rem = srem i32 %a, 15
+  ret i32 %rem
+}
+; CHECK-V1-NOT:   r[[#]] s%= 15
+; CHECK-V4-NOT:   w[[#]] s%= 15
+
+define dso_local i64 @foo4(i64 noundef %a) local_unnamed_addr {
+entry:
+  %div = udiv exact i64 %a, 15
+  ret i64 %div
+}
+; CHECK-V1-NOT:   r[[#]] /= 15
+; CHECK-V4-NOT:   w[[#]] /= 15
diff --git a/llvm/test/CodeGen/BPF/srem_error.ll b/llvm/test/CodeGen/BPF/srem_error.ll
index 883e72d32dc7e..8245a47b1ddd2 100644
--- a/llvm/test/CodeGen/BPF/srem_error.ll
+++ b/llvm/test/CodeGen/BPF/srem_error.ll
@@ -2,7 +2,7 @@
 ; RUN: FileCheck %s < %t1
 ; CHECK: unsupported signed division
 
-define i32 @test(i32 %len) {
-  %1 = srem i32 %len, 15
-  ret i32 %1
+define i64 @test(i64 %len) {
+  %1 = srem i64 %len, 15
+  ret i64 %1
 }
diff --git a/llvm/test/CodeGen/DirectX/abs.ll b/llvm/test/CodeGen/DirectX/abs.ll
index 85090a51c55a4..34464e9db14cb 100644
--- a/llvm/test/CodeGen/DirectX/abs.ll
+++ b/llvm/test/CodeGen/DirectX/abs.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S  -dxil-intrinsic-expansion  -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,EXPCHECK
-; RUN: opt -S  -dxil-op-lower  -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK
+; RUN: opt -S  -dxil-intrinsic-expansion -dxil-op-lower  -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK
 
 ; Make sure dxil operation function calls for abs are generated for int16_t/int/int64_t.
 
diff --git a/llvm/test/CodeGen/DirectX/atan2_error.ll b/llvm/test/CodeGen/DirectX/atan2_error.ll
index 5b3077f85f5d4..372934098b7ca 100644
--- a/llvm/test/CodeGen/DirectX/atan2_error.ll
+++ b/llvm/test/CodeGen/DirectX/atan2_error.ll
@@ -1,4 +1,4 @@
-; RUN: not opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s 2>&1 | FileCheck %s
+; RUN: not opt -S -dxil-intrinsic-expansion -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s 2>&1 | FileCheck %s
 
 ; DXIL operation atan does not support double overload type
 ; CHECK: in function atan2_double
diff --git a/llvm/test/CodeGen/DirectX/clamp.ll b/llvm/test/CodeGen/DirectX/clamp.ll
index 2f29e4479f9ca..23aadf893e8ca 100644
--- a/llvm/test/CodeGen/DirectX/clamp.ll
+++ b/llvm/test/CodeGen/DirectX/clamp.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
+; RUN: opt -S -dxil-intrinsic-expansion -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
 
 ; Make sure dxil operation function calls for clamp/uclamp are generated for half/float/double/i16/i32/i64.
 
diff --git a/llvm/test/CodeGen/DirectX/cross.ll b/llvm/test/CodeGen/DirectX/cross.ll
new file mode 100644
index 0000000000000..6ec3ec4d3594a
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/cross.ll
@@ -0,0 +1,56 @@
+; RUN: opt -S  -dxil-intrinsic-expansion  < %s | FileCheck %s
+
+; Make sure dxil operation function calls for cross are generated for half/float.
+
+declare <3 x half> @llvm.dx.cross.v3f16(<3 x half>, <3 x half>)
+declare <3 x float> @llvm.dx.cross.v3f32(<3 x float>, <3 x float>)
+
+define noundef <3 x half> @test_cross_half3(<3 x half> noundef %p0, <3 x half> noundef %p1) {
+entry:
+  ; CHECK: %x0 = extractelement <3 x half> %p0, i64 0
+  ; CHECK: %x1 = extractelement <3 x half> %p0, i64 1
+  ; CHECK: %x2 = extractelement <3 x half> %p0, i64 2
+  ; CHECK: %y0 = extractelement <3 x half> %p1, i64 0
+  ; CHECK: %y1 = extractelement <3 x half> %p1, i64 1
+  ; CHECK: %y2 = extractelement <3 x half> %p1, i64 2
+  ; CHECK: %0 = fmul half %x1, %y2
+  ; CHECK: %1 = fmul half %x2, %y1
+  ; CHECK: %hlsl.cross1 = fsub half %0, %1
+  ; CHECK: %2 = fmul half %x2, %y0
+  ; CHECK: %3 = fmul half %x0, %y2
+  ; CHECK: %hlsl.cross2 = fsub half %2, %3
+  ; CHECK: %4 = fmul half %x0, %y1
+  ; CHECK: %5 = fmul half %x1, %y0
+  ; CHECK: %hlsl.cross3 = fsub half %4, %5
+  ; CHECK: %6 = insertelement <3 x half> undef, half %hlsl.cross1, i64 0
+  ; CHECK: %7 = insertelement <3 x half> %6, half %hlsl.cross2, i64 1
+  ; CHECK: %8 = insertelement <3 x half> %7, half %hlsl.cross3, i64 2
+  ; CHECK: ret <3 x half> %8
+  %hlsl.cross = call <3 x half> @llvm.dx.cross.v3f16(<3 x half> %p0, <3 x half> %p1)
+  ret <3 x half> %hlsl.cross
+}
+
+define noundef <3 x float> @test_cross_float3(<3 x float> noundef %p0, <3 x float> noundef %p1) {
+entry:
+  ; CHECK: %x0 = extractelement <3 x float> %p0, i64 0
+  ; CHECK: %x1 = extractelement <3 x float> %p0, i64 1
+  ; CHECK: %x2 = extractelement <3 x float> %p0, i64 2
+  ; CHECK: %y0 = extractelement <3 x float> %p1, i64 0
+  ; CHECK: %y1 = extractelement <3 x float> %p1, i64 1
+  ; CHECK: %y2 = extractelement <3 x float> %p1, i64 2
+  ; CHECK: %0 = fmul float %x1, %y2
+  ; CHECK: %1 = fmul float %x2, %y1
+  ; CHECK: %hlsl.cross1 = fsub float %0, %1
+  ; CHECK: %2 = fmul float %x2, %y0
+  ; CHECK: %3 = fmul float %x0, %y2
+  ; CHECK: %hlsl.cross2 = fsub float %2, %3
+  ; CHECK: %4 = fmul float %x0, %y1
+  ; CHECK: %5 = fmul float %x1, %y0
+  ; CHECK: %hlsl.cross3 = fsub float %4, %5
+  ; CHECK: %6 = insertelement <3 x float> undef, float %hlsl.cross1, i64 0
+  ; CHECK: %7 = insertelement <3 x float> %6, float %hlsl.cross2, i64 1
+  ; CHECK: %8 = insertelement <3 x float> %7, float %hlsl.cross3, i64 2
+  ; CHECK: ret <3 x float> %8
+  %hlsl.cross = call <3 x float> @llvm.dx.cross.v3f32(<3 x float> %p0, <3 x float> %p1)
+  ret <3 x float> %hlsl.cross
+}
diff --git a/llvm/test/CodeGen/DirectX/exp.ll b/llvm/test/CodeGen/DirectX/exp.ll
index f67e2744c4ee3..c2d9938d27ecd 100644
--- a/llvm/test/CodeGen/DirectX/exp.ll
+++ b/llvm/test/CodeGen/DirectX/exp.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
+; RUN: opt -S -dxil-intrinsic-expansion -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
 
 ; Make sure dxil operation function calls for exp are generated for float and half.
 
diff --git a/llvm/test/CodeGen/DirectX/fdot.ll b/llvm/test/CodeGen/DirectX/fdot.ll
index aa1b15972e266..78e111c41feef 100644
--- a/llvm/test/CodeGen/DirectX/fdot.ll
+++ b/llvm/test/CodeGen/DirectX/fdot.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S  -dxil-intrinsic-expansion -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,EXPCHECK
-; RUN: opt -S  -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK
+; RUN: opt -S  -dxil-intrinsic-expansion -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK
 
 ; Make sure dxil operation function calls for dot are generated for float type vectors.
 
diff --git a/llvm/test/CodeGen/DirectX/idot.ll b/llvm/test/CodeGen/DirectX/idot.ll
index 5848868ed0556..26e7ff395bd16 100644
--- a/llvm/test/CodeGen/DirectX/idot.ll
+++ b/llvm/test/CodeGen/DirectX/idot.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S  -dxil-intrinsic-expansion -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,EXPCHECK
-; RUN: opt -S  -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK
+; RUN: opt -S  -dxil-intrinsic-expansion -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK
 
 ; Make sure dxil operation function calls for dot are generated for int/uint vectors.
 
diff --git a/llvm/test/CodeGen/DirectX/length.ll b/llvm/test/CodeGen/DirectX/length.ll
index d12fcf4722a07..fc5868a7f6e82 100644
--- a/llvm/test/CodeGen/DirectX/length.ll
+++ b/llvm/test/CodeGen/DirectX/length.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S  -dxil-intrinsic-expansion  < %s | FileCheck %s --check-prefixes=CHECK,EXPCHECK
-; RUN: opt -S  -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library < %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK
+; RUN: opt -S  -dxil-intrinsic-expansion -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library < %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK
 
 ; Make sure dxil operation function calls for length are generated for half/float.
 
diff --git a/llvm/test/CodeGen/DirectX/length_error.ll b/llvm/test/CodeGen/DirectX/length_error.ll
index e1899c8deb515..143b41fc506e1 100644
--- a/llvm/test/CodeGen/DirectX/length_error.ll
+++ b/llvm/test/CodeGen/DirectX/length_error.ll
@@ -1,4 +1,4 @@
-; RUN: not opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s 2>&1 | FileCheck %s
+; RUN: not opt -S -dxil-intrinsic-expansion -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s 2>&1 | FileCheck %s
 
 ; DXIL operation length does not support double overload type
 ; CHECK: Cannot create Sqrt operation: Invalid overload type
diff --git a/llvm/test/CodeGen/DirectX/length_invalid_intrinsic_error.ll b/llvm/test/CodeGen/DirectX/length_invalid_intrinsic_error.ll
index 277fbaa462b0f..f722de2f9029e 100644
--- a/llvm/test/CodeGen/DirectX/length_invalid_intrinsic_error.ll
+++ b/llvm/test/CodeGen/DirectX/length_invalid_intrinsic_error.ll
@@ -1,4 +1,4 @@
-; RUN: not opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s 2>&1 | FileCheck %s
+; RUN: not opt -S -dxil-intrinsic-expansion -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s 2>&1 | FileCheck %s
 
 ; DXIL operation length does not support 1-element vector types.
 ; CHECK: LLVM ERROR: Invalid input type for length intrinsic
diff --git a/llvm/test/CodeGen/DirectX/lerp.ll b/llvm/test/CodeGen/DirectX/lerp.ll
index 9f5e4c7710d21..9ea0861cff265 100644
--- a/llvm/test/CodeGen/DirectX/lerp.ll
+++ b/llvm/test/CodeGen/DirectX/lerp.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.0-library < %s | FileCheck %s
+; RUN: opt -S -dxil-intrinsic-expansion -dxil-op-lower -mtriple=dxil-pc-shadermodel6.0-library < %s | FileCheck %s
 
 ; Make sure dxil operation function calls for lerp are generated for float and half.
 
diff --git a/llvm/test/CodeGen/DirectX/llc-pipeline.ll b/llvm/test/CodeGen/DirectX/llc-pipeline.ll
index 102748508b4ad..224037cfe7fbe 100644
--- a/llvm/test/CodeGen/DirectX/llc-pipeline.ll
+++ b/llvm/test/CodeGen/DirectX/llc-pipeline.ll
@@ -12,7 +12,6 @@
 ; CHECK-NEXT:   FunctionPass Manager
 ; CHECK-NEXT:     Dominator Tree Construction
 ; CHECK-NEXT:     Scalarize vector operations
-; CHECK-NEXT:   DXIL Intrinsic Expansion
 ; CHECK-NEXT:   DXIL Resource analysis
 ; CHECK-NEXT:   DXIL Op Lowering
 ; CHECK-NEXT:   DXIL Finalize Linkage
diff --git a/llvm/test/CodeGen/DirectX/log.ll b/llvm/test/CodeGen/DirectX/log.ll
index ee289088d243b..195713309cd44 100644
--- a/llvm/test/CodeGen/DirectX/log.ll
+++ b/llvm/test/CodeGen/DirectX/log.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S  -dxil-intrinsic-expansion  -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,EXPCHECK
-; RUN: opt -S  -dxil-op-lower  -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK
+; RUN: opt -S  -dxil-intrinsic-expansion -dxil-op-lower  -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK
 
 ; Make sure dxil operation function calls for log are generated.
 
diff --git a/llvm/test/CodeGen/DirectX/log10.ll b/llvm/test/CodeGen/DirectX/log10.ll
index a69f270f9dc88..f3acccce7e451 100644
--- a/llvm/test/CodeGen/DirectX/log10.ll
+++ b/llvm/test/CodeGen/DirectX/log10.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S  -dxil-intrinsic-expansion  -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,EXPCHECK
-; RUN: opt -S  -dxil-op-lower  -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK
+; RUN: opt -S  -dxil-intrinsic-expansion -dxil-op-lower  -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK
 
 ; Make sure dxil operation function calls for log10 are generated.
 
diff --git a/llvm/test/CodeGen/DirectX/normalize.ll b/llvm/test/CodeGen/DirectX/normalize.ll
index e2c8a5d4656a6..2aba9d5f74d78 100644
--- a/llvm/test/CodeGen/DirectX/normalize.ll
+++ b/llvm/test/CodeGen/DirectX/normalize.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S  -dxil-intrinsic-expansion  < %s | FileCheck %s --check-prefixes=CHECK,EXPCHECK
-; RUN: opt -S  -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library < %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK
+; RUN: opt -S  -dxil-intrinsic-expansion -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library < %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK
 
 ; Make sure dxil operation function calls for normalize are generated for half/float.
 
diff --git a/llvm/test/CodeGen/DirectX/normalize_error.ll b/llvm/test/CodeGen/DirectX/normalize_error.ll
index da0905690236d..35a91c0cdc24d 100644
--- a/llvm/test/CodeGen/DirectX/normalize_error.ll
+++ b/llvm/test/CodeGen/DirectX/normalize_error.ll
@@ -1,4 +1,4 @@
-; RUN: not opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s 2>&1 | FileCheck %s
+; RUN: not opt -S -dxil-intrinsic-expansion -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s 2>&1 | FileCheck %s
 
 ; DXIL operation normalize does not support double overload type
 ; CHECK: Cannot create Dot2 operation: Invalid overload type
diff --git a/llvm/test/CodeGen/DirectX/pow.ll b/llvm/test/CodeGen/DirectX/pow.ll
index 0e83c4ff8add6..378649ec119d5 100644
--- a/llvm/test/CodeGen/DirectX/pow.ll
+++ b/llvm/test/CodeGen/DirectX/pow.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S  -dxil-intrinsic-expansion -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,EXPCHECK
-; RUN: opt -S  -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK
+; RUN: opt -S  -dxil-intrinsic-expansion -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK
 
 ; Make sure dxil operation function calls for pow are generated.
 
diff --git a/llvm/test/CodeGen/DirectX/rsqrt.ll b/llvm/test/CodeGen/DirectX/rsqrt.ll
index 26b22e19635af..612b6222e7594 100644
--- a/llvm/test/CodeGen/DirectX/rsqrt.ll
+++ b/llvm/test/CodeGen/DirectX/rsqrt.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -S -scalarizer -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
+; RUN: opt -S -passes='function(scalarizer<load-store>),dxil-op-lower' -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
 
 ; Make sure dxil operation function calls for rsqrt are generated for float and half.
 
diff --git a/llvm/test/CodeGen/DirectX/scalar-data.ll b/llvm/test/CodeGen/DirectX/scalar-data.ll
index 4438604a3a879..c436f1eae4425 100644
--- a/llvm/test/CodeGen/DirectX/scalar-data.ll
+++ b/llvm/test/CodeGen/DirectX/scalar-data.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -dxil-data-scalarization -scalarizer -scalarize-load-store -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
+; RUN: opt -S -passes='dxil-data-scalarization,function(scalarizer<load-store>),dxil-op-lower' -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
 ; RUN: llc %s -mtriple=dxil-pc-shadermodel6.3-library --filetype=asm -o - | FileCheck %s
 
 ; Make sure we don't touch arrays without vectors and that can recurse multiple-dimension arrays of vectors
diff --git a/llvm/test/CodeGen/DirectX/scalar-load.ll b/llvm/test/CodeGen/DirectX/scalar-load.ll
index 11678f48a5e01..612e5646f5a08 100644
--- a/llvm/test/CodeGen/DirectX/scalar-load.ll
+++ b/llvm/test/CodeGen/DirectX/scalar-load.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -dxil-data-scalarization -scalarizer -scalarize-load-store -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
+; RUN: opt -S -passes='dxil-data-scalarization,function(scalarizer<load-store>),dxil-op-lower' -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
 ; RUN: llc %s -mtriple=dxil-pc-shadermodel6.3-library --filetype=asm -o - | FileCheck %s
 
 ; Make sure we can load groupshared, static vectors and arrays of vectors
diff --git a/llvm/test/CodeGen/DirectX/scalar-store.ll b/llvm/test/CodeGen/DirectX/scalar-store.ll
index 08d8a2c57c6c3..7734d32bca58c 100644
--- a/llvm/test/CodeGen/DirectX/scalar-store.ll
+++ b/llvm/test/CodeGen/DirectX/scalar-store.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -dxil-data-scalarization -scalarizer -scalarize-load-store -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
+; RUN: opt -S -passes='dxil-data-scalarization,scalarizer<load-store>,dxil-op-lower' -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
 ; RUN: llc %s -mtriple=dxil-pc-shadermodel6.3-library --filetype=asm -o - | FileCheck %s
 
 ; Make sure we can store groupshared, static vectors and arrays of vectors
diff --git a/llvm/test/CodeGen/DirectX/sign.ll b/llvm/test/CodeGen/DirectX/sign.ll
index 0a1631bd5ddc1..47e51b28d2084 100644
--- a/llvm/test/CodeGen/DirectX/sign.ll
+++ b/llvm/test/CodeGen/DirectX/sign.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -S  -dxil-op-lower  -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
+; RUN: opt -S  -dxil-intrinsic-expansion -dxil-op-lower  -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
 
 
 define noundef i32 @sign_half(half noundef %a) {
diff --git a/llvm/test/CodeGen/DirectX/step.ll b/llvm/test/CodeGen/DirectX/step.ll
index 0393c1533cb93..1c9894026c62e 100644
--- a/llvm/test/CodeGen/DirectX/step.ll
+++ b/llvm/test/CodeGen/DirectX/step.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S  -dxil-intrinsic-expansion  < %s | FileCheck %s --check-prefix=CHECK
-; RUN: opt -S  -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library < %s | FileCheck %s --check-prefix=CHECK
+; RUN: opt -S  -dxil-intrinsic-expansion -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library < %s | FileCheck %s --check-prefix=CHECK
 
 ; Make sure dxil operation function calls for step are generated for half/float.
 
diff --git a/llvm/test/CodeGen/MIR/Generic/machine-function-hasfakeuses-conflict.mir b/llvm/test/CodeGen/MIR/Generic/machine-function-hasfakeuses-conflict.mir
new file mode 100644
index 0000000000000..56175bd8bad6a
--- /dev/null
+++ b/llvm/test/CodeGen/MIR/Generic/machine-function-hasfakeuses-conflict.mir
@@ -0,0 +1,14 @@
+# RUN: not llc -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+
+# Test that computed properties are not conflicting with explicitly set
+# properties
+
+---
+# CHECK: error: {{.*}}: TestHasFakeUsesOverrideConflict has explicit property hasFakeUses=false, but contains fake uses
+name:            TestHasFakeUsesOverrideConflict
+hasFakeUses: false
+body: |
+  bb.0:
+    %0:_(s32) = G_IMPLICIT_DEF
+    FAKE_USE %0
+...
diff --git a/llvm/test/CodeGen/MIR/Generic/machine-function-optionally-computed-properties.mir b/llvm/test/CodeGen/MIR/Generic/machine-function-optionally-computed-properties.mir
index 858bbc8394bb3..8ac6b2e6ecafd 100644
--- a/llvm/test/CodeGen/MIR/Generic/machine-function-optionally-computed-properties.mir
+++ b/llvm/test/CodeGen/MIR/Generic/machine-function-optionally-computed-properties.mir
@@ -62,3 +62,23 @@ noVRegs: false
 name:            TestNoVRegsOverrideTrue
 noVRegs: true
 ...
+---
+# CHECK-LABEL: name: TestHasFakeUses
+# CHECK: hasFakeUses: false
+# CHECK: ...
+name:            TestHasFakeUses
+...
+---
+# CHECK-LABEL: name: TestHasFakeUsesOverride
+# CHECK: hasFakeUses: false
+# CHECK: ...
+name:            TestHasFakeUsesOverride
+hasFakeUses: false
+...
+---
+# CHECK-LABEL: name: TestHasFakeUsesOverrideTrue
+# CHECK: hasFakeUses: true
+# CHECK: ...
+name:            TestHasFakeUsesOverrideTrue
+hasFakeUses: true
+...
diff --git a/llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg.ll b/llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg.ll
index ccf89aac2d540..5ffa852d3d57f 100644
--- a/llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg.ll
+++ b/llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg.ll
@@ -63,7 +63,7 @@
 ; 64BIT-DAG:     STD renamable $x11, 0, %stack.1.arg2 :: (store (s64) into %ir.arg2)
 ; 64BIT-DAG:     renamable $x6 = LD 0, %stack.1.arg2 :: (load (s64) from %ir.arg2)
 ; 64BIT-DAG:     renamable $x9 = ADDI8 renamable $x6, 4
-; 64BIT-DAG:     renamable $x7 = ADDI8 %fixed-stack.0, 4
+; 64BIT-DAG:     renamable $x7 = disjoint ADDI8 %fixed-stack.0, 4
 ; 64BIT-DAG:     renamable $r8 = LWZ 0, %fixed-stack.0 :: (load (s32) from %fixed-stack.0, align 8)
 ; 64BIT-DAG:     STD killed renamable $x11, 0, %stack.0.arg1 :: (store (s64) into %ir.arg1)
 ; 64BIT-DAG:     STD killed renamable $x7, 0, %stack.0.arg1 :: (store (s64) into %ir.arg1)
diff --git a/llvm/test/CodeGen/PowerPC/p10-spill-crlt.ll b/llvm/test/CodeGen/PowerPC/p10-spill-crlt.ll
index 4b032781c3764..c733a01950603 100644
--- a/llvm/test/CodeGen/PowerPC/p10-spill-crlt.ll
+++ b/llvm/test/CodeGen/PowerPC/p10-spill-crlt.ll
@@ -30,14 +30,16 @@ define dso_local void @P10_Spill_CR_LT() local_unnamed_addr {
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    std r0, 16(r1)
 ; CHECK-NEXT:    stw r12, 8(r1)
-; CHECK-NEXT:    stdu r1, -48(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    stdu r1, -64(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
 ; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset r29, -24
 ; CHECK-NEXT:    .cfi_offset r30, -16
 ; CHECK-NEXT:    .cfi_offset cr2, 8
 ; CHECK-NEXT:    .cfi_offset cr3, 8
 ; CHECK-NEXT:    .cfi_offset cr4, 8
-; CHECK-NEXT:    std r30, 32(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r29, 40(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r30, 48(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    bl call_2@notoc
 ; CHECK-NEXT:    bc 12, 4*cr5+lt, .LBB0_13
 ; CHECK-NEXT:  # %bb.1: # %bb
@@ -65,10 +67,11 @@ define dso_local void @P10_Spill_CR_LT() local_unnamed_addr {
 ; CHECK-NEXT:    bc 12, 4*cr3+eq, .LBB0_11
 ; CHECK-NEXT:  # %bb.6: # %bb32
 ; CHECK-NEXT:    #
-; CHECK-NEXT:    rlwinm r30, r30, 0, 24, 22
 ; CHECK-NEXT:    andi. r3, r30, 2
+; CHECK-NEXT:    rlwinm r29, r30, 0, 24, 22
 ; CHECK-NEXT:    mcrf cr2, cr0
 ; CHECK-NEXT:    bl call_4@notoc
+; CHECK-NEXT:    mr r30, r29
 ; CHECK-NEXT:    beq+ cr2, .LBB0_3
 ; CHECK-NEXT:  # %bb.7: # %bb37
 ; CHECK-NEXT:  .LBB0_8: # %bb22
@@ -89,11 +92,13 @@ define dso_local void @P10_Spill_CR_LT() local_unnamed_addr {
 ; CHECK-BE-NEXT:    stdu r1, -144(r1)
 ; CHECK-BE-NEXT:    .cfi_def_cfa_offset 144
 ; CHECK-BE-NEXT:    .cfi_offset lr, 16
+; CHECK-BE-NEXT:    .cfi_offset r28, -32
 ; CHECK-BE-NEXT:    .cfi_offset r29, -24
 ; CHECK-BE-NEXT:    .cfi_offset r30, -16
 ; CHECK-BE-NEXT:    .cfi_offset cr2, 8
 ; CHECK-BE-NEXT:    .cfi_offset cr2, 8
 ; CHECK-BE-NEXT:    .cfi_offset cr2, 8
+; CHECK-BE-NEXT:    std r28, 112(r1) # 8-byte Folded Spill
 ; CHECK-BE-NEXT:    std r29, 120(r1) # 8-byte Folded Spill
 ; CHECK-BE-NEXT:    std r30, 128(r1) # 8-byte Folded Spill
 ; CHECK-BE-NEXT:    bl call_2
@@ -126,11 +131,12 @@ define dso_local void @P10_Spill_CR_LT() local_unnamed_addr {
 ; CHECK-BE-NEXT:    bc 12, 4*cr3+eq, .LBB0_11
 ; CHECK-BE-NEXT:  # %bb.6: # %bb32
 ; CHECK-BE-NEXT:    #
-; CHECK-BE-NEXT:    rlwinm r29, r29, 0, 24, 22
 ; CHECK-BE-NEXT:    andi. r3, r29, 2
+; CHECK-BE-NEXT:    rlwinm r28, r29, 0, 24, 22
 ; CHECK-BE-NEXT:    mcrf cr2, cr0
 ; CHECK-BE-NEXT:    bl call_4
 ; CHECK-BE-NEXT:    nop
+; CHECK-BE-NEXT:    mr r29, r28
 ; CHECK-BE-NEXT:    beq+ cr2, .LBB0_3
 ; CHECK-BE-NEXT:  # %bb.7: # %bb37
 ; CHECK-BE-NEXT:  .LBB0_8: # %bb22
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/alu-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/alu-rv64.mir
index 5cc6ea9f74d27..be12333e1499b 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/alu-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/alu-rv64.mir
@@ -245,8 +245,8 @@ body:             |
     ; RV64I-NEXT: PseudoRET implicit $x10
     %0:gprb(s64) = COPY $x10
     %1:gprb(s32) = G_TRUNC %0(s64)
-    %2:gprb(s64) = G_CONSTANT i64 31
-    %3:gprb(s32) = G_SHL %1, %2(s64)
+    %2:gprb(s32) = G_CONSTANT i32 31
+    %3:gprb(s32) = G_SHL %1, %2
     %4:gprb(s64) = G_ANYEXT %3(s32)
     $x10 = COPY %4(s64)
     PseudoRET implicit $x10
@@ -297,8 +297,8 @@ body:             |
     ; RV64I-NEXT: PseudoRET implicit $x10
     %0:gprb(s64) = COPY $x10
     %1:gprb(s32) = G_TRUNC %0(s64)
-    %2:gprb(s64) = G_CONSTANT i64 31
-    %3:gprb(s32) = G_ASHR %1, %2(s64)
+    %2:gprb(s32) = G_CONSTANT i32 31
+    %3:gprb(s32) = G_ASHR %1, %2
     %4:gprb(s64) = G_ANYEXT %3(s32)
     $x10 = COPY %4(s64)
     PseudoRET implicit $x10
@@ -349,8 +349,8 @@ body:             |
     ; RV64I-NEXT: PseudoRET implicit $x10
     %0:gprb(s64) = COPY $x10
     %1:gprb(s32) = G_TRUNC %0(s64)
-    %2:gprb(s64) = G_CONSTANT i64 31
-    %3:gprb(s32) = G_LSHR %1, %2(s64)
+    %2:gprb(s32) = G_CONSTANT i32 31
+    %3:gprb(s32) = G_LSHR %1, %2
     %4:gprb(s64) = G_ANYEXT %3(s32)
     $x10 = COPY %4(s64)
     PseudoRET implicit $x10
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rotate-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rotate-rv64.mir
index b9e9f36f766fb..b75e926bb50c4 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rotate-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rotate-rv64.mir
@@ -24,9 +24,8 @@ body:             |
     %0:gprb(s64) = COPY $x10
     %1:gprb(s32) = G_TRUNC %0(s64)
     %2:gprb(s64) = COPY $x11
-    %7:gprb(s64) = G_CONSTANT i64 4294967295
-    %6:gprb(s64) = G_AND %2, %7
-    %4:gprb(s32) = G_ROTL %1, %6(s64)
+    %6:gprb(s32) = G_TRUNC %2(s64)
+    %4:gprb(s32) = G_ROTL %1, %6(s32)
     %5:gprb(s64) = G_ANYEXT %4(s32)
     $x10 = COPY %5(s64)
     PseudoRET implicit $x10
@@ -75,9 +74,8 @@ body:             |
     %0:gprb(s64) = COPY $x10
     %1:gprb(s32) = G_TRUNC %0(s64)
     %2:gprb(s64) = COPY $x11
-    %7:gprb(s64) = G_CONSTANT i64 4294967295
-    %6:gprb(s64) = G_AND %2, %7
-    %4:gprb(s32) = G_ROTR %1, %6(s64)
+    %6:gprb(s32) = G_TRUNC %2(s64)
+    %4:gprb(s32) = G_ROTR %1, %6(s32)
     %5:gprb(s64) = G_ANYEXT %4(s32)
     $x10 = COPY %5(s64)
     PseudoRET implicit $x10
@@ -119,13 +117,13 @@ body:             |
     ; CHECK: liveins: $x10
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
-    ; CHECK-NEXT: [[RORIW:%[0-9]+]]:gpr = RORIW [[COPY]], 17
+    ; CHECK-NEXT: [[RORIW:%[0-9]+]]:gpr = RORIW [[COPY]], 15
     ; CHECK-NEXT: $x10 = COPY [[RORIW]]
     ; CHECK-NEXT: PseudoRET implicit $x10
     %0:gprb(s64) = COPY $x10
     %1:gprb(s32) = G_TRUNC %0(s64)
-    %2:gprb(s64) = G_CONSTANT i64 15
-    %3:gprb(s32) = G_ROTL %1, %2(s64)
+    %2:gprb(s32) = G_CONSTANT i32 15
+    %3:gprb(s32) = G_ROTL %1, %2(s32)
     %4:gprb(s64) = G_ANYEXT %3(s32)
     $x10 = COPY %4(s64)
     PseudoRET implicit $x10
@@ -167,13 +165,14 @@ body:             |
     ; CHECK: liveins: $x10
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
-    ; CHECK-NEXT: [[RORIW:%[0-9]+]]:gpr = RORIW [[COPY]], 15
-    ; CHECK-NEXT: $x10 = COPY [[RORIW]]
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 15
+    ; CHECK-NEXT: [[RORW:%[0-9]+]]:gpr = RORW [[COPY]], [[ADDI]]
+    ; CHECK-NEXT: $x10 = COPY [[RORW]]
     ; CHECK-NEXT: PseudoRET implicit $x10
     %0:gprb(s64) = COPY $x10
     %1:gprb(s32) = G_TRUNC %0(s64)
-    %2:gprb(s64) = G_CONSTANT i64 15
-    %3:gprb(s32) = G_ROTR %1, %2(s64)
+    %2:gprb(s32) = G_CONSTANT i32 15
+    %3:gprb(s32) = G_ROTR %1, %2(s32)
     %4:gprb(s64) = G_ANYEXT %3(s32)
     $x10 = COPY %4(s64)
     PseudoRET implicit $x10
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/render-vlop-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/render-vlop-rv32.mir
new file mode 100644
index 0000000000000..5600e351aa398
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/render-vlop-rv32.mir
@@ -0,0 +1,76 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=riscv32 -mattr=+v,+m -run-pass=instruction-select \
+# RUN:   -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            negative_vl
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: negative_vl
+    ; CHECK: [[ADDI:%[0-9]+]]:gprnox0 = ADDI $x0, -2
+    ; CHECK-NEXT: [[PseudoVMCLR_M_B1_:%[0-9]+]]:vr = PseudoVMCLR_M_B1 [[ADDI]], 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B1_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s32) = G_CONSTANT i32 -2
+    %1:vrb(<vscale x 1 x s1>) = G_VMCLR_VL %0(s32)
+    $v0 = COPY %1(<vscale x 1 x s1>)
+    PseudoRET implicit $v0
+...
+---
+name:            nonconst_vl
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: nonconst_vl
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprnox0 = COPY $x10
+    ; CHECK-NEXT: [[PseudoVMCLR_M_B1_:%[0-9]+]]:vr = PseudoVMCLR_M_B1 [[COPY]], 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B1_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s32) = COPY $x10
+    %1:vrb(<vscale x 1 x s1>) = G_VMCLR_VL %0(s32)
+    $v0 = COPY %1(<vscale x 1 x s1>)
+    PseudoRET implicit $v0
+...
+
+---
+name:            nonzero_vl
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: nonzero_vl
+    ; CHECK: [[PseudoVMCLR_M_B1_:%[0-9]+]]:vr = PseudoVMCLR_M_B1 1, 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B1_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s32) = G_CONSTANT i32 1
+    %1:vrb(<vscale x 1 x s1>) = G_VMCLR_VL %0(s32)
+    $v0 = COPY %1(<vscale x 1 x s1>)
+    PseudoRET implicit $v0
+...
+
+---
+name:            zero_vl
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: zero_vl
+    ; CHECK: [[PseudoVMCLR_M_B1_:%[0-9]+]]:vr = PseudoVMCLR_M_B1 0, 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B1_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s32) = G_CONSTANT i32 0
+    %1:vrb(<vscale x 1 x s1>) = G_VMCLR_VL %0(s32)
+    $v0 = COPY %1(<vscale x 1 x s1>)
+    PseudoRET implicit $v0
+...
+
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/render-vlop-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/render-vlop-rv64.mir
new file mode 100644
index 0000000000000..c2c0ed72be7b7
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/render-vlop-rv64.mir
@@ -0,0 +1,76 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=riscv64 -mattr=+v,+m -run-pass=instruction-select \
+# RUN:   -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            negative_vl
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: negative_vl
+    ; CHECK: [[ADDI:%[0-9]+]]:gprnox0 = ADDI $x0, -2
+    ; CHECK-NEXT: [[PseudoVMCLR_M_B1_:%[0-9]+]]:vr = PseudoVMCLR_M_B1 [[ADDI]], 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B1_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s64) = G_CONSTANT i64 -2
+    %1:vrb(<vscale x 1 x s1>) = G_VMCLR_VL %0(s64)
+    $v0 = COPY %1(<vscale x 1 x s1>)
+    PseudoRET implicit $v0
+...
+---
+name:            nonconst_vl
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $x10
+    ; CHECK-LABEL: name: nonconst_vl
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprnox0 = COPY $x10
+    ; CHECK-NEXT: [[PseudoVMCLR_M_B1_:%[0-9]+]]:vr = PseudoVMCLR_M_B1 [[COPY]], 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B1_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s64) = COPY $x10
+    %1:vrb(<vscale x 1 x s1>) = G_VMCLR_VL %0(s64)
+    $v0 = COPY %1(<vscale x 1 x s1>)
+    PseudoRET implicit $v0
+...
+
+---
+name:            nonzero_vl
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: nonzero_vl
+    ; CHECK: [[PseudoVMCLR_M_B1_:%[0-9]+]]:vr = PseudoVMCLR_M_B1 1, 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B1_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s64) = G_CONSTANT i64 1
+    %1:vrb(<vscale x 1 x s1>) = G_VMCLR_VL %0(s64)
+    $v0 = COPY %1(<vscale x 1 x s1>)
+    PseudoRET implicit $v0
+...
+
+---
+name:            zero_vl
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: zero_vl
+    ; CHECK: [[PseudoVMCLR_M_B1_:%[0-9]+]]:vr = PseudoVMCLR_M_B1 0, 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B1_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s64) = G_CONSTANT i64 0
+    %1:vrb(<vscale x 1 x s1>) = G_VMCLR_VL %0(s64)
+    $v0 = COPY %1(<vscale x 1 x s1>)
+    PseudoRET implicit $v0
+...
+
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vmclr-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vmclr-rv32.mir
new file mode 100644
index 0000000000000..1ef1312cc17c0
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vmclr-rv32.mir
@@ -0,0 +1,124 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+v,+m -run-pass=instruction-select \
+# RUN:   -simplify-mir -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            splat_zero_nxv1i1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv1i1
+    ; CHECK: [[PseudoVMCLR_M_B1_:%[0-9]+]]:vr = PseudoVMCLR_M_B1 -1, 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B1_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s32) = G_CONSTANT i32 -1
+    %1:vrb(<vscale x 1 x s1>) = G_VMCLR_VL %0(s32)
+    $v0 = COPY %1(<vscale x 1 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv2i1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv2i1
+    ; CHECK: [[PseudoVMCLR_M_B2_:%[0-9]+]]:vr = PseudoVMCLR_M_B2 -1, 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B2_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s32) = G_CONSTANT i32 -1
+    %1:vrb(<vscale x 2 x s1>) = G_VMCLR_VL %0(s32)
+    $v0 = COPY %1(<vscale x 2 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv4i1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv4i1
+    ; CHECK: [[PseudoVMCLR_M_B4_:%[0-9]+]]:vr = PseudoVMCLR_M_B4 -1, 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B4_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s32) = G_CONSTANT i32 -1
+    %1:vrb(<vscale x 4 x s1>) = G_VMCLR_VL %0(s32)
+    $v0 = COPY %1(<vscale x 4 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv8i1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv8i1
+    ; CHECK: [[PseudoVMCLR_M_B8_:%[0-9]+]]:vr = PseudoVMCLR_M_B8 -1, 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B8_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s32) = G_CONSTANT i32 -1
+    %1:vrb(<vscale x 8 x s1>) = G_VMCLR_VL %0(s32)
+    $v0 = COPY %1(<vscale x 8 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv16i1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv16i1
+    ; CHECK: [[PseudoVMCLR_M_B16_:%[0-9]+]]:vr = PseudoVMCLR_M_B16 -1, 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B16_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s32) = G_CONSTANT i32 -1
+    %1:vrb(<vscale x 16 x s1>) = G_VMCLR_VL %0(s32)
+    $v0 = COPY %1(<vscale x 16 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv32i1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv32i1
+    ; CHECK: [[PseudoVMCLR_M_B32_:%[0-9]+]]:vr = PseudoVMCLR_M_B32 -1, 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B32_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s32) = G_CONSTANT i32 -1
+    %1:vrb(<vscale x 32 x s1>) = G_VMCLR_VL %0(s32)
+    $v0 = COPY %1(<vscale x 32 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv64i1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv64i1
+    ; CHECK: [[PseudoVMCLR_M_B64_:%[0-9]+]]:vr = PseudoVMCLR_M_B64 -1, 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B64_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s32) = G_CONSTANT i32 -1
+    %1:vrb(<vscale x 64 x s1>) = G_VMCLR_VL %0(s32)
+    $v0 = COPY %1(<vscale x 64 x s1>)
+    PseudoRET implicit $v0
+
+...
+
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vmclr-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vmclr-rv64.mir
new file mode 100644
index 0000000000000..b7541cd4e96fb
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vmclr-rv64.mir
@@ -0,0 +1,124 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -mattr=+v,+m -run-pass=instruction-select \
+# RUN:   -simplify-mir -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            splat_zero_nxv1i1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv1i1
+    ; CHECK: [[PseudoVMCLR_M_B1_:%[0-9]+]]:vr = PseudoVMCLR_M_B1 -1, 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B1_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s64) = G_CONSTANT i64 -1
+    %1:vrb(<vscale x 1 x s1>) = G_VMCLR_VL %0(s64)
+    $v0 = COPY %1(<vscale x 1 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv2i1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv2i1
+    ; CHECK: [[PseudoVMCLR_M_B2_:%[0-9]+]]:vr = PseudoVMCLR_M_B2 -1, 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B2_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s64) = G_CONSTANT i64 -1
+    %1:vrb(<vscale x 2 x s1>) = G_VMCLR_VL %0(s64)
+    $v0 = COPY %1(<vscale x 2 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv4i1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv4i1
+    ; CHECK: [[PseudoVMCLR_M_B4_:%[0-9]+]]:vr = PseudoVMCLR_M_B4 -1, 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B4_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s64) = G_CONSTANT i64 -1
+    %1:vrb(<vscale x 4 x s1>) = G_VMCLR_VL %0(s64)
+    $v0 = COPY %1(<vscale x 4 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv8i1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv8i1
+    ; CHECK: [[PseudoVMCLR_M_B8_:%[0-9]+]]:vr = PseudoVMCLR_M_B8 -1, 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B8_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s64) = G_CONSTANT i64 -1
+    %1:vrb(<vscale x 8 x s1>) = G_VMCLR_VL %0(s64)
+    $v0 = COPY %1(<vscale x 8 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv16i1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv16i1
+    ; CHECK: [[PseudoVMCLR_M_B16_:%[0-9]+]]:vr = PseudoVMCLR_M_B16 -1, 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B16_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s64) = G_CONSTANT i64 -1
+    %1:vrb(<vscale x 16 x s1>) = G_VMCLR_VL %0(s64)
+    $v0 = COPY %1(<vscale x 16 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv32i1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv32i1
+    ; CHECK: [[PseudoVMCLR_M_B32_:%[0-9]+]]:vr = PseudoVMCLR_M_B32 -1, 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B32_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s64) = G_CONSTANT i64 -1
+    %1:vrb(<vscale x 32 x s1>) = G_VMCLR_VL %0(s64)
+    $v0 = COPY %1(<vscale x 32 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv64i1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv64i1
+    ; CHECK: [[PseudoVMCLR_M_B64_:%[0-9]+]]:vr = PseudoVMCLR_M_B64 -1, 0 /* e8 */
+    ; CHECK-NEXT: $v0 = COPY [[PseudoVMCLR_M_B64_]]
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:gprb(s64) = G_CONSTANT i64 -1
+    %1:vrb(<vscale x 64 x s1>) = G_VMCLR_VL %0(s64)
+    $v0 = COPY %1(<vscale x 64 x s1>)
+    PseudoRET implicit $v0
+
+...
+
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/zba-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/zba-rv64.mir
index f656f13b59d42..ade3d987244e0 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/zba-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/zba-rv64.mir
@@ -269,7 +269,7 @@ body:             |
     %1:gprb(s64) = COPY $x11
     %2:gprb(s32) = G_TRUNC %0
     %3:gprb(s32) = G_TRUNC %1
-    %4:gprb(s64) = G_CONSTANT i64 1
+    %4:gprb(s32) = G_CONSTANT i32 1
     %5:gprb(s32) = G_SHL %2, %4
     %6:gprb(s32) = G_ADD %5, %3
     %7:gprb(s64) = G_ANYEXT %6
@@ -295,7 +295,7 @@ body:             |
     %1:gprb(s64) = COPY $x11
     %2:gprb(s32) = G_TRUNC %0
     %3:gprb(s32) = G_TRUNC %1
-    %4:gprb(s64) = G_CONSTANT i64 2
+    %4:gprb(s32) = G_CONSTANT i32 2
     %5:gprb(s32) = G_SHL %2, %4
     %6:gprb(s32) = G_ADD %5, %3
     %7:gprb(s64) = G_ANYEXT %6
@@ -321,7 +321,7 @@ body:             |
     %1:gprb(s64) = COPY $x11
     %2:gprb(s32) = G_TRUNC %0
     %3:gprb(s32) = G_TRUNC %1
-    %4:gprb(s64) = G_CONSTANT i64 3
+    %4:gprb(s32) = G_CONSTANT i32 3
     %5:gprb(s32) = G_SHL %2, %4
     %6:gprb(s32) = G_ADD %5, %3
     %7:gprb(s64) = G_ANYEXT %6
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-half.ll b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-half.ll
index 63bc43ae20e7b..51809d0069910 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-half.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-half.ll
@@ -870,31 +870,31 @@ define half @caller_half_return_stack2(half %x, half %y) nounwind {
   ; RV32I-NEXT:   ADJCALLSTACKDOWN 4, 0, implicit-def $x2, implicit $x2
   ; RV32I-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
   ; RV32I-NEXT:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[C]](s16)
-  ; RV32I-NEXT:   [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
-  ; RV32I-NEXT:   [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[C1]](s16)
-  ; RV32I-NEXT:   [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
-  ; RV32I-NEXT:   [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
-  ; RV32I-NEXT:   [[ANYEXT6:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
-  ; RV32I-NEXT:   [[ANYEXT7:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
-  ; RV32I-NEXT:   [[ANYEXT8:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
-  ; RV32I-NEXT:   [[COPY2:%[0-9]+]]:_(p0) = COPY $x2
+  ; RV32I-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY [[ANYEXT]](s32)
+  ; RV32I-NEXT:   [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[C1]](s16)
+  ; RV32I-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY [[ANYEXT]](s32)
+  ; RV32I-NEXT:   [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
+  ; RV32I-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY [[ANYEXT3]](s32)
+  ; RV32I-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY [[ANYEXT3]](s32)
+  ; RV32I-NEXT:   [[COPY6:%[0-9]+]]:_(s32) = COPY [[ANYEXT]](s32)
+  ; RV32I-NEXT:   [[COPY7:%[0-9]+]]:_(p0) = COPY $x2
   ; RV32I-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; RV32I-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY2]], [[C2]](s32)
-  ; RV32I-NEXT:   G_STORE [[ANYEXT8]](s32), [[PTR_ADD]](p0) :: (store (s32) into stack, align 16)
+  ; RV32I-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY7]], [[C2]](s32)
+  ; RV32I-NEXT:   G_STORE [[COPY6]](s32), [[PTR_ADD]](p0) :: (store (s32) into stack, align 16)
   ; RV32I-NEXT:   $x10 = COPY [[ANYEXT]](s32)
   ; RV32I-NEXT:   $x11 = COPY [[ANYEXT1]](s32)
-  ; RV32I-NEXT:   $x12 = COPY [[ANYEXT2]](s32)
-  ; RV32I-NEXT:   $x13 = COPY [[ANYEXT3]](s32)
-  ; RV32I-NEXT:   $x14 = COPY [[ANYEXT4]](s32)
-  ; RV32I-NEXT:   $x15 = COPY [[ANYEXT5]](s32)
-  ; RV32I-NEXT:   $x16 = COPY [[ANYEXT6]](s32)
-  ; RV32I-NEXT:   $x17 = COPY [[ANYEXT7]](s32)
+  ; RV32I-NEXT:   $x12 = COPY [[COPY2]](s32)
+  ; RV32I-NEXT:   $x13 = COPY [[ANYEXT2]](s32)
+  ; RV32I-NEXT:   $x14 = COPY [[COPY3]](s32)
+  ; RV32I-NEXT:   $x15 = COPY [[ANYEXT3]](s32)
+  ; RV32I-NEXT:   $x16 = COPY [[COPY4]](s32)
+  ; RV32I-NEXT:   $x17 = COPY [[COPY5]](s32)
   ; RV32I-NEXT:   PseudoCALL target-flags(riscv-call) @callee_half_return_stack2, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit-def $x10
   ; RV32I-NEXT:   ADJCALLSTACKUP 4, 0, implicit-def $x2, implicit $x2
-  ; RV32I-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $x10
-  ; RV32I-NEXT:   [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY3]](s32)
-  ; RV32I-NEXT:   [[ANYEXT9:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s16)
-  ; RV32I-NEXT:   $x10 = COPY [[ANYEXT9]](s32)
+  ; RV32I-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY8]](s32)
+  ; RV32I-NEXT:   [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s16)
+  ; RV32I-NEXT:   $x10 = COPY [[ANYEXT4]](s32)
   ; RV32I-NEXT:   PseudoRET implicit $x10
   ;
   ; RV32IF-LABEL: name: caller_half_return_stack2
@@ -910,28 +910,28 @@ define half @caller_half_return_stack2(half %x, half %y) nounwind {
   ; RV32IF-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
   ; RV32IF-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
   ; RV32IF-NEXT:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[C]](s16)
-  ; RV32IF-NEXT:   [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
-  ; RV32IF-NEXT:   [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[C1]](s16)
-  ; RV32IF-NEXT:   [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
-  ; RV32IF-NEXT:   [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
-  ; RV32IF-NEXT:   [[ANYEXT6:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
-  ; RV32IF-NEXT:   [[ANYEXT7:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
-  ; RV32IF-NEXT:   [[ANYEXT8:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
+  ; RV32IF-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY [[ANYEXT]](s32)
+  ; RV32IF-NEXT:   [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[C1]](s16)
+  ; RV32IF-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY [[ANYEXT]](s32)
+  ; RV32IF-NEXT:   [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
+  ; RV32IF-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY [[ANYEXT3]](s32)
+  ; RV32IF-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY [[ANYEXT3]](s32)
+  ; RV32IF-NEXT:   [[COPY6:%[0-9]+]]:_(s32) = COPY [[ANYEXT]](s32)
   ; RV32IF-NEXT:   $f10_f = COPY [[ANYEXT]](s32)
   ; RV32IF-NEXT:   $f11_f = COPY [[ANYEXT1]](s32)
-  ; RV32IF-NEXT:   $f12_f = COPY [[ANYEXT2]](s32)
-  ; RV32IF-NEXT:   $f13_f = COPY [[ANYEXT3]](s32)
-  ; RV32IF-NEXT:   $f14_f = COPY [[ANYEXT4]](s32)
-  ; RV32IF-NEXT:   $f15_f = COPY [[ANYEXT5]](s32)
-  ; RV32IF-NEXT:   $f16_f = COPY [[ANYEXT6]](s32)
-  ; RV32IF-NEXT:   $f17_f = COPY [[ANYEXT7]](s32)
-  ; RV32IF-NEXT:   $x10 = COPY [[ANYEXT8]](s32)
+  ; RV32IF-NEXT:   $f12_f = COPY [[COPY2]](s32)
+  ; RV32IF-NEXT:   $f13_f = COPY [[ANYEXT2]](s32)
+  ; RV32IF-NEXT:   $f14_f = COPY [[COPY3]](s32)
+  ; RV32IF-NEXT:   $f15_f = COPY [[ANYEXT3]](s32)
+  ; RV32IF-NEXT:   $f16_f = COPY [[COPY4]](s32)
+  ; RV32IF-NEXT:   $f17_f = COPY [[COPY5]](s32)
+  ; RV32IF-NEXT:   $x10 = COPY [[COPY6]](s32)
   ; RV32IF-NEXT:   PseudoCALL target-flags(riscv-call) @callee_half_return_stack2, csr_ilp32f_lp64f, implicit-def $x1, implicit $f10_f, implicit $f11_f, implicit $f12_f, implicit $f13_f, implicit $f14_f, implicit $f15_f, implicit $f16_f, implicit $f17_f, implicit $x10, implicit-def $f10_f
   ; RV32IF-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-  ; RV32IF-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $f10_f
-  ; RV32IF-NEXT:   [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
-  ; RV32IF-NEXT:   [[ANYEXT9:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s16)
-  ; RV32IF-NEXT:   $f10_f = COPY [[ANYEXT9]](s32)
+  ; RV32IF-NEXT:   [[COPY7:%[0-9]+]]:_(s32) = COPY $f10_f
+  ; RV32IF-NEXT:   [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY7]](s32)
+  ; RV32IF-NEXT:   [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s16)
+  ; RV32IF-NEXT:   $f10_f = COPY [[ANYEXT4]](s32)
   ; RV32IF-NEXT:   PseudoRET implicit $f10_f
   ;
   ; RV32IZFH-LABEL: name: caller_half_return_stack2
@@ -972,31 +972,31 @@ define half @caller_half_return_stack2(half %x, half %y) nounwind {
   ; RV64I-NEXT:   ADJCALLSTACKDOWN 8, 0, implicit-def $x2, implicit $x2
   ; RV64I-NEXT:   [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC]](s16)
   ; RV64I-NEXT:   [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C]](s16)
-  ; RV64I-NEXT:   [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC]](s16)
-  ; RV64I-NEXT:   [[ANYEXT3:%[0-9]+]]:_(s64) = G_ANYEXT [[C1]](s16)
-  ; RV64I-NEXT:   [[ANYEXT4:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC]](s16)
-  ; RV64I-NEXT:   [[ANYEXT5:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC1]](s16)
-  ; RV64I-NEXT:   [[ANYEXT6:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC1]](s16)
-  ; RV64I-NEXT:   [[ANYEXT7:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC1]](s16)
-  ; RV64I-NEXT:   [[ANYEXT8:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC]](s16)
-  ; RV64I-NEXT:   [[COPY2:%[0-9]+]]:_(p0) = COPY $x2
+  ; RV64I-NEXT:   [[COPY2:%[0-9]+]]:_(s64) = COPY [[ANYEXT]](s64)
+  ; RV64I-NEXT:   [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[C1]](s16)
+  ; RV64I-NEXT:   [[COPY3:%[0-9]+]]:_(s64) = COPY [[ANYEXT]](s64)
+  ; RV64I-NEXT:   [[ANYEXT3:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC1]](s16)
+  ; RV64I-NEXT:   [[COPY4:%[0-9]+]]:_(s64) = COPY [[ANYEXT3]](s64)
+  ; RV64I-NEXT:   [[COPY5:%[0-9]+]]:_(s64) = COPY [[ANYEXT3]](s64)
+  ; RV64I-NEXT:   [[COPY6:%[0-9]+]]:_(s64) = COPY [[ANYEXT]](s64)
+  ; RV64I-NEXT:   [[COPY7:%[0-9]+]]:_(p0) = COPY $x2
   ; RV64I-NEXT:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; RV64I-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY2]], [[C2]](s64)
-  ; RV64I-NEXT:   G_STORE [[ANYEXT8]](s64), [[PTR_ADD]](p0) :: (store (s64) into stack, align 16)
+  ; RV64I-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY7]], [[C2]](s64)
+  ; RV64I-NEXT:   G_STORE [[COPY6]](s64), [[PTR_ADD]](p0) :: (store (s64) into stack, align 16)
   ; RV64I-NEXT:   $x10 = COPY [[ANYEXT]](s64)
   ; RV64I-NEXT:   $x11 = COPY [[ANYEXT1]](s64)
-  ; RV64I-NEXT:   $x12 = COPY [[ANYEXT2]](s64)
-  ; RV64I-NEXT:   $x13 = COPY [[ANYEXT3]](s64)
-  ; RV64I-NEXT:   $x14 = COPY [[ANYEXT4]](s64)
-  ; RV64I-NEXT:   $x15 = COPY [[ANYEXT5]](s64)
-  ; RV64I-NEXT:   $x16 = COPY [[ANYEXT6]](s64)
-  ; RV64I-NEXT:   $x17 = COPY [[ANYEXT7]](s64)
+  ; RV64I-NEXT:   $x12 = COPY [[COPY2]](s64)
+  ; RV64I-NEXT:   $x13 = COPY [[ANYEXT2]](s64)
+  ; RV64I-NEXT:   $x14 = COPY [[COPY3]](s64)
+  ; RV64I-NEXT:   $x15 = COPY [[ANYEXT3]](s64)
+  ; RV64I-NEXT:   $x16 = COPY [[COPY4]](s64)
+  ; RV64I-NEXT:   $x17 = COPY [[COPY5]](s64)
   ; RV64I-NEXT:   PseudoCALL target-flags(riscv-call) @callee_half_return_stack2, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit-def $x10
   ; RV64I-NEXT:   ADJCALLSTACKUP 8, 0, implicit-def $x2, implicit $x2
-  ; RV64I-NEXT:   [[COPY3:%[0-9]+]]:_(s64) = COPY $x10
-  ; RV64I-NEXT:   [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY3]](s64)
-  ; RV64I-NEXT:   [[ANYEXT9:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC2]](s16)
-  ; RV64I-NEXT:   $x10 = COPY [[ANYEXT9]](s64)
+  ; RV64I-NEXT:   [[COPY8:%[0-9]+]]:_(s64) = COPY $x10
+  ; RV64I-NEXT:   [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY8]](s64)
+  ; RV64I-NEXT:   [[ANYEXT4:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC2]](s16)
+  ; RV64I-NEXT:   $x10 = COPY [[ANYEXT4]](s64)
   ; RV64I-NEXT:   PseudoRET implicit $x10
   ;
   ; RV64IF-LABEL: name: caller_half_return_stack2
@@ -1012,28 +1012,28 @@ define half @caller_half_return_stack2(half %x, half %y) nounwind {
   ; RV64IF-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
   ; RV64IF-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
   ; RV64IF-NEXT:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[C]](s16)
-  ; RV64IF-NEXT:   [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
-  ; RV64IF-NEXT:   [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[C1]](s16)
-  ; RV64IF-NEXT:   [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
-  ; RV64IF-NEXT:   [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
-  ; RV64IF-NEXT:   [[ANYEXT6:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
-  ; RV64IF-NEXT:   [[ANYEXT7:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
+  ; RV64IF-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY [[ANYEXT]](s32)
+  ; RV64IF-NEXT:   [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[C1]](s16)
+  ; RV64IF-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY [[ANYEXT]](s32)
+  ; RV64IF-NEXT:   [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s16)
+  ; RV64IF-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY [[ANYEXT3]](s32)
+  ; RV64IF-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY [[ANYEXT3]](s32)
   ; RV64IF-NEXT:   $f10_f = COPY [[ANYEXT]](s32)
   ; RV64IF-NEXT:   $f11_f = COPY [[ANYEXT1]](s32)
-  ; RV64IF-NEXT:   $f12_f = COPY [[ANYEXT2]](s32)
-  ; RV64IF-NEXT:   $f13_f = COPY [[ANYEXT3]](s32)
-  ; RV64IF-NEXT:   $f14_f = COPY [[ANYEXT4]](s32)
-  ; RV64IF-NEXT:   $f15_f = COPY [[ANYEXT5]](s32)
-  ; RV64IF-NEXT:   $f16_f = COPY [[ANYEXT6]](s32)
-  ; RV64IF-NEXT:   $f17_f = COPY [[ANYEXT7]](s32)
-  ; RV64IF-NEXT:   [[ANYEXT8:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC]](s16)
-  ; RV64IF-NEXT:   $x10 = COPY [[ANYEXT8]](s64)
+  ; RV64IF-NEXT:   $f12_f = COPY [[COPY2]](s32)
+  ; RV64IF-NEXT:   $f13_f = COPY [[ANYEXT2]](s32)
+  ; RV64IF-NEXT:   $f14_f = COPY [[COPY3]](s32)
+  ; RV64IF-NEXT:   $f15_f = COPY [[ANYEXT3]](s32)
+  ; RV64IF-NEXT:   $f16_f = COPY [[COPY4]](s32)
+  ; RV64IF-NEXT:   $f17_f = COPY [[COPY5]](s32)
+  ; RV64IF-NEXT:   [[ANYEXT4:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC]](s16)
+  ; RV64IF-NEXT:   $x10 = COPY [[ANYEXT4]](s64)
   ; RV64IF-NEXT:   PseudoCALL target-flags(riscv-call) @callee_half_return_stack2, csr_ilp32f_lp64f, implicit-def $x1, implicit $f10_f, implicit $f11_f, implicit $f12_f, implicit $f13_f, implicit $f14_f, implicit $f15_f, implicit $f16_f, implicit $f17_f, implicit $x10, implicit-def $f10_f
   ; RV64IF-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-  ; RV64IF-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $f10_f
-  ; RV64IF-NEXT:   [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
-  ; RV64IF-NEXT:   [[ANYEXT9:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s16)
-  ; RV64IF-NEXT:   $f10_f = COPY [[ANYEXT9]](s32)
+  ; RV64IF-NEXT:   [[COPY6:%[0-9]+]]:_(s32) = COPY $f10_f
+  ; RV64IF-NEXT:   [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY6]](s32)
+  ; RV64IF-NEXT:   [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s16)
+  ; RV64IF-NEXT:   $f10_f = COPY [[ANYEXT5]](s32)
   ; RV64IF-NEXT:   PseudoRET implicit $f10_f
   ;
   ; RV64IZFH-LABEL: name: caller_half_return_stack2
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-ilp32-ilp32f-ilp32d-common.ll b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-ilp32-ilp32f-ilp32d-common.ll
index b7eb4574cc4dd..fb00e4860ddf2 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-ilp32-ilp32f-ilp32d-common.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-ilp32-ilp32f-ilp32d-common.ll
@@ -1091,8 +1091,8 @@ define %struct.large2 @callee_large_struct_ret2() nounwind {
   ; RV32I-NEXT: {{  $}}
   ; RV32I-NEXT:   [[COPY:%[0-9]+]]:_(p0) = COPY $x10
   ; RV32I-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; RV32I-NEXT:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; RV32I-NEXT:   [[DEF2:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
+  ; RV32I-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY [[DEF]](s32)
+  ; RV32I-NEXT:   [[DEF1:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; RV32I-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
   ; RV32I-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 2.000000e+00
   ; RV32I-NEXT:   [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 3
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-lp64-lp64f-lp64d-common.ll b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-lp64-lp64f-lp64d-common.ll
index 6750954a53708..3a0831b3c5214 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-lp64-lp64f-lp64d-common.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-lp64-lp64f-lp64d-common.ll
@@ -747,7 +747,7 @@ define %struct.large2 @callee_large_struct_ret2() nounwind {
   ; RV64I-NEXT:   [[COPY:%[0-9]+]]:_(p0) = COPY $x10
   ; RV64I-NEXT:   [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
   ; RV64I-NEXT:   [[DEF1:%[0-9]+]]:_(s128) = G_IMPLICIT_DEF
-  ; RV64I-NEXT:   [[DEF2:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
+  ; RV64I-NEXT:   [[COPY1:%[0-9]+]]:_(s64) = COPY [[DEF]](s64)
   ; RV64I-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
   ; RV64I-NEXT:   [[C1:%[0-9]+]]:_(s128) = G_CONSTANT i128 2
   ; RV64I-NEXT:   [[C2:%[0-9]+]]:_(s64) = G_FCONSTANT double 3.000000e+00
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-abs-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-abs-rv32.mir
index cbafa76ed4cd4..e64e7866103ff 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-abs-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-abs-rv32.mir
@@ -110,16 +110,14 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[COPY1]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[COPY1]], [[C1]](s32)
     ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[ASHR]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD]](s32), [[ASHR]]
     ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[ADD]](s32)
-    ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[COPY1]], [[ASHR1]]
+    ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[COPY1]], [[ASHR]]
     ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[ICMP]]
     ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[ADD2]](s32)
     ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY2]], [[ASHR]]
-    ; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[COPY3]], [[ASHR1]]
+    ; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[COPY3]], [[ASHR]]
     ; CHECK-NEXT: $x10 = COPY [[XOR]](s32)
     ; CHECK-NEXT: $x11 = COPY [[XOR1]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-abs-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-abs-rv64.mir
index 81da754b7ecc5..1c94f129b06b3 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-abs-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-abs-rv64.mir
@@ -11,19 +11,18 @@ body:             |
     ; RV64I-LABEL: name: abs_i8
     ; RV64I: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; RV64I-NEXT: [[ASSERT_ZEXT:%[0-9]+]]:_(s64) = G_ASSERT_ZEXT [[COPY]], 8
+    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ASSERT_ZEXT]](s64)
-    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; RV64I-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; RV64I-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 7
-    ; RV64I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[C2]](s64)
-    ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[ASSERT_ZEXT]](s64)
-    ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[TRUNC1]], [[ASHR1]]
+    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; RV64I-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C1]](s32)
+    ; RV64I-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s32)
+    ; RV64I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[C]](s32)
+    ; RV64I-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
+    ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY1]], [[ASHR1]]
     ; RV64I-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[ADD]], [[ASHR1]]
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[XOR]](s32)
-    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C3]]
+    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C2]]
     ; RV64I-NEXT: $x10 = COPY [[AND]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
     ;
@@ -53,11 +52,11 @@ body:             |
     ; RV64I-LABEL: name: abs_i16
     ; RV64I: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; RV64I-NEXT: [[ASSERT_SEXT:%[0-9]+]]:_(s64) = G_ASSERT_SEXT [[COPY]], 16
+    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ASSERT_SEXT]](s64)
-    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 15
-    ; RV64I-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[TRUNC]], [[C]](s64)
-    ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[ASSERT_SEXT]](s64)
-    ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[TRUNC1]], [[ASHR]]
+    ; RV64I-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[TRUNC]], [[C]](s32)
+    ; RV64I-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
+    ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY1]], [[ASHR]]
     ; RV64I-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[ADD]], [[ASHR]]
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[XOR]](s32)
     ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
@@ -91,8 +90,8 @@ body:             |
     ; RV64I: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; RV64I-NEXT: [[ASSERT_SEXT:%[0-9]+]]:_(s64) = G_ASSERT_SEXT [[COPY]], 32
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ASSERT_SEXT]](s64)
-    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 31
-    ; RV64I-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[TRUNC]], [[C]](s64)
+    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; RV64I-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[TRUNC]], [[C]](s32)
     ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[TRUNC]], [[ASHR]]
     ; RV64I-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[ADD]], [[ASHR]]
     ; RV64I-NEXT: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[XOR]](s32)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-addo-subo-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-addo-subo-rv32.mir
index 9227e6530221c..3a292093aa607 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-addo-subo-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-addo-subo-rv32.mir
@@ -16,13 +16,11 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[ASHR]], [[ASHR1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ADD]], [[C2]](s32)
-    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C2]](s32)
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ADD]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[ADD]](s32), [[ASHR2]]
     ; CHECK-NEXT: $x10 = COPY [[ADD]](s32)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
@@ -53,13 +51,11 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[ASHR]], [[ASHR1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ADD]], [[C2]](s32)
-    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C2]](s32)
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ADD]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[ADD]](s32), [[ASHR2]]
     ; CHECK-NEXT: $x10 = COPY [[ADD]](s32)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
@@ -123,15 +119,14 @@ body:             |
     ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ADD]](s32)
     ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[COPY1]], [[COPY3]]
     ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[ICMP]]
-    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY [[ADD2]](s32)
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY [[ADD2]](s32)
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[COPY5]](s32), [[COPY1]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[COPY5]](s32), [[COPY1]]
     ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[COPY4]](s32), [[COPY]]
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[ICMP3]], [[ICMP1]]
-    ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[COPY3]](s32), [[C1]]
-    ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[COPY3]](s32), [[C1]]
+    ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[COPY3]](s32), [[C]]
+    ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[COPY3]](s32), [[C]]
     ; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[COPY2]](s32), [[C]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s32), [[ICMP6]], [[ICMP4]]
     ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[SELECT]]
@@ -168,13 +163,11 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[ASHR]], [[ASHR1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[SUB]], [[C2]](s32)
-    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C2]](s32)
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[SUB]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[SUB]](s32), [[ASHR2]]
     ; CHECK-NEXT: $x10 = COPY [[SUB]](s32)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
@@ -205,13 +198,11 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[ASHR]], [[ASHR1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[SUB]], [[C2]](s32)
-    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C2]](s32)
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[SUB]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[SUB]](s32), [[ASHR2]]
     ; CHECK-NEXT: $x10 = COPY [[SUB]](s32)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
@@ -275,13 +266,12 @@ body:             |
     ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[COPY1]], [[COPY3]]
     ; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[ICMP]]
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[SUB2]](s32), [[COPY1]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[SUB2]](s32), [[COPY1]]
     ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[SUB]](s32), [[COPY]]
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[ICMP3]], [[ICMP1]]
-    ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s32) = G_ICMP intpred(sgt), [[COPY3]](s32), [[C1]]
-    ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[COPY3]](s32), [[C1]]
+    ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s32) = G_ICMP intpred(sgt), [[COPY3]](s32), [[C]]
+    ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[COPY3]](s32), [[C]]
     ; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s32) = G_ICMP intpred(ugt), [[COPY2]](s32), [[C]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s32), [[ICMP6]], [[ICMP4]]
     ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[SELECT]]
@@ -318,8 +308,7 @@ body:             |
     ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[COPY1]]
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ADD]](s32)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
@@ -350,8 +339,7 @@ body:             |
     ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[COPY1]]
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ADD]](s32)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
@@ -450,8 +438,7 @@ body:             |
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[COPY1]]
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[SUB]](s32)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
@@ -482,8 +469,7 @@ body:             |
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[COPY1]]
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[SUB]](s32)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-addo-subo-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-addo-subo-rv64.mir
index 8acaff5dbb25a..d7f19b5fb0277 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-addo-subo-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-addo-subo-rv64.mir
@@ -16,13 +16,11 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[ASHR]], [[ASHR1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ADD]], [[C2]](s64)
-    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C2]](s64)
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ADD]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[ADD]](s64), [[ASHR2]]
     ; CHECK-NEXT: $x10 = COPY [[ADD]](s64)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
@@ -53,13 +51,11 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[ASHR]], [[ASHR1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ADD]], [[C2]](s64)
-    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C2]](s64)
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ADD]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[ADD]](s64), [[ASHR2]]
     ; CHECK-NEXT: $x10 = COPY [[ADD]](s64)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
@@ -153,13 +149,11 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s64) = G_SUB [[ASHR]], [[ASHR1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[SUB]], [[C2]](s64)
-    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C2]](s64)
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[SUB]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[SUB]](s64), [[ASHR2]]
     ; CHECK-NEXT: $x10 = COPY [[SUB]](s64)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
@@ -190,13 +184,11 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s64) = G_SUB [[ASHR]], [[ASHR1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[SUB]], [[C2]](s64)
-    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C2]](s64)
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[SUB]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[SUB]](s64), [[ASHR2]]
     ; CHECK-NEXT: $x10 = COPY [[SUB]](s64)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
@@ -293,11 +285,10 @@ body:             |
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ADD]](s32)
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
-    ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[ADD]](s32)
-    ; CHECK-NEXT: $x10 = COPY [[ANYEXT1]](s64)
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[ANYEXT]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[COPY2]](s64)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
     %2:_(s64) = COPY $x10
@@ -329,11 +320,10 @@ body:             |
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ADD]](s32)
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
-    ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[ADD]](s32)
-    ; CHECK-NEXT: $x10 = COPY [[ANYEXT1]](s64)
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[ANYEXT]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[COPY2]](s64)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
     %2:_(s64) = COPY $x10
@@ -425,8 +415,7 @@ body:             |
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[TRUNC1]]
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SUB]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -460,8 +449,7 @@ body:             |
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[TRUNC1]]
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SUB]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -495,8 +483,7 @@ body:             |
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[TRUNC1]]
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SUB]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ashr-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ashr-rv32.mir
index e24c7ad539133..e5ac59c6c4481 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ashr-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ashr-rv32.mir
@@ -101,21 +101,21 @@ body:             |
     ; CHECK: %xlo:_(s32) = COPY $x10
     ; CHECK-NEXT: %xhi:_(s32) = COPY $x11
     ; CHECK-NEXT: %ylo:_(s32) = COPY $x12
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %xhi, [[C]](s32)
-    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB %ylo, [[C1]]
-    ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C1]], %ylo
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), %ylo(s32), [[C1]]
-    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), %ylo(s32), [[C2]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %xhi, [[C1]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s32)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB %ylo, [[C2]]
+    ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C2]], %ylo
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), %ylo(s32), [[C2]]
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), %ylo(s32), [[C3]]
     ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], %ylo(s32)
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR %xlo, %ylo(s32)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ASHR]], [[SUB1]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR]], [[SHL1]]
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
-    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[C3]](s32)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[C]](s32)
     ; CHECK-NEXT: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[SUB]](s32)
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[OR]], [[ASHR3]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s32), %xlo, [[SELECT]]
@@ -189,72 +189,59 @@ body:             |
     ; CHECK-NEXT: %hi1:_(s32) = COPY $x12
     ; CHECK-NEXT: %lo2:_(s32) = COPY $x13
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR %hi1, [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
-    ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB %lo2, [[C1]]
-    ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C1]], %lo2
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), %lo2(s32), [[C1]]
-    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), %lo2(s32), [[C2]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB %lo2, [[C2]]
+    ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C2]], %lo2
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), %lo2(s32), [[C2]]
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), %lo2(s32), [[C1]]
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
     ; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB %lo2, [[C3]]
     ; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C3]], %lo2
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), %lo2(s32), [[C3]]
-    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), %lo2(s32), [[C4]]
+    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), %lo2(s32), [[C1]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR %hi1, %lo2(s32)
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ASHR]], [[SUB3]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR]], [[SHL]]
     ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[SUB2]](s32)
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[OR]], [[ASHR1]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s32), %hi1, [[SELECT]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; CHECK-NEXT: [[SUB4:%[0-9]+]]:_(s32) = G_SUB %lo2, [[C5]]
-    ; CHECK-NEXT: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C5]], %lo2
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), %lo2(s32), [[C5]]
-    ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), %lo2(s32), [[C6]]
+    ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), %lo2(s32), [[C3]]
+    ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), %lo2(s32), [[C1]]
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR %mid1, %lo2(s32)
     ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR %lo1, %lo2(s32)
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %mid1, [[SUB5]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %mid1, [[SUB3]](s32)
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[LSHR2]], [[SHL1]]
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR %mid1, [[SUB4]](s32)
+    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR %mid1, [[SUB2]](s32)
     ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP4]](s32), [[OR1]], [[LSHR3]]
     ; CHECK-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s32), %lo1, [[SELECT2]]
-    ; CHECK-NEXT: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP4]](s32), [[LSHR1]], [[C7]]
-    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; CHECK-NEXT: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[C8]]
-    ; CHECK-NEXT: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[C8]], [[SUB1]]
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[SUB1]](s32), [[C8]]
-    ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[SUB1]](s32), [[C9]]
+    ; CHECK-NEXT: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP4]](s32), [[LSHR1]], [[C1]]
+    ; CHECK-NEXT: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[C3]]
+    ; CHECK-NEXT: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[SUB1]]
+    ; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[SUB1]](s32), [[C3]]
+    ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[SUB1]](s32), [[C1]]
     ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL %hi1, [[SUB1]](s32)
-    ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR %hi1, [[SUB7]](s32)
+    ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR %hi1, [[SUB5]](s32)
     ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ASHR]], [[SUB1]](s32)
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[LSHR4]], [[SHL3]]
-    ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL %hi1, [[SUB6]](s32)
-    ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP6]](s32), [[SHL2]], [[C10]]
+    ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL %hi1, [[SUB4]](s32)
+    ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP6]](s32), [[SHL2]], [[C1]]
     ; CHECK-NEXT: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[ICMP6]](s32), [[OR2]], [[SHL4]]
     ; CHECK-NEXT: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP7]](s32), [[ASHR]], [[SELECT6]]
     ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SELECT3]], [[SELECT5]]
     ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SELECT4]], [[SELECT7]]
-    ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
-    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[C11]](s32)
-    ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; CHECK-NEXT: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SUB]], [[C12]]
-    ; CHECK-NEXT: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[C12]], [[SUB]]
-    ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ICMP8:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[SUB]](s32), [[C12]]
-    ; CHECK-NEXT: [[ICMP9:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[SUB]](s32), [[C13]]
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[C]](s32)
+    ; CHECK-NEXT: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB]], [[C3]]
+    ; CHECK-NEXT: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[SUB]]
+    ; CHECK-NEXT: [[ICMP8:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[SUB]](s32), [[C3]]
+    ; CHECK-NEXT: [[ICMP9:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[SUB]](s32), [[C1]]
     ; CHECK-NEXT: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[SUB]](s32)
     ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR %hi1, [[SUB]](s32)
-    ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ASHR]], [[SUB9]](s32)
+    ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ASHR]], [[SUB7]](s32)
     ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[LSHR5]], [[SHL5]]
-    ; CHECK-NEXT: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
-    ; CHECK-NEXT: [[ASHR4:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[C14]](s32)
-    ; CHECK-NEXT: [[ASHR5:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[SUB8]](s32)
+    ; CHECK-NEXT: [[ASHR4:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR5:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[SUB6]](s32)
     ; CHECK-NEXT: [[SELECT8:%[0-9]+]]:_(s32) = G_SELECT [[ICMP8]](s32), [[OR5]], [[ASHR5]]
     ; CHECK-NEXT: [[SELECT9:%[0-9]+]]:_(s32) = G_SELECT [[ICMP9]](s32), %hi1, [[SELECT8]]
     ; CHECK-NEXT: [[SELECT10:%[0-9]+]]:_(s32) = G_SELECT [[ICMP8]](s32), [[ASHR3]], [[ASHR4]]
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ashr-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ashr-rv64.mir
index 01662f3893120..72ad9d70b7468 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ashr-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ashr-rv64.mir
@@ -12,10 +12,9 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C1]](s64)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C2]](s64)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C1]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s32)
     ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[AND]](s32)
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ASHR1]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -41,10 +40,9 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 17
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C1]](s64)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 17
-    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C2]](s64)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 17
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C1]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s32)
     ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[AND]](s32)
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ASHR1]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -70,10 +68,9 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C1]](s64)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C2]](s64)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C1]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s32)
     ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[AND]](s32)
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ASHR1]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -136,21 +133,21 @@ body:             |
     ; CHECK: %x00:_(s64) = COPY $x10
     ; CHECK-NEXT: %x01:_(s64) = COPY $x11
     ; CHECK-NEXT: %y00:_(s64) = COPY $x12
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL %x01, [[C]](s64)
-    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
-    ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s64) = G_SUB %y00, [[C1]]
-    ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s64) = G_SUB [[C1]], %y00
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), %y00(s64), [[C1]]
-    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), %y00(s64), [[C2]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL %x01, [[C1]](s64)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
+    ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s64) = G_SUB %y00, [[C2]]
+    ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s64) = G_SUB [[C2]], %y00
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), %y00(s64), [[C2]]
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), %y00(s64), [[C3]]
     ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[ASHR]], %y00(s64)
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR %x00, %y00(s64)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[ASHR]], [[SUB1]](s64)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL1]]
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
-    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[ASHR]], [[C3]](s64)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[ASHR]], [[C]](s64)
     ; CHECK-NEXT: [[ASHR3:%[0-9]+]]:_(s64) = G_ASHR [[ASHR]], [[SUB]](s64)
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[OR]], [[ASHR3]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s64), %x00, [[SELECT]]
@@ -234,62 +231,49 @@ body:             |
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
     ; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s64) = G_SUB %lo2, [[C3]]
     ; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(s64) = G_SUB [[C3]], %lo2
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), %lo2(s64), [[C3]]
-    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), %lo2(s64), [[C4]]
+    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), %lo2(s64), [[C2]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR %hi1, %lo2(s64)
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[ASHR]], [[SUB3]](s64)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
     ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[ASHR]], [[SUB2]](s64)
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s64), [[OR]], [[ASHR1]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s64), %hi1, [[SELECT]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
-    ; CHECK-NEXT: [[SUB4:%[0-9]+]]:_(s64) = G_SUB %lo2, [[C5]]
-    ; CHECK-NEXT: [[SUB5:%[0-9]+]]:_(s64) = G_SUB [[C5]], %lo2
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), %lo2(s64), [[C5]]
-    ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), %lo2(s64), [[C6]]
+    ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), %lo2(s64), [[C3]]
+    ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), %lo2(s64), [[C2]]
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR %mid1, %lo2(s64)
     ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR %lo1, %lo2(s64)
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %mid1, [[SUB5]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %mid1, [[SUB3]](s64)
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s64) = G_OR [[LSHR2]], [[SHL1]]
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s64) = G_LSHR %mid1, [[SUB4]](s64)
+    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s64) = G_LSHR %mid1, [[SUB2]](s64)
     ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s64), [[OR1]], [[LSHR3]]
     ; CHECK-NEXT: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP5]](s64), %lo1, [[SELECT2]]
-    ; CHECK-NEXT: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s64), [[LSHR1]], [[C7]]
-    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
-    ; CHECK-NEXT: [[SUB6:%[0-9]+]]:_(s64) = G_SUB [[SUB1]], [[C8]]
-    ; CHECK-NEXT: [[SUB7:%[0-9]+]]:_(s64) = G_SUB [[C8]], [[SUB1]]
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[SUB1]](s64), [[C8]]
-    ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[SUB1]](s64), [[C9]]
+    ; CHECK-NEXT: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s64), [[LSHR1]], [[C2]]
+    ; CHECK-NEXT: [[SUB4:%[0-9]+]]:_(s64) = G_SUB [[SUB1]], [[C3]]
+    ; CHECK-NEXT: [[SUB5:%[0-9]+]]:_(s64) = G_SUB [[C3]], [[SUB1]]
+    ; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[SUB1]](s64), [[C3]]
+    ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[SUB1]](s64), [[C2]]
     ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL %hi1, [[SUB1]](s64)
-    ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s64) = G_LSHR %hi1, [[SUB7]](s64)
+    ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s64) = G_LSHR %hi1, [[SUB5]](s64)
     ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s64) = G_SHL [[ASHR]], [[SUB1]](s64)
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s64) = G_OR [[LSHR4]], [[SHL3]]
-    ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s64) = G_SHL %hi1, [[SUB6]](s64)
-    ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s64), [[SHL2]], [[C10]]
+    ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s64) = G_SHL %hi1, [[SUB4]](s64)
+    ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s64), [[SHL2]], [[C2]]
     ; CHECK-NEXT: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s64), [[OR2]], [[SHL4]]
     ; CHECK-NEXT: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s64), [[ASHR]], [[SELECT6]]
     ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s64) = G_OR [[SELECT3]], [[SELECT5]]
     ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s64) = G_OR [[SELECT4]], [[SELECT7]]
-    ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
-    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[ASHR]], [[C11]](s64)
-    ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
-    ; CHECK-NEXT: [[SUB8:%[0-9]+]]:_(s64) = G_SUB [[SUB]], [[C12]]
-    ; CHECK-NEXT: [[SUB9:%[0-9]+]]:_(s64) = G_SUB [[C12]], [[SUB]]
-    ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[ICMP8:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[SUB]](s64), [[C12]]
-    ; CHECK-NEXT: [[ICMP9:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[SUB]](s64), [[C13]]
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[ASHR]], [[C]](s64)
+    ; CHECK-NEXT: [[SUB6:%[0-9]+]]:_(s64) = G_SUB [[SUB]], [[C3]]
+    ; CHECK-NEXT: [[SUB7:%[0-9]+]]:_(s64) = G_SUB [[C3]], [[SUB]]
+    ; CHECK-NEXT: [[ICMP8:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[SUB]](s64), [[C3]]
+    ; CHECK-NEXT: [[ICMP9:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[SUB]](s64), [[C2]]
     ; CHECK-NEXT: [[ASHR3:%[0-9]+]]:_(s64) = G_ASHR [[ASHR]], [[SUB]](s64)
     ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(s64) = G_LSHR %hi1, [[SUB]](s64)
-    ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[ASHR]], [[SUB9]](s64)
+    ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[ASHR]], [[SUB7]](s64)
     ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s64) = G_OR [[LSHR5]], [[SHL5]]
-    ; CHECK-NEXT: [[C14:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
-    ; CHECK-NEXT: [[ASHR4:%[0-9]+]]:_(s64) = G_ASHR [[ASHR]], [[C14]](s64)
-    ; CHECK-NEXT: [[ASHR5:%[0-9]+]]:_(s64) = G_ASHR [[ASHR]], [[SUB8]](s64)
+    ; CHECK-NEXT: [[ASHR4:%[0-9]+]]:_(s64) = G_ASHR [[ASHR]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR5:%[0-9]+]]:_(s64) = G_ASHR [[ASHR]], [[SUB6]](s64)
     ; CHECK-NEXT: [[SELECT8:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s64), [[OR5]], [[ASHR5]]
     ; CHECK-NEXT: [[SELECT9:%[0-9]+]]:_(s64) = G_SELECT [[ICMP9]](s64), %hi1, [[SELECT8]]
     ; CHECK-NEXT: [[SELECT10:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s64), [[ASHR3]], [[ASHR4]]
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bitreverse-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bitreverse-rv32.mir
index 7625a5c2d568a..2f1ca52c6cf0d 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bitreverse-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bitreverse-rv32.mir
@@ -20,29 +20,26 @@ body:             |
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -16
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[AND1]], [[C4]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[AND1]], [[C1]]
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C2]](s32)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[OR]], [[C2]](s32)
     ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SHL1]], [[C3]]
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[LSHR1]], [[AND3]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 -52
-    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C6]]
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[AND4]], [[C7]]
-    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C5]](s32)
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C5]](s32)
-    ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SHL2]], [[C6]]
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 -52
+    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C5]]
+    ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[AND4]], [[C1]]
+    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C4]](s32)
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C4]](s32)
+    ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SHL2]], [[C5]]
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[LSHR2]], [[AND6]]
-    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 -86
-    ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C9]]
-    ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[AND7]], [[C10]]
-    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND8]], [[C8]](s32)
-    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[OR2]], [[C8]](s32)
-    ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[SHL3]], [[C9]]
+    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 -86
+    ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C7]]
+    ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[AND7]], [[C1]]
+    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND8]], [[C6]](s32)
+    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[OR2]], [[C6]](s32)
+    ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[SHL3]], [[C7]]
     ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[LSHR3]], [[AND9]]
     ; CHECK-NEXT: $x10 = COPY [[OR3]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -73,29 +70,26 @@ body:             |
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -3856
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[AND1]], [[C4]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[AND1]], [[C1]]
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C2]](s32)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[OR]], [[C2]](s32)
     ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SHL1]], [[C3]]
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[LSHR1]], [[AND3]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 -13108
-    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C6]]
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[AND4]], [[C7]]
-    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C5]](s32)
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C5]](s32)
-    ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SHL2]], [[C6]]
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 -13108
+    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C5]]
+    ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[AND4]], [[C1]]
+    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C4]](s32)
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C4]](s32)
+    ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SHL2]], [[C5]]
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[LSHR2]], [[AND6]]
-    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 -21846
-    ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C9]]
-    ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[AND7]], [[C10]]
-    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND8]], [[C8]](s32)
-    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[OR2]], [[C8]](s32)
-    ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[SHL3]], [[C9]]
+    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 -21846
+    ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C7]]
+    ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[AND7]], [[C1]]
+    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND8]], [[C6]](s32)
+    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[OR2]], [[C6]](s32)
+    ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[SHL3]], [[C7]]
     ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[LSHR3]], [[AND9]]
     ; CHECK-NEXT: $x10 = COPY [[OR3]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -202,38 +196,29 @@ body:             |
     ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C7]](s32)
     ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[SHL4]], [[C8]]
     ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[LSHR4]], [[AND7]]
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C9]](s32)
-    ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C9]](s32)
+    ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
+    ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[LSHR5]], [[SHL5]]
-    ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 65280
-    ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C10]]
-    ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND8]], [[C11]](s32)
+    ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C1]]
+    ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND8]], [[C2]](s32)
     ; CHECK-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[OR6]], [[SHL6]]
-    ; CHECK-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C11]](s32)
-    ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C10]]
+    ; CHECK-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C1]]
     ; CHECK-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[OR7]], [[AND9]]
-    ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 -252645136
-    ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[OR8]], [[C13]]
-    ; CHECK-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[AND10]], [[C12]](s32)
-    ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[OR8]], [[C12]](s32)
-    ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[SHL7]], [[C13]]
+    ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[OR8]], [[C4]]
+    ; CHECK-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[AND10]], [[C3]](s32)
+    ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[OR8]], [[C3]](s32)
+    ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[SHL7]], [[C4]]
     ; CHECK-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[LSHR7]], [[AND11]]
-    ; CHECK-NEXT: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; CHECK-NEXT: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 -858993460
-    ; CHECK-NEXT: [[AND12:%[0-9]+]]:_(s32) = G_AND [[OR9]], [[C15]]
-    ; CHECK-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[AND12]], [[C14]](s32)
-    ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR9]], [[C14]](s32)
-    ; CHECK-NEXT: [[AND13:%[0-9]+]]:_(s32) = G_AND [[SHL8]], [[C15]]
+    ; CHECK-NEXT: [[AND12:%[0-9]+]]:_(s32) = G_AND [[OR9]], [[C6]]
+    ; CHECK-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[AND12]], [[C5]](s32)
+    ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[OR9]], [[C5]](s32)
+    ; CHECK-NEXT: [[AND13:%[0-9]+]]:_(s32) = G_AND [[SHL8]], [[C6]]
     ; CHECK-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[LSHR8]], [[AND13]]
-    ; CHECK-NEXT: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1431655766
-    ; CHECK-NEXT: [[AND14:%[0-9]+]]:_(s32) = G_AND [[OR10]], [[C17]]
-    ; CHECK-NEXT: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[AND14]], [[C16]](s32)
-    ; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C16]](s32)
-    ; CHECK-NEXT: [[AND15:%[0-9]+]]:_(s32) = G_AND [[SHL9]], [[C17]]
+    ; CHECK-NEXT: [[AND14:%[0-9]+]]:_(s32) = G_AND [[OR10]], [[C8]]
+    ; CHECK-NEXT: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[AND14]], [[C7]](s32)
+    ; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[OR10]], [[C7]](s32)
+    ; CHECK-NEXT: [[AND15:%[0-9]+]]:_(s32) = G_AND [[SHL9]], [[C8]]
     ; CHECK-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[LSHR9]], [[AND15]]
     ; CHECK-NEXT: $x10 = COPY [[OR5]](s32)
     ; CHECK-NEXT: $x11 = COPY [[OR11]](s32)
@@ -262,12 +247,10 @@ body:             |
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SHL]], [[C1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C3]]
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C2]](s32)
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C4]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C2]]
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C]](s32)
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C]]
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[AND2]]
     ; CHECK-NEXT: $x10 = COPY [[OR]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -297,16 +280,12 @@ body:             |
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C3]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C2]](s32)
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C4]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C]]
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[AND2]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
-    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C6]]
-    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C5]](s32)
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C7]]
-    ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[AND4]]
+    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C]](s32)
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C4]]
+    ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[AND3]]
     ; CHECK-NEXT: $x10 = COPY [[OR1]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %1:_(s32) = COPY $x10
@@ -336,20 +315,15 @@ body:             |
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SHL1]], [[C3]]
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[AND1]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C5]]
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C4]](s32)
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C6]]
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C4]]
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C2]](s32)
+    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C5]]
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[AND3]]
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
-    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
-    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C8]]
-    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[C7]](s32)
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C9]]
-    ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[AND5]]
+    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C]](s32)
+    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C2]]
+    ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[AND4]]
     ; CHECK-NEXT: $x10 = COPY [[OR2]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %1:_(s32) = COPY $x10
@@ -391,27 +365,16 @@ body:             |
     ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
     ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C8]]
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[AND4]]
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
-    ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C10]]
-    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C9]](s32)
-    ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C11]]
-    ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[OR2]], [[AND6]]
-    ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
-    ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C13]]
-    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND7]], [[C12]](s32)
-    ; CHECK-NEXT: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[LSHR2]], [[C14]]
-    ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[AND8]]
-    ; CHECK-NEXT: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
-    ; CHECK-NEXT: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
-    ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C16]]
-    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND9]], [[C15]](s32)
-    ; CHECK-NEXT: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C17]]
-    ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[OR4]], [[AND10]]
+    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C4]](s32)
+    ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C2]]
+    ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[OR2]], [[AND5]]
+    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C2]](s32)
+    ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR2]], [[C4]]
+    ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[AND6]]
+    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C]](s32)
+    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C9]]
+    ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[OR4]], [[AND7]]
     ; CHECK-NEXT: $x10 = COPY [[OR5]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %1:_(s32) = COPY $x10
@@ -441,29 +404,26 @@ body:             |
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -986896
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16777215
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[AND1]], [[C4]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[AND1]], [[C1]]
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C2]](s32)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[OR]], [[C2]](s32)
     ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SHL1]], [[C3]]
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[LSHR1]], [[AND3]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 -3355444
-    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C6]]
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16777215
-    ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[AND4]], [[C7]]
-    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C5]](s32)
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C5]](s32)
-    ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SHL2]], [[C6]]
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 -3355444
+    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C5]]
+    ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[AND4]], [[C1]]
+    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C4]](s32)
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C4]](s32)
+    ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SHL2]], [[C5]]
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[LSHR2]], [[AND6]]
-    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 -5592406
-    ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C9]]
-    ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 16777215
-    ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[AND7]], [[C10]]
-    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND8]], [[C8]](s32)
-    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[OR2]], [[C8]](s32)
-    ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[SHL3]], [[C9]]
+    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 -5592406
+    ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C7]]
+    ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[AND7]], [[C1]]
+    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND8]], [[C6]](s32)
+    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[OR2]], [[C6]](s32)
+    ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[SHL3]], [[C7]]
     ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[LSHR3]], [[AND9]]
     ; CHECK-NEXT: $x10 = COPY [[OR3]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -496,19 +456,13 @@ body:             |
     ; CHECK-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s4>) = G_BUILD_VECTOR [[C3]](s4), [[C3]](s4)
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(<2 x s4>) = G_AND [[SHL1]], [[BUILD_VECTOR3]]
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(<2 x s4>) = G_OR [[AND]], [[AND1]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s4) = G_CONSTANT i4 1
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(<2 x s4>) = G_LSHR [[TRUNC]], [[BUILD_VECTOR2]](<2 x s4>)
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s4) = G_CONSTANT i4 2
     ; CHECK-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s4>) = G_BUILD_VECTOR [[C4]](s4), [[C4]](s4)
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(<2 x s4>) = G_LSHR [[TRUNC]], [[BUILD_VECTOR4]](<2 x s4>)
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s4) = G_CONSTANT i4 2
-    ; CHECK-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s4>) = G_BUILD_VECTOR [[C5]](s4), [[C5]](s4)
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(<2 x s4>) = G_AND [[LSHR]], [[BUILD_VECTOR5]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(<2 x s4>) = G_AND [[LSHR]], [[BUILD_VECTOR4]]
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(<2 x s4>) = G_OR [[OR]], [[AND2]]
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s4) = G_CONSTANT i4 3
-    ; CHECK-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x s4>) = G_BUILD_VECTOR [[C6]](s4), [[C6]](s4)
-    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(<2 x s4>) = G_LSHR [[TRUNC]], [[BUILD_VECTOR6]](<2 x s4>)
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s4) = G_CONSTANT i4 1
-    ; CHECK-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x s4>) = G_BUILD_VECTOR [[C7]](s4), [[C7]](s4)
-    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(<2 x s4>) = G_AND [[LSHR1]], [[BUILD_VECTOR7]]
+    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(<2 x s4>) = G_LSHR [[TRUNC]], [[BUILD_VECTOR]](<2 x s4>)
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(<2 x s4>) = G_AND [[LSHR1]], [[BUILD_VECTOR2]]
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(<2 x s4>) = G_OR [[OR1]], [[AND3]]
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s4>) = COPY [[OR2]](<2 x s4>)
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(<2 x s32>) = G_ANYEXT [[COPY1]](<2 x s4>)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bitreverse-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bitreverse-rv64.mir
index 71583f15cd5cd..357b7ec47df51 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bitreverse-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bitreverse-rv64.mir
@@ -11,44 +11,36 @@ body:             |
     ; CHECK: liveins: $x10
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C2]](s64)
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR]], [[SHL]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -16
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[AND1]], [[C4]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C5]](s64)
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[OR]], [[C6]](s64)
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[AND1]], [[C1]]
+    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C2]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[OR]], [[C2]](s32)
     ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SHL1]], [[C3]]
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[LSHR1]], [[AND3]]
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 -52
-    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C7]]
-    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[AND4]], [[C8]]
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C9]](s64)
-    ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C10]](s64)
-    ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SHL2]], [[C7]]
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 -52
+    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C5]]
+    ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[AND4]], [[C1]]
+    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C4]](s32)
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C4]](s32)
+    ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SHL2]], [[C5]]
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[LSHR2]], [[AND6]]
-    ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 -86
-    ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C11]]
-    ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[AND7]], [[C12]]
-    ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND8]], [[C13]](s64)
-    ; CHECK-NEXT: [[C14:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[OR2]], [[C14]](s64)
-    ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[SHL3]], [[C11]]
+    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 -86
+    ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C7]]
+    ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[AND7]], [[C1]]
+    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND8]], [[C6]](s32)
+    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[OR2]], [[C6]](s32)
+    ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[SHL3]], [[C7]]
     ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[LSHR3]], [[AND9]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR3]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -71,44 +63,36 @@ body:             |
     ; CHECK: liveins: $x10
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C2]](s64)
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR]], [[SHL]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -3856
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[AND1]], [[C4]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C5]](s64)
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[OR]], [[C6]](s64)
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[AND1]], [[C1]]
+    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C2]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[OR]], [[C2]](s32)
     ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SHL1]], [[C3]]
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[LSHR1]], [[AND3]]
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 -13108
-    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C7]]
-    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[AND4]], [[C8]]
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C9]](s64)
-    ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C10]](s64)
-    ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SHL2]], [[C7]]
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 -13108
+    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C5]]
+    ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[AND4]], [[C1]]
+    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C4]](s32)
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C4]](s32)
+    ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SHL2]], [[C5]]
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[LSHR2]], [[AND6]]
-    ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 -21846
-    ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C11]]
-    ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[AND7]], [[C12]]
-    ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND8]], [[C13]](s64)
-    ; CHECK-NEXT: [[C14:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[OR2]], [[C14]](s64)
-    ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[SHL3]], [[C11]]
+    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 -21846
+    ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C7]]
+    ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[AND7]], [[C1]]
+    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND8]], [[C6]](s32)
+    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[OR2]], [[C6]](s32)
+    ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[SHL3]], [[C7]]
     ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[LSHR3]], [[AND9]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR3]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -132,43 +116,38 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC]], [[C1]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC]], [[C]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR]], [[SHL]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65280
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C2]]
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C3]](s64)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65280
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C2]](s32)
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC]], [[C4]](s64)
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C2]]
+    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC]], [[C2]](s32)
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C1]]
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[AND1]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 -252645136
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C5]]
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C6]](s64)
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR2]], [[C7]](s64)
-    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SHL2]], [[C5]]
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 -252645136
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C4]]
+    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C3]](s32)
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR2]], [[C3]](s32)
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SHL2]], [[C4]]
     ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[LSHR2]], [[AND3]]
-    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 -858993460
-    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C8]]
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[C9]](s64)
-    ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[OR3]], [[C10]](s64)
-    ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SHL3]], [[C8]]
+    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 -858993460
+    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C6]]
+    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[C5]](s32)
+    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[OR3]], [[C5]](s32)
+    ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SHL3]], [[C6]]
     ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[LSHR3]], [[AND5]]
-    ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1431655766
-    ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[OR4]], [[C11]]
-    ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[C12]](s64)
-    ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C13]](s64)
-    ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[SHL4]], [[C11]]
+    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1431655766
+    ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[OR4]], [[C8]]
+    ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[C7]](s32)
+    ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C7]](s32)
+    ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[SHL4]], [[C8]]
     ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[LSHR4]], [[AND7]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -261,18 +240,15 @@ body:             |
     ; CHECK: liveins: $x10
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SHL]], [[C1]]
-    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C2]]
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C3]](s64)
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C4]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C2]]
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C]](s32)
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C]]
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[AND2]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -295,27 +271,21 @@ body:             |
     ; CHECK: liveins: $x10
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SHL]], [[C1]]
-    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C2]]
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C3]](s64)
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C4]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C3]]
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C2]](s32)
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C]]
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[AND2]]
-    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
-    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[TRUNC2]], [[C5]]
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C6]](s64)
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C7]]
-    ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[AND4]]
+    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C]](s32)
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C4]]
+    ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[AND3]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR1]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -337,33 +307,25 @@ body:             |
     ; CHECK: liveins: $x10
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SHL]], [[C1]]
-    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C2]](s64)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C2]](s32)
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SHL1]], [[C3]]
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[AND1]]
-    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
     ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[TRUNC2]], [[C4]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C5]](s64)
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C6]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C4]]
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C2]](s32)
+    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C5]]
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[AND3]]
-    ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
-    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[TRUNC3]], [[C7]]
-    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
-    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[C8]](s64)
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C9]]
-    ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[AND5]]
+    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C]](s32)
+    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C2]]
+    ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[AND4]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR2]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -385,55 +347,38 @@ body:             |
     ; CHECK: liveins: $x10
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 6
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SHL]], [[C1]]
-    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C2]](s64)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C2]](s32)
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SHL1]], [[C3]]
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[AND1]]
-    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[TRUNC2]], [[C4]](s64)
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C4]](s32)
     ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SHL2]], [[C5]]
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[AND2]]
-    ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
-    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[TRUNC3]], [[C6]]
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C7]](s64)
+    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C7]]
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C6]](s32)
     ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
     ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C8]]
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[AND4]]
-    ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
-    ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[TRUNC4]], [[C9]]
-    ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C10]](s64)
-    ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C11]]
-    ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[OR2]], [[AND6]]
-    ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
-    ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[TRUNC5]], [[C12]]
-    ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND7]], [[C13]](s64)
-    ; CHECK-NEXT: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[LSHR2]], [[C14]]
-    ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[AND8]]
-    ; CHECK-NEXT: [[TRUNC6:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
-    ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[TRUNC6]], [[C15]]
-    ; CHECK-NEXT: [[C16:%[0-9]+]]:_(s64) = G_CONSTANT i64 6
-    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND9]], [[C16]](s64)
-    ; CHECK-NEXT: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C17]]
-    ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[OR4]], [[AND10]]
+    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C4]](s32)
+    ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C2]]
+    ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[OR2]], [[AND5]]
+    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C2]](s32)
+    ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR2]], [[C4]]
+    ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[AND6]]
+    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C]](s32)
+    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C9]]
+    ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[OR4]], [[AND7]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR5]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -455,44 +400,36 @@ body:             |
     ; CHECK: liveins: $x10
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16777215
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C2]](s64)
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR]], [[SHL]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -986896
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16777215
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[AND1]], [[C4]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C5]](s64)
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[OR]], [[C6]](s64)
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[AND1]], [[C1]]
+    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C2]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[OR]], [[C2]](s32)
     ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SHL1]], [[C3]]
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[LSHR1]], [[AND3]]
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 -3355444
-    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C7]]
-    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 16777215
-    ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[AND4]], [[C8]]
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C9]](s64)
-    ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C10]](s64)
-    ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SHL2]], [[C7]]
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 -3355444
+    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C5]]
+    ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[AND4]], [[C1]]
+    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C4]](s32)
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C4]](s32)
+    ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SHL2]], [[C5]]
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[LSHR2]], [[AND6]]
-    ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 -5592406
-    ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C11]]
-    ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 16777215
-    ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[AND7]], [[C12]]
-    ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND8]], [[C13]](s64)
-    ; CHECK-NEXT: [[C14:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[OR2]], [[C14]](s64)
-    ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[SHL3]], [[C11]]
+    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 -5592406
+    ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C7]]
+    ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[AND7]], [[C1]]
+    ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND8]], [[C6]](s32)
+    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[OR2]], [[C6]](s32)
+    ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[SHL3]], [[C7]]
     ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[LSHR3]], [[AND9]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR3]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -526,19 +463,13 @@ body:             |
     ; CHECK-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s4>) = G_BUILD_VECTOR [[C3]](s4), [[C3]](s4)
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(<2 x s4>) = G_AND [[SHL1]], [[BUILD_VECTOR3]]
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(<2 x s4>) = G_OR [[AND]], [[AND1]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s4) = G_CONSTANT i4 1
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(<2 x s4>) = G_LSHR [[TRUNC]], [[BUILD_VECTOR2]](<2 x s4>)
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s4) = G_CONSTANT i4 2
     ; CHECK-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s4>) = G_BUILD_VECTOR [[C4]](s4), [[C4]](s4)
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(<2 x s4>) = G_LSHR [[TRUNC]], [[BUILD_VECTOR4]](<2 x s4>)
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s4) = G_CONSTANT i4 2
-    ; CHECK-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s4>) = G_BUILD_VECTOR [[C5]](s4), [[C5]](s4)
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(<2 x s4>) = G_AND [[LSHR]], [[BUILD_VECTOR5]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(<2 x s4>) = G_AND [[LSHR]], [[BUILD_VECTOR4]]
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(<2 x s4>) = G_OR [[OR]], [[AND2]]
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s4) = G_CONSTANT i4 3
-    ; CHECK-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x s4>) = G_BUILD_VECTOR [[C6]](s4), [[C6]](s4)
-    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(<2 x s4>) = G_LSHR [[TRUNC]], [[BUILD_VECTOR6]](<2 x s4>)
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s4) = G_CONSTANT i4 1
-    ; CHECK-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x s4>) = G_BUILD_VECTOR [[C7]](s4), [[C7]](s4)
-    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(<2 x s4>) = G_AND [[LSHR1]], [[BUILD_VECTOR7]]
+    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(<2 x s4>) = G_LSHR [[TRUNC]], [[BUILD_VECTOR]](<2 x s4>)
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(<2 x s4>) = G_AND [[LSHR1]], [[BUILD_VECTOR2]]
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(<2 x s4>) = G_OR [[OR1]], [[AND3]]
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s4>) = COPY [[OR2]](<2 x s4>)
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(<2 x s32>) = G_ANYEXT [[COPY1]](<2 x s4>)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bswap-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bswap-rv32.mir
index d6598c89b39af..d31a4484e7a17 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bswap-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bswap-rv32.mir
@@ -101,17 +101,14 @@ body:             |
     ; RV32I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C2]](s32)
     ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C1]]
     ; RV32I-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[AND1]]
-    ; RV32I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; RV32I-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C3]](s32)
-    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C3]](s32)
+    ; RV32I-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
+    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
     ; RV32I-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[LSHR2]], [[SHL2]]
-    ; RV32I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65280
-    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C4]]
-    ; RV32I-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C5]](s32)
+    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C1]]
+    ; RV32I-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
     ; RV32I-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[SHL3]]
-    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C5]](s32)
-    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C4]]
+    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C1]]
     ; RV32I-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[OR4]], [[AND3]]
     ; RV32I-NEXT: $x10 = COPY [[OR2]](s32)
     ; RV32I-NEXT: $x11 = COPY [[OR5]](s32)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bswap-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bswap-rv64.mir
index 61a0de9739842..d5d182ad18734 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bswap-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bswap-rv64.mir
@@ -16,16 +16,14 @@ body:             |
     ; RV64I-NEXT: {{  $}}
     ; RV64I-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; RV64I-NEXT: [[ASSERT_ZEXT:%[0-9]+]]:_(s64) = G_ASSERT_ZEXT [[COPY]], 16
+    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ASSERT_ZEXT]](s64)
-    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; RV64I-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[ASSERT_ZEXT]](s64)
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC1]], [[C1]](s64)
+    ; RV64I-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
+    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC]], [[C]](s32)
     ; RV64I-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR]], [[SHL]]
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR]](s32)
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C2]]
+    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C1]]
     ; RV64I-NEXT: $x10 = COPY [[AND]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
     ;
@@ -58,19 +56,17 @@ body:             |
     ; RV64I-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; RV64I-NEXT: [[ASSERT_ZEXT:%[0-9]+]]:_(s64) = G_ASSERT_ZEXT [[COPY]], 32
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ASSERT_ZEXT]](s64)
-    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; RV64I-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC]], [[C1]](s64)
+    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; RV64I-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
+    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC]], [[C]](s32)
     ; RV64I-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR]], [[SHL]]
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65280
-    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C2]]
-    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; RV64I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C3]](s64)
+    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65280
+    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
+    ; RV64I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C2]](s32)
     ; RV64I-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
-    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC]], [[C4]](s64)
-    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C2]]
+    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC]], [[C2]](s32)
+    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C1]]
     ; RV64I-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[AND1]]
     ; RV64I-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[OR2]](s32)
     ; RV64I-NEXT: $x10 = COPY [[ZEXT]](s64)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-const-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-const-rv64.mir
index 6389fd6fcd077..10f891538e1fe 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-const-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-const-rv64.mir
@@ -116,10 +116,9 @@ body:             |
     ; CHECK-LABEL: name: const_i192
     ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -214772364921
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
     ; CHECK-NEXT: $x10 = COPY [[C]](s64)
     ; CHECK-NEXT: $x11 = COPY [[C1]](s64)
-    ; CHECK-NEXT: $x12 = COPY [[C2]](s64)
+    ; CHECK-NEXT: $x12 = COPY [[C1]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11, implicit $x12
     %0:_(s192) = G_CONSTANT i192 -214772364921
     %1:_(s64), %2:_(s64), %3:_(s64) = G_UNMERGE_VALUES %0(s192)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-constbarrier-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-constbarrier-rv32.mir
index bbe8ef4b092d3..b1630d9c5f6e0 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-constbarrier-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-constbarrier-rv32.mir
@@ -52,8 +52,9 @@ name:            constbarrier_nxv2i1
 body:             |
   bb.0.entry:
     ; CHECK-LABEL: name: constbarrier_nxv2i1
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMCLR_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL [[C]](s32)
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMCLR_VL [[C]](s32)
     ; CHECK-NEXT: [[CONSTANT_FOLD_BARRIER:%[0-9]+]]:_(<vscale x 2 x s1>) = G_CONSTANT_FOLD_BARRIER [[VMCLR_VL]]
     ; CHECK-NEXT: $v8 = COPY [[CONSTANT_FOLD_BARRIER]](<vscale x 2 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v8
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-constbarrier-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-constbarrier-rv64.mir
index 96b1aa53d46ea..029205cc6bc22 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-constbarrier-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-constbarrier-rv64.mir
@@ -70,8 +70,9 @@ name:            constbarrier_nxv2i1
 body:             |
   bb.0.entry:
     ; CHECK-LABEL: name: constbarrier_nxv2i1
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMCLR_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL [[C]](s64)
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMCLR_VL [[C]](s64)
     ; CHECK-NEXT: [[CONSTANT_FOLD_BARRIER:%[0-9]+]]:_(<vscale x 2 x s1>) = G_CONSTANT_FOLD_BARRIER [[VMCLR_VL]]
     ; CHECK-NEXT: $v8 = COPY [[CONSTANT_FOLD_BARRIER]](<vscale x 2 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v8
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ctlz-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ctlz-rv32.mir
index 354fc109a4638..c2db4049580e1 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ctlz-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ctlz-rv32.mir
@@ -20,41 +20,33 @@ body:             |
     ; RV32I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C]](s32)
     ; RV32I-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY]], [[LSHR]]
     ; RV32I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
+    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C1]]
     ; RV32I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C2]](s32)
     ; RV32I-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[LSHR1]]
-    ; RV32I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C5]]
-    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C4]](s32)
+    ; RV32I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C1]]
+    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C3]](s32)
     ; RV32I-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[LSHR2]]
-    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C7]]
-    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C6]](s32)
-    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 85
-    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C8]]
+    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C1]]
+    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C]](s32)
+    ; RV32I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 85
+    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C4]]
     ; RV32I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[OR2]], [[AND4]]
-    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C10]]
-    ; RV32I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C9]](s32)
-    ; RV32I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 51
-    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C11]]
-    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C11]]
+    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
+    ; RV32I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C2]](s32)
+    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 51
+    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C5]]
+    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C5]]
     ; RV32I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND6]], [[AND7]]
-    ; RV32I-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C12]](s32)
+    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C3]](s32)
     ; RV32I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR5]], [[ADD]]
-    ; RV32I-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
-    ; RV32I-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C13]]
-    ; RV32I-NEXT: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32I-NEXT: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND8]], [[C14]]
-    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C15]](s32)
-    ; RV32I-NEXT: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; RV32I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C16]], [[LSHR6]]
+    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; RV32I-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C6]]
+    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND8]], [[C]]
+    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C7]](s32)
+    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; RV32I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C8]], [[LSHR6]]
     ; RV32I-NEXT: $x10 = COPY [[SUB1]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10
     ;
@@ -94,48 +86,38 @@ body:             |
     ; RV32I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C]](s32)
     ; RV32I-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY]], [[LSHR]]
     ; RV32I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
+    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C1]]
     ; RV32I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C2]](s32)
     ; RV32I-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[LSHR1]]
-    ; RV32I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C5]]
-    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C4]](s32)
+    ; RV32I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C1]]
+    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C3]](s32)
     ; RV32I-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[LSHR2]]
-    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C7]]
-    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C6]](s32)
+    ; RV32I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C1]]
+    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C4]](s32)
     ; RV32I-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[OR2]], [[LSHR3]]
-    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C9]]
-    ; RV32I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[C8]](s32)
-    ; RV32I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 21845
-    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C10]]
+    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C1]]
+    ; RV32I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[C]](s32)
+    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 21845
+    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C5]]
     ; RV32I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[OR3]], [[AND5]]
-    ; RV32I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C12]]
-    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[C11]](s32)
-    ; RV32I-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 13107
-    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C13]]
-    ; RV32I-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C13]]
+    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
+    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[C2]](s32)
+    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 13107
+    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C6]]
+    ; RV32I-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C6]]
     ; RV32I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND7]], [[AND8]]
-    ; RV32I-NEXT: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C14]](s32)
+    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C3]](s32)
     ; RV32I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR6]], [[ADD]]
-    ; RV32I-NEXT: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 3855
-    ; RV32I-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C15]]
-    ; RV32I-NEXT: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 257
-    ; RV32I-NEXT: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND9]], [[C16]]
-    ; RV32I-NEXT: [[C18:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C18]]
-    ; RV32I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[AND10]], [[C17]](s32)
-    ; RV32I-NEXT: [[C19:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; RV32I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C19]], [[LSHR7]]
+    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 3855
+    ; RV32I-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C7]]
+    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 257
+    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND9]], [[C8]]
+    ; RV32I-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C1]]
+    ; RV32I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[AND10]], [[C4]](s32)
+    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; RV32I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C9]], [[LSHR7]]
     ; RV32I-NEXT: $x10 = COPY [[SUB1]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10
     ;
@@ -184,28 +166,25 @@ body:             |
     ; RV32I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; RV32I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[OR3]], [[C4]](s32)
     ; RV32I-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[LSHR4]]
-    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[OR4]], [[C5]](s32)
-    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
-    ; RV32I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C6]]
+    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[OR4]], [[C]](s32)
+    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
+    ; RV32I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C5]]
     ; RV32I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[OR4]], [[AND]]
-    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C7]](s32)
-    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
-    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C8]]
-    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C8]]
+    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C1]](s32)
+    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
+    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C6]]
+    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C6]]
     ; RV32I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND1]], [[AND2]]
-    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C9]](s32)
+    ; RV32I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C2]](s32)
     ; RV32I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR7]], [[ADD]]
-    ; RV32I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
-    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C10]]
-    ; RV32I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
-    ; RV32I-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[C11]]
-    ; RV32I-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C12]](s32)
-    ; RV32I-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; RV32I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C13]], [[LSHR8]]
+    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
+    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C7]]
+    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
+    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[C8]]
+    ; RV32I-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C9]](s32)
+    ; RV32I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; RV32I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C10]], [[LSHR8]]
     ; RV32I-NEXT: $x10 = COPY [[SUB1]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10
     ;
@@ -250,71 +229,52 @@ body:             |
     ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; RV32I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[OR3]], [[C5]](s32)
     ; RV32I-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[LSHR4]]
-    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[OR4]], [[C6]](s32)
-    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
-    ; RV32I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C7]]
+    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[OR4]], [[C1]](s32)
+    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
+    ; RV32I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C6]]
     ; RV32I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[OR4]], [[AND]]
-    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C8]](s32)
-    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
-    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C9]]
-    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C9]]
+    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C2]](s32)
+    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
+    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C7]]
+    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C7]]
     ; RV32I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND1]], [[AND2]]
-    ; RV32I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C10]](s32)
+    ; RV32I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C3]](s32)
     ; RV32I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR7]], [[ADD]]
-    ; RV32I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
-    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C11]]
-    ; RV32I-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
-    ; RV32I-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[C12]]
-    ; RV32I-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C13]](s32)
-    ; RV32I-NEXT: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; RV32I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C14]], [[LSHR8]]
-    ; RV32I-NEXT: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; RV32I-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SUB1]], [[C15]]
-    ; RV32I-NEXT: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32I-NEXT: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C16]](s32)
+    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
+    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C8]]
+    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
+    ; RV32I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[C9]]
+    ; RV32I-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C10]](s32)
+    ; RV32I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; RV32I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C11]], [[LSHR8]]
+    ; RV32I-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SUB1]], [[C11]]
+    ; RV32I-NEXT: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s32)
     ; RV32I-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[COPY1]], [[LSHR9]]
-    ; RV32I-NEXT: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[LSHR10:%[0-9]+]]:_(s32) = G_LSHR [[OR5]], [[C17]](s32)
+    ; RV32I-NEXT: [[LSHR10:%[0-9]+]]:_(s32) = G_LSHR [[OR5]], [[C2]](s32)
     ; RV32I-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[OR5]], [[LSHR10]]
-    ; RV32I-NEXT: [[C18:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR11:%[0-9]+]]:_(s32) = G_LSHR [[OR6]], [[C18]](s32)
+    ; RV32I-NEXT: [[LSHR11:%[0-9]+]]:_(s32) = G_LSHR [[OR6]], [[C3]](s32)
     ; RV32I-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[OR6]], [[LSHR11]]
-    ; RV32I-NEXT: [[C19:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; RV32I-NEXT: [[LSHR12:%[0-9]+]]:_(s32) = G_LSHR [[OR7]], [[C19]](s32)
+    ; RV32I-NEXT: [[LSHR12:%[0-9]+]]:_(s32) = G_LSHR [[OR7]], [[C4]](s32)
     ; RV32I-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[OR7]], [[LSHR12]]
-    ; RV32I-NEXT: [[C20:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; RV32I-NEXT: [[LSHR13:%[0-9]+]]:_(s32) = G_LSHR [[OR8]], [[C20]](s32)
+    ; RV32I-NEXT: [[LSHR13:%[0-9]+]]:_(s32) = G_LSHR [[OR8]], [[C5]](s32)
     ; RV32I-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[OR8]], [[LSHR13]]
-    ; RV32I-NEXT: [[C21:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32I-NEXT: [[LSHR14:%[0-9]+]]:_(s32) = G_LSHR [[OR9]], [[C21]](s32)
-    ; RV32I-NEXT: [[C22:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
-    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR14]], [[C22]]
+    ; RV32I-NEXT: [[LSHR14:%[0-9]+]]:_(s32) = G_LSHR [[OR9]], [[C1]](s32)
+    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR14]], [[C6]]
     ; RV32I-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[OR9]], [[AND4]]
-    ; RV32I-NEXT: [[C23:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[LSHR15:%[0-9]+]]:_(s32) = G_LSHR [[SUB2]], [[C23]](s32)
-    ; RV32I-NEXT: [[C24:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
-    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR15]], [[C24]]
-    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SUB2]], [[C24]]
+    ; RV32I-NEXT: [[LSHR15:%[0-9]+]]:_(s32) = G_LSHR [[SUB2]], [[C2]](s32)
+    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR15]], [[C7]]
+    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SUB2]], [[C7]]
     ; RV32I-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[AND5]], [[AND6]]
-    ; RV32I-NEXT: [[C25:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR16:%[0-9]+]]:_(s32) = G_LSHR [[ADD3]], [[C25]](s32)
+    ; RV32I-NEXT: [[LSHR16:%[0-9]+]]:_(s32) = G_LSHR [[ADD3]], [[C3]](s32)
     ; RV32I-NEXT: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[LSHR16]], [[ADD3]]
-    ; RV32I-NEXT: [[C26:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
-    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ADD4]], [[C26]]
-    ; RV32I-NEXT: [[C27:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
-    ; RV32I-NEXT: [[C28:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; RV32I-NEXT: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[AND7]], [[C27]]
-    ; RV32I-NEXT: [[LSHR17:%[0-9]+]]:_(s32) = G_LSHR [[MUL1]], [[C28]](s32)
-    ; RV32I-NEXT: [[C29:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; RV32I-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C29]], [[LSHR17]]
+    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ADD4]], [[C8]]
+    ; RV32I-NEXT: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[AND7]], [[C9]]
+    ; RV32I-NEXT: [[LSHR17:%[0-9]+]]:_(s32) = G_LSHR [[MUL1]], [[C10]](s32)
+    ; RV32I-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C11]], [[LSHR17]]
     ; RV32I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[ADD2]], [[SUB3]]
-    ; RV32I-NEXT: [[C30:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; RV32I-NEXT: $x10 = COPY [[SELECT]](s32)
-    ; RV32I-NEXT: $x11 = COPY [[C30]](s32)
+    ; RV32I-NEXT: $x11 = COPY [[C]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10, implicit $x11
     ;
     ; RV32ZBB-LABEL: name: ctlz_i64
@@ -329,9 +289,8 @@ body:             |
     ; RV32ZBB-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[CTLZ]], [[C1]]
     ; RV32ZBB-NEXT: [[CTLZ1:%[0-9]+]]:_(s32) = G_CTLZ [[COPY1]](s32)
     ; RV32ZBB-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[ADD]], [[CTLZ1]]
-    ; RV32ZBB-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; RV32ZBB-NEXT: $x10 = COPY [[SELECT]](s32)
-    ; RV32ZBB-NEXT: $x11 = COPY [[C2]](s32)
+    ; RV32ZBB-NEXT: $x11 = COPY [[C]](s32)
     ; RV32ZBB-NEXT: PseudoRET implicit $x10, implicit $x11
     %1:_(s32) = COPY $x10
     %2:_(s32) = COPY $x11
@@ -359,41 +318,33 @@ body:             |
     ; RV32I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C]](s32)
     ; RV32I-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY]], [[LSHR]]
     ; RV32I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
+    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C1]]
     ; RV32I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C2]](s32)
     ; RV32I-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[LSHR1]]
-    ; RV32I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C5]]
-    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C4]](s32)
+    ; RV32I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C1]]
+    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C3]](s32)
     ; RV32I-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[LSHR2]]
-    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C7]]
-    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C6]](s32)
-    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 85
-    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C8]]
+    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C1]]
+    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C]](s32)
+    ; RV32I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 85
+    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C4]]
     ; RV32I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[OR2]], [[AND4]]
-    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C10]]
-    ; RV32I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C9]](s32)
-    ; RV32I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 51
-    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C11]]
-    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C11]]
+    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
+    ; RV32I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C2]](s32)
+    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 51
+    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C5]]
+    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C5]]
     ; RV32I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND6]], [[AND7]]
-    ; RV32I-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C12]](s32)
+    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C3]](s32)
     ; RV32I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR5]], [[ADD]]
-    ; RV32I-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
-    ; RV32I-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C13]]
-    ; RV32I-NEXT: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32I-NEXT: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND8]], [[C14]]
-    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C15]](s32)
-    ; RV32I-NEXT: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; RV32I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C16]], [[LSHR6]]
+    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; RV32I-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C6]]
+    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND8]], [[C]]
+    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C7]](s32)
+    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; RV32I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C8]], [[LSHR6]]
     ; RV32I-NEXT: $x10 = COPY [[SUB1]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10
     ;
@@ -433,48 +384,38 @@ body:             |
     ; RV32I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C]](s32)
     ; RV32I-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY]], [[LSHR]]
     ; RV32I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
+    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C1]]
     ; RV32I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C2]](s32)
     ; RV32I-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[LSHR1]]
-    ; RV32I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C5]]
-    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C4]](s32)
+    ; RV32I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C1]]
+    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C3]](s32)
     ; RV32I-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[LSHR2]]
-    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C7]]
-    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C6]](s32)
+    ; RV32I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C1]]
+    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C4]](s32)
     ; RV32I-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[OR2]], [[LSHR3]]
-    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C9]]
-    ; RV32I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[C8]](s32)
-    ; RV32I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 21845
-    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C10]]
+    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C1]]
+    ; RV32I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[C]](s32)
+    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 21845
+    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C5]]
     ; RV32I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[OR3]], [[AND5]]
-    ; RV32I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C12]]
-    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[C11]](s32)
-    ; RV32I-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 13107
-    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C13]]
-    ; RV32I-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C13]]
+    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
+    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[C2]](s32)
+    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 13107
+    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C6]]
+    ; RV32I-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C6]]
     ; RV32I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND7]], [[AND8]]
-    ; RV32I-NEXT: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C14]](s32)
+    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C3]](s32)
     ; RV32I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR6]], [[ADD]]
-    ; RV32I-NEXT: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 3855
-    ; RV32I-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C15]]
-    ; RV32I-NEXT: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 257
-    ; RV32I-NEXT: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND9]], [[C16]]
-    ; RV32I-NEXT: [[C18:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C18]]
-    ; RV32I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[AND10]], [[C17]](s32)
-    ; RV32I-NEXT: [[C19:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; RV32I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C19]], [[LSHR7]]
+    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 3855
+    ; RV32I-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C7]]
+    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 257
+    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND9]], [[C8]]
+    ; RV32I-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C1]]
+    ; RV32I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[AND10]], [[C4]](s32)
+    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; RV32I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C9]], [[LSHR7]]
     ; RV32I-NEXT: $x10 = COPY [[SUB1]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10
     ;
@@ -523,28 +464,25 @@ body:             |
     ; RV32I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; RV32I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[OR3]], [[C4]](s32)
     ; RV32I-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[LSHR4]]
-    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[OR4]], [[C5]](s32)
-    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
-    ; RV32I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C6]]
+    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[OR4]], [[C]](s32)
+    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
+    ; RV32I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C5]]
     ; RV32I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[OR4]], [[AND]]
-    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C7]](s32)
-    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
-    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C8]]
-    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C8]]
+    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C1]](s32)
+    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
+    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C6]]
+    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C6]]
     ; RV32I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND1]], [[AND2]]
-    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C9]](s32)
+    ; RV32I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C2]](s32)
     ; RV32I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR7]], [[ADD]]
-    ; RV32I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
-    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C10]]
-    ; RV32I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
-    ; RV32I-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[C11]]
-    ; RV32I-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C12]](s32)
-    ; RV32I-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; RV32I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C13]], [[LSHR8]]
+    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
+    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C7]]
+    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
+    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[C8]]
+    ; RV32I-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C9]](s32)
+    ; RV32I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; RV32I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C10]], [[LSHR8]]
     ; RV32I-NEXT: $x10 = COPY [[SUB1]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10
     ;
@@ -589,71 +527,52 @@ body:             |
     ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; RV32I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[OR3]], [[C5]](s32)
     ; RV32I-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[LSHR4]]
-    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[OR4]], [[C6]](s32)
-    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
-    ; RV32I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C7]]
+    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[OR4]], [[C1]](s32)
+    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
+    ; RV32I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C6]]
     ; RV32I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[OR4]], [[AND]]
-    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C8]](s32)
-    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
-    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C9]]
-    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C9]]
+    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C2]](s32)
+    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
+    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C7]]
+    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C7]]
     ; RV32I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND1]], [[AND2]]
-    ; RV32I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C10]](s32)
+    ; RV32I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C3]](s32)
     ; RV32I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR7]], [[ADD]]
-    ; RV32I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
-    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C11]]
-    ; RV32I-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
-    ; RV32I-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[C12]]
-    ; RV32I-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C13]](s32)
-    ; RV32I-NEXT: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; RV32I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C14]], [[LSHR8]]
-    ; RV32I-NEXT: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; RV32I-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SUB1]], [[C15]]
-    ; RV32I-NEXT: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32I-NEXT: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C16]](s32)
+    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
+    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C8]]
+    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
+    ; RV32I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[C9]]
+    ; RV32I-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C10]](s32)
+    ; RV32I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; RV32I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C11]], [[LSHR8]]
+    ; RV32I-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SUB1]], [[C11]]
+    ; RV32I-NEXT: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s32)
     ; RV32I-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[COPY1]], [[LSHR9]]
-    ; RV32I-NEXT: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[LSHR10:%[0-9]+]]:_(s32) = G_LSHR [[OR5]], [[C17]](s32)
+    ; RV32I-NEXT: [[LSHR10:%[0-9]+]]:_(s32) = G_LSHR [[OR5]], [[C2]](s32)
     ; RV32I-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[OR5]], [[LSHR10]]
-    ; RV32I-NEXT: [[C18:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR11:%[0-9]+]]:_(s32) = G_LSHR [[OR6]], [[C18]](s32)
+    ; RV32I-NEXT: [[LSHR11:%[0-9]+]]:_(s32) = G_LSHR [[OR6]], [[C3]](s32)
     ; RV32I-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[OR6]], [[LSHR11]]
-    ; RV32I-NEXT: [[C19:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; RV32I-NEXT: [[LSHR12:%[0-9]+]]:_(s32) = G_LSHR [[OR7]], [[C19]](s32)
+    ; RV32I-NEXT: [[LSHR12:%[0-9]+]]:_(s32) = G_LSHR [[OR7]], [[C4]](s32)
     ; RV32I-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[OR7]], [[LSHR12]]
-    ; RV32I-NEXT: [[C20:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; RV32I-NEXT: [[LSHR13:%[0-9]+]]:_(s32) = G_LSHR [[OR8]], [[C20]](s32)
+    ; RV32I-NEXT: [[LSHR13:%[0-9]+]]:_(s32) = G_LSHR [[OR8]], [[C5]](s32)
     ; RV32I-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[OR8]], [[LSHR13]]
-    ; RV32I-NEXT: [[C21:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32I-NEXT: [[LSHR14:%[0-9]+]]:_(s32) = G_LSHR [[OR9]], [[C21]](s32)
-    ; RV32I-NEXT: [[C22:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
-    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR14]], [[C22]]
+    ; RV32I-NEXT: [[LSHR14:%[0-9]+]]:_(s32) = G_LSHR [[OR9]], [[C1]](s32)
+    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR14]], [[C6]]
     ; RV32I-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[OR9]], [[AND4]]
-    ; RV32I-NEXT: [[C23:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[LSHR15:%[0-9]+]]:_(s32) = G_LSHR [[SUB2]], [[C23]](s32)
-    ; RV32I-NEXT: [[C24:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
-    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR15]], [[C24]]
-    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SUB2]], [[C24]]
+    ; RV32I-NEXT: [[LSHR15:%[0-9]+]]:_(s32) = G_LSHR [[SUB2]], [[C2]](s32)
+    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR15]], [[C7]]
+    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SUB2]], [[C7]]
     ; RV32I-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[AND5]], [[AND6]]
-    ; RV32I-NEXT: [[C25:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR16:%[0-9]+]]:_(s32) = G_LSHR [[ADD3]], [[C25]](s32)
+    ; RV32I-NEXT: [[LSHR16:%[0-9]+]]:_(s32) = G_LSHR [[ADD3]], [[C3]](s32)
     ; RV32I-NEXT: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[LSHR16]], [[ADD3]]
-    ; RV32I-NEXT: [[C26:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
-    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ADD4]], [[C26]]
-    ; RV32I-NEXT: [[C27:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
-    ; RV32I-NEXT: [[C28:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; RV32I-NEXT: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[AND7]], [[C27]]
-    ; RV32I-NEXT: [[LSHR17:%[0-9]+]]:_(s32) = G_LSHR [[MUL1]], [[C28]](s32)
-    ; RV32I-NEXT: [[C29:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; RV32I-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C29]], [[LSHR17]]
+    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ADD4]], [[C8]]
+    ; RV32I-NEXT: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[AND7]], [[C9]]
+    ; RV32I-NEXT: [[LSHR17:%[0-9]+]]:_(s32) = G_LSHR [[MUL1]], [[C10]](s32)
+    ; RV32I-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C11]], [[LSHR17]]
     ; RV32I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[ADD2]], [[SUB3]]
-    ; RV32I-NEXT: [[C30:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; RV32I-NEXT: $x10 = COPY [[SELECT]](s32)
-    ; RV32I-NEXT: $x11 = COPY [[C30]](s32)
+    ; RV32I-NEXT: $x11 = COPY [[C]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10, implicit $x11
     ;
     ; RV32ZBB-LABEL: name: ctlz_zero_undef_i64
@@ -668,9 +587,8 @@ body:             |
     ; RV32ZBB-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[CTLZ]], [[C1]]
     ; RV32ZBB-NEXT: [[CTLZ1:%[0-9]+]]:_(s32) = G_CTLZ [[COPY1]](s32)
     ; RV32ZBB-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[ADD]], [[CTLZ1]]
-    ; RV32ZBB-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; RV32ZBB-NEXT: $x10 = COPY [[SELECT]](s32)
-    ; RV32ZBB-NEXT: $x11 = COPY [[C2]](s32)
+    ; RV32ZBB-NEXT: $x11 = COPY [[C]](s32)
     ; RV32ZBB-NEXT: PseudoRET implicit $x10, implicit $x11
     %1:_(s32) = COPY $x10
     %2:_(s32) = COPY $x11
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ctlz-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ctlz-rv64.mir
index 38a4b9c6dae38..bc6aafb1e3b2c 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ctlz-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ctlz-rv64.mir
@@ -14,49 +14,41 @@ body:             |
     ; RV64I: liveins: $x10
     ; RV64I-NEXT: {{  $}}
     ; RV64I-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C1]](s64)
-    ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; RV64I-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[TRUNC1]], [[LSHR]]
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C2]]
-    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C3]](s64)
+    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
+    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C]](s32)
+    ; RV64I-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
+    ; RV64I-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY1]], [[LSHR]]
+    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C1]]
+    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C2]](s32)
     ; RV64I-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[LSHR1]]
-    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C4]]
-    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C5]](s64)
+    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C1]]
+    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C3]](s32)
     ; RV64I-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[LSHR2]]
-    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C6]]
-    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C7]](s64)
-    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 85
-    ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C8]]
+    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C1]]
+    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C]](s32)
+    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 85
+    ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C4]]
     ; RV64I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[OR2]], [[AND4]]
-    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV64I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C9]]
-    ; RV64I-NEXT: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C10]](s64)
-    ; RV64I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 51
-    ; RV64I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C11]]
-    ; RV64I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C11]]
+    ; RV64I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
+    ; RV64I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C2]](s32)
+    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 51
+    ; RV64I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C5]]
+    ; RV64I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C5]]
     ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND6]], [[AND7]]
-    ; RV64I-NEXT: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C12]](s64)
+    ; RV64I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C3]](s32)
     ; RV64I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR5]], [[ADD]]
-    ; RV64I-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
-    ; RV64I-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C13]]
-    ; RV64I-NEXT: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND8]], [[C14]]
-    ; RV64I-NEXT: [[C15:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; RV64I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C15]](s64)
-    ; RV64I-NEXT: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; RV64I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C16]], [[LSHR6]]
+    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; RV64I-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C6]]
+    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND8]], [[C]]
+    ; RV64I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C7]](s32)
+    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; RV64I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C8]], [[LSHR6]]
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SUB1]](s32)
     ; RV64I-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
@@ -93,56 +85,46 @@ body:             |
     ; RV64I: liveins: $x10
     ; RV64I-NEXT: {{  $}}
     ; RV64I-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C1]](s64)
-    ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; RV64I-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[TRUNC1]], [[LSHR]]
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C2]]
-    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C3]](s64)
+    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
+    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C]](s32)
+    ; RV64I-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
+    ; RV64I-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY1]], [[LSHR]]
+    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C1]]
+    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C2]](s32)
     ; RV64I-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[LSHR1]]
-    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C4]]
-    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C5]](s64)
+    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C1]]
+    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C3]](s32)
     ; RV64I-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[LSHR2]]
-    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C6]]
-    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C7]](s64)
+    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C1]]
+    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C4]](s32)
     ; RV64I-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[OR2]], [[LSHR3]]
-    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C8]]
-    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[C9]](s64)
-    ; RV64I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 21845
-    ; RV64I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C10]]
+    ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C1]]
+    ; RV64I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[C]](s32)
+    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 21845
+    ; RV64I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C5]]
     ; RV64I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[OR3]], [[AND5]]
-    ; RV64I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C11]]
-    ; RV64I-NEXT: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[C12]](s64)
-    ; RV64I-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 13107
-    ; RV64I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C13]]
-    ; RV64I-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C13]]
+    ; RV64I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
+    ; RV64I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[C2]](s32)
+    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 13107
+    ; RV64I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C6]]
+    ; RV64I-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C6]]
     ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND7]], [[AND8]]
-    ; RV64I-NEXT: [[C14:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C14]](s64)
+    ; RV64I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C3]](s32)
     ; RV64I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR6]], [[ADD]]
-    ; RV64I-NEXT: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 3855
-    ; RV64I-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C15]]
-    ; RV64I-NEXT: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 257
-    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND9]], [[C16]]
-    ; RV64I-NEXT: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C17]]
-    ; RV64I-NEXT: [[C18:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; RV64I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[AND10]], [[C18]](s64)
-    ; RV64I-NEXT: [[C19:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; RV64I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C19]], [[LSHR7]]
+    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 3855
+    ; RV64I-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C7]]
+    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 257
+    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND9]], [[C8]]
+    ; RV64I-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C1]]
+    ; RV64I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[AND10]], [[C4]](s32)
+    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; RV64I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C9]], [[LSHR7]]
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SUB1]](s32)
     ; RV64I-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
@@ -180,43 +162,40 @@ body:             |
     ; RV64I-NEXT: {{  $}}
     ; RV64I-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC]], [[C]](s64)
+    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC]], [[C]](s32)
     ; RV64I-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[TRUNC]], [[LSHR]]
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C1]](s64)
+    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C1]](s32)
     ; RV64I-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[LSHR1]]
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[OR1]], [[C2]](s64)
+    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[OR1]], [[C2]](s32)
     ; RV64I-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[LSHR2]]
-    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[OR2]], [[C3]](s64)
+    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[OR2]], [[C3]](s32)
     ; RV64I-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[OR2]], [[LSHR3]]
-    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; RV64I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[OR3]], [[C4]](s64)
+    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; RV64I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[OR3]], [[C4]](s32)
     ; RV64I-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[LSHR4]]
-    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[OR4]], [[C5]](s64)
-    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
-    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C6]]
+    ; RV64I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[OR4]], [[C]](s32)
+    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
+    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C5]]
     ; RV64I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[OR4]], [[AND]]
-    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C7]](s64)
-    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
-    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C8]]
-    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C8]]
+    ; RV64I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C1]](s32)
+    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
+    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C6]]
+    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C6]]
     ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND1]], [[AND2]]
-    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C9]](s64)
+    ; RV64I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C2]](s32)
     ; RV64I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR7]], [[ADD]]
-    ; RV64I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
-    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C10]]
-    ; RV64I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
-    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[C11]]
-    ; RV64I-NEXT: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; RV64I-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C12]](s64)
-    ; RV64I-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; RV64I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C13]], [[LSHR8]]
+    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
+    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C7]]
+    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
+    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[C8]]
+    ; RV64I-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C9]](s32)
+    ; RV64I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; RV64I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C10]], [[LSHR8]]
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SUB1]](s32)
     ; RV64I-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
@@ -266,28 +245,25 @@ body:             |
     ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
     ; RV64I-NEXT: [[LSHR5:%[0-9]+]]:_(s64) = G_LSHR [[OR4]], [[C5]](s64)
     ; RV64I-NEXT: [[OR5:%[0-9]+]]:_(s64) = G_OR [[OR4]], [[LSHR5]]
-    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR6:%[0-9]+]]:_(s64) = G_LSHR [[OR5]], [[C6]](s64)
-    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 6148914691236517205
-    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[LSHR6]], [[C7]]
+    ; RV64I-NEXT: [[LSHR6:%[0-9]+]]:_(s64) = G_LSHR [[OR5]], [[C]](s64)
+    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 6148914691236517205
+    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[LSHR6]], [[C6]]
     ; RV64I-NEXT: [[SUB:%[0-9]+]]:_(s64) = G_SUB [[OR5]], [[AND]]
-    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR7:%[0-9]+]]:_(s64) = G_LSHR [[SUB]], [[C8]](s64)
-    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 3689348814741910323
-    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[LSHR7]], [[C9]]
-    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[SUB]], [[C9]]
+    ; RV64I-NEXT: [[LSHR7:%[0-9]+]]:_(s64) = G_LSHR [[SUB]], [[C1]](s64)
+    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 3689348814741910323
+    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[LSHR7]], [[C7]]
+    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[SUB]], [[C7]]
     ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[AND1]], [[AND2]]
-    ; RV64I-NEXT: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR8:%[0-9]+]]:_(s64) = G_LSHR [[ADD]], [[C10]](s64)
+    ; RV64I-NEXT: [[LSHR8:%[0-9]+]]:_(s64) = G_LSHR [[ADD]], [[C2]](s64)
     ; RV64I-NEXT: [[ADD1:%[0-9]+]]:_(s64) = G_ADD [[LSHR8]], [[ADD]]
-    ; RV64I-NEXT: [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 1085102592571150095
-    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND [[ADD1]], [[C11]]
-    ; RV64I-NEXT: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 72340172838076673
-    ; RV64I-NEXT: [[C13:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[AND3]], [[C12]]
-    ; RV64I-NEXT: [[LSHR9:%[0-9]+]]:_(s64) = G_LSHR [[MUL]], [[C13]](s64)
-    ; RV64I-NEXT: [[C14:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
-    ; RV64I-NEXT: [[SUB1:%[0-9]+]]:_(s64) = G_SUB [[C14]], [[LSHR9]]
+    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 1085102592571150095
+    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND [[ADD1]], [[C8]]
+    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 72340172838076673
+    ; RV64I-NEXT: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
+    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[AND3]], [[C9]]
+    ; RV64I-NEXT: [[LSHR9:%[0-9]+]]:_(s64) = G_LSHR [[MUL]], [[C10]](s64)
+    ; RV64I-NEXT: [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
+    ; RV64I-NEXT: [[SUB1:%[0-9]+]]:_(s64) = G_SUB [[C11]], [[LSHR9]]
     ; RV64I-NEXT: $x10 = COPY [[SUB1]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
     ;
@@ -314,49 +290,41 @@ body:             |
     ; RV64I: liveins: $x10
     ; RV64I-NEXT: {{  $}}
     ; RV64I-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C1]](s64)
-    ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; RV64I-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[TRUNC1]], [[LSHR]]
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C2]]
-    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C3]](s64)
+    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
+    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C]](s32)
+    ; RV64I-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
+    ; RV64I-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY1]], [[LSHR]]
+    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C1]]
+    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C2]](s32)
     ; RV64I-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[LSHR1]]
-    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C4]]
-    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C5]](s64)
+    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C1]]
+    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C3]](s32)
     ; RV64I-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[LSHR2]]
-    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C6]]
-    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C7]](s64)
-    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 85
-    ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C8]]
+    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C1]]
+    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C]](s32)
+    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 85
+    ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C4]]
     ; RV64I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[OR2]], [[AND4]]
-    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV64I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C9]]
-    ; RV64I-NEXT: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C10]](s64)
-    ; RV64I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 51
-    ; RV64I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C11]]
-    ; RV64I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C11]]
+    ; RV64I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
+    ; RV64I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C2]](s32)
+    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 51
+    ; RV64I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C5]]
+    ; RV64I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C5]]
     ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND6]], [[AND7]]
-    ; RV64I-NEXT: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C12]](s64)
+    ; RV64I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C3]](s32)
     ; RV64I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR5]], [[ADD]]
-    ; RV64I-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
-    ; RV64I-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C13]]
-    ; RV64I-NEXT: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND8]], [[C14]]
-    ; RV64I-NEXT: [[C15:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; RV64I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C15]](s64)
-    ; RV64I-NEXT: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; RV64I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C16]], [[LSHR6]]
+    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; RV64I-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C6]]
+    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND8]], [[C]]
+    ; RV64I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C7]](s32)
+    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; RV64I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C8]], [[LSHR6]]
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SUB1]](s32)
     ; RV64I-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
@@ -393,56 +361,46 @@ body:             |
     ; RV64I: liveins: $x10
     ; RV64I-NEXT: {{  $}}
     ; RV64I-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C1]](s64)
-    ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; RV64I-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[TRUNC1]], [[LSHR]]
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C2]]
-    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C3]](s64)
+    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
+    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C]](s32)
+    ; RV64I-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
+    ; RV64I-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY1]], [[LSHR]]
+    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C1]]
+    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C2]](s32)
     ; RV64I-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[LSHR1]]
-    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C4]]
-    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C5]](s64)
+    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C1]]
+    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C3]](s32)
     ; RV64I-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[LSHR2]]
-    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C6]]
-    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C7]](s64)
+    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C1]]
+    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C4]](s32)
     ; RV64I-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[OR2]], [[LSHR3]]
-    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C8]]
-    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[C9]](s64)
-    ; RV64I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 21845
-    ; RV64I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C10]]
+    ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C1]]
+    ; RV64I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND4]], [[C]](s32)
+    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 21845
+    ; RV64I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C5]]
     ; RV64I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[OR3]], [[AND5]]
-    ; RV64I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C11]]
-    ; RV64I-NEXT: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[C12]](s64)
-    ; RV64I-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 13107
-    ; RV64I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C13]]
-    ; RV64I-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C13]]
+    ; RV64I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
+    ; RV64I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[C2]](s32)
+    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 13107
+    ; RV64I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C6]]
+    ; RV64I-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C6]]
     ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND7]], [[AND8]]
-    ; RV64I-NEXT: [[C14:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C14]](s64)
+    ; RV64I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C3]](s32)
     ; RV64I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR6]], [[ADD]]
-    ; RV64I-NEXT: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 3855
-    ; RV64I-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C15]]
-    ; RV64I-NEXT: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 257
-    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND9]], [[C16]]
-    ; RV64I-NEXT: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C17]]
-    ; RV64I-NEXT: [[C18:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; RV64I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[AND10]], [[C18]](s64)
-    ; RV64I-NEXT: [[C19:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; RV64I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C19]], [[LSHR7]]
+    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 3855
+    ; RV64I-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C7]]
+    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 257
+    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND9]], [[C8]]
+    ; RV64I-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C1]]
+    ; RV64I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[AND10]], [[C4]](s32)
+    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; RV64I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C9]], [[LSHR7]]
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SUB1]](s32)
     ; RV64I-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
@@ -480,43 +438,40 @@ body:             |
     ; RV64I-NEXT: {{  $}}
     ; RV64I-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC]], [[C]](s64)
+    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC]], [[C]](s32)
     ; RV64I-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[TRUNC]], [[LSHR]]
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C1]](s64)
+    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[OR]], [[C1]](s32)
     ; RV64I-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[LSHR1]]
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[OR1]], [[C2]](s64)
+    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[OR1]], [[C2]](s32)
     ; RV64I-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[LSHR2]]
-    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[OR2]], [[C3]](s64)
+    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[OR2]], [[C3]](s32)
     ; RV64I-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[OR2]], [[LSHR3]]
-    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; RV64I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[OR3]], [[C4]](s64)
+    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; RV64I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[OR3]], [[C4]](s32)
     ; RV64I-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[LSHR4]]
-    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[OR4]], [[C5]](s64)
-    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
-    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C6]]
+    ; RV64I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[OR4]], [[C]](s32)
+    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
+    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C5]]
     ; RV64I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[OR4]], [[AND]]
-    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C7]](s64)
-    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
-    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C8]]
-    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C8]]
+    ; RV64I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C1]](s32)
+    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
+    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C6]]
+    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C6]]
     ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND1]], [[AND2]]
-    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C9]](s64)
+    ; RV64I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C2]](s32)
     ; RV64I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR7]], [[ADD]]
-    ; RV64I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
-    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C10]]
-    ; RV64I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
-    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[C11]]
-    ; RV64I-NEXT: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; RV64I-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C12]](s64)
-    ; RV64I-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; RV64I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C13]], [[LSHR8]]
+    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
+    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C7]]
+    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
+    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[C8]]
+    ; RV64I-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C9]](s32)
+    ; RV64I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; RV64I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C10]], [[LSHR8]]
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SUB1]](s32)
     ; RV64I-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
@@ -566,28 +521,25 @@ body:             |
     ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
     ; RV64I-NEXT: [[LSHR5:%[0-9]+]]:_(s64) = G_LSHR [[OR4]], [[C5]](s64)
     ; RV64I-NEXT: [[OR5:%[0-9]+]]:_(s64) = G_OR [[OR4]], [[LSHR5]]
-    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR6:%[0-9]+]]:_(s64) = G_LSHR [[OR5]], [[C6]](s64)
-    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 6148914691236517205
-    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[LSHR6]], [[C7]]
+    ; RV64I-NEXT: [[LSHR6:%[0-9]+]]:_(s64) = G_LSHR [[OR5]], [[C]](s64)
+    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 6148914691236517205
+    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[LSHR6]], [[C6]]
     ; RV64I-NEXT: [[SUB:%[0-9]+]]:_(s64) = G_SUB [[OR5]], [[AND]]
-    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR7:%[0-9]+]]:_(s64) = G_LSHR [[SUB]], [[C8]](s64)
-    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 3689348814741910323
-    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[LSHR7]], [[C9]]
-    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[SUB]], [[C9]]
+    ; RV64I-NEXT: [[LSHR7:%[0-9]+]]:_(s64) = G_LSHR [[SUB]], [[C1]](s64)
+    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 3689348814741910323
+    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[LSHR7]], [[C7]]
+    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[SUB]], [[C7]]
     ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[AND1]], [[AND2]]
-    ; RV64I-NEXT: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR8:%[0-9]+]]:_(s64) = G_LSHR [[ADD]], [[C10]](s64)
+    ; RV64I-NEXT: [[LSHR8:%[0-9]+]]:_(s64) = G_LSHR [[ADD]], [[C2]](s64)
     ; RV64I-NEXT: [[ADD1:%[0-9]+]]:_(s64) = G_ADD [[LSHR8]], [[ADD]]
-    ; RV64I-NEXT: [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 1085102592571150095
-    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND [[ADD1]], [[C11]]
-    ; RV64I-NEXT: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 72340172838076673
-    ; RV64I-NEXT: [[C13:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[AND3]], [[C12]]
-    ; RV64I-NEXT: [[LSHR9:%[0-9]+]]:_(s64) = G_LSHR [[MUL]], [[C13]](s64)
-    ; RV64I-NEXT: [[C14:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
-    ; RV64I-NEXT: [[SUB1:%[0-9]+]]:_(s64) = G_SUB [[C14]], [[LSHR9]]
+    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 1085102592571150095
+    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND [[ADD1]], [[C8]]
+    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 72340172838076673
+    ; RV64I-NEXT: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
+    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[AND3]], [[C9]]
+    ; RV64I-NEXT: [[LSHR9:%[0-9]+]]:_(s64) = G_LSHR [[MUL]], [[C10]](s64)
+    ; RV64I-NEXT: [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
+    ; RV64I-NEXT: [[SUB1:%[0-9]+]]:_(s64) = G_SUB [[C11]], [[LSHR9]]
     ; RV64I-NEXT: $x10 = COPY [[SUB1]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
     ;
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ctpop-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ctpop-rv32.mir
index c64669cb7341e..7fdcd79d811aa 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ctpop-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ctpop-rv32.mir
@@ -22,22 +22,20 @@ body:             |
     ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C2]]
     ; RV32I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[AND1]]
     ; RV32I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C4]]
+    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
     ; RV32I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C3]](s32)
-    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 51
-    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C5]]
-    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C5]]
+    ; RV32I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 51
+    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C4]]
+    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C4]]
     ; RV32I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND3]], [[AND4]]
-    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C6]](s32)
+    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C5]](s32)
     ; RV32I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR2]], [[ADD]]
-    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
-    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C7]]
-    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND5]], [[C8]]
-    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C9]](s32)
+    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C6]]
+    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND5]], [[C]]
+    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C7]](s32)
     ; RV32I-NEXT: $x10 = COPY [[LSHR3]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10
     ;
@@ -77,24 +75,22 @@ body:             |
     ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C2]]
     ; RV32I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[AND1]]
     ; RV32I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C4]]
+    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
     ; RV32I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C3]](s32)
-    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 13107
-    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C5]]
-    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C5]]
+    ; RV32I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 13107
+    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C4]]
+    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C4]]
     ; RV32I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND3]], [[AND4]]
-    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C6]](s32)
+    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C5]](s32)
     ; RV32I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR2]], [[ADD]]
-    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 3855
-    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C7]]
-    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 257
-    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND5]], [[C8]]
-    ; RV32I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C10]]
-    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[C9]](s32)
+    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 3855
+    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C6]]
+    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 257
+    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND5]], [[C7]]
+    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C1]]
+    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[C8]](s32)
     ; RV32I-NEXT: $x10 = COPY [[LSHR3]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10
     ;
@@ -193,30 +189,22 @@ body:             |
     ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[C6]]
     ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C7]](s32)
-    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C8]](s32)
-    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
-    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C9]]
+    ; RV32I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
+    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C1]]
     ; RV32I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[COPY1]], [[AND4]]
-    ; RV32I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SUB1]], [[C10]](s32)
-    ; RV32I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
-    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C11]]
-    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SUB1]], [[C11]]
+    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SUB1]], [[C2]](s32)
+    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C3]]
+    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SUB1]], [[C3]]
     ; RV32I-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[AND5]], [[AND6]]
-    ; RV32I-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD2]], [[C12]](s32)
+    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD2]], [[C4]](s32)
     ; RV32I-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[LSHR6]], [[ADD2]]
-    ; RV32I-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
-    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ADD3]], [[C13]]
-    ; RV32I-NEXT: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
-    ; RV32I-NEXT: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; RV32I-NEXT: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[AND7]], [[C14]]
-    ; RV32I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[MUL1]], [[C15]](s32)
+    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ADD3]], [[C5]]
+    ; RV32I-NEXT: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[AND7]], [[C6]]
+    ; RV32I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[MUL1]], [[C7]](s32)
     ; RV32I-NEXT: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[LSHR7]], [[LSHR3]]
-    ; RV32I-NEXT: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; RV32I-NEXT: $x10 = COPY [[ADD4]](s32)
-    ; RV32I-NEXT: $x11 = COPY [[C16]](s32)
+    ; RV32I-NEXT: $x11 = COPY [[C8]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10, implicit $x11
     ;
     ; RV32ZBB-LABEL: name: ctpop_i64
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ctpop-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ctpop-rv64.mir
index 196b367e59271..ec885c170b5b6 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ctpop-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-ctpop-rv64.mir
@@ -14,32 +14,30 @@ body:             |
     ; RV64I: liveins: $x10
     ; RV64I-NEXT: {{  $}}
     ; RV64I-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C1]](s64)
+    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
+    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C]](s32)
     ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 85
     ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C2]]
-    ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; RV64I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC1]], [[AND1]]
-    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C3]]
-    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C4]](s64)
-    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 51
-    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C5]]
-    ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C5]]
+    ; RV64I-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
+    ; RV64I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY1]], [[AND1]]
+    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
+    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C3]](s32)
+    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 51
+    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C4]]
+    ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C4]]
     ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND3]], [[AND4]]
-    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C6]](s64)
+    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C5]](s32)
     ; RV64I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR2]], [[ADD]]
-    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
-    ; RV64I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C7]]
-    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND5]], [[C8]]
-    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C9]](s64)
+    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; RV64I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C6]]
+    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND5]], [[C]]
+    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C7]](s32)
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LSHR3]](s32)
     ; RV64I-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
@@ -74,34 +72,32 @@ body:             |
     ; RV64I: liveins: $x10
     ; RV64I-NEXT: {{  $}}
     ; RV64I-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C1]](s64)
+    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
+    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C]](s32)
     ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 21845
     ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C2]]
-    ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; RV64I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC1]], [[AND1]]
-    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C3]]
-    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C4]](s64)
-    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 13107
-    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C5]]
-    ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C5]]
+    ; RV64I-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
+    ; RV64I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY1]], [[AND1]]
+    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
+    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C3]](s32)
+    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 13107
+    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C4]]
+    ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C4]]
     ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND3]], [[AND4]]
-    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C6]](s64)
+    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C5]](s32)
     ; RV64I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR2]], [[ADD]]
-    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 3855
-    ; RV64I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C7]]
-    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 257
-    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND5]], [[C8]]
-    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C9]]
-    ; RV64I-NEXT: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[C10]](s64)
+    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 3855
+    ; RV64I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C6]]
+    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 257
+    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND5]], [[C7]]
+    ; RV64I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C1]]
+    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND6]], [[C8]](s32)
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LSHR3]](s32)
     ; RV64I-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
@@ -137,26 +133,26 @@ body:             |
     ; RV64I-NEXT: {{  $}}
     ; RV64I-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC]], [[C]](s64)
+    ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC]], [[C]](s32)
     ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
     ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C1]]
     ; RV64I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[AND]]
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C2]](s64)
+    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C2]](s32)
     ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
     ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C3]]
     ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C3]]
     ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND1]], [[AND2]]
-    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C4]](s64)
+    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C4]](s32)
     ; RV64I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR2]], [[ADD]]
     ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
     ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C5]]
     ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
+    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[C6]]
-    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C7]](s64)
+    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C7]](s32)
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LSHR3]](s32)
     ; RV64I-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-cttz-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-cttz-rv32.mir
index 372becaf08d94..6b2475be8a007 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-cttz-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-cttz-rv32.mir
@@ -26,22 +26,20 @@ body:             |
     ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C3]]
     ; RV32I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[AND2]]
     ; RV32I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C5]]
+    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C2]]
     ; RV32I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C4]](s32)
-    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 51
-    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C6]]
-    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C6]]
+    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 51
+    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C5]]
+    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C5]]
     ; RV32I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[AND4]], [[AND5]]
-    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C7]](s32)
+    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C6]](s32)
     ; RV32I-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[LSHR2]], [[ADD1]]
-    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
-    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C8]]
-    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND6]], [[C9]]
-    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C10]](s32)
+    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C7]]
+    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND6]], [[C1]]
+    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C8]](s32)
     ; RV32I-NEXT: $x10 = COPY [[LSHR3]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10
     ;
@@ -85,24 +83,22 @@ body:             |
     ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C3]]
     ; RV32I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[AND2]]
     ; RV32I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C5]]
+    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C2]]
     ; RV32I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C4]](s32)
-    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 13107
-    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C6]]
-    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C6]]
+    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 13107
+    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C5]]
+    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C5]]
     ; RV32I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[AND4]], [[AND5]]
-    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C7]](s32)
+    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C6]](s32)
     ; RV32I-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[LSHR2]], [[ADD1]]
-    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 3855
-    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C8]]
-    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 257
-    ; RV32I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND6]], [[C9]]
-    ; RV32I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C11]]
-    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND7]], [[C10]](s32)
+    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 3855
+    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C7]]
+    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 257
+    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND6]], [[C8]]
+    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C2]]
+    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND7]], [[C9]](s32)
     ; RV32I-NEXT: $x10 = COPY [[LSHR3]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10
     ;
@@ -213,34 +209,24 @@ body:             |
     ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C9]](s32)
     ; RV32I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
     ; RV32I-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[LSHR3]], [[C10]]
-    ; RV32I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV32I-NEXT: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[COPY]], [[C11]]
-    ; RV32I-NEXT: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[C11]]
+    ; RV32I-NEXT: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[COPY]], [[C1]]
+    ; RV32I-NEXT: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[C1]]
     ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[XOR1]], [[ADD4]]
-    ; RV32I-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C12]](s32)
-    ; RV32I-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
-    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C13]]
+    ; RV32I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C2]](s32)
+    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C3]]
     ; RV32I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND5]], [[AND6]]
-    ; RV32I-NEXT: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SUB1]], [[C14]](s32)
-    ; RV32I-NEXT: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
-    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C15]]
-    ; RV32I-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[SUB1]], [[C15]]
+    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SUB1]], [[C4]](s32)
+    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C5]]
+    ; RV32I-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[SUB1]], [[C5]]
     ; RV32I-NEXT: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[AND7]], [[AND8]]
-    ; RV32I-NEXT: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD5]], [[C16]](s32)
+    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD5]], [[C6]](s32)
     ; RV32I-NEXT: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[LSHR6]], [[ADD5]]
-    ; RV32I-NEXT: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
-    ; RV32I-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ADD6]], [[C17]]
-    ; RV32I-NEXT: [[C18:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
-    ; RV32I-NEXT: [[C19:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; RV32I-NEXT: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[AND9]], [[C18]]
-    ; RV32I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[MUL1]], [[C19]](s32)
+    ; RV32I-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ADD6]], [[C7]]
+    ; RV32I-NEXT: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[AND9]], [[C8]]
+    ; RV32I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[MUL1]], [[C9]](s32)
     ; RV32I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[ADD3]], [[LSHR7]]
-    ; RV32I-NEXT: [[C20:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; RV32I-NEXT: $x10 = COPY [[SELECT]](s32)
-    ; RV32I-NEXT: $x11 = COPY [[C20]](s32)
+    ; RV32I-NEXT: $x11 = COPY [[C]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10, implicit $x11
     ;
     ; RV32ZBB-LABEL: name: cttz_i64
@@ -255,9 +241,8 @@ body:             |
     ; RV32ZBB-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[CTTZ]], [[C1]]
     ; RV32ZBB-NEXT: [[CTTZ1:%[0-9]+]]:_(s32) = G_CTTZ [[COPY]](s32)
     ; RV32ZBB-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[ADD]], [[CTTZ1]]
-    ; RV32ZBB-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; RV32ZBB-NEXT: $x10 = COPY [[SELECT]](s32)
-    ; RV32ZBB-NEXT: $x11 = COPY [[C2]](s32)
+    ; RV32ZBB-NEXT: $x11 = COPY [[C]](s32)
     ; RV32ZBB-NEXT: PseudoRET implicit $x10, implicit $x11
     %1:_(s32) = COPY $x10
     %2:_(s32) = COPY $x11
@@ -291,22 +276,20 @@ body:             |
     ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C3]]
     ; RV32I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[AND2]]
     ; RV32I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C5]]
+    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C2]]
     ; RV32I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C4]](s32)
-    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 51
-    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C6]]
-    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C6]]
+    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 51
+    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C5]]
+    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C5]]
     ; RV32I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[AND4]], [[AND5]]
-    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C7]](s32)
+    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C6]](s32)
     ; RV32I-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[LSHR2]], [[ADD1]]
-    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
-    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C8]]
-    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND6]], [[C9]]
-    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C10]](s32)
+    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C7]]
+    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND6]], [[C1]]
+    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C8]](s32)
     ; RV32I-NEXT: $x10 = COPY [[LSHR3]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10
     ;
@@ -350,24 +333,22 @@ body:             |
     ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C3]]
     ; RV32I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[AND2]]
     ; RV32I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C5]]
+    ; RV32I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C2]]
     ; RV32I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C4]](s32)
-    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 13107
-    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C6]]
-    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C6]]
+    ; RV32I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 13107
+    ; RV32I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C5]]
+    ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C5]]
     ; RV32I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[AND4]], [[AND5]]
-    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C7]](s32)
+    ; RV32I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV32I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C6]](s32)
     ; RV32I-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[LSHR2]], [[ADD1]]
-    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 3855
-    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C8]]
-    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 257
-    ; RV32I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND6]], [[C9]]
-    ; RV32I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C11]]
-    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND7]], [[C10]](s32)
+    ; RV32I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 3855
+    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C7]]
+    ; RV32I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 257
+    ; RV32I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; RV32I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND6]], [[C8]]
+    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C2]]
+    ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND7]], [[C9]](s32)
     ; RV32I-NEXT: $x10 = COPY [[LSHR3]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10
     ;
@@ -478,34 +459,24 @@ body:             |
     ; RV32I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C9]](s32)
     ; RV32I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
     ; RV32I-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[LSHR3]], [[C10]]
-    ; RV32I-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV32I-NEXT: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[COPY]], [[C11]]
-    ; RV32I-NEXT: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[C11]]
+    ; RV32I-NEXT: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[COPY]], [[C1]]
+    ; RV32I-NEXT: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[C1]]
     ; RV32I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[XOR1]], [[ADD4]]
-    ; RV32I-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C12]](s32)
-    ; RV32I-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
-    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C13]]
+    ; RV32I-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND5]], [[C2]](s32)
+    ; RV32I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C3]]
     ; RV32I-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND5]], [[AND6]]
-    ; RV32I-NEXT: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SUB1]], [[C14]](s32)
-    ; RV32I-NEXT: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
-    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C15]]
-    ; RV32I-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[SUB1]], [[C15]]
+    ; RV32I-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SUB1]], [[C4]](s32)
+    ; RV32I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C5]]
+    ; RV32I-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[SUB1]], [[C5]]
     ; RV32I-NEXT: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[AND7]], [[AND8]]
-    ; RV32I-NEXT: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD5]], [[C16]](s32)
+    ; RV32I-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD5]], [[C6]](s32)
     ; RV32I-NEXT: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[LSHR6]], [[ADD5]]
-    ; RV32I-NEXT: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
-    ; RV32I-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ADD6]], [[C17]]
-    ; RV32I-NEXT: [[C18:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
-    ; RV32I-NEXT: [[C19:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; RV32I-NEXT: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[AND9]], [[C18]]
-    ; RV32I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[MUL1]], [[C19]](s32)
+    ; RV32I-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ADD6]], [[C7]]
+    ; RV32I-NEXT: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[AND9]], [[C8]]
+    ; RV32I-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[MUL1]], [[C9]](s32)
     ; RV32I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[ADD3]], [[LSHR7]]
-    ; RV32I-NEXT: [[C20:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; RV32I-NEXT: $x10 = COPY [[SELECT]](s32)
-    ; RV32I-NEXT: $x11 = COPY [[C20]](s32)
+    ; RV32I-NEXT: $x11 = COPY [[C]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10, implicit $x11
     ;
     ; RV32ZBB-LABEL: name: cttz_zero_undef_i64
@@ -520,9 +491,8 @@ body:             |
     ; RV32ZBB-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[CTTZ]], [[C1]]
     ; RV32ZBB-NEXT: [[CTTZ1:%[0-9]+]]:_(s32) = G_CTTZ [[COPY]](s32)
     ; RV32ZBB-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[ADD]], [[CTTZ1]]
-    ; RV32ZBB-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; RV32ZBB-NEXT: $x10 = COPY [[SELECT]](s32)
-    ; RV32ZBB-NEXT: $x11 = COPY [[C2]](s32)
+    ; RV32ZBB-NEXT: $x11 = COPY [[C]](s32)
     ; RV32ZBB-NEXT: PseudoRET implicit $x10, implicit $x11
     %1:_(s32) = COPY $x10
     %2:_(s32) = COPY $x11
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-cttz-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-cttz-rv64.mir
index e51a2143efd02..f8285d609875b 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-cttz-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-cttz-rv64.mir
@@ -17,33 +17,30 @@ body:             |
     ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
     ; RV64I-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[TRUNC]], [[C]]
-    ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[TRUNC1]], [[C]]
+    ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[TRUNC]], [[C]]
     ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[XOR]], [[ADD]]
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C1]]
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C2]](s64)
+    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C2]]
+    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C1]](s32)
     ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 85
     ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C3]]
     ; RV64I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[AND2]]
-    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C4]]
-    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C5]](s64)
-    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 51
-    ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C6]]
-    ; RV64I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C6]]
+    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C2]]
+    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C4]](s32)
+    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 51
+    ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C5]]
+    ; RV64I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C5]]
     ; RV64I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[AND4]], [[AND5]]
-    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C7]](s64)
+    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C6]](s32)
     ; RV64I-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[LSHR2]], [[ADD1]]
-    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
-    ; RV64I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C8]]
-    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND6]], [[C9]]
-    ; RV64I-NEXT: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C10]](s64)
+    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; RV64I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C7]]
+    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND6]], [[C1]]
+    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C8]](s32)
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LSHR3]](s32)
     ; RV64I-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
@@ -81,35 +78,32 @@ body:             |
     ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
     ; RV64I-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[TRUNC]], [[C]]
-    ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[TRUNC1]], [[C]]
+    ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[TRUNC]], [[C]]
     ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[XOR]], [[ADD]]
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C1]]
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C2]](s64)
+    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C2]]
+    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C1]](s32)
     ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 21845
     ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C3]]
     ; RV64I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[AND2]]
-    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C4]]
-    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C5]](s64)
-    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 13107
-    ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C6]]
-    ; RV64I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C6]]
+    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C2]]
+    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C4]](s32)
+    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 13107
+    ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C5]]
+    ; RV64I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C5]]
     ; RV64I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[AND4]], [[AND5]]
-    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C7]](s64)
+    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C6]](s32)
     ; RV64I-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[LSHR2]], [[ADD1]]
-    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 3855
-    ; RV64I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C8]]
-    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 257
-    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND6]], [[C9]]
-    ; RV64I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C10]]
-    ; RV64I-NEXT: [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND7]], [[C11]](s64)
+    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 3855
+    ; RV64I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C7]]
+    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 257
+    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND6]], [[C8]]
+    ; RV64I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C2]]
+    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND7]], [[C9]](s32)
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LSHR3]](s32)
     ; RV64I-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
@@ -149,26 +143,26 @@ body:             |
     ; RV64I-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[TRUNC]], [[C]]
     ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[TRUNC]], [[C]]
     ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[XOR]], [[ADD]]
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C1]](s64)
+    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C1]](s32)
     ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
     ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C2]]
     ; RV64I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[AND1]]
-    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C3]](s64)
+    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C3]](s32)
     ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
     ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C4]]
     ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C4]]
     ; RV64I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[AND2]], [[AND3]]
-    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C5]](s64)
+    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C5]](s32)
     ; RV64I-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[LSHR2]], [[ADD1]]
     ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
     ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C6]]
     ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
+    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND4]], [[C7]]
-    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C8]](s64)
+    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C8]](s32)
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LSHR3]](s32)
     ; RV64I-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
@@ -253,33 +247,30 @@ body:             |
     ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
     ; RV64I-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[TRUNC]], [[C]]
-    ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[TRUNC1]], [[C]]
+    ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[TRUNC]], [[C]]
     ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[XOR]], [[ADD]]
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C1]]
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C2]](s64)
+    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C2]]
+    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C1]](s32)
     ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 85
     ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C3]]
     ; RV64I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[AND2]]
-    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C4]]
-    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C5]](s64)
-    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 51
-    ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C6]]
-    ; RV64I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C6]]
+    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C2]]
+    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C4]](s32)
+    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 51
+    ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C5]]
+    ; RV64I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C5]]
     ; RV64I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[AND4]], [[AND5]]
-    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C7]](s64)
+    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C6]](s32)
     ; RV64I-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[LSHR2]], [[ADD1]]
-    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
-    ; RV64I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C8]]
-    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND6]], [[C9]]
-    ; RV64I-NEXT: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C10]](s64)
+    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; RV64I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C7]]
+    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND6]], [[C1]]
+    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C8]](s32)
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LSHR3]](s32)
     ; RV64I-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
@@ -317,35 +308,32 @@ body:             |
     ; RV64I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
     ; RV64I-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[TRUNC]], [[C]]
-    ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[TRUNC1]], [[C]]
+    ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[TRUNC]], [[C]]
     ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[XOR]], [[ADD]]
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C1]]
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C2]](s64)
+    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C2]]
+    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[C1]](s32)
     ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 21845
     ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C3]]
     ; RV64I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[AND2]]
-    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C4]]
-    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C5]](s64)
-    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 13107
-    ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C6]]
-    ; RV64I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C6]]
+    ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C2]]
+    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[C4]](s32)
+    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 13107
+    ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C5]]
+    ; RV64I-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C5]]
     ; RV64I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[AND4]], [[AND5]]
-    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C7]](s64)
+    ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C6]](s32)
     ; RV64I-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[LSHR2]], [[ADD1]]
-    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 3855
-    ; RV64I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C8]]
-    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 257
-    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND6]], [[C9]]
-    ; RV64I-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV64I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C10]]
-    ; RV64I-NEXT: [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND7]], [[C11]](s64)
+    ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 3855
+    ; RV64I-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C7]]
+    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 257
+    ; RV64I-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND6]], [[C8]]
+    ; RV64I-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C2]]
+    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[AND7]], [[C9]](s32)
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LSHR3]](s32)
     ; RV64I-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
@@ -385,26 +373,26 @@ body:             |
     ; RV64I-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[TRUNC]], [[C]]
     ; RV64I-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[TRUNC]], [[C]]
     ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[XOR]], [[ADD]]
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C1]](s64)
+    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV64I-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C1]](s32)
     ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
     ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C2]]
     ; RV64I-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[AND1]]
-    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C3]](s64)
+    ; RV64I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; RV64I-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C3]](s32)
     ; RV64I-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
     ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C4]]
     ; RV64I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C4]]
     ; RV64I-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[AND2]], [[AND3]]
-    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C5]](s64)
+    ; RV64I-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; RV64I-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C5]](s32)
     ; RV64I-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[LSHR2]], [[ADD1]]
     ; RV64I-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
     ; RV64I-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C6]]
     ; RV64I-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
+    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; RV64I-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND4]], [[C7]]
-    ; RV64I-NEXT: [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C8]](s64)
+    ; RV64I-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C8]](s32)
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LSHR3]](s32)
     ; RV64I-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-div-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-div-rv32.mir
index 26d8785afb470..3733a92a47e19 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-div-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-div-rv32.mir
@@ -13,9 +13,8 @@ body:             |
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
     ; CHECK-I-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-I-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
-    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[ASHR]](s32)
     ; CHECK-I-NEXT: $x11 = COPY [[ASHR1]](s32)
@@ -31,9 +30,8 @@ body:             |
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
     ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-M-NEXT: [[SDIV:%[0-9]+]]:_(s32) = G_SDIV [[ASHR]], [[ASHR1]]
     ; CHECK-M-NEXT: $x10 = COPY [[SDIV]](s32)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
@@ -57,9 +55,8 @@ body:             |
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; CHECK-I-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-I-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[ASHR]](s32)
     ; CHECK-I-NEXT: $x11 = COPY [[ASHR1]](s32)
@@ -75,9 +72,8 @@ body:             |
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-M-NEXT: [[SDIV:%[0-9]+]]:_(s32) = G_SDIV [[ASHR]], [[ASHR1]]
     ; CHECK-M-NEXT: $x10 = COPY [[SDIV]](s32)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
@@ -101,9 +97,8 @@ body:             |
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-I-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-I-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[ASHR]](s32)
     ; CHECK-I-NEXT: $x11 = COPY [[ASHR1]](s32)
@@ -119,9 +114,8 @@ body:             |
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-M-NEXT: [[SDIV:%[0-9]+]]:_(s32) = G_SDIV [[ASHR]], [[ASHR1]]
     ; CHECK-M-NEXT: $x10 = COPY [[SDIV]](s32)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
@@ -176,9 +170,8 @@ body:             |
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-I-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %xhi, [[C]](s32)
     ; CHECK-I-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %yhi, [[C1]](s32)
-    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %yhi, [[C]](s32)
+    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY %xlo(s32)
     ; CHECK-I-NEXT: $x11 = COPY [[ASHR]](s32)
@@ -200,9 +193,8 @@ body:             |
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %xhi, [[C]](s32)
     ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %yhi, [[C1]](s32)
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %yhi, [[C]](s32)
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-M-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-M-NEXT: $x10 = COPY %xlo(s32)
     ; CHECK-M-NEXT: $x11 = COPY [[ASHR]](s32)
@@ -292,8 +284,7 @@ body:             |
     ; CHECK-I-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
     ; CHECK-I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
-    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[AND]](s32)
     ; CHECK-I-NEXT: $x11 = COPY [[AND1]](s32)
@@ -308,8 +299,7 @@ body:             |
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
     ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
-    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-M-NEXT: [[UDIV:%[0-9]+]]:_(s32) = G_UDIV [[AND]], [[AND1]]
     ; CHECK-M-NEXT: $x10 = COPY [[UDIV]](s32)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
@@ -332,8 +322,7 @@ body:             |
     ; CHECK-I-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[AND]](s32)
     ; CHECK-I-NEXT: $x11 = COPY [[AND1]](s32)
@@ -348,8 +337,7 @@ body:             |
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-M-NEXT: [[UDIV:%[0-9]+]]:_(s32) = G_UDIV [[AND]], [[AND1]]
     ; CHECK-M-NEXT: $x10 = COPY [[UDIV]](s32)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
@@ -372,8 +360,7 @@ body:             |
     ; CHECK-I-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[AND]](s32)
     ; CHECK-I-NEXT: $x11 = COPY [[AND1]](s32)
@@ -388,8 +375,7 @@ body:             |
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-M-NEXT: [[UDIV:%[0-9]+]]:_(s32) = G_UDIV [[AND]], [[AND1]]
     ; CHECK-M-NEXT: $x10 = COPY [[UDIV]](s32)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
@@ -445,10 +431,8 @@ body:             |
     ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND %xlo, [[C]]
     ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND %xhi, [[C1]]
-    ; CHECK-I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; CHECK-I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND %ylo, [[C2]]
-    ; CHECK-I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND %yhi, [[C3]]
+    ; CHECK-I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND %ylo, [[C]]
+    ; CHECK-I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND %yhi, [[C1]]
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[AND]](s32)
     ; CHECK-I-NEXT: $x11 = COPY [[AND1]](s32)
@@ -471,10 +455,8 @@ body:             |
     ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND %xlo, [[C]]
     ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND %xhi, [[C1]]
-    ; CHECK-M-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; CHECK-M-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-M-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND %ylo, [[C2]]
-    ; CHECK-M-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND %yhi, [[C3]]
+    ; CHECK-M-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND %ylo, [[C]]
+    ; CHECK-M-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND %yhi, [[C1]]
     ; CHECK-M-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-M-NEXT: $x10 = COPY [[AND]](s32)
     ; CHECK-M-NEXT: $x11 = COPY [[AND1]](s32)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-div-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-div-rv64.mir
index bbbe38f695d2e..183f5e5928239 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-div-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-div-rv64.mir
@@ -13,9 +13,8 @@ body:             |
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
     ; CHECK-I-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-I-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[ASHR]](s64)
     ; CHECK-I-NEXT: $x11 = COPY [[ASHR1]](s64)
@@ -29,15 +28,12 @@ body:             |
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
+    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
     ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; CHECK-M-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C2]](s64)
-    ; CHECK-M-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C3]](s64)
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C]](s32)
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-M-NEXT: [[SDIV:%[0-9]+]]:_(s32) = G_SDIV [[ASHR]], [[ASHR1]]
     ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SDIV]](s32)
     ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -62,9 +58,8 @@ body:             |
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 49
     ; CHECK-I-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-I-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 49
-    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[ASHR]](s64)
     ; CHECK-I-NEXT: $x11 = COPY [[ASHR1]](s64)
@@ -78,15 +73,12 @@ body:             |
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 17
-    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 17
-    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 17
+    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
+    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
     ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; CHECK-M-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 17
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C2]](s64)
-    ; CHECK-M-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 17
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C3]](s64)
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C]](s32)
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-M-NEXT: [[SDIV:%[0-9]+]]:_(s32) = G_SDIV [[ASHR]], [[ASHR1]]
     ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SDIV]](s32)
     ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -111,9 +103,8 @@ body:             |
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
     ; CHECK-I-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-I-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[ASHR]](s64)
     ; CHECK-I-NEXT: $x11 = COPY [[ASHR1]](s64)
@@ -127,15 +118,12 @@ body:             |
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
+    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
     ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; CHECK-M-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C2]](s64)
-    ; CHECK-M-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C3]](s64)
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C]](s32)
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-M-NEXT: [[SDIV:%[0-9]+]]:_(s32) = G_SDIV [[ASHR]], [[ASHR1]]
     ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SDIV]](s32)
     ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -228,9 +216,8 @@ body:             |
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
     ; CHECK-I-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL %xhi, [[C]](s64)
     ; CHECK-I-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %yhi, [[C1]](s64)
-    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %yhi, [[C]](s64)
+    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY %xlo(s64)
     ; CHECK-I-NEXT: $x11 = COPY [[ASHR]](s64)
@@ -252,9 +239,8 @@ body:             |
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
     ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL %xhi, [[C]](s64)
     ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %yhi, [[C1]](s64)
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %yhi, [[C]](s64)
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-M-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-M-NEXT: $x10 = COPY %xlo(s64)
     ; CHECK-M-NEXT: $x11 = COPY [[ASHR]](s64)
@@ -344,8 +330,7 @@ body:             |
     ; CHECK-I-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[AND]](s64)
     ; CHECK-I-NEXT: $x11 = COPY [[AND1]](s64)
@@ -362,8 +347,7 @@ body:             |
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
+    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C]]
     ; CHECK-M-NEXT: [[UDIV:%[0-9]+]]:_(s32) = G_UDIV [[AND]], [[AND1]]
     ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UDIV]](s32)
     ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -387,8 +371,7 @@ body:             |
     ; CHECK-I-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 32767
     ; CHECK-I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32767
-    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[AND]](s64)
     ; CHECK-I-NEXT: $x11 = COPY [[AND1]](s64)
@@ -405,8 +388,7 @@ body:             |
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
     ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
-    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
+    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C]]
     ; CHECK-M-NEXT: [[UDIV:%[0-9]+]]:_(s32) = G_UDIV [[AND]], [[AND1]]
     ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UDIV]](s32)
     ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -430,8 +412,7 @@ body:             |
     ; CHECK-I-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
     ; CHECK-I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[AND]](s64)
     ; CHECK-I-NEXT: $x11 = COPY [[AND1]](s64)
@@ -448,8 +429,7 @@ body:             |
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
+    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C]]
     ; CHECK-M-NEXT: [[UDIV:%[0-9]+]]:_(s32) = G_UDIV [[AND]], [[AND1]]
     ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UDIV]](s32)
     ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -473,8 +453,7 @@ body:             |
     ; CHECK-I-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
     ; CHECK-I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[AND]](s64)
     ; CHECK-I-NEXT: $x11 = COPY [[AND1]](s64)
@@ -545,10 +524,8 @@ body:             |
     ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND %xlo, [[C]]
     ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND %xhi, [[C1]]
-    ; CHECK-I-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK-I-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-I-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND %ylo, [[C2]]
-    ; CHECK-I-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND %yhi, [[C3]]
+    ; CHECK-I-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND %ylo, [[C]]
+    ; CHECK-I-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND %yhi, [[C1]]
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[AND]](s64)
     ; CHECK-I-NEXT: $x11 = COPY [[AND1]](s64)
@@ -571,10 +548,8 @@ body:             |
     ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND %xlo, [[C]]
     ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND %xhi, [[C1]]
-    ; CHECK-M-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK-M-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-M-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND %ylo, [[C2]]
-    ; CHECK-M-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND %yhi, [[C3]]
+    ; CHECK-M-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND %ylo, [[C]]
+    ; CHECK-M-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND %yhi, [[C1]]
     ; CHECK-M-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-M-NEXT: $x10 = COPY [[AND]](s64)
     ; CHECK-M-NEXT: $x11 = COPY [[AND1]](s64)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fshl-fshr-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fshl-fshr-rv32.mir
index 9dd0811507e24..0745d7bb55017 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fshl-fshr-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fshl-fshr-rv32.mir
@@ -125,46 +125,42 @@ body:             |
     ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13
     ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY4]], [[C1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY4]], [[C2]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[XOR]], [[C]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[C2]]
-    ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND]]
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[C2]]
-    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND]](s32), [[C3]]
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[C4]]
+    ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C4]], [[AND]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[C4]]
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND]](s32), [[C1]]
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[AND]](s32)
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[SUB1]](s32)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[AND]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR]], [[SHL1]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[SUB]](s32)
-    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[SHL]], [[C4]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[SHL]], [[C1]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[OR]], [[SHL2]]
     ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s32), [[COPY1]], [[SELECT1]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C5]](s32)
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
-    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C6]](s32)
+    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C3]](s32)
+    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C5]](s32)
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[LSHR1]], [[SHL3]]
-    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C5]](s32)
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND1]], [[C7]]
-    ; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C7]], [[AND1]]
-    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND1]](s32), [[C7]]
-    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND1]](s32), [[C8]]
+    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C3]](s32)
+    ; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND1]], [[C4]]
+    ; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C4]], [[AND1]]
+    ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND1]](s32), [[C4]]
+    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND1]](s32), [[C1]]
     ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[LSHR2]], [[AND1]](s32)
     ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[OR1]], [[AND1]](s32)
     ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[SUB3]](s32)
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[LSHR4]], [[SHL4]]
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[LSHR2]], [[SUB2]](s32)
     ; CHECK-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[OR2]], [[LSHR5]]
     ; CHECK-NEXT: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s32), [[OR1]], [[SELECT3]]
-    ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[LSHR3]], [[C9]]
+    ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[LSHR3]], [[C1]]
     ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SELECT]], [[SELECT4]]
     ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SELECT2]], [[SELECT5]]
     ; CHECK-NEXT: $x10 = COPY [[OR3]](s32)
@@ -309,13 +305,13 @@ body:             |
     ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13
     ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY4]], [[C1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY4]], [[C2]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[XOR]], [[C]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C2]](s32)
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C3]](s32)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C3]](s32)
     ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C4]](s32)
@@ -323,33 +319,28 @@ body:             |
     ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[AND1]], [[C5]]
     ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C5]], [[AND1]]
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND1]](s32), [[C5]]
-    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND1]](s32), [[C6]]
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND1]](s32), [[C1]]
     ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[SHL]], [[AND1]](s32)
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[SHL]], [[SUB1]](s32)
     ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[OR]], [[AND1]](s32)
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[LSHR1]], [[SHL3]]
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[SHL]], [[SUB]](s32)
-    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[SHL2]], [[C7]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[SHL2]], [[C1]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[OR1]], [[SHL4]]
     ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s32), [[OR]], [[SELECT1]]
-    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[C8]]
-    ; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C8]], [[AND]]
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[C8]]
-    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND]](s32), [[C9]]
+    ; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[C5]]
+    ; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C5]], [[AND]]
+    ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[C5]]
+    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND]](s32), [[C1]]
     ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[AND]](s32)
     ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[AND]](s32)
     ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[SUB3]](s32)
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[LSHR3]], [[SHL5]]
-    ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[SUB2]](s32)
     ; CHECK-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[OR2]], [[LSHR4]]
     ; CHECK-NEXT: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s32), [[COPY2]], [[SELECT3]]
-    ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[LSHR2]], [[C10]]
+    ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[LSHR2]], [[C1]]
     ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SELECT]], [[SELECT4]]
     ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SELECT2]], [[SELECT5]]
     ; CHECK-NEXT: $x10 = COPY [[OR3]](s32)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fshl-fshr-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fshl-fshr-rv64.mir
index 6d67521a28768..98f2d1d19596c 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fshl-fshr-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-fshl-fshr-rv64.mir
@@ -18,16 +18,15 @@ body:             |
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
-    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[TRUNC1]], [[C1]]
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[TRUNC]], [[C1]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[XOR]], [[C]]
-    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC2]], [[AND]](s32)
-    ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[TRUNC3]], [[C2]]
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C3]](s64)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[AND]](s32)
+    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[TRUNC2]], [[C3]]
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C2]](s32)
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[AND1]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR1]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR]](s32)
@@ -61,16 +60,15 @@ body:             |
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
-    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[TRUNC1]], [[C1]]
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[TRUNC]], [[C1]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[XOR]], [[C]]
-    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC2]], [[AND]](s32)
-    ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[TRUNC3]], [[C2]]
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C3]](s64)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[AND]](s32)
+    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[TRUNC2]], [[C3]]
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[C2]](s32)
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[AND1]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR1]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR]](s32)
@@ -108,9 +106,9 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
     ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[TRUNC2]], [[C1]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[XOR]], [[C]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[AND]](s32)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC1]], [[C2]](s64)
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC1]], [[C2]](s32)
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[AND1]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR1]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR]](s32)
@@ -176,16 +174,15 @@ body:             |
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
-    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[TRUNC1]], [[C1]]
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[TRUNC]], [[C1]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[XOR]], [[C]]
-    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC2]], [[C2]](s64)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C2]](s32)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[SHL]], [[AND1]](s32)
-    ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[TRUNC3]], [[C3]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[TRUNC2]], [[C3]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[AND]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[LSHR]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR]](s32)
@@ -219,16 +216,15 @@ body:             |
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
-    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[TRUNC1]], [[C1]]
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[TRUNC]], [[C1]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[XOR]], [[C]]
-    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC2]], [[C2]](s64)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C2]](s32)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[SHL]], [[AND1]](s32)
-    ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[TRUNC3]], [[C3]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[TRUNC2]], [[C3]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[AND]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[LSHR]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR]](s32)
@@ -266,8 +262,8 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
     ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[TRUNC2]], [[C1]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[XOR]], [[C]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C2]](s64)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C2]](s32)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[SHL]], [[AND1]](s32)
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC1]], [[AND]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[LSHR]]
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-icmp-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-icmp-rv32.mir
index 894a6a0aa954a..413dbe275dfde 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-icmp-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-icmp-rv32.mir
@@ -11,9 +11,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sgt), [[ASHR]](s32), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -37,9 +36,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sgt), [[ASHR]](s32), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -63,9 +61,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sgt), [[ASHR]](s32), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -109,9 +106,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %xlo, [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %ylo, [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %ylo, [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sgt), [[ASHR]](s32), [[ASHR1]]
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[ASHR]](s32), [[ASHR1]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ugt), %xhi(s32), %yhi
@@ -169,9 +165,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[ASHR]](s32), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -195,9 +190,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[ASHR]](s32), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -221,9 +215,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[ASHR]](s32), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -267,9 +260,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %xlo, [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %ylo, [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %ylo, [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[ASHR]](s32), [[ASHR1]]
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[ASHR]](s32), [[ASHR1]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), %xhi(s32), %yhi
@@ -327,9 +319,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sge), [[ASHR]](s32), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -353,9 +344,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sge), [[ASHR]](s32), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -379,9 +369,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sge), [[ASHR]](s32), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -425,9 +414,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %xlo, [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %ylo, [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %ylo, [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sge), [[ASHR]](s32), [[ASHR1]]
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[ASHR]](s32), [[ASHR1]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(uge), %xhi(s32), %yhi
@@ -485,9 +473,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sle), [[ASHR]](s32), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -511,9 +498,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sle), [[ASHR]](s32), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -537,9 +523,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sle), [[ASHR]](s32), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -583,9 +568,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %xlo, [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %ylo, [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %ylo, [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sle), [[ASHR]](s32), [[ASHR1]]
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[ASHR]](s32), [[ASHR1]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ule), %xhi(s32), %yhi
@@ -642,8 +626,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ugt), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -666,8 +649,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ugt), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -690,8 +672,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ugt), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -736,10 +717,8 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND %xhi, [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND %xlo, [[C1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND %yhi, [[C2]]
-    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND %ylo, [[C3]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND %yhi, [[C]]
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND %ylo, [[C1]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ugt), [[AND1]](s32), [[AND3]]
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND1]](s32), [[AND3]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ugt), [[AND]](s32), [[AND2]]
@@ -796,8 +775,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -820,8 +798,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -844,8 +821,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -890,10 +866,8 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND %xhi, [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND %xlo, [[C1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND %yhi, [[C2]]
-    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND %ylo, [[C3]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND %yhi, [[C]]
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND %ylo, [[C1]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND1]](s32), [[AND3]]
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND1]](s32), [[AND3]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[AND2]]
@@ -950,8 +924,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(uge), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -974,8 +947,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(uge), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -998,8 +970,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(uge), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1044,10 +1015,8 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND %xhi, [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND %xlo, [[C1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND %yhi, [[C2]]
-    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND %ylo, [[C3]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND %yhi, [[C]]
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND %ylo, [[C1]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(uge), [[AND1]](s32), [[AND3]]
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND1]](s32), [[AND3]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(uge), [[AND]](s32), [[AND2]]
@@ -1104,8 +1073,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ule), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1128,8 +1096,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ule), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1152,8 +1119,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ule), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1198,10 +1164,8 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND %xhi, [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND %xlo, [[C1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND %yhi, [[C2]]
-    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND %ylo, [[C3]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND %yhi, [[C]]
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND %ylo, [[C1]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ule), [[AND1]](s32), [[AND3]]
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND1]](s32), [[AND3]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ule), [[AND]](s32), [[AND2]]
@@ -1258,8 +1222,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1282,8 +1245,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1306,8 +1268,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1352,15 +1313,13 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND %xhi, [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND %xlo, [[C1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND %yhi, [[C2]]
-    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND %ylo, [[C3]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND %yhi, [[C]]
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND %ylo, [[C1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[AND]], [[AND2]]
     ; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[AND1]], [[AND3]]
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[XOR]], [[XOR1]]
-    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[OR]](s32), [[C4]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[OR]](s32), [[C2]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %xhi:_(s32) = COPY $x10
@@ -1414,8 +1373,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1438,8 +1396,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1462,8 +1419,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[AND]](s32), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1508,15 +1464,13 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND %xhi, [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND %xlo, [[C1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND %yhi, [[C2]]
-    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND %ylo, [[C3]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND %yhi, [[C]]
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND %ylo, [[C1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[AND]], [[AND2]]
     ; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[AND1]], [[AND3]]
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[XOR]], [[XOR1]]
-    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[OR]](s32), [[C4]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[OR]](s32), [[C2]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %xhi:_(s32) = COPY $x10
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-icmp-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-icmp-rv64.mir
index ef4166f0e5a95..27ac60ae14313 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-icmp-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-icmp-rv64.mir
@@ -11,9 +11,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(sgt), [[ASHR]](s64), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -37,9 +36,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 49
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 49
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(sgt), [[ASHR]](s64), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -63,9 +61,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(sgt), [[ASHR]](s64), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -131,9 +128,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL %xlo, [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %ylo, [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %ylo, [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(sgt), [[ASHR]](s64), [[ASHR1]]
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[ASHR]](s64), [[ASHR1]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s64) = G_ICMP intpred(ugt), %xhi(s64), %yhi
@@ -197,9 +193,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(slt), [[ASHR]](s64), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -223,9 +218,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 49
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 49
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(slt), [[ASHR]](s64), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -249,9 +243,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(slt), [[ASHR]](s64), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -317,9 +310,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL %xlo, [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %ylo, [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %ylo, [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(sge), [[ASHR]](s64), [[ASHR1]]
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[ASHR]](s64), [[ASHR1]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s64) = G_ICMP intpred(uge), %xhi(s64), %yhi
@@ -383,9 +375,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(sge), [[ASHR]](s64), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -409,9 +400,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 49
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 49
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(sge), [[ASHR]](s64), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -435,9 +425,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(sge), [[ASHR]](s64), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -503,9 +492,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL %xlo, [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %ylo, [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %ylo, [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(sge), [[ASHR]](s64), [[ASHR1]]
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[ASHR]](s64), [[ASHR1]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s64) = G_ICMP intpred(uge), %xhi(s64), %yhi
@@ -569,9 +557,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(sle), [[ASHR]](s64), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -595,9 +582,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 49
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 49
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(sle), [[ASHR]](s64), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -621,9 +607,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(sle), [[ASHR]](s64), [[ASHR1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -689,9 +674,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL %xlo, [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %ylo, [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %ylo, [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(sle), [[ASHR]](s64), [[ASHR1]]
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[ASHR]](s64), [[ASHR1]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s64) = G_ICMP intpred(ule), %xhi(s64), %yhi
@@ -754,8 +738,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ugt), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -778,8 +761,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 32767
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32767
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ugt), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -802,8 +784,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ugt), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -826,8 +807,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ugt), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -872,10 +852,8 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND %xhi, [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND %xlo, [[C1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND %yhi, [[C2]]
-    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND %ylo, [[C3]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND %yhi, [[C]]
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND %ylo, [[C1]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ugt), [[AND1]](s64), [[AND3]]
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[AND1]](s64), [[AND3]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s64) = G_ICMP intpred(ugt), [[AND]](s64), [[AND2]]
@@ -938,8 +916,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -962,8 +939,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 32767
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32767
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -986,8 +962,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1010,8 +985,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1056,10 +1030,8 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND %xhi, [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND %xlo, [[C1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND %yhi, [[C2]]
-    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND %ylo, [[C3]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND %yhi, [[C]]
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND %ylo, [[C1]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND1]](s64), [[AND3]]
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[AND1]](s64), [[AND3]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND2]]
@@ -1122,8 +1094,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(uge), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1146,8 +1117,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 32767
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32767
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(uge), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1170,8 +1140,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(uge), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1194,8 +1163,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(uge), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1240,10 +1208,8 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND %xhi, [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND %xlo, [[C1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND %yhi, [[C2]]
-    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND %ylo, [[C3]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND %yhi, [[C]]
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND %ylo, [[C1]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(uge), [[AND1]](s64), [[AND3]]
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[AND1]](s64), [[AND3]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s64) = G_ICMP intpred(uge), [[AND]](s64), [[AND2]]
@@ -1306,8 +1272,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ule), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1330,8 +1295,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 32767
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32767
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ule), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1354,8 +1318,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ule), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1378,8 +1341,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ule), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1424,10 +1386,8 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND %xhi, [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND %xlo, [[C1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND %yhi, [[C2]]
-    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND %ylo, [[C3]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND %yhi, [[C]]
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND %ylo, [[C1]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ule), [[AND1]](s64), [[AND3]]
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[AND1]](s64), [[AND3]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s64) = G_ICMP intpred(ule), [[AND]](s64), [[AND2]]
@@ -1490,8 +1450,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1514,8 +1473,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 32767
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32767
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1538,8 +1496,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1562,8 +1519,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1608,15 +1564,13 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND %xhi, [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND %xlo, [[C1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND %yhi, [[C2]]
-    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND %ylo, [[C3]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND %yhi, [[C]]
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND %ylo, [[C1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
     ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[AND]], [[AND2]]
     ; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[AND1]], [[AND3]]
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[XOR]], [[XOR1]]
-    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[OR]](s64), [[C4]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[OR]](s64), [[C2]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %xhi:_(s64) = COPY $x10
@@ -1670,8 +1624,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1694,8 +1647,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 32767
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32767
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1718,8 +1670,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1742,8 +1693,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[AND]](s64), [[AND1]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -1788,15 +1738,13 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND %xhi, [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND %xlo, [[C1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND %yhi, [[C2]]
-    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND %ylo, [[C3]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND %yhi, [[C]]
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND %ylo, [[C1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
     ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[AND]], [[AND2]]
     ; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[AND1]], [[AND3]]
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[XOR]], [[XOR1]]
-    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[OR]](s64), [[C4]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[OR]](s64), [[C2]]
     ; CHECK-NEXT: $x10 = COPY [[ICMP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %xhi:_(s64) = COPY $x10
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-implicit-def-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-implicit-def-rv32.mir
index 1662ba03489ba..b6a7bffee569a 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-implicit-def-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-implicit-def-rv32.mir
@@ -62,9 +62,8 @@ body:             |
   bb.0.entry:
     ; CHECK-LABEL: name: implicit_def_i48
     ; CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
     ; CHECK-NEXT: $x10 = COPY [[DEF]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[DEF1]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[DEF]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
     %0:_(s48) = G_IMPLICIT_DEF
     %1:_(s64) = G_ANYEXT %0(s48)
@@ -80,9 +79,8 @@ body:             |
   bb.0.entry:
     ; CHECK-LABEL: name: implicit_def_i64
     ; CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
     ; CHECK-NEXT: $x10 = COPY [[DEF]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[DEF1]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[DEF]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
     %0:_(s64) = G_IMPLICIT_DEF
     %1:_(s32), %2:_(s32) = G_UNMERGE_VALUES %0(s64)
@@ -97,11 +95,9 @@ body:             |
   bb.0.entry:
     ; CHECK-LABEL: name: implicit_def_i96
     ; CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; CHECK-NEXT: [[DEF2:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
     ; CHECK-NEXT: $x10 = COPY [[DEF]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[DEF1]](s32)
-    ; CHECK-NEXT: $x12 = COPY [[DEF2]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[DEF]](s32)
+    ; CHECK-NEXT: $x12 = COPY [[DEF]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11, implicit $x12
     %0:_(s96) = G_IMPLICIT_DEF
     %1:_(s32), %2:_(s32), %3:_(s32) = G_UNMERGE_VALUES %0(s96)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-implicit-def-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-implicit-def-rv64.mir
index 09f1d521b2bca..c024033e07d8c 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-implicit-def-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-implicit-def-rv64.mir
@@ -76,9 +76,8 @@ body:             |
   bb.0.entry:
     ; CHECK-LABEL: name: implicit_def_i72
     ; CHECK: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
-    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
     ; CHECK-NEXT: $x10 = COPY [[DEF]](s64)
-    ; CHECK-NEXT: $x11 = COPY [[DEF1]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[DEF]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
     %0:_(s72) = G_IMPLICIT_DEF
     %1:_(s128) = G_ANYEXT %0(s72)
@@ -94,9 +93,8 @@ body:             |
   bb.0.entry:
     ; CHECK-LABEL: name: implicit_def_i128
     ; CHECK: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
-    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
     ; CHECK-NEXT: $x10 = COPY [[DEF]](s64)
-    ; CHECK-NEXT: $x11 = COPY [[DEF1]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[DEF]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
     %0:_(s128) = G_IMPLICIT_DEF
     %1:_(s64), %2:_(s64) = G_UNMERGE_VALUES %0(s128)
@@ -111,11 +109,9 @@ body:             |
   bb.0.entry:
     ; CHECK-LABEL: name: implicit_def_i192
     ; CHECK: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
-    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
-    ; CHECK-NEXT: [[DEF2:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
     ; CHECK-NEXT: $x10 = COPY [[DEF]](s64)
-    ; CHECK-NEXT: $x11 = COPY [[DEF1]](s64)
-    ; CHECK-NEXT: $x12 = COPY [[DEF2]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[DEF]](s64)
+    ; CHECK-NEXT: $x12 = COPY [[DEF]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11, implicit $x12
     %0:_(s192) = G_IMPLICIT_DEF
     %1:_(s64), %2:_(s64), %3:_(s64) = G_UNMERGE_VALUES %0(s192)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-itofp-f16-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-itofp-f16-rv64.mir
index 797e79b69da5f..52c69d1acbffc 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-itofp-f16-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-itofp-f16-rv64.mir
@@ -14,10 +14,9 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 31
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 31
-    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
     ; CHECK-NEXT: [[SITOFP:%[0-9]+]]:_(s16) = G_SITOFP [[ASHR]](s32)
     ; CHECK-NEXT: $f10_h = COPY [[SITOFP]](s16)
     ; CHECK-NEXT: PseudoRET implicit $f10_h
@@ -64,10 +63,9 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
     ; CHECK-NEXT: [[SITOFP:%[0-9]+]]:_(s16) = G_SITOFP [[ASHR]](s32)
     ; CHECK-NEXT: $f10_h = COPY [[SITOFP]](s16)
     ; CHECK-NEXT: PseudoRET implicit $f10_h
@@ -114,10 +112,9 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
     ; CHECK-NEXT: [[SITOFP:%[0-9]+]]:_(s16) = G_SITOFP [[ASHR]](s32)
     ; CHECK-NEXT: $f10_h = COPY [[SITOFP]](s16)
     ; CHECK-NEXT: PseudoRET implicit $f10_h
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-itofp-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-itofp-rv64.mir
index b400e8502ed7c..bc09a44dee2e0 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-itofp-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-itofp-rv64.mir
@@ -14,10 +14,9 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 31
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 31
-    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
     ; CHECK-NEXT: [[SITOFP:%[0-9]+]]:_(s32) = G_SITOFP [[ASHR]](s32)
     ; CHECK-NEXT: $f10_f = COPY [[SITOFP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $f10_f
@@ -64,10 +63,9 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
     ; CHECK-NEXT: [[SITOFP:%[0-9]+]]:_(s32) = G_SITOFP [[ASHR]](s32)
     ; CHECK-NEXT: $f10_f = COPY [[SITOFP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $f10_f
@@ -114,10 +112,9 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
     ; CHECK-NEXT: [[SITOFP:%[0-9]+]]:_(s32) = G_SITOFP [[ASHR]](s32)
     ; CHECK-NEXT: $f10_f = COPY [[SITOFP]](s32)
     ; CHECK-NEXT: PseudoRET implicit $f10_f
@@ -248,10 +245,9 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 31
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 31
-    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
     ; CHECK-NEXT: [[SITOFP:%[0-9]+]]:_(s64) = G_SITOFP [[ASHR]](s32)
     ; CHECK-NEXT: $f10_d = COPY [[SITOFP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $f10_d
@@ -298,10 +294,9 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
     ; CHECK-NEXT: [[SITOFP:%[0-9]+]]:_(s64) = G_SITOFP [[ASHR]](s32)
     ; CHECK-NEXT: $f10_d = COPY [[SITOFP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $f10_d
@@ -348,10 +343,9 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
     ; CHECK-NEXT: [[SITOFP:%[0-9]+]]:_(s64) = G_SITOFP [[ASHR]](s32)
     ; CHECK-NEXT: $f10_d = COPY [[SITOFP]](s64)
     ; CHECK-NEXT: PseudoRET implicit $f10_d
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-load-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-load-rv32.mir
index bed44eb657da9..3260725ac6f5d 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-load-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-load-rv32.mir
@@ -280,14 +280,12 @@ body:             |
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
     ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s32)
     ; CHECK-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C3]](s32)
+    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
     ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C4]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C5]](s32)
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
     ; CHECK-NEXT: $x10 = COPY [[OR2]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -380,35 +378,27 @@ body:             |
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
     ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s32)
     ; CHECK-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C3]](s32)
+    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
     ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C4]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C5]](s32)
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; CHECK-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C6]](s32)
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; CHECK-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s32)
     ; CHECK-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C7]](s32)
+    ; CHECK-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
     ; CHECK-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C8]](s32)
+    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD4]], [[C1]](s32)
     ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; CHECK-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C9]](s32)
+    ; CHECK-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
     ; CHECK-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C10]](s32)
+    ; CHECK-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32)
     ; CHECK-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C11]](s32)
+    ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
     ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
-    ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C12]](s32)
+    ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[OR4]], [[C3]](s32)
     ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
     ; CHECK-NEXT: $x10 = COPY [[OR2]](s32)
     ; CHECK-NEXT: $x11 = COPY [[OR5]](s32)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-load-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-load-rv64.mir
index 491e4a358b1ad..3374a4a602ba9 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-load-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-load-rv64.mir
@@ -275,8 +275,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
     ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s64)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -322,20 +322,18 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
     ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD]](p0) :: (load (s8) from unknown-address + 1)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s64)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXTLOAD1]], [[C1]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
     ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
     ; CHECK-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C3]](s64)
+    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
     ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C4]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C5]](s64)
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[OR1]], [[C3]](s32)
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR2]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -381,8 +379,8 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
     ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s64)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LOAD]], [[C1]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -433,38 +431,30 @@ body:             |
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
     ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
     ; CHECK-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s64) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C3]](s64)
+    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
     ; CHECK-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s64) = G_ZEXTLOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[ZEXTLOAD3]], [[C4]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[ZEXTLOAD3]], [[C1]](s64)
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[OR1]], [[C5]](s64)
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[OR1]], [[C3]](s64)
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s64) = G_OR [[SHL2]], [[OR]]
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; CHECK-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C6]](s64)
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+    ; CHECK-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
     ; CHECK-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s64) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C7]](s64)
+    ; CHECK-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
     ; CHECK-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s64) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s64) = G_SHL [[ZEXTLOAD5]], [[C8]](s64)
+    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s64) = G_SHL [[ZEXTLOAD5]], [[C1]](s64)
     ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s64) = G_OR [[SHL3]], [[ZEXTLOAD4]]
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; CHECK-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C9]](s64)
+    ; CHECK-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
     ; CHECK-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s64) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C10]](s64)
+    ; CHECK-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
     ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s64) = G_SHL [[LOAD]], [[C11]](s64)
+    ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s64) = G_SHL [[LOAD]], [[C1]](s64)
     ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s64) = G_OR [[SHL4]], [[ZEXTLOAD6]]
-    ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[OR4]], [[C12]](s64)
+    ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[OR4]], [[C3]](s64)
     ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s64) = G_OR [[SHL5]], [[OR3]]
-    ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-    ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[OR5]], [[C13]](s64)
+    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[OR5]], [[C5]](s64)
     ; CHECK-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[OR2]]
     ; CHECK-NEXT: $x10 = COPY [[OR6]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -512,14 +502,12 @@ body:             |
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
     ; CHECK-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s64) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C3]](s64)
+    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
     ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[LOAD]], [[C4]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[LOAD]], [[C1]](s64)
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[OR1]], [[C5]](s64)
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[OR1]], [[C3]](s64)
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s64) = G_OR [[SHL2]], [[OR]]
     ; CHECK-NEXT: $x10 = COPY [[OR2]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -569,83 +557,60 @@ body:             |
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
     ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
     ; CHECK-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(s64) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s8) from unknown-address + 2)
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C3]](s64)
+    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64)
     ; CHECK-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(s64) = G_ZEXTLOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[ZEXTLOAD3]], [[C4]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[ZEXTLOAD3]], [[C1]](s64)
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[ZEXTLOAD2]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[OR1]], [[C5]](s64)
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[OR1]], [[C3]](s64)
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s64) = G_OR [[SHL2]], [[OR]]
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; CHECK-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C6]](s64)
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+    ; CHECK-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
     ; CHECK-NEXT: [[ZEXTLOAD4:%[0-9]+]]:_(s64) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C7]](s64)
+    ; CHECK-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64)
     ; CHECK-NEXT: [[ZEXTLOAD5:%[0-9]+]]:_(s64) = G_ZEXTLOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
-    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s64) = G_SHL [[ZEXTLOAD5]], [[C8]](s64)
+    ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s64) = G_SHL [[ZEXTLOAD5]], [[C1]](s64)
     ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s64) = G_OR [[SHL3]], [[ZEXTLOAD4]]
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; CHECK-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C9]](s64)
+    ; CHECK-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s64)
     ; CHECK-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(s64) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s8) from unknown-address + 6)
-    ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C10]](s64)
+    ; CHECK-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64)
     ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
-    ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s64) = G_SHL [[LOAD]], [[C11]](s64)
+    ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s64) = G_SHL [[LOAD]], [[C1]](s64)
     ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s64) = G_OR [[SHL4]], [[ZEXTLOAD6]]
-    ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[OR4]], [[C12]](s64)
+    ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[OR4]], [[C3]](s64)
     ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s64) = G_OR [[SHL5]], [[OR3]]
-    ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-    ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[OR5]], [[C13]](s64)
+    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[OR5]], [[C5]](s64)
     ; CHECK-NEXT: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL6]], [[OR2]]
-    ; CHECK-NEXT: [[C14:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C14]](s64)
+    ; CHECK-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C1]](s64)
     ; CHECK-NEXT: [[ZEXTLOAD7:%[0-9]+]]:_(s64) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
-    ; CHECK-NEXT: [[C15:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C15]](s64)
+    ; CHECK-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C]](s64)
     ; CHECK-NEXT: [[ZEXTLOAD8:%[0-9]+]]:_(s64) = G_ZEXTLOAD [[PTR_ADD8]](p0) :: (load (s8) from unknown-address + 9)
-    ; CHECK-NEXT: [[C16:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(s64) = G_SHL [[ZEXTLOAD8]], [[C16]](s64)
+    ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(s64) = G_SHL [[ZEXTLOAD8]], [[C1]](s64)
     ; CHECK-NEXT: [[OR7:%[0-9]+]]:_(s64) = G_OR [[SHL7]], [[ZEXTLOAD7]]
-    ; CHECK-NEXT: [[C17:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; CHECK-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C17]](s64)
+    ; CHECK-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s64)
     ; CHECK-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(s64) = G_ZEXTLOAD [[PTR_ADD9]](p0) :: (load (s8) from unknown-address + 10)
-    ; CHECK-NEXT: [[C18:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD9]], [[C18]](s64)
+    ; CHECK-NEXT: [[PTR_ADD10:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD9]], [[C]](s64)
     ; CHECK-NEXT: [[ZEXTLOAD10:%[0-9]+]]:_(s64) = G_ZEXTLOAD [[PTR_ADD10]](p0) :: (load (s8) from unknown-address + 11)
-    ; CHECK-NEXT: [[C19:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(s64) = G_SHL [[ZEXTLOAD10]], [[C19]](s64)
+    ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(s64) = G_SHL [[ZEXTLOAD10]], [[C1]](s64)
     ; CHECK-NEXT: [[OR8:%[0-9]+]]:_(s64) = G_OR [[SHL8]], [[ZEXTLOAD9]]
-    ; CHECK-NEXT: [[C20:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(s64) = G_SHL [[OR8]], [[C20]](s64)
+    ; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(s64) = G_SHL [[OR8]], [[C3]](s64)
     ; CHECK-NEXT: [[OR9:%[0-9]+]]:_(s64) = G_OR [[SHL9]], [[OR7]]
-    ; CHECK-NEXT: [[C21:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; CHECK-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C21]](s64)
+    ; CHECK-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD7]], [[C4]](s64)
     ; CHECK-NEXT: [[ZEXTLOAD11:%[0-9]+]]:_(s64) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
-    ; CHECK-NEXT: [[C22:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C22]](s64)
+    ; CHECK-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C]](s64)
     ; CHECK-NEXT: [[ZEXTLOAD12:%[0-9]+]]:_(s64) = G_ZEXTLOAD [[PTR_ADD12]](p0) :: (load (s8) from unknown-address + 13)
-    ; CHECK-NEXT: [[C23:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(s64) = G_SHL [[ZEXTLOAD12]], [[C23]](s64)
+    ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(s64) = G_SHL [[ZEXTLOAD12]], [[C1]](s64)
     ; CHECK-NEXT: [[OR10:%[0-9]+]]:_(s64) = G_OR [[SHL10]], [[ZEXTLOAD11]]
-    ; CHECK-NEXT: [[C24:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; CHECK-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C24]](s64)
+    ; CHECK-NEXT: [[PTR_ADD13:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD11]], [[C2]](s64)
     ; CHECK-NEXT: [[ZEXTLOAD13:%[0-9]+]]:_(s64) = G_ZEXTLOAD [[PTR_ADD13]](p0) :: (load (s8) from unknown-address + 14)
-    ; CHECK-NEXT: [[C25:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD13]], [[C25]](s64)
+    ; CHECK-NEXT: [[PTR_ADD14:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD13]], [[C]](s64)
     ; CHECK-NEXT: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD14]](p0) :: (load (s8) from unknown-address + 15)
-    ; CHECK-NEXT: [[C26:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(s64) = G_SHL [[LOAD1]], [[C26]](s64)
+    ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(s64) = G_SHL [[LOAD1]], [[C1]](s64)
     ; CHECK-NEXT: [[OR11:%[0-9]+]]:_(s64) = G_OR [[SHL11]], [[ZEXTLOAD13]]
-    ; CHECK-NEXT: [[C27:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[SHL12:%[0-9]+]]:_(s64) = G_SHL [[OR11]], [[C27]](s64)
+    ; CHECK-NEXT: [[SHL12:%[0-9]+]]:_(s64) = G_SHL [[OR11]], [[C3]](s64)
     ; CHECK-NEXT: [[OR12:%[0-9]+]]:_(s64) = G_OR [[SHL12]], [[OR10]]
-    ; CHECK-NEXT: [[C28:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-    ; CHECK-NEXT: [[SHL13:%[0-9]+]]:_(s64) = G_SHL [[OR12]], [[C28]](s64)
+    ; CHECK-NEXT: [[SHL13:%[0-9]+]]:_(s64) = G_SHL [[OR12]], [[C5]](s64)
     ; CHECK-NEXT: [[OR13:%[0-9]+]]:_(s64) = G_OR [[SHL13]], [[OR9]]
     ; CHECK-NEXT: $x10 = COPY [[OR6]](s64)
     ; CHECK-NEXT: $x11 = COPY [[OR13]](s64)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-lshr-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-lshr-rv32.mir
index 604ae2a4b7928..71d7e839691dd 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-lshr-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-lshr-rv32.mir
@@ -10,8 +10,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[AND]](s32)
     ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -34,8 +33,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[AND]](s32)
     ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -58,8 +56,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[AND]](s32)
     ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -112,11 +109,10 @@ body:             |
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND]], %ylo(s32)
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[SUB1]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR1]], [[SHL]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[SUB]](s32)
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[OR]], [[LSHR2]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s32), [[AND]], [[SELECT]]
-    ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[LSHR]], [[C4]]
+    ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[LSHR]], [[C3]]
     ; CHECK-NEXT: $x10 = COPY [[SELECT1]](s32)
     ; CHECK-NEXT: $x11 = COPY [[SELECT2]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
@@ -154,11 +150,10 @@ body:             |
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR %lo1, %lo2(s32)
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %hi1, [[SUB1]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR1]], [[SHL]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR %hi1, [[SUB]](s32)
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[OR]], [[LSHR2]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s32), %lo1, [[SELECT]]
-    ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[LSHR]], [[C2]]
+    ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[LSHR]], [[C1]]
     ; CHECK-NEXT: $x10 = COPY [[SELECT1]](s32)
     ; CHECK-NEXT: $x11 = COPY [[SELECT2]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
@@ -188,74 +183,60 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB %lo2, [[C1]]
     ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C1]], %lo2
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), %lo2(s32), [[C1]]
-    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), %lo2(s32), [[C2]]
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB %lo2, [[C3]]
-    ; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C3]], %lo2
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), %lo2(s32), [[C3]]
-    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), %lo2(s32), [[C4]]
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), %lo2(s32), [[C]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB %lo2, [[C2]]
+    ; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C2]], %lo2
+    ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), %lo2(s32), [[C2]]
+    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), %lo2(s32), [[C]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR %hi1, %lo2(s32)
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[SUB3]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR]], [[SHL]]
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[C]], [[SUB2]](s32)
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[OR]], [[LSHR1]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s32), %hi1, [[SELECT]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; CHECK-NEXT: [[SUB4:%[0-9]+]]:_(s32) = G_SUB %lo2, [[C5]]
-    ; CHECK-NEXT: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C5]], %lo2
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), %lo2(s32), [[C5]]
-    ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), %lo2(s32), [[C6]]
+    ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), %lo2(s32), [[C2]]
+    ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), %lo2(s32), [[C]]
     ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR %mid1, %lo2(s32)
     ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR %lo1, %lo2(s32)
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %mid1, [[SUB5]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %mid1, [[SUB3]](s32)
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[LSHR3]], [[SHL1]]
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR %mid1, [[SUB4]](s32)
+    ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR %mid1, [[SUB2]](s32)
     ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP4]](s32), [[OR1]], [[LSHR4]]
     ; CHECK-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s32), %lo1, [[SELECT2]]
-    ; CHECK-NEXT: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP4]](s32), [[LSHR2]], [[C7]]
-    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; CHECK-NEXT: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[C8]]
-    ; CHECK-NEXT: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[C8]], [[SUB1]]
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[SUB1]](s32), [[C8]]
-    ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[SUB1]](s32), [[C9]]
+    ; CHECK-NEXT: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP4]](s32), [[LSHR2]], [[C]]
+    ; CHECK-NEXT: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[C2]]
+    ; CHECK-NEXT: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SUB1]]
+    ; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[SUB1]](s32), [[C2]]
+    ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[SUB1]](s32), [[C]]
     ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL %hi1, [[SUB1]](s32)
-    ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR %hi1, [[SUB7]](s32)
+    ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR %hi1, [[SUB5]](s32)
     ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C]], [[SUB1]](s32)
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[LSHR5]], [[SHL3]]
-    ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL %hi1, [[SUB6]](s32)
-    ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP6]](s32), [[SHL2]], [[C10]]
+    ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL %hi1, [[SUB4]](s32)
+    ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP6]](s32), [[SHL2]], [[C]]
     ; CHECK-NEXT: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[ICMP6]](s32), [[OR2]], [[SHL4]]
     ; CHECK-NEXT: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP7]](s32), [[C]], [[SELECT6]]
     ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SELECT3]], [[SELECT5]]
     ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SELECT4]], [[SELECT7]]
-    ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; CHECK-NEXT: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SUB]], [[C12]]
-    ; CHECK-NEXT: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[C12]], [[SUB]]
-    ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ICMP8:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[SUB]](s32), [[C12]]
-    ; CHECK-NEXT: [[ICMP9:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[SUB]](s32), [[C13]]
+    ; CHECK-NEXT: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB]], [[C2]]
+    ; CHECK-NEXT: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SUB]]
+    ; CHECK-NEXT: [[ICMP8:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[SUB]](s32), [[C2]]
+    ; CHECK-NEXT: [[ICMP9:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[SUB]](s32), [[C]]
     ; CHECK-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[C]], [[SUB]](s32)
     ; CHECK-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR %hi1, [[SUB]](s32)
-    ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C]], [[SUB9]](s32)
+    ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C]], [[SUB7]](s32)
     ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[LSHR7]], [[SHL5]]
-    ; CHECK-NEXT: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[C]], [[SUB8]](s32)
+    ; CHECK-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[C]], [[SUB6]](s32)
     ; CHECK-NEXT: [[SELECT8:%[0-9]+]]:_(s32) = G_SELECT [[ICMP8]](s32), [[OR5]], [[LSHR8]]
     ; CHECK-NEXT: [[SELECT9:%[0-9]+]]:_(s32) = G_SELECT [[ICMP9]](s32), %hi1, [[SELECT8]]
-    ; CHECK-NEXT: [[SELECT10:%[0-9]+]]:_(s32) = G_SELECT [[ICMP8]](s32), [[LSHR6]], [[C14]]
+    ; CHECK-NEXT: [[SELECT10:%[0-9]+]]:_(s32) = G_SELECT [[ICMP8]](s32), [[LSHR6]], [[C]]
     ; CHECK-NEXT: [[SELECT11:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[OR3]], [[SELECT9]]
     ; CHECK-NEXT: [[SELECT12:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[OR4]], [[SELECT10]]
     ; CHECK-NEXT: [[SELECT13:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s32), %lo1, [[SELECT11]]
     ; CHECK-NEXT: [[SELECT14:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s32), %mid1, [[SELECT12]]
-    ; CHECK-NEXT: [[SELECT15:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[SELECT1]], [[C11]]
+    ; CHECK-NEXT: [[SELECT15:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[SELECT1]], [[C]]
     ; CHECK-NEXT: $x10 = COPY [[SELECT13]](s32)
     ; CHECK-NEXT: $x11 = COPY [[SELECT14]](s32)
     ; CHECK-NEXT: $x12 = COPY [[SELECT15]](s32)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-lshr-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-lshr-rv64.mir
index 43318118f09c5..d08fde01d029c 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-lshr-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-lshr-rv64.mir
@@ -12,8 +12,7 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[AND]](s32)
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LSHR]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -39,8 +38,7 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[AND]](s32)
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LSHR]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -66,8 +64,7 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[AND]](s32)
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LSHR]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -144,11 +141,10 @@ body:             |
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[AND]], %y00(s64)
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[AND1]], [[SUB1]](s64)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR1]], [[SHL]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
     ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[AND1]], [[SUB]](s64)
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[OR]], [[LSHR2]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s64), [[AND]], [[SELECT]]
-    ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[LSHR]], [[C4]]
+    ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[LSHR]], [[C3]]
     ; CHECK-NEXT: $x10 = COPY [[SELECT1]](s64)
     ; CHECK-NEXT: $x11 = COPY [[SELECT2]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
@@ -186,11 +182,10 @@ body:             |
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR %lo1, %lo2(s64)
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL %hi1, [[SUB1]](s64)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR1]], [[SHL]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
     ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR %hi1, [[SUB]](s64)
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[OR]], [[LSHR2]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s64), %lo1, [[SELECT]]
-    ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[LSHR]], [[C2]]
+    ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[LSHR]], [[C1]]
     ; CHECK-NEXT: $x10 = COPY [[SELECT1]](s64)
     ; CHECK-NEXT: $x11 = COPY [[SELECT2]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
@@ -220,74 +215,60 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 128
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s64) = G_SUB %lo2, [[C1]]
     ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s64) = G_SUB [[C1]], %lo2
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), %lo2(s64), [[C1]]
-    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), %lo2(s64), [[C2]]
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
-    ; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s64) = G_SUB %lo2, [[C3]]
-    ; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(s64) = G_SUB [[C3]], %lo2
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), %lo2(s64), [[C3]]
-    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), %lo2(s64), [[C4]]
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), %lo2(s64), [[C]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
+    ; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s64) = G_SUB %lo2, [[C2]]
+    ; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(s64) = G_SUB [[C2]], %lo2
+    ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), %lo2(s64), [[C2]]
+    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), %lo2(s64), [[C]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR %hi1, %lo2(s64)
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[C]], [[SUB3]](s64)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[C]], [[SUB2]](s64)
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s64), [[OR]], [[LSHR1]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s64), %hi1, [[SELECT]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
-    ; CHECK-NEXT: [[SUB4:%[0-9]+]]:_(s64) = G_SUB %lo2, [[C5]]
-    ; CHECK-NEXT: [[SUB5:%[0-9]+]]:_(s64) = G_SUB [[C5]], %lo2
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), %lo2(s64), [[C5]]
-    ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), %lo2(s64), [[C6]]
+    ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), %lo2(s64), [[C2]]
+    ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), %lo2(s64), [[C]]
     ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR %mid1, %lo2(s64)
     ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s64) = G_LSHR %lo1, %lo2(s64)
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %mid1, [[SUB5]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %mid1, [[SUB3]](s64)
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s64) = G_OR [[LSHR3]], [[SHL1]]
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s64) = G_LSHR %mid1, [[SUB4]](s64)
+    ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(s64) = G_LSHR %mid1, [[SUB2]](s64)
     ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s64), [[OR1]], [[LSHR4]]
     ; CHECK-NEXT: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP5]](s64), %lo1, [[SELECT2]]
-    ; CHECK-NEXT: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s64), [[LSHR2]], [[C7]]
-    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
-    ; CHECK-NEXT: [[SUB6:%[0-9]+]]:_(s64) = G_SUB [[SUB1]], [[C8]]
-    ; CHECK-NEXT: [[SUB7:%[0-9]+]]:_(s64) = G_SUB [[C8]], [[SUB1]]
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[SUB1]](s64), [[C8]]
-    ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[SUB1]](s64), [[C9]]
+    ; CHECK-NEXT: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s64), [[LSHR2]], [[C]]
+    ; CHECK-NEXT: [[SUB4:%[0-9]+]]:_(s64) = G_SUB [[SUB1]], [[C2]]
+    ; CHECK-NEXT: [[SUB5:%[0-9]+]]:_(s64) = G_SUB [[C2]], [[SUB1]]
+    ; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[SUB1]](s64), [[C2]]
+    ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[SUB1]](s64), [[C]]
     ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL %hi1, [[SUB1]](s64)
-    ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(s64) = G_LSHR %hi1, [[SUB7]](s64)
+    ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(s64) = G_LSHR %hi1, [[SUB5]](s64)
     ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s64) = G_SHL [[C]], [[SUB1]](s64)
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s64) = G_OR [[LSHR5]], [[SHL3]]
-    ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s64) = G_SHL %hi1, [[SUB6]](s64)
-    ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s64), [[SHL2]], [[C10]]
+    ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s64) = G_SHL %hi1, [[SUB4]](s64)
+    ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s64), [[SHL2]], [[C]]
     ; CHECK-NEXT: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s64), [[OR2]], [[SHL4]]
     ; CHECK-NEXT: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s64), [[C]], [[SELECT6]]
     ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s64) = G_OR [[SELECT3]], [[SELECT5]]
     ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s64) = G_OR [[SELECT4]], [[SELECT7]]
-    ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
-    ; CHECK-NEXT: [[SUB8:%[0-9]+]]:_(s64) = G_SUB [[SUB]], [[C12]]
-    ; CHECK-NEXT: [[SUB9:%[0-9]+]]:_(s64) = G_SUB [[C12]], [[SUB]]
-    ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[ICMP8:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[SUB]](s64), [[C12]]
-    ; CHECK-NEXT: [[ICMP9:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[SUB]](s64), [[C13]]
+    ; CHECK-NEXT: [[SUB6:%[0-9]+]]:_(s64) = G_SUB [[SUB]], [[C2]]
+    ; CHECK-NEXT: [[SUB7:%[0-9]+]]:_(s64) = G_SUB [[C2]], [[SUB]]
+    ; CHECK-NEXT: [[ICMP8:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[SUB]](s64), [[C2]]
+    ; CHECK-NEXT: [[ICMP9:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[SUB]](s64), [[C]]
     ; CHECK-NEXT: [[LSHR6:%[0-9]+]]:_(s64) = G_LSHR [[C]], [[SUB]](s64)
     ; CHECK-NEXT: [[LSHR7:%[0-9]+]]:_(s64) = G_LSHR %hi1, [[SUB]](s64)
-    ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[C]], [[SUB9]](s64)
+    ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[C]], [[SUB7]](s64)
     ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s64) = G_OR [[LSHR7]], [[SHL5]]
-    ; CHECK-NEXT: [[C14:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[LSHR8:%[0-9]+]]:_(s64) = G_LSHR [[C]], [[SUB8]](s64)
+    ; CHECK-NEXT: [[LSHR8:%[0-9]+]]:_(s64) = G_LSHR [[C]], [[SUB6]](s64)
     ; CHECK-NEXT: [[SELECT8:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s64), [[OR5]], [[LSHR8]]
     ; CHECK-NEXT: [[SELECT9:%[0-9]+]]:_(s64) = G_SELECT [[ICMP9]](s64), %hi1, [[SELECT8]]
-    ; CHECK-NEXT: [[SELECT10:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s64), [[LSHR6]], [[C14]]
+    ; CHECK-NEXT: [[SELECT10:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s64), [[LSHR6]], [[C]]
     ; CHECK-NEXT: [[SELECT11:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[OR3]], [[SELECT9]]
     ; CHECK-NEXT: [[SELECT12:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[OR4]], [[SELECT10]]
     ; CHECK-NEXT: [[SELECT13:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s64), %lo1, [[SELECT11]]
     ; CHECK-NEXT: [[SELECT14:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s64), %mid1, [[SELECT12]]
-    ; CHECK-NEXT: [[SELECT15:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[SELECT1]], [[C11]]
+    ; CHECK-NEXT: [[SELECT15:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[SELECT1]], [[C]]
     ; CHECK-NEXT: $x10 = COPY [[SELECT13]](s64)
     ; CHECK-NEXT: $x11 = COPY [[SELECT14]](s64)
     ; CHECK-NEXT: $x12 = COPY [[SELECT15]](s64)
@@ -321,8 +302,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 281474976710655
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 281474976710655
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[AND]], [[AND1]](s64)
     ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -346,9 +326,9 @@ body:             |
     ; CHECK: liveins: $x10
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC]], [[C]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC]], [[C]](s32)
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LSHR]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mul-ext-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mul-ext-rv32.mir
index ec2dc568a5ec3..04a6aa0966281 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mul-ext-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mul-ext-rv32.mir
@@ -212,12 +212,10 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[ASHR]], [[ASHR1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[MUL]], [[C2]](s32)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[MUL]], [[C]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ASHR2]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %0:_(s32) = COPY $x10
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mul-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mul-rv32.mir
index 1af5b686a5267..70de06b6e772d 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mul-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mul-rv32.mir
@@ -141,8 +141,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[COPY]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[COPY1]], [[C1]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[COPY1]], [[C]](s32)
     ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
     ; CHECK-NEXT: $x11 = COPY [[ASHR]](s32)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mul-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mul-rv64.mir
index 478a652dbf82b..401483cae98f1 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mul-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mul-rv64.mir
@@ -166,8 +166,7 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[COPY1]], [[C1]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[COPY1]], [[C]](s64)
     ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
     ; CHECK-NEXT: $x11 = COPY [[ASHR]](s64)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mulo-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mulo-rv32.mir
index 2e46893b8cf97..42de607ca3113 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mulo-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mulo-rv32.mir
@@ -20,13 +20,11 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[ASHR]], [[ASHR1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[MUL]], [[C2]](s32)
-    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C2]](s32)
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[MUL]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[MUL]](s32), [[ASHR2]]
     ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
@@ -40,18 +38,16 @@ body:             |
     ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; LIBCALL-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; LIBCALL-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; LIBCALL-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; LIBCALL-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; LIBCALL-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: $x10 = COPY [[ASHR]](s32)
     ; LIBCALL-NEXT: $x11 = COPY [[ASHR1]](s32)
     ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
     ; LIBCALL-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
-    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; LIBCALL-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C2]](s32)
-    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C2]](s32)
+    ; LIBCALL-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C]](s32)
     ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[COPY2]](s32), [[ASHR2]]
     ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s32)
     ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s32)
@@ -82,13 +78,11 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[ASHR]], [[ASHR1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[MUL]], [[C2]](s32)
-    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C2]](s32)
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[MUL]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[MUL]](s32), [[ASHR2]]
     ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
@@ -102,18 +96,16 @@ body:             |
     ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; LIBCALL-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; LIBCALL-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; LIBCALL-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; LIBCALL-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; LIBCALL-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: $x10 = COPY [[ASHR]](s32)
     ; LIBCALL-NEXT: $x11 = COPY [[ASHR1]](s32)
     ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
     ; LIBCALL-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
-    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; LIBCALL-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C2]](s32)
-    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C2]](s32)
+    ; LIBCALL-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C]](s32)
     ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[COPY2]](s32), [[ASHR2]]
     ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s32)
     ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s32)
@@ -156,9 +148,9 @@ body:             |
     ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
     ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; LIBCALL-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[COPY]], [[C]](s32)
-    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
-    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[COPY1]], [[C1]](s32)
+    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[COPY1]], [[C]](s32)
     ; LIBCALL-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: $x10 = COPY [[COPY]](s32)
     ; LIBCALL-NEXT: $x11 = COPY [[ASHR]](s32)
@@ -168,13 +160,11 @@ body:             |
     ; LIBCALL-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
     ; LIBCALL-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x11
-    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
-    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[COPY2]], [[C2]](s32)
-    ; LIBCALL-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[COPY2]], [[C]](s32)
     ; LIBCALL-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY2]], [[COPY2]]
     ; LIBCALL-NEXT: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[COPY3]], [[ASHR2]]
     ; LIBCALL-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[XOR]], [[XOR1]]
-    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[OR]](s32), [[C3]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[OR]](s32), [[C1]]
     ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s32)
     ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s32)
     ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
@@ -200,11 +190,9 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND]], [[AND1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C2]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[MUL]](s32), [[AND2]]
     ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
@@ -217,16 +205,14 @@ body:             |
     ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; LIBCALL-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: $x10 = COPY [[AND]](s32)
     ; LIBCALL-NEXT: $x11 = COPY [[AND1]](s32)
     ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
     ; LIBCALL-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
-    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; LIBCALL-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; LIBCALL-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
     ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[COPY2]](s32), [[AND2]]
     ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s32)
     ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s32)
@@ -256,11 +242,9 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND]], [[AND1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C2]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[MUL]](s32), [[AND2]]
     ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
@@ -273,16 +257,14 @@ body:             |
     ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; LIBCALL-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: $x10 = COPY [[AND]](s32)
     ; LIBCALL-NEXT: $x11 = COPY [[AND1]](s32)
     ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
     ; LIBCALL-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
-    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; LIBCALL-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; LIBCALL-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
     ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[COPY2]](s32), [[AND2]]
     ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s32)
     ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s32)
@@ -324,25 +306,22 @@ body:             |
     ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
     ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; LIBCALL-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: $x10 = COPY [[COPY]](s32)
     ; LIBCALL-NEXT: $x11 = COPY [[C]](s32)
     ; LIBCALL-NEXT: $x12 = COPY [[COPY1]](s32)
-    ; LIBCALL-NEXT: $x13 = COPY [[C1]](s32)
+    ; LIBCALL-NEXT: $x13 = COPY [[C]](s32)
     ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit-def $x10, implicit-def $x11
     ; LIBCALL-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
     ; LIBCALL-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x11
-    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; LIBCALL-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
-    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C3]]
-    ; LIBCALL-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
     ; LIBCALL-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY2]], [[AND]]
     ; LIBCALL-NEXT: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[COPY3]], [[AND1]]
     ; LIBCALL-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[XOR]], [[XOR1]]
-    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[OR]](s32), [[C4]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[OR]](s32), [[C]]
     ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s32)
     ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s32)
     ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mulo-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mulo-rv64.mir
index 29f4458d5f7fe..996885ba12aeb 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mulo-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-mulo-rv64.mir
@@ -20,13 +20,11 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[ASHR]], [[ASHR1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[MUL]], [[C2]](s64)
-    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C2]](s64)
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[MUL]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[MUL]](s64), [[ASHR2]]
     ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
@@ -40,18 +38,16 @@ body:             |
     ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
     ; LIBCALL-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; LIBCALL-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; LIBCALL-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; LIBCALL-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; LIBCALL-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: $x10 = COPY [[ASHR]](s64)
     ; LIBCALL-NEXT: $x11 = COPY [[ASHR1]](s64)
     ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
     ; LIBCALL-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
-    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; LIBCALL-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[COPY2]], [[C2]](s64)
-    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C2]](s64)
+    ; LIBCALL-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[COPY2]], [[C]](s64)
+    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C]](s64)
     ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[ASHR2]]
     ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
     ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
@@ -82,13 +78,11 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[ASHR]], [[ASHR1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[MUL]], [[C2]](s64)
-    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C2]](s64)
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[MUL]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[MUL]](s64), [[ASHR2]]
     ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
@@ -102,18 +96,16 @@ body:             |
     ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
     ; LIBCALL-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; LIBCALL-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; LIBCALL-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; LIBCALL-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; LIBCALL-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: $x10 = COPY [[ASHR]](s64)
     ; LIBCALL-NEXT: $x11 = COPY [[ASHR1]](s64)
     ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
     ; LIBCALL-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
-    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; LIBCALL-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[COPY2]], [[C2]](s64)
-    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C2]](s64)
+    ; LIBCALL-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[COPY2]], [[C]](s64)
+    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C]](s64)
     ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[ASHR2]]
     ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
     ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
@@ -207,8 +199,7 @@ body:             |
     ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
     ; LIBCALL-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[C]](s64)
-    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
-    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[COPY1]], [[C1]](s64)
+    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[COPY1]], [[C]](s64)
     ; LIBCALL-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: $x10 = COPY [[COPY]](s64)
     ; LIBCALL-NEXT: $x11 = COPY [[ASHR]](s64)
@@ -218,13 +209,12 @@ body:             |
     ; LIBCALL-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
     ; LIBCALL-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x11
-    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
-    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[COPY2]], [[C2]](s64)
-    ; LIBCALL-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[COPY2]], [[C]](s64)
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
     ; LIBCALL-NEXT: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[COPY2]], [[COPY2]]
     ; LIBCALL-NEXT: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[COPY3]], [[ASHR2]]
     ; LIBCALL-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[XOR]], [[XOR1]]
-    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[OR]](s64), [[C3]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[OR]](s64), [[C1]]
     ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
     ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
     ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
@@ -250,11 +240,9 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[AND]], [[AND1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[MUL]], [[C2]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[MUL]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[MUL]](s64), [[AND2]]
     ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
@@ -267,16 +255,14 @@ body:             |
     ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; LIBCALL-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: $x10 = COPY [[AND]](s64)
     ; LIBCALL-NEXT: $x11 = COPY [[AND1]](s64)
     ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
     ; LIBCALL-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
-    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; LIBCALL-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C2]]
+    ; LIBCALL-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C]]
     ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[AND2]]
     ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
     ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
@@ -306,11 +292,9 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[AND]], [[AND1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[MUL]], [[C2]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[MUL]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[MUL]](s64), [[AND2]]
     ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
@@ -323,16 +307,14 @@ body:             |
     ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
     ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; LIBCALL-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: $x10 = COPY [[AND]](s64)
     ; LIBCALL-NEXT: $x11 = COPY [[AND1]](s64)
     ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
     ; LIBCALL-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
-    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; LIBCALL-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C2]]
+    ; LIBCALL-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C]]
     ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[AND2]]
     ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
     ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
@@ -362,11 +344,9 @@ body:             |
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[AND]], [[AND1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[MUL]], [[C2]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[MUL]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[MUL]](s64), [[AND2]]
     ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
     ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
@@ -379,16 +359,14 @@ body:             |
     ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
     ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; LIBCALL-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: $x10 = COPY [[AND]](s64)
     ; LIBCALL-NEXT: $x11 = COPY [[AND1]](s64)
     ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
     ; LIBCALL-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
-    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; LIBCALL-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C2]]
+    ; LIBCALL-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C]]
     ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[AND2]]
     ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
     ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
@@ -430,25 +408,22 @@ body:             |
     ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
     ; LIBCALL-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: $x10 = COPY [[COPY]](s64)
     ; LIBCALL-NEXT: $x11 = COPY [[C]](s64)
     ; LIBCALL-NEXT: $x12 = COPY [[COPY1]](s64)
-    ; LIBCALL-NEXT: $x13 = COPY [[C1]](s64)
+    ; LIBCALL-NEXT: $x13 = COPY [[C]](s64)
     ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__multi3, csr_ilp32_lp64, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit-def $x10, implicit-def $x11
     ; LIBCALL-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
     ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
     ; LIBCALL-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x11
-    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; LIBCALL-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C2]]
-    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY3]], [[C3]]
-    ; LIBCALL-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C1]]
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY3]], [[C]]
     ; LIBCALL-NEXT: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[COPY2]], [[AND]]
     ; LIBCALL-NEXT: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[COPY3]], [[AND1]]
     ; LIBCALL-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[XOR]], [[XOR1]]
-    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[OR]](s64), [[C4]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[OR]](s64), [[C]]
     ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
     ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
     ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-rem-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-rem-rv32.mir
index 99ca07d954ffc..090acca126234 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-rem-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-rem-rv32.mir
@@ -13,9 +13,8 @@ body:             |
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
     ; CHECK-I-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-I-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
-    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[ASHR]](s32)
     ; CHECK-I-NEXT: $x11 = COPY [[ASHR1]](s32)
@@ -31,9 +30,8 @@ body:             |
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
     ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-M-NEXT: [[SREM:%[0-9]+]]:_(s32) = G_SREM [[ASHR]], [[ASHR1]]
     ; CHECK-M-NEXT: $x10 = COPY [[SREM]](s32)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
@@ -57,9 +55,8 @@ body:             |
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; CHECK-I-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-I-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[ASHR]](s32)
     ; CHECK-I-NEXT: $x11 = COPY [[ASHR1]](s32)
@@ -75,9 +72,8 @@ body:             |
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-M-NEXT: [[SREM:%[0-9]+]]:_(s32) = G_SREM [[ASHR]], [[ASHR1]]
     ; CHECK-M-NEXT: $x10 = COPY [[SREM]](s32)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
@@ -101,9 +97,8 @@ body:             |
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-I-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-I-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[ASHR]](s32)
     ; CHECK-I-NEXT: $x11 = COPY [[ASHR1]](s32)
@@ -119,9 +114,8 @@ body:             |
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-M-NEXT: [[SREM:%[0-9]+]]:_(s32) = G_SREM [[ASHR]], [[ASHR1]]
     ; CHECK-M-NEXT: $x10 = COPY [[SREM]](s32)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
@@ -176,9 +170,8 @@ body:             |
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-I-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %xhi, [[C]](s32)
     ; CHECK-I-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %yhi, [[C1]](s32)
-    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %yhi, [[C]](s32)
+    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY %xlo(s32)
     ; CHECK-I-NEXT: $x11 = COPY [[ASHR]](s32)
@@ -200,9 +193,8 @@ body:             |
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %xhi, [[C]](s32)
     ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %yhi, [[C1]](s32)
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %yhi, [[C]](s32)
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-M-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-M-NEXT: $x10 = COPY %xlo(s32)
     ; CHECK-M-NEXT: $x11 = COPY [[ASHR]](s32)
@@ -292,8 +284,7 @@ body:             |
     ; CHECK-I-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
     ; CHECK-I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
-    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[AND]](s32)
     ; CHECK-I-NEXT: $x11 = COPY [[AND1]](s32)
@@ -308,8 +299,7 @@ body:             |
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
     ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
-    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-M-NEXT: [[UREM:%[0-9]+]]:_(s32) = G_UREM [[AND]], [[AND1]]
     ; CHECK-M-NEXT: $x10 = COPY [[UREM]](s32)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
@@ -332,8 +322,7 @@ body:             |
     ; CHECK-I-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[AND]](s32)
     ; CHECK-I-NEXT: $x11 = COPY [[AND1]](s32)
@@ -348,8 +337,7 @@ body:             |
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-M-NEXT: [[UREM:%[0-9]+]]:_(s32) = G_UREM [[AND]], [[AND1]]
     ; CHECK-M-NEXT: $x10 = COPY [[UREM]](s32)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
@@ -372,8 +360,7 @@ body:             |
     ; CHECK-I-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[AND]](s32)
     ; CHECK-I-NEXT: $x11 = COPY [[AND1]](s32)
@@ -388,8 +375,7 @@ body:             |
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-M-NEXT: [[UREM:%[0-9]+]]:_(s32) = G_UREM [[AND]], [[AND1]]
     ; CHECK-M-NEXT: $x10 = COPY [[UREM]](s32)
     ; CHECK-M-NEXT: PseudoRET implicit $x10
@@ -445,10 +431,8 @@ body:             |
     ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND %xlo, [[C]]
     ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND %xhi, [[C1]]
-    ; CHECK-I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; CHECK-I-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND %ylo, [[C2]]
-    ; CHECK-I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND %yhi, [[C3]]
+    ; CHECK-I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND %ylo, [[C]]
+    ; CHECK-I-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND %yhi, [[C1]]
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[AND]](s32)
     ; CHECK-I-NEXT: $x11 = COPY [[AND1]](s32)
@@ -471,10 +455,8 @@ body:             |
     ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND %xlo, [[C]]
     ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND %xhi, [[C1]]
-    ; CHECK-M-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; CHECK-M-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-M-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND %ylo, [[C2]]
-    ; CHECK-M-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND %yhi, [[C3]]
+    ; CHECK-M-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND %ylo, [[C]]
+    ; CHECK-M-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND %yhi, [[C1]]
     ; CHECK-M-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-M-NEXT: $x10 = COPY [[AND]](s32)
     ; CHECK-M-NEXT: $x11 = COPY [[AND1]](s32)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-rem-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-rem-rv64.mir
index 64458c40f4464..cd951688843ee 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-rem-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-rem-rv64.mir
@@ -13,9 +13,8 @@ body:             |
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
     ; CHECK-I-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-I-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[ASHR]](s64)
     ; CHECK-I-NEXT: $x11 = COPY [[ASHR1]](s64)
@@ -29,15 +28,12 @@ body:             |
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
+    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
     ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; CHECK-M-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C2]](s64)
-    ; CHECK-M-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C3]](s64)
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C]](s32)
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-M-NEXT: [[SREM:%[0-9]+]]:_(s32) = G_SREM [[ASHR]], [[ASHR1]]
     ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SREM]](s32)
     ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -62,9 +58,8 @@ body:             |
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 49
     ; CHECK-I-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-I-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 49
-    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[ASHR]](s64)
     ; CHECK-I-NEXT: $x11 = COPY [[ASHR1]](s64)
@@ -78,15 +73,12 @@ body:             |
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 17
-    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 17
-    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 17
+    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
+    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
     ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; CHECK-M-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 17
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C2]](s64)
-    ; CHECK-M-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 17
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C3]](s64)
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C]](s32)
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-M-NEXT: [[SREM:%[0-9]+]]:_(s32) = G_SREM [[ASHR]], [[ASHR1]]
     ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SREM]](s32)
     ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -111,9 +103,8 @@ body:             |
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
     ; CHECK-I-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; CHECK-I-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[ASHR]](s64)
     ; CHECK-I-NEXT: $x11 = COPY [[ASHR1]](s64)
@@ -127,15 +118,12 @@ body:             |
     ; CHECK-M: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-M-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-M-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s64)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s64)
+    ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]](s32)
+    ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
     ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; CHECK-M-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C2]](s64)
-    ; CHECK-M-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C3]](s64)
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C]](s32)
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; CHECK-M-NEXT: [[SREM:%[0-9]+]]:_(s32) = G_SREM [[ASHR]], [[ASHR1]]
     ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SREM]](s32)
     ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -228,9 +216,8 @@ body:             |
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
     ; CHECK-I-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL %xhi, [[C]](s64)
     ; CHECK-I-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %yhi, [[C1]](s64)
-    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %yhi, [[C]](s64)
+    ; CHECK-I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY %xlo(s64)
     ; CHECK-I-NEXT: $x11 = COPY [[ASHR]](s64)
@@ -252,9 +239,8 @@ body:             |
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
     ; CHECK-M-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL %xhi, [[C]](s64)
     ; CHECK-M-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %yhi, [[C1]](s64)
-    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-M-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %yhi, [[C]](s64)
+    ; CHECK-M-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; CHECK-M-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-M-NEXT: $x10 = COPY %xlo(s64)
     ; CHECK-M-NEXT: $x11 = COPY [[ASHR]](s64)
@@ -344,8 +330,7 @@ body:             |
     ; CHECK-I-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[AND]](s64)
     ; CHECK-I-NEXT: $x11 = COPY [[AND1]](s64)
@@ -362,8 +347,7 @@ body:             |
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
+    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C]]
     ; CHECK-M-NEXT: [[UREM:%[0-9]+]]:_(s32) = G_UREM [[AND]], [[AND1]]
     ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UREM]](s32)
     ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -387,8 +371,7 @@ body:             |
     ; CHECK-I-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 32767
     ; CHECK-I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32767
-    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[AND]](s64)
     ; CHECK-I-NEXT: $x11 = COPY [[AND1]](s64)
@@ -405,8 +388,7 @@ body:             |
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
     ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
-    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
+    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C]]
     ; CHECK-M-NEXT: [[UREM:%[0-9]+]]:_(s32) = G_UREM [[AND]], [[AND1]]
     ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UREM]](s32)
     ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -430,8 +412,7 @@ body:             |
     ; CHECK-I-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
     ; CHECK-I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[AND]](s64)
     ; CHECK-I-NEXT: $x11 = COPY [[AND1]](s64)
@@ -448,8 +429,7 @@ body:             |
     ; CHECK-M-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-M-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
+    ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C]]
     ; CHECK-M-NEXT: [[UREM:%[0-9]+]]:_(s32) = G_UREM [[AND]], [[AND1]]
     ; CHECK-M-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UREM]](s32)
     ; CHECK-M-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -473,8 +453,7 @@ body:             |
     ; CHECK-I-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; CHECK-I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
     ; CHECK-I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[AND]](s64)
     ; CHECK-I-NEXT: $x11 = COPY [[AND1]](s64)
@@ -545,10 +524,8 @@ body:             |
     ; CHECK-I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND %xlo, [[C]]
     ; CHECK-I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND %xhi, [[C1]]
-    ; CHECK-I-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK-I-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-I-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND %ylo, [[C2]]
-    ; CHECK-I-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND %yhi, [[C3]]
+    ; CHECK-I-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND %ylo, [[C]]
+    ; CHECK-I-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND %yhi, [[C1]]
     ; CHECK-I-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-I-NEXT: $x10 = COPY [[AND]](s64)
     ; CHECK-I-NEXT: $x11 = COPY [[AND1]](s64)
@@ -571,10 +548,8 @@ body:             |
     ; CHECK-M-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-M-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND %xlo, [[C]]
     ; CHECK-M-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND %xhi, [[C1]]
-    ; CHECK-M-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK-M-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-M-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND %ylo, [[C2]]
-    ; CHECK-M-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND %yhi, [[C3]]
+    ; CHECK-M-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND %ylo, [[C]]
+    ; CHECK-M-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND %yhi, [[C1]]
     ; CHECK-M-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
     ; CHECK-M-NEXT: $x10 = COPY [[AND]](s64)
     ; CHECK-M-NEXT: $x11 = COPY [[AND1]](s64)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-rotate-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-rotate-rv32.mir
index cb7ffdf10c191..fc94c2aaf2a1a 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-rotate-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-rotate-rv32.mir
@@ -21,11 +21,12 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[COPY1]]
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[AND]](s32)
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C2]]
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[AND1]](s32)
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C2]]
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[AND1]](s32)
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C2]]
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[AND2]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR]]
     ; CHECK-NEXT: $x10 = COPY [[OR]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -54,11 +55,12 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[COPY1]]
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[AND]](s32)
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C2]]
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[AND1]](s32)
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C2]]
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[AND1]](s32)
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C2]]
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[AND2]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR]]
     ; CHECK-NEXT: $x10 = COPY [[OR]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -128,34 +130,29 @@ body:             |
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
     ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[C2]]
     ; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND]]
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[C2]]
-    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND]](s32), [[C3]]
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND]](s32), [[C]]
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[AND]](s32)
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[SUB2]](s32)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[AND]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR]], [[SHL1]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[SUB1]](s32)
-    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[SHL]], [[C4]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[SHL]], [[C]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[OR]], [[SHL2]]
     ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s32), [[COPY1]], [[SELECT1]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[AND1]], [[C5]]
-    ; CHECK-NEXT: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C5]], [[AND1]]
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND1]](s32), [[C5]]
-    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND1]](s32), [[C6]]
+    ; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[AND1]], [[C2]]
+    ; CHECK-NEXT: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND1]]
+    ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND1]](s32), [[C2]]
+    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND1]](s32), [[C]]
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[AND1]](s32)
     ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[AND1]](s32)
     ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[SUB4]](s32)
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[LSHR2]], [[SHL3]]
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[SUB3]](s32)
     ; CHECK-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[OR1]], [[LSHR3]]
     ; CHECK-NEXT: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s32), [[COPY]], [[SELECT3]]
-    ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[LSHR1]], [[C7]]
+    ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[LSHR1]], [[C]]
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SELECT]], [[SELECT4]]
     ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SELECT2]], [[SELECT5]]
     ; CHECK-NEXT: $x10 = COPY [[OR2]](s32)
@@ -190,10 +187,11 @@ body:             |
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[COPY1]]
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C2]]
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[AND]](s32)
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[AND2]](s32)
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C2]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C2]]
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[AND1]](s32)
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[AND3]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR]], [[SHL]]
     ; CHECK-NEXT: $x10 = COPY [[OR]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -223,10 +221,11 @@ body:             |
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[COPY1]]
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C2]]
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[AND]](s32)
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[AND2]](s32)
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C2]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C2]]
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[AND1]](s32)
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[AND3]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR]], [[SHL]]
     ; CHECK-NEXT: $x10 = COPY [[OR]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -296,32 +295,27 @@ body:             |
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
     ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[C2]]
     ; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND]]
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[C2]]
-    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND]](s32), [[C3]]
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND]](s32), [[C]]
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[AND]](s32)
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[AND]](s32)
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[SUB2]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR1]], [[SHL]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[SUB1]](s32)
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[OR]], [[LSHR2]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s32), [[COPY]], [[SELECT]]
-    ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[LSHR]], [[C4]]
+    ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[LSHR]], [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[AND1]], [[C5]]
-    ; CHECK-NEXT: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C5]], [[AND1]]
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND1]](s32), [[C5]]
-    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND1]](s32), [[C6]]
+    ; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[AND1]], [[C2]]
+    ; CHECK-NEXT: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[AND1]]
+    ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND1]](s32), [[C2]]
+    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND1]](s32), [[C]]
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[AND1]](s32)
     ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[SUB4]](s32)
     ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[AND1]](s32)
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[LSHR3]], [[SHL2]]
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[SUB3]](s32)
-    ; CHECK-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[SHL1]], [[C7]]
+    ; CHECK-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[SHL1]], [[C]]
     ; CHECK-NEXT: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[OR1]], [[SHL3]]
     ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s32), [[COPY1]], [[SELECT4]]
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SELECT1]], [[SELECT3]]
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-rotate-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-rotate-rv64.mir
index b9d7b838c3b97..2334fe1015e2f 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-rotate-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-rotate-rv64.mir
@@ -21,15 +21,14 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
-    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
-    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC2]], [[AND]](s32)
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
-    ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[TRUNC3]], [[C2]]
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[AND1]](s32)
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C2]]
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[AND1]](s32)
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C2]]
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[AND2]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -59,15 +58,14 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
-    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
-    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC2]], [[AND]](s32)
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
-    ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[TRUNC3]], [[C2]]
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[AND1]](s32)
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C2]]
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[AND1]](s32)
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C2]]
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[AND2]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[LSHR]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -113,9 +111,8 @@ body:             |
     ; RV64ZBB_OR_RV64ZBKB-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; RV64ZBB_OR_RV64ZBKB-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
     ; RV64ZBB_OR_RV64ZBKB-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
-    ; RV64ZBB_OR_RV64ZBKB-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; RV64ZBB_OR_RV64ZBKB-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
-    ; RV64ZBB_OR_RV64ZBKB-NEXT: [[ROTL:%[0-9]+]]:_(s32) = G_ROTL [[TRUNC]], [[AND]](s64)
+    ; RV64ZBB_OR_RV64ZBKB-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; RV64ZBB_OR_RV64ZBKB-NEXT: [[ROTL:%[0-9]+]]:_(s32) = G_ROTL [[TRUNC]], [[TRUNC1]](s32)
     ; RV64ZBB_OR_RV64ZBKB-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ROTL]](s32)
     ; RV64ZBB_OR_RV64ZBKB-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; RV64ZBB_OR_RV64ZBKB-NEXT: PseudoRET implicit $x10
@@ -181,15 +178,15 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
-    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
-    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC2]], [[C2]]
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[AND]](s32)
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
-    ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC3]], [[AND2]](s32)
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C2]]
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C2]]
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[AND1]](s32)
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[TRUNC1]](s32)
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[AND3]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR]], [[SHL]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -219,15 +216,15 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
-    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
-    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC2]], [[C2]]
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[AND]](s32)
-    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
-    ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC3]], [[AND2]](s32)
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C2]]
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C2]]
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[AND1]](s32)
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[TRUNC1]](s32)
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[AND3]](s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR]], [[SHL]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[OR]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
@@ -273,9 +270,8 @@ body:             |
     ; RV64ZBB_OR_RV64ZBKB-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; RV64ZBB_OR_RV64ZBKB-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
     ; RV64ZBB_OR_RV64ZBKB-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
-    ; RV64ZBB_OR_RV64ZBKB-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; RV64ZBB_OR_RV64ZBKB-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
-    ; RV64ZBB_OR_RV64ZBKB-NEXT: [[ROTR:%[0-9]+]]:_(s32) = G_ROTR [[TRUNC]], [[AND]](s64)
+    ; RV64ZBB_OR_RV64ZBKB-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; RV64ZBB_OR_RV64ZBKB-NEXT: [[ROTR:%[0-9]+]]:_(s32) = G_ROTR [[TRUNC]], [[TRUNC1]](s32)
     ; RV64ZBB_OR_RV64ZBKB-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ROTR]](s32)
     ; RV64ZBB_OR_RV64ZBKB-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; RV64ZBB_OR_RV64ZBKB-NEXT: PseudoRET implicit $x10
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-sat-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-sat-rv32.mir
index 08aa92e0207bc..4932b1aebdec6 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-sat-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-sat-rv32.mir
@@ -64,9 +64,8 @@ body:             |
     ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[COPY4]](s32), [[COPY2]]
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[ICMP3]], [[ICMP1]]
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[SELECT]](s32), [[C]], [[COPY4]]
-    ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[SELECT]](s32), [[C1]], [[COPY5]]
+    ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[SELECT]](s32), [[C]], [[COPY5]]
     ; CHECK-NEXT: $x10 = COPY [[SELECT1]](s32)
     ; CHECK-NEXT: $x11 = COPY [[SELECT2]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
@@ -149,28 +148,24 @@ body:             |
     ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ADD]](s32)
     ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[COPY1]], [[COPY3]]
     ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[ICMP]]
-    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY [[ADD2]](s32)
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY [[ADD2]](s32)
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[COPY5]](s32), [[COPY1]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[COPY5]](s32), [[COPY1]]
     ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[COPY4]](s32), [[COPY]]
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[ICMP3]], [[ICMP1]]
-    ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[COPY3]](s32), [[C1]]
-    ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[COPY3]](s32), [[C1]]
+    ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[COPY3]](s32), [[C]]
+    ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[COPY3]](s32), [[C]]
     ; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[COPY2]](s32), [[C]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s32), [[ICMP6]], [[ICMP4]]
     ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[SELECT]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
-    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[COPY5]], [[C2]](s32)
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[COPY5]], [[C3]](s32)
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
-    ; CHECK-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ASHR]], [[C4]]
-    ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD3]](s32), [[C4]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[COPY5]], [[C1]](s32)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; CHECK-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ASHR]], [[C]]
+    ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD3]](s32), [[C]]
     ; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY [[ADD3]](s32)
-    ; CHECK-NEXT: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ASHR1]], [[C5]]
+    ; CHECK-NEXT: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ASHR]], [[C2]]
     ; CHECK-NEXT: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[ADD4]], [[ICMP7]]
     ; CHECK-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY [[ADD5]](s32)
     ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[XOR]](s32), [[COPY6]], [[COPY4]]
@@ -241,14 +236,13 @@ body:             |
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[COPY]](s32), [[COPY2]]
     ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[COPY1]], [[COPY3]]
     ; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[ICMP]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[COPY1]](s32), [[COPY3]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[COPY1]](s32), [[COPY3]]
     ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[COPY]](s32), [[COPY2]]
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[ICMP3]], [[ICMP1]]
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[SELECT]](s32), [[C]], [[SUB]]
-    ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[SELECT]](s32), [[C1]], [[SUB2]]
+    ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[SELECT]](s32), [[C]], [[SUB2]]
     ; CHECK-NEXT: $x10 = COPY [[SELECT1]](s32)
     ; CHECK-NEXT: $x11 = COPY [[SELECT2]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
@@ -331,26 +325,22 @@ body:             |
     ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[COPY1]], [[COPY3]]
     ; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[ICMP]]
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[SUB2]](s32), [[COPY1]]
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[SUB2]](s32), [[COPY1]]
     ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[SUB]](s32), [[COPY]]
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[ICMP3]], [[ICMP1]]
-    ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s32) = G_ICMP intpred(sgt), [[COPY3]](s32), [[C1]]
-    ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[COPY3]](s32), [[C1]]
+    ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s32) = G_ICMP intpred(sgt), [[COPY3]](s32), [[C]]
+    ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[COPY3]](s32), [[C]]
     ; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s32) = G_ICMP intpred(ugt), [[COPY2]](s32), [[C]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s32), [[ICMP6]], [[ICMP4]]
     ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[SELECT1]], [[SELECT]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
-    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SUB2]], [[C2]](s32)
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
-    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SUB2]], [[C3]](s32)
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
-    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[ASHR]], [[C4]]
-    ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD]](s32), [[C4]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SUB2]], [[C1]](s32)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[ASHR]], [[C]]
+    ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD]](s32), [[C]]
     ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ADD]](s32)
-    ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ASHR1]], [[C5]]
+    ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ASHR]], [[C2]]
     ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[ICMP7]]
     ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY [[ADD2]](s32)
     ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[XOR]](s32), [[COPY4]], [[SUB]]
@@ -385,11 +375,10 @@ body:             |
     ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[COPY1]]
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[AND1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[C2]], [[ADD]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[C1]], [[ADD]]
     ; CHECK-NEXT: $x10 = COPY [[SELECT]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %2:_(s32) = COPY $x10
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-sat-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-sat-rv64.mir
index 5eaf8b37fe981..fc4a497fa2132 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-sat-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-sat-rv64.mir
@@ -89,14 +89,15 @@ body:             |
     ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY]], 32
     ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY1]], 32
     ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[SEXT_INREG]], [[SEXT_INREG1]]
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ADD]](s64)
     ; CHECK-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s64) = G_SEXT_INREG [[ADD]], 32
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[ADD]](s64), [[SEXT_INREG2]]
-    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ADD]](s64)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 31
-    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[TRUNC]], [[C]](s64)
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[COPY2]], [[C]](s32)
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
     ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ASHR]], [[C1]]
-    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s64), [[ADD1]], [[TRUNC]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s64), [[ADD1]], [[COPY2]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SELECT]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -180,11 +181,10 @@ body:             |
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[TRUNC1]]
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s64), [[C2]], [[SUB]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s64), [[C1]], [[SUB]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SELECT]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -246,14 +246,15 @@ body:             |
     ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY]], 32
     ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY1]], 32
     ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s64) = G_SUB [[SEXT_INREG]], [[SEXT_INREG1]]
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[SUB]](s64)
     ; CHECK-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s64) = G_SEXT_INREG [[SUB]], 32
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[SUB]](s64), [[SEXT_INREG2]]
-    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[SUB]](s64)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 31
-    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[TRUNC]], [[C]](s64)
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[COPY2]], [[C]](s32)
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
     ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[ASHR]], [[C1]]
-    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s64), [[ADD]], [[TRUNC]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s64), [[ADD]], [[COPY2]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SELECT]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
@@ -338,11 +339,10 @@ body:             |
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ADD]](s32)
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s64), [[C2]], [[ADD]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s64), [[C1]], [[ADD]]
     ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[SELECT]](s32)
     ; CHECK-NEXT: $x10 = COPY [[ANYEXT1]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-select-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-select-rv32.mir
index a85a32d1ddd2a..9498c9265f75a 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-select-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-select-rv32.mir
@@ -111,9 +111,7 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND %c0, [[C]]
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s32), %xhi, %yhi
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND %c0, [[C1]]
-    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s32), %xlo, %ylo
+    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s32), %xlo, %ylo
     ; CHECK-NEXT: $x10 = COPY [[SELECT]](s32)
     ; CHECK-NEXT: $x11 = COPY [[SELECT1]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
@@ -148,9 +146,7 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND %c0, [[C]]
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s32), %xhi, %yhi
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND %c0, [[C1]]
-    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s32), %xlo, %ylo
+    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s32), %xlo, %ylo
     ; CHECK-NEXT: $x10 = COPY [[SELECT]](s32)
     ; CHECK-NEXT: $x11 = COPY [[SELECT1]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-select-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-select-rv64.mir
index 5bad3d4c2aae1..7ecb79d13d370 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-select-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-select-rv64.mir
@@ -148,9 +148,7 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND %c0, [[C]]
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[AND]](s64), %xhi, %yhi
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND %c0, [[C1]]
-    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[AND1]](s64), %xlo, %ylo
+    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[AND]](s64), %xlo, %ylo
     ; CHECK-NEXT: $x10 = COPY [[SELECT]](s64)
     ; CHECK-NEXT: $x11 = COPY [[SELECT1]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
@@ -185,9 +183,7 @@ body:             |
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND %c0, [[C]]
     ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[AND]](s64), %xhi, %yhi
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND %c0, [[C1]]
-    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[AND1]](s64), %xlo, %ylo
+    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[AND]](s64), %xlo, %ylo
     ; CHECK-NEXT: $x10 = COPY [[SELECT]](s64)
     ; CHECK-NEXT: $x11 = COPY [[SELECT1]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-shl-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-shl-rv32.mir
index ce00388c61dbe..8b0728b335093 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-shl-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-shl-rv32.mir
@@ -102,9 +102,8 @@ body:             |
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR %xlo, [[SUB1]](s32)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %xhi, %ylo(s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR]], [[SHL1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL %xlo, [[SUB]](s32)
-    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[SHL]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[SHL]], [[C1]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[OR]], [[SHL2]]
     ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s32), %xhi, [[SELECT1]]
     ; CHECK-NEXT: $x10 = COPY [[SELECT]](s32)
@@ -144,9 +143,8 @@ body:             |
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR %lo1, [[SUB1]](s32)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %hi1, %lo2(s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR]], [[SHL1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL %lo1, [[SUB]](s32)
-    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[SHL]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[SHL]], [[C1]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[OR]], [[SHL2]]
     ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s32), %hi1, [[SELECT1]]
     ; CHECK-NEXT: $x10 = COPY [[SELECT]](s32)
@@ -183,45 +181,35 @@ body:             |
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
     ; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB %lo2, [[C2]]
     ; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C2]], %lo2
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), %lo2(s32), [[C2]]
-    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), %lo2(s32), [[C3]]
+    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), %lo2(s32), [[C1]]
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL %lo1, %lo2(s32)
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR %lo1, [[SUB3]](s32)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL %mid1, %lo2(s32)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[LSHR]], [[SHL1]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL %lo1, [[SUB2]](s32)
-    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[SHL]], [[C4]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[SHL]], [[C1]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s32), [[OR]], [[SHL2]]
     ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s32), %mid1, [[SELECT1]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; CHECK-NEXT: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[C5]]
-    ; CHECK-NEXT: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C5]], [[SUB1]]
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[SUB1]](s32), [[C5]]
-    ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[SUB1]](s32), [[C6]]
+    ; CHECK-NEXT: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[C2]]
+    ; CHECK-NEXT: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SUB1]]
+    ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[SUB1]](s32), [[C2]]
+    ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[SUB1]](s32), [[C1]]
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR %lo1, [[SUB1]](s32)
     ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL %mid1, [[SUB5]](s32)
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[LSHR1]], [[SHL3]]
     ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR %mid1, [[SUB4]](s32)
     ; CHECK-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP4]](s32), [[OR1]], [[LSHR2]]
     ; CHECK-NEXT: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s32), %lo1, [[SELECT3]]
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), %lo2(s32), [[C7]]
+    ; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), %lo2(s32), [[C2]]
     ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL %hi1, %lo2(s32)
-    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP6]](s32), [[SHL4]], [[C8]]
+    ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP6]](s32), [[SHL4]], [[C1]]
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SELECT4]], [[SELECT5]]
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[SUB]](s32), [[C11]]
+    ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[SUB]](s32), [[C2]]
     ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL %lo1, [[SUB]](s32)
-    ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[ICMP7]](s32), [[SHL5]], [[C12]]
-    ; CHECK-NEXT: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[SELECT]], [[C9]]
-    ; CHECK-NEXT: [[SELECT8:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[SELECT2]], [[C10]]
+    ; CHECK-NEXT: [[SELECT6:%[0-9]+]]:_(s32) = G_SELECT [[ICMP7]](s32), [[SHL5]], [[C1]]
+    ; CHECK-NEXT: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[SELECT]], [[C1]]
+    ; CHECK-NEXT: [[SELECT8:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[SELECT2]], [[C1]]
     ; CHECK-NEXT: [[SELECT9:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[OR2]], [[SELECT6]]
     ; CHECK-NEXT: [[SELECT10:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s32), %hi1, [[SELECT9]]
     ; CHECK-NEXT: $x10 = COPY [[SELECT7]](s32)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-shl-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-shl-rv64.mir
index 057cbcb9972ac..5319a103540c1 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-shl-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-shl-rv64.mir
@@ -134,9 +134,8 @@ body:             |
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR %x00, [[SUB1]](s64)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %x01, %y00(s64)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
     ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL %x00, [[SUB]](s64)
-    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[SHL]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[SHL]], [[C1]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[OR]], [[SHL2]]
     ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s64), %x01, [[SELECT1]]
     ; CHECK-NEXT: $x10 = COPY [[SELECT]](s64)
@@ -176,9 +175,8 @@ body:             |
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR %lo1, [[SUB1]](s64)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %hi1, %lo2(s64)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL1]]
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
     ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL %lo1, [[SUB]](s64)
-    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[SHL]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[SHL]], [[C1]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[OR]], [[SHL2]]
     ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s64), %hi1, [[SELECT1]]
     ; CHECK-NEXT: $x10 = COPY [[SELECT]](s64)
@@ -215,45 +213,35 @@ body:             |
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
     ; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s64) = G_SUB %lo2, [[C2]]
     ; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(s64) = G_SUB [[C2]], %lo2
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), %lo2(s64), [[C2]]
-    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), %lo2(s64), [[C3]]
+    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), %lo2(s64), [[C1]]
     ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL %lo1, %lo2(s64)
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR %lo1, [[SUB3]](s64)
     ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL %mid1, %lo2(s64)
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL1]]
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
     ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL %lo1, [[SUB2]](s64)
-    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s64), [[SHL]], [[C4]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s64), [[SHL]], [[C1]]
     ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s64), [[OR]], [[SHL2]]
     ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s64), %mid1, [[SELECT1]]
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
-    ; CHECK-NEXT: [[SUB4:%[0-9]+]]:_(s64) = G_SUB [[SUB1]], [[C5]]
-    ; CHECK-NEXT: [[SUB5:%[0-9]+]]:_(s64) = G_SUB [[C5]], [[SUB1]]
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[SUB1]](s64), [[C5]]
-    ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[SUB1]](s64), [[C6]]
+    ; CHECK-NEXT: [[SUB4:%[0-9]+]]:_(s64) = G_SUB [[SUB1]], [[C2]]
+    ; CHECK-NEXT: [[SUB5:%[0-9]+]]:_(s64) = G_SUB [[C2]], [[SUB1]]
+    ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[SUB1]](s64), [[C2]]
+    ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[SUB1]](s64), [[C1]]
     ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR %lo1, [[SUB1]](s64)
     ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s64) = G_SHL %mid1, [[SUB5]](s64)
     ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s64) = G_OR [[LSHR1]], [[SHL3]]
     ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR %mid1, [[SUB4]](s64)
     ; CHECK-NEXT: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s64), [[OR1]], [[LSHR2]]
     ; CHECK-NEXT: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP5]](s64), %lo1, [[SELECT3]]
-    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
-    ; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), %lo2(s64), [[C7]]
+    ; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), %lo2(s64), [[C2]]
     ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s64) = G_SHL %hi1, %lo2(s64)
-    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s64), [[SHL4]], [[C8]]
+    ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s64), [[SHL4]], [[C1]]
     ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s64) = G_OR [[SELECT4]], [[SELECT5]]
-    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
-    ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[SUB]](s64), [[C11]]
+    ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[SUB]](s64), [[C2]]
     ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s64) = G_SHL %lo1, [[SUB]](s64)
-    ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s64), [[SHL5]], [[C12]]
-    ; CHECK-NEXT: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[SELECT]], [[C9]]
-    ; CHECK-NEXT: [[SELECT8:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[SELECT2]], [[C10]]
+    ; CHECK-NEXT: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s64), [[SHL5]], [[C1]]
+    ; CHECK-NEXT: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[SELECT]], [[C1]]
+    ; CHECK-NEXT: [[SELECT8:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[SELECT2]], [[C1]]
     ; CHECK-NEXT: [[SELECT9:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s64), [[OR2]], [[SELECT6]]
     ; CHECK-NEXT: [[SELECT10:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s64), %hi1, [[SELECT9]]
     ; CHECK-NEXT: $x10 = COPY [[SELECT7]](s64)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-smax-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-smax-rv32.mir
index 4f66e8c8f5abf..e7804dd0f18ad 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-smax-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-smax-rv32.mir
@@ -13,14 +13,12 @@ body:             |
     ; RV32I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; RV32I-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; RV32I-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; RV32I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; RV32I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; RV32I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; RV32I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; RV32I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; RV32I-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sgt), [[ASHR]](s32), [[ASHR1]]
     ; RV32I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[COPY]], [[COPY1]]
-    ; RV32I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; RV32I-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[SELECT]], [[C2]](s32)
-    ; RV32I-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C2]](s32)
+    ; RV32I-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[SELECT]], [[C]](s32)
+    ; RV32I-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C]](s32)
     ; RV32I-NEXT: $x10 = COPY [[ASHR2]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10
     ;
@@ -53,14 +51,12 @@ body:             |
     ; RV32I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; RV32I-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; RV32I-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; RV32I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; RV32I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; RV32I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; RV32I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; RV32I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; RV32I-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sgt), [[ASHR]](s32), [[ASHR1]]
     ; RV32I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[COPY]], [[COPY1]]
-    ; RV32I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; RV32I-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[SELECT]], [[C2]](s32)
-    ; RV32I-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C2]](s32)
+    ; RV32I-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[SELECT]], [[C]](s32)
+    ; RV32I-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C]](s32)
     ; RV32I-NEXT: $x10 = COPY [[ASHR2]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10
     ;
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-smax-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-smax-rv64.mir
index 6e82d1b38b362..30a74dcac0c19 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-smax-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-smax-rv64.mir
@@ -13,17 +13,15 @@ body:             |
     ; RV64I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
     ; RV64I-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; RV64I-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; RV64I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; RV64I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; RV64I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; RV64I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; RV64I-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(sgt), [[ASHR]](s64), [[ASHR1]]
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
     ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
     ; RV64I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s64), [[TRUNC]], [[TRUNC1]]
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SELECT]](s32)
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; RV64I-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C2]](s64)
-    ; RV64I-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C2]](s64)
+    ; RV64I-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C]](s64)
+    ; RV64I-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C]](s64)
     ; RV64I-NEXT: $x10 = COPY [[ASHR2]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
     ;
@@ -56,17 +54,15 @@ body:             |
     ; RV64I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
     ; RV64I-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; RV64I-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; RV64I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; RV64I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; RV64I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; RV64I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; RV64I-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(sgt), [[ASHR]](s64), [[ASHR1]]
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
     ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
     ; RV64I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s64), [[TRUNC]], [[TRUNC1]]
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SELECT]](s32)
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; RV64I-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C2]](s64)
-    ; RV64I-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C2]](s64)
+    ; RV64I-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C]](s64)
+    ; RV64I-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C]](s64)
     ; RV64I-NEXT: $x10 = COPY [[ASHR2]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
     ;
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-smin-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-smin-rv32.mir
index c02df74bd25e3..7a048c5ca57ae 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-smin-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-smin-rv32.mir
@@ -13,14 +13,12 @@ body:             |
     ; RV32I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; RV32I-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; RV32I-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; RV32I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; RV32I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; RV32I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; RV32I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; RV32I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; RV32I-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[ASHR]](s32), [[ASHR1]]
     ; RV32I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[COPY]], [[COPY1]]
-    ; RV32I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; RV32I-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[SELECT]], [[C2]](s32)
-    ; RV32I-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C2]](s32)
+    ; RV32I-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[SELECT]], [[C]](s32)
+    ; RV32I-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C]](s32)
     ; RV32I-NEXT: $x10 = COPY [[ASHR2]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10
     ;
@@ -53,14 +51,12 @@ body:             |
     ; RV32I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; RV32I-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
     ; RV32I-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
-    ; RV32I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; RV32I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
-    ; RV32I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; RV32I-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; RV32I-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
     ; RV32I-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[ASHR]](s32), [[ASHR1]]
     ; RV32I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[COPY]], [[COPY1]]
-    ; RV32I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; RV32I-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[SELECT]], [[C2]](s32)
-    ; RV32I-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C2]](s32)
+    ; RV32I-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[SELECT]], [[C]](s32)
+    ; RV32I-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C]](s32)
     ; RV32I-NEXT: $x10 = COPY [[ASHR2]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10
     ;
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-smin-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-smin-rv64.mir
index bb7f32bc2eaa0..c247e74bfed41 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-smin-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-smin-rv64.mir
@@ -13,17 +13,15 @@ body:             |
     ; RV64I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
     ; RV64I-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; RV64I-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; RV64I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; RV64I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; RV64I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; RV64I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; RV64I-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(slt), [[ASHR]](s64), [[ASHR1]]
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
     ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
     ; RV64I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s64), [[TRUNC]], [[TRUNC1]]
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SELECT]](s32)
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
-    ; RV64I-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C2]](s64)
-    ; RV64I-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C2]](s64)
+    ; RV64I-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C]](s64)
+    ; RV64I-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C]](s64)
     ; RV64I-NEXT: $x10 = COPY [[ASHR2]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
     ;
@@ -56,17 +54,15 @@ body:             |
     ; RV64I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
     ; RV64I-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
     ; RV64I-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; RV64I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
-    ; RV64I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; RV64I-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C]](s64)
+    ; RV64I-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C]](s64)
     ; RV64I-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(slt), [[ASHR]](s64), [[ASHR1]]
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
     ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
     ; RV64I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s64), [[TRUNC]], [[TRUNC1]]
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SELECT]](s32)
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; RV64I-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C2]](s64)
-    ; RV64I-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C2]](s64)
+    ; RV64I-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C]](s64)
+    ; RV64I-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C]](s64)
     ; RV64I-NEXT: $x10 = COPY [[ASHR2]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
     ;
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-store-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-store-rv32.mir
index 791bdb30c490f..30b9e75a58ea3 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-store-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-store-rv32.mir
@@ -287,8 +287,7 @@ body:             |
     ; CHECK-NEXT: G_STORE [[LSHR1]](s32), [[PTR_ADD1]](p0) :: (store (s8) into unknown-address + 1)
     ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
     ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[C5]](s32)
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C6]](s32)
+    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C4]](s32)
     ; CHECK-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p0) :: (store (s8) into unknown-address + 2)
     ; CHECK-NEXT: G_STORE [[LSHR2]](s32), [[PTR_ADD2]](p0) :: (store (s8) into unknown-address + 3)
     ; CHECK-NEXT: PseudoRET
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-store-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-store-rv64.mir
index 860bc932d8560..9a6c53beff3bb 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-store-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-store-rv64.mir
@@ -261,10 +261,10 @@ body:             |
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x11
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C1]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C]](s32)
     ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
     ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C2]](s64)
     ; CHECK-NEXT: G_STORE [[TRUNC]](s32), [[COPY1]](p0) :: (store (s8))
@@ -310,22 +310,21 @@ body:             |
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x11
     ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C]](s32)
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
     ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s64)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
-    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C3]](s64)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C3]]
+    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C2]](s32)
     ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
     ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C4]](s64)
     ; CHECK-NEXT: G_STORE [[COPY2]](s32), [[COPY1]](p0) :: (store (s8))
     ; CHECK-NEXT: G_STORE [[LSHR1]](s32), [[PTR_ADD1]](p0) :: (store (s8) into unknown-address + 1)
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[C5]](s64)
-    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C6]](s64)
+    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[C5]](s32)
+    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C4]](s64)
     ; CHECK-NEXT: G_STORE [[LSHR]](s32), [[PTR_ADD]](p0) :: (store (s8) into unknown-address + 2)
     ; CHECK-NEXT: G_STORE [[LSHR2]](s32), [[PTR_ADD2]](p0) :: (store (s8) into unknown-address + 3)
     ; CHECK-NEXT: PseudoRET
@@ -369,8 +368,8 @@ body:             |
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x11
     ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C]](s32)
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
     ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s64)
     ; CHECK-NEXT: G_STORE [[COPY2]](s32), [[COPY1]](p0) :: (store (s16))
@@ -419,17 +418,15 @@ body:             |
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s64)
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC]], [[C2]](s64)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC]], [[C2]](s32)
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
     ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C3]](s64)
     ; CHECK-NEXT: G_STORE [[TRUNC]](s32), [[COPY1]](p0) :: (store (s16))
     ; CHECK-NEXT: G_STORE [[LSHR1]](s32), [[PTR_ADD1]](p0) :: (store (s16) into unknown-address + 2)
     ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[LSHR]](s64)
-    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC1]], [[C4]](s64)
-    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C5]](s64)
+    ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC1]], [[C2]](s32)
+    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C3]](s64)
     ; CHECK-NEXT: G_STORE [[TRUNC1]](s32), [[PTR_ADD]](p0) :: (store (s16) into unknown-address + 4)
     ; CHECK-NEXT: G_STORE [[LSHR2]](s32), [[PTR_ADD2]](p0) :: (store (s16) into unknown-address + 6)
     ; CHECK-NEXT: PseudoRET
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-umax-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-umax-rv32.mir
index bc3226b010617..fdb890a112e54 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-umax-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-umax-rv32.mir
@@ -12,12 +12,10 @@ body:             |
     ; RV32I-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; RV32I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; RV32I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; RV32I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; RV32I-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ugt), [[AND]](s32), [[AND1]]
     ; RV32I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[COPY]], [[COPY1]]
-    ; RV32I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SELECT]], [[C2]]
+    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SELECT]], [[C]]
     ; RV32I-NEXT: $x10 = COPY [[AND2]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10
     ;
@@ -26,8 +24,7 @@ body:             |
     ; RV32ZBB-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; RV32ZBB-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; RV32ZBB-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; RV32ZBB-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV32ZBB-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; RV32ZBB-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; RV32ZBB-NEXT: [[UMAX:%[0-9]+]]:_(s32) = G_UMAX [[AND]], [[AND1]]
     ; RV32ZBB-NEXT: $x10 = COPY [[UMAX]](s32)
     ; RV32ZBB-NEXT: PseudoRET implicit $x10
@@ -50,12 +47,10 @@ body:             |
     ; RV32I-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; RV32I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; RV32I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; RV32I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; RV32I-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ugt), [[AND]](s32), [[AND1]]
     ; RV32I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[COPY]], [[COPY1]]
-    ; RV32I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SELECT]], [[C2]]
+    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SELECT]], [[C]]
     ; RV32I-NEXT: $x10 = COPY [[AND2]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10
     ;
@@ -64,8 +59,7 @@ body:             |
     ; RV32ZBB-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; RV32ZBB-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; RV32ZBB-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; RV32ZBB-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32ZBB-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; RV32ZBB-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; RV32ZBB-NEXT: [[UMAX:%[0-9]+]]:_(s32) = G_UMAX [[AND]], [[AND1]]
     ; RV32ZBB-NEXT: $x10 = COPY [[UMAX]](s32)
     ; RV32ZBB-NEXT: PseudoRET implicit $x10
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-umax-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-umax-rv64.mir
index 1953007690741..17f99ecc1e98a 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-umax-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-umax-rv64.mir
@@ -12,15 +12,13 @@ body:             |
     ; RV64I-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; RV64I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; RV64I-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ugt), [[AND]](s64), [[AND1]]
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
     ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
     ; RV64I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s64), [[TRUNC]], [[TRUNC1]]
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SELECT]](s32)
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C2]]
+    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C]]
     ; RV64I-NEXT: $x10 = COPY [[AND2]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
     ;
@@ -29,8 +27,7 @@ body:             |
     ; RV64ZBB-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; RV64ZBB-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; RV64ZBB-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; RV64ZBB-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; RV64ZBB-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; RV64ZBB-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; RV64ZBB-NEXT: [[UMAX:%[0-9]+]]:_(s64) = G_UMAX [[AND]], [[AND1]]
     ; RV64ZBB-NEXT: $x10 = COPY [[UMAX]](s64)
     ; RV64ZBB-NEXT: PseudoRET implicit $x10
@@ -53,15 +50,13 @@ body:             |
     ; RV64I-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; RV64I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
     ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; RV64I-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ugt), [[AND]](s64), [[AND1]]
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
     ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
     ; RV64I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s64), [[TRUNC]], [[TRUNC1]]
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SELECT]](s32)
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C2]]
+    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C]]
     ; RV64I-NEXT: $x10 = COPY [[AND2]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
     ;
@@ -70,8 +65,7 @@ body:             |
     ; RV64ZBB-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; RV64ZBB-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
     ; RV64ZBB-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; RV64ZBB-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; RV64ZBB-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; RV64ZBB-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; RV64ZBB-NEXT: [[UMAX:%[0-9]+]]:_(s64) = G_UMAX [[AND]], [[AND1]]
     ; RV64ZBB-NEXT: $x10 = COPY [[UMAX]](s64)
     ; RV64ZBB-NEXT: PseudoRET implicit $x10
@@ -96,8 +90,7 @@ body:             |
     ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
     ; RV64I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
     ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; RV64I-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ugt), [[AND]](s64), [[AND1]]
     ; RV64I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s64), [[TRUNC]], [[TRUNC1]]
     ; RV64I-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[SELECT]](s32)
@@ -109,8 +102,7 @@ body:             |
     ; RV64ZBB-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; RV64ZBB-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
     ; RV64ZBB-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; RV64ZBB-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; RV64ZBB-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; RV64ZBB-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; RV64ZBB-NEXT: [[UMAX:%[0-9]+]]:_(s64) = G_UMAX [[AND]], [[AND1]]
     ; RV64ZBB-NEXT: $x10 = COPY [[UMAX]](s64)
     ; RV64ZBB-NEXT: PseudoRET implicit $x10
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-umin-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-umin-rv32.mir
index fe090f30f417b..0c5830f5f1cd8 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-umin-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-umin-rv32.mir
@@ -12,12 +12,10 @@ body:             |
     ; RV32I-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; RV32I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; RV32I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; RV32I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; RV32I-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[AND1]]
     ; RV32I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[COPY]], [[COPY1]]
-    ; RV32I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SELECT]], [[C2]]
+    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SELECT]], [[C]]
     ; RV32I-NEXT: $x10 = COPY [[AND2]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10
     ;
@@ -26,8 +24,7 @@ body:             |
     ; RV32ZBB-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; RV32ZBB-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
     ; RV32ZBB-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; RV32ZBB-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; RV32ZBB-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; RV32ZBB-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; RV32ZBB-NEXT: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[AND]], [[AND1]]
     ; RV32ZBB-NEXT: $x10 = COPY [[UMIN]](s32)
     ; RV32ZBB-NEXT: PseudoRET implicit $x10
@@ -50,12 +47,10 @@ body:             |
     ; RV32I-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; RV32I-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; RV32I-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; RV32I-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; RV32I-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; RV32I-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[AND1]]
     ; RV32I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[COPY]], [[COPY1]]
-    ; RV32I-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SELECT]], [[C2]]
+    ; RV32I-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SELECT]], [[C]]
     ; RV32I-NEXT: $x10 = COPY [[AND2]](s32)
     ; RV32I-NEXT: PseudoRET implicit $x10
     ;
@@ -64,8 +59,7 @@ body:             |
     ; RV32ZBB-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
     ; RV32ZBB-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; RV32ZBB-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
-    ; RV32ZBB-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; RV32ZBB-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; RV32ZBB-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; RV32ZBB-NEXT: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[AND]], [[AND1]]
     ; RV32ZBB-NEXT: $x10 = COPY [[UMIN]](s32)
     ; RV32ZBB-NEXT: PseudoRET implicit $x10
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-umin-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-umin-rv64.mir
index c3cc5287194cf..33fac8ec13d64 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-umin-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-umin-rv64.mir
@@ -12,15 +12,13 @@ body:             |
     ; RV64I-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; RV64I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; RV64I-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
     ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
     ; RV64I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s64), [[TRUNC]], [[TRUNC1]]
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SELECT]](s32)
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C2]]
+    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C]]
     ; RV64I-NEXT: $x10 = COPY [[AND2]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
     ;
@@ -29,8 +27,7 @@ body:             |
     ; RV64ZBB-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; RV64ZBB-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
     ; RV64ZBB-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; RV64ZBB-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
-    ; RV64ZBB-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; RV64ZBB-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; RV64ZBB-NEXT: [[UMIN:%[0-9]+]]:_(s64) = G_UMIN [[AND]], [[AND1]]
     ; RV64ZBB-NEXT: $x10 = COPY [[UMIN]](s64)
     ; RV64ZBB-NEXT: PseudoRET implicit $x10
@@ -53,15 +50,13 @@ body:             |
     ; RV64I-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; RV64I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
     ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; RV64I-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
     ; RV64I-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
     ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
     ; RV64I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s64), [[TRUNC]], [[TRUNC1]]
     ; RV64I-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[SELECT]](s32)
-    ; RV64I-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C2]]
+    ; RV64I-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C]]
     ; RV64I-NEXT: $x10 = COPY [[AND2]](s64)
     ; RV64I-NEXT: PseudoRET implicit $x10
     ;
@@ -70,8 +65,7 @@ body:             |
     ; RV64ZBB-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; RV64ZBB-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
     ; RV64ZBB-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; RV64ZBB-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
-    ; RV64ZBB-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; RV64ZBB-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; RV64ZBB-NEXT: [[UMIN:%[0-9]+]]:_(s64) = G_UMIN [[AND]], [[AND1]]
     ; RV64ZBB-NEXT: $x10 = COPY [[UMIN]](s64)
     ; RV64ZBB-NEXT: PseudoRET implicit $x10
@@ -96,8 +90,7 @@ body:             |
     ; RV64I-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
     ; RV64I-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
     ; RV64I-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; RV64I-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; RV64I-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; RV64I-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
     ; RV64I-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s64), [[TRUNC]], [[TRUNC1]]
     ; RV64I-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[SELECT]](s32)
@@ -109,8 +102,7 @@ body:             |
     ; RV64ZBB-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
     ; RV64ZBB-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
     ; RV64ZBB-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
-    ; RV64ZBB-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
-    ; RV64ZBB-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; RV64ZBB-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
     ; RV64ZBB-NEXT: [[UMIN:%[0-9]+]]:_(s64) = G_UMIN [[AND]], [[AND1]]
     ; RV64ZBB-NEXT: $x10 = COPY [[UMIN]](s64)
     ; RV64ZBB-NEXT: PseudoRET implicit $x10
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-anyext.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-anyext.mir
index 3a2d40fe19313..c74a999354b60 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-anyext.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-anyext.mir
@@ -126,12 +126,10 @@ body:             |
     ; RV32-NEXT: {{  $}}
     ; RV32-NEXT: [[COPY:%[0-9]+]]:_(<vscale x 1 x s1>) = COPY $v0
     ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C1]](s32)
+    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 1 x s64>) = G_SPLAT_VECTOR [[MV]](s64)
-    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C2]](s32), [[C3]](s32)
+    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C1]](s32), [[C]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 1 x s64>) = G_SPLAT_VECTOR [[MV1]](s64)
     ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 1 x s64>) = G_SELECT [[COPY]](<vscale x 1 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
     ; RV32-NEXT: $v8 = COPY [[SELECT]](<vscale x 1 x s64>)
@@ -276,12 +274,10 @@ body:             |
     ; RV32-NEXT: {{  $}}
     ; RV32-NEXT: [[COPY:%[0-9]+]]:_(<vscale x 2 x s1>) = COPY $v0
     ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C1]](s32)
+    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 2 x s64>) = G_SPLAT_VECTOR [[MV]](s64)
-    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C2]](s32), [[C3]](s32)
+    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C1]](s32), [[C]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 2 x s64>) = G_SPLAT_VECTOR [[MV1]](s64)
     ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 2 x s64>) = G_SELECT [[COPY]](<vscale x 2 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
     ; RV32-NEXT: $v8m2 = COPY [[SELECT]](<vscale x 2 x s64>)
@@ -426,12 +422,10 @@ body:             |
     ; RV32-NEXT: {{  $}}
     ; RV32-NEXT: [[COPY:%[0-9]+]]:_(<vscale x 4 x s1>) = COPY $v0
     ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C1]](s32)
+    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 4 x s64>) = G_SPLAT_VECTOR [[MV]](s64)
-    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C2]](s32), [[C3]](s32)
+    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C1]](s32), [[C]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 4 x s64>) = G_SPLAT_VECTOR [[MV1]](s64)
     ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 4 x s64>) = G_SELECT [[COPY]](<vscale x 4 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
     ; RV32-NEXT: $v8m4 = COPY [[SELECT]](<vscale x 4 x s64>)
@@ -576,12 +570,10 @@ body:             |
     ; RV32-NEXT: {{  $}}
     ; RV32-NEXT: [[COPY:%[0-9]+]]:_(<vscale x 8 x s1>) = COPY $v0
     ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C1]](s32)
+    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 8 x s64>) = G_SPLAT_VECTOR [[MV]](s64)
-    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C2]](s32), [[C3]](s32)
+    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C1]](s32), [[C]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 8 x s64>) = G_SPLAT_VECTOR [[MV1]](s64)
     ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 8 x s64>) = G_SELECT [[COPY]](<vscale x 8 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
     ; RV32-NEXT: $v8m8 = COPY [[SELECT]](<vscale x 8 x s64>)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-extract-subvector.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-extract-subvector.mir
new file mode 100644
index 0000000000000..ca03482eee0d1
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-extract-subvector.mir
@@ -0,0 +1,493 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+v -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,RV32
+# RUN: llc -mtriple=riscv64 -mattr=+v -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,RV64
+
+# Special handling for i1-element vectors with non-zero index
+---
+name:            extract_subvector_nxv2i1_nxv4i1
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32-LABEL: name: extract_subvector_nxv2i1_nxv4i1
+    ; RV32: [[DEF:%[0-9]+]]:_(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SPLAT_VECTOR [[C]](s32)
+    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SPLAT_VECTOR [[C1]](s32)
+    ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SELECT [[DEF]](<vscale x 4 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
+    ; RV32-NEXT: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; RV32-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C2]](s32)
+    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; RV32-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL [[C3]](s32)
+    ; RV32-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 4 x s8>) = G_IMPLICIT_DEF
+    ; RV32-NEXT: [[VSLIDEDOWN_VL:%[0-9]+]]:_(<vscale x 4 x s8>) = G_VSLIDEDOWN_VL [[DEF1]], [[SELECT]], [[LSHR]](s32), [[VMSET_VL]](<vscale x 2 x s1>), [[C3]], 3
+    ; RV32-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 2 x s8>) = G_EXTRACT_SUBVECTOR [[VSLIDEDOWN_VL]](<vscale x 4 x s8>), 0
+    ; RV32-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV32-NEXT: [[SPLAT_VECTOR2:%[0-9]+]]:_(<vscale x 2 x s8>) = G_SPLAT_VECTOR [[C4]](s32)
+    ; RV32-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 2 x s1>) = G_ICMP intpred(ne), [[EXTRACT_SUBVECTOR]](<vscale x 2 x s8>), [[SPLAT_VECTOR2]]
+    ; RV32-NEXT: $v8 = COPY [[ICMP]](<vscale x 2 x s1>)
+    ; RV32-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64-LABEL: name: extract_subvector_nxv2i1_nxv4i1
+    ; RV64: [[DEF:%[0-9]+]]:_(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    ; RV64-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV64-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[C]](s32)
+    ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; RV64-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV64-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C1]](s32)
+    ; RV64-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SPLAT_VECTOR [[ANYEXT1]](s64)
+    ; RV64-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SELECT [[DEF]](<vscale x 4 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
+    ; RV64-NEXT: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; RV64-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; RV64-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[READ_VLENB]], [[C2]](s64)
+    ; RV64-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; RV64-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL [[C3]](s64)
+    ; RV64-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 4 x s8>) = G_IMPLICIT_DEF
+    ; RV64-NEXT: [[VSLIDEDOWN_VL:%[0-9]+]]:_(<vscale x 4 x s8>) = G_VSLIDEDOWN_VL [[DEF1]], [[SELECT]], [[LSHR]](s64), [[VMSET_VL]](<vscale x 2 x s1>), [[C3]], 3
+    ; RV64-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 2 x s8>) = G_EXTRACT_SUBVECTOR [[VSLIDEDOWN_VL]](<vscale x 4 x s8>), 0
+    ; RV64-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV64-NEXT: [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[C4]](s32)
+    ; RV64-NEXT: [[SPLAT_VECTOR2:%[0-9]+]]:_(<vscale x 2 x s8>) = G_SPLAT_VECTOR [[ANYEXT2]](s64)
+    ; RV64-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 2 x s1>) = G_ICMP intpred(ne), [[EXTRACT_SUBVECTOR]](<vscale x 2 x s8>), [[SPLAT_VECTOR2]]
+    ; RV64-NEXT: $v8 = COPY [[ICMP]](<vscale x 2 x s1>)
+    ; RV64-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    %1:_(<vscale x 2 x s1>) = G_EXTRACT_SUBVECTOR %0(<vscale x 4 x s1>), 2
+    $v8 = COPY %1(<vscale x 2 x s1>)
+    PseudoRET implicit $v8
+...
+---
+name:            extract_subvector_nxv2i1_nxv8i1
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32-LABEL: name: extract_subvector_nxv2i1_nxv8i1
+    ; RV32: [[DEF:%[0-9]+]]:_(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SPLAT_VECTOR [[C]](s32)
+    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SPLAT_VECTOR [[C1]](s32)
+    ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SELECT [[DEF]](<vscale x 8 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
+    ; RV32-NEXT: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; RV32-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C2]](s32)
+    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; RV32-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL [[C3]](s32)
+    ; RV32-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 8 x s8>) = G_IMPLICIT_DEF
+    ; RV32-NEXT: [[VSLIDEDOWN_VL:%[0-9]+]]:_(<vscale x 8 x s8>) = G_VSLIDEDOWN_VL [[DEF1]], [[SELECT]], [[LSHR]](s32), [[VMSET_VL]](<vscale x 2 x s1>), [[C3]], 3
+    ; RV32-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 2 x s8>) = G_EXTRACT_SUBVECTOR [[VSLIDEDOWN_VL]](<vscale x 8 x s8>), 0
+    ; RV32-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV32-NEXT: [[SPLAT_VECTOR2:%[0-9]+]]:_(<vscale x 2 x s8>) = G_SPLAT_VECTOR [[C4]](s32)
+    ; RV32-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 2 x s1>) = G_ICMP intpred(ne), [[EXTRACT_SUBVECTOR]](<vscale x 2 x s8>), [[SPLAT_VECTOR2]]
+    ; RV32-NEXT: $v8 = COPY [[ICMP]](<vscale x 2 x s1>)
+    ; RV32-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64-LABEL: name: extract_subvector_nxv2i1_nxv8i1
+    ; RV64: [[DEF:%[0-9]+]]:_(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    ; RV64-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV64-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[C]](s32)
+    ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; RV64-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV64-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C1]](s32)
+    ; RV64-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SPLAT_VECTOR [[ANYEXT1]](s64)
+    ; RV64-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SELECT [[DEF]](<vscale x 8 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
+    ; RV64-NEXT: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; RV64-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; RV64-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[READ_VLENB]], [[C2]](s64)
+    ; RV64-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; RV64-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL [[C3]](s64)
+    ; RV64-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 8 x s8>) = G_IMPLICIT_DEF
+    ; RV64-NEXT: [[VSLIDEDOWN_VL:%[0-9]+]]:_(<vscale x 8 x s8>) = G_VSLIDEDOWN_VL [[DEF1]], [[SELECT]], [[LSHR]](s64), [[VMSET_VL]](<vscale x 2 x s1>), [[C3]], 3
+    ; RV64-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 2 x s8>) = G_EXTRACT_SUBVECTOR [[VSLIDEDOWN_VL]](<vscale x 8 x s8>), 0
+    ; RV64-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV64-NEXT: [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[C4]](s32)
+    ; RV64-NEXT: [[SPLAT_VECTOR2:%[0-9]+]]:_(<vscale x 2 x s8>) = G_SPLAT_VECTOR [[ANYEXT2]](s64)
+    ; RV64-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 2 x s1>) = G_ICMP intpred(ne), [[EXTRACT_SUBVECTOR]](<vscale x 2 x s8>), [[SPLAT_VECTOR2]]
+    ; RV64-NEXT: $v8 = COPY [[ICMP]](<vscale x 2 x s1>)
+    ; RV64-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    %1:_(<vscale x 2 x s1>) = G_EXTRACT_SUBVECTOR %0(<vscale x 8 x s1>), 2
+    $v8 = COPY %1(<vscale x 2 x s1>)
+    PseudoRET implicit $v8
+...
+---
+name:            extract_subvector_nxv4i1_nxv64i1
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32-LABEL: name: extract_subvector_nxv4i1_nxv64i1
+    ; RV32: [[DEF:%[0-9]+]]:_(<vscale x 64 x s1>) = G_IMPLICIT_DEF
+    ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SPLAT_VECTOR [[C]](s32)
+    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SPLAT_VECTOR [[C1]](s32)
+    ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SELECT [[DEF]](<vscale x 64 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
+    ; RV32-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 4 x s8>) = G_EXTRACT_SUBVECTOR [[SELECT]](<vscale x 64 x s8>), 16
+    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV32-NEXT: [[SPLAT_VECTOR2:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SPLAT_VECTOR [[C2]](s32)
+    ; RV32-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 4 x s1>) = G_ICMP intpred(ne), [[EXTRACT_SUBVECTOR]](<vscale x 4 x s8>), [[SPLAT_VECTOR2]]
+    ; RV32-NEXT: $v8 = COPY [[ICMP]](<vscale x 4 x s1>)
+    ; RV32-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64-LABEL: name: extract_subvector_nxv4i1_nxv64i1
+    ; RV64: [[DEF:%[0-9]+]]:_(<vscale x 64 x s1>) = G_IMPLICIT_DEF
+    ; RV64-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV64-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[C]](s32)
+    ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; RV64-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV64-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C1]](s32)
+    ; RV64-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SPLAT_VECTOR [[ANYEXT1]](s64)
+    ; RV64-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SELECT [[DEF]](<vscale x 64 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
+    ; RV64-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 4 x s8>) = G_EXTRACT_SUBVECTOR [[SELECT]](<vscale x 64 x s8>), 16
+    ; RV64-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; RV64-NEXT: [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[C2]](s32)
+    ; RV64-NEXT: [[SPLAT_VECTOR2:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SPLAT_VECTOR [[ANYEXT2]](s64)
+    ; RV64-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 4 x s1>) = G_ICMP intpred(ne), [[EXTRACT_SUBVECTOR]](<vscale x 4 x s8>), [[SPLAT_VECTOR2]]
+    ; RV64-NEXT: $v8 = COPY [[ICMP]](<vscale x 4 x s1>)
+    ; RV64-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 64 x s1>) = G_IMPLICIT_DEF
+    %1:_(<vscale x 4 x s1>) = G_EXTRACT_SUBVECTOR %0(<vscale x 64 x s1>), 16
+    $v8 = COPY %1(<vscale x 4 x s1>)
+    PseudoRET implicit $v8
+...
+---
+name:            extract_subvector_nxv32i1_nxv64i1
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32-LABEL: name: extract_subvector_nxv32i1_nxv64i1
+    ; RV32: [[DEF:%[0-9]+]]:_(<vscale x 64 x s1>) = G_IMPLICIT_DEF
+    ; RV32-NEXT: [[BITCAST:%[0-9]+]]:_(<vscale x 8 x s8>) = G_BITCAST [[DEF]](<vscale x 64 x s1>)
+    ; RV32-NEXT: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV32-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; RV32-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 4 x s1>) = G_VMSET_VL [[C1]](s32)
+    ; RV32-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 8 x s8>) = G_IMPLICIT_DEF
+    ; RV32-NEXT: [[VSLIDEDOWN_VL:%[0-9]+]]:_(<vscale x 8 x s8>) = G_VSLIDEDOWN_VL [[DEF1]], [[BITCAST]], [[LSHR]](s32), [[VMSET_VL]](<vscale x 4 x s1>), [[C1]], 3
+    ; RV32-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 4 x s8>) = G_EXTRACT_SUBVECTOR [[VSLIDEDOWN_VL]](<vscale x 8 x s8>), 0
+    ; RV32-NEXT: [[BITCAST1:%[0-9]+]]:_(<vscale x 32 x s1>) = G_BITCAST [[EXTRACT_SUBVECTOR]](<vscale x 4 x s8>)
+    ; RV32-NEXT: $v8 = COPY [[BITCAST1]](<vscale x 32 x s1>)
+    ; RV32-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64-LABEL: name: extract_subvector_nxv32i1_nxv64i1
+    ; RV64: [[DEF:%[0-9]+]]:_(<vscale x 64 x s1>) = G_IMPLICIT_DEF
+    ; RV64-NEXT: [[BITCAST:%[0-9]+]]:_(<vscale x 8 x s8>) = G_BITCAST [[DEF]](<vscale x 64 x s1>)
+    ; RV64-NEXT: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; RV64-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
+    ; RV64-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; RV64-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 4 x s1>) = G_VMSET_VL [[C1]](s64)
+    ; RV64-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 8 x s8>) = G_IMPLICIT_DEF
+    ; RV64-NEXT: [[VSLIDEDOWN_VL:%[0-9]+]]:_(<vscale x 8 x s8>) = G_VSLIDEDOWN_VL [[DEF1]], [[BITCAST]], [[LSHR]](s64), [[VMSET_VL]](<vscale x 4 x s1>), [[C1]], 3
+    ; RV64-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 4 x s8>) = G_EXTRACT_SUBVECTOR [[VSLIDEDOWN_VL]](<vscale x 8 x s8>), 0
+    ; RV64-NEXT: [[BITCAST1:%[0-9]+]]:_(<vscale x 32 x s1>) = G_BITCAST [[EXTRACT_SUBVECTOR]](<vscale x 4 x s8>)
+    ; RV64-NEXT: $v8 = COPY [[BITCAST1]](<vscale x 32 x s1>)
+    ; RV64-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 64 x s1>) = G_IMPLICIT_DEF
+    %1:_(<vscale x 32 x s1>) = G_EXTRACT_SUBVECTOR %0(<vscale x 64 x s1>), 32
+    $v8 = COPY %1(<vscale x 32 x s1>)
+    PseudoRET implicit $v8
+...
+
+# i1-element vectors with zero index
+---
+name:            extract_subvector_nxv1i1_nxv4i1_zero
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: extract_subvector_nxv1i1_nxv4i1_zero
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 1 x s1>) = G_EXTRACT_SUBVECTOR [[DEF]](<vscale x 4 x s1>), 0
+    ; CHECK-NEXT: $v8 = COPY [[EXTRACT_SUBVECTOR]](<vscale x 1 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    %1:_(<vscale x 1 x s1>) = G_EXTRACT_SUBVECTOR %0(<vscale x 4 x s1>), 0
+    $v8 = COPY %1(<vscale x 1 x s1>)
+    PseudoRET implicit $v8
+...
+---
+name:            extract_subvector_nxv4i1_nxv8i1_zero
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: extract_subvector_nxv4i1_nxv8i1_zero
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 4 x s1>) = G_EXTRACT_SUBVECTOR [[DEF]](<vscale x 8 x s1>), 0
+    ; CHECK-NEXT: $v8 = COPY [[EXTRACT_SUBVECTOR]](<vscale x 4 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    %1:_(<vscale x 4 x s1>) = G_EXTRACT_SUBVECTOR %0(<vscale x 8 x s1>), 0
+    $v8 = COPY %1(<vscale x 4 x s1>)
+    PseudoRET implicit $v8
+...
+---
+name:            extract_subvector_nxv32i1_nxv64i1_zero
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: extract_subvector_nxv32i1_nxv64i1_zero
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 64 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<vscale x 8 x s8>) = G_BITCAST [[DEF]](<vscale x 64 x s1>)
+    ; CHECK-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 4 x s8>) = G_EXTRACT_SUBVECTOR [[BITCAST]](<vscale x 8 x s8>), 0
+    ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<vscale x 32 x s1>) = G_BITCAST [[EXTRACT_SUBVECTOR]](<vscale x 4 x s8>)
+    ; CHECK-NEXT: $v8 = COPY [[BITCAST1]](<vscale x 32 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 64 x s1>) = G_IMPLICIT_DEF
+    %1:_(<vscale x 32 x s1>) = G_EXTRACT_SUBVECTOR %0(<vscale x 64 x s1>), 0
+    $v8 = COPY %1(<vscale x 32 x s1>)
+    PseudoRET implicit $v8
+...
+
+# Extract with zero index
+---
+name:            extract_subvector_nxv1i8_nxv2i8_zero
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: extract_subvector_nxv1i8_nxv2i8_zero
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 2 x s8>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 1 x s8>) = G_EXTRACT_SUBVECTOR [[DEF]](<vscale x 2 x s8>), 0
+    ; CHECK-NEXT: $v8 = COPY [[EXTRACT_SUBVECTOR]](<vscale x 1 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 2 x s8>) = G_IMPLICIT_DEF
+    %1:_(<vscale x 1 x s8>) = G_EXTRACT_SUBVECTOR %0(<vscale x 2 x s8>), 0
+    $v8 = COPY %1(<vscale x 1 x s8>)
+    PseudoRET implicit $v8
+...
+---
+name:            extract_subvector_nxv2i16_nxv4i16_zero
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: extract_subvector_nxv2i16_nxv4i16_zero
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 2 x s16>) = G_EXTRACT_SUBVECTOR [[DEF]](<vscale x 4 x s16>), 0
+    ; CHECK-NEXT: $v8 = COPY [[EXTRACT_SUBVECTOR]](<vscale x 2 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 4 x s16>) = G_IMPLICIT_DEF
+    %1:_(<vscale x 2 x s16>) = G_EXTRACT_SUBVECTOR %0(<vscale x 4 x s16>), 0
+    $v8 = COPY %1(<vscale x 2 x s16>)
+    PseudoRET implicit $v8
+...
+---
+name:            extract_subvector_nxv4i32_nxv8i32_zero
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: extract_subvector_nxv4i32_nxv8i32_zero
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 8 x s32>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 4 x s32>) = G_EXTRACT_SUBVECTOR [[DEF]](<vscale x 8 x s32>), 0
+    ; CHECK-NEXT: $v8 = COPY [[EXTRACT_SUBVECTOR]](<vscale x 4 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 8 x s32>) = G_IMPLICIT_DEF
+    %1:_(<vscale x 4 x s32>) = G_EXTRACT_SUBVECTOR %0(<vscale x 8 x s32>), 0
+    $v8 = COPY %1(<vscale x 4 x s32>)
+    PseudoRET implicit $v8
+...
+---
+name:            extract_subvector_nxv4i8_nxv8i64_zero
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: extract_subvector_nxv4i8_nxv8i64_zero
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 8 x s64>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 4 x s64>) = G_EXTRACT_SUBVECTOR [[DEF]](<vscale x 8 x s64>), 0
+    ; CHECK-NEXT: $v8 = COPY [[EXTRACT_SUBVECTOR]](<vscale x 4 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 8 x s64>) = G_IMPLICIT_DEF
+    %1:_(<vscale x 4 x s64>) = G_EXTRACT_SUBVECTOR %0(<vscale x 8 x s64>), 0
+    $v8 = COPY %1(<vscale x 4 x s64>)
+    PseudoRET implicit $v8
+...
+
+# Extract with non-zero index
+---
+name:            extract_subvector_nxv1i8_nxv2i8
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32-LABEL: name: extract_subvector_nxv1i8_nxv2i8
+    ; RV32: [[DEF:%[0-9]+]]:_(<vscale x 2 x s8>) = G_IMPLICIT_DEF
+    ; RV32-NEXT: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; RV32-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; RV32-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL [[C1]](s32)
+    ; RV32-NEXT: [[VSLIDEDOWN_VL:%[0-9]+]]:_(<vscale x 2 x s8>) = G_VSLIDEDOWN_VL [[DEF]], [[DEF]], [[LSHR]](s32), [[VMSET_VL]](<vscale x 1 x s1>), [[C1]], 3
+    ; RV32-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 1 x s8>) = G_EXTRACT_SUBVECTOR [[VSLIDEDOWN_VL]](<vscale x 2 x s8>), 0
+    ; RV32-NEXT: $v8 = COPY [[EXTRACT_SUBVECTOR]](<vscale x 1 x s8>)
+    ; RV32-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64-LABEL: name: extract_subvector_nxv1i8_nxv2i8
+    ; RV64: [[DEF:%[0-9]+]]:_(<vscale x 2 x s8>) = G_IMPLICIT_DEF
+    ; RV64-NEXT: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
+    ; RV64-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
+    ; RV64-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; RV64-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL [[C1]](s64)
+    ; RV64-NEXT: [[VSLIDEDOWN_VL:%[0-9]+]]:_(<vscale x 2 x s8>) = G_VSLIDEDOWN_VL [[DEF]], [[DEF]], [[LSHR]](s64), [[VMSET_VL]](<vscale x 1 x s1>), [[C1]], 3
+    ; RV64-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 1 x s8>) = G_EXTRACT_SUBVECTOR [[VSLIDEDOWN_VL]](<vscale x 2 x s8>), 0
+    ; RV64-NEXT: $v8 = COPY [[EXTRACT_SUBVECTOR]](<vscale x 1 x s8>)
+    ; RV64-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 2 x s8>) = G_IMPLICIT_DEF
+    %1:_(<vscale x 1 x s8>) = G_EXTRACT_SUBVECTOR %0(<vscale x 2 x s8>), 1
+    $v8 = COPY %1(<vscale x 1 x s8>)
+    PseudoRET implicit $v8
+...
+---
+name:            extract_subvector_nxv1i16_nxv4i16
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32-LABEL: name: extract_subvector_nxv1i16_nxv4i16
+    ; RV32: [[DEF:%[0-9]+]]:_(<vscale x 4 x s16>) = G_IMPLICIT_DEF
+    ; RV32-NEXT: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; RV32-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; RV32-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL [[C1]](s32)
+    ; RV32-NEXT: [[VSLIDEDOWN_VL:%[0-9]+]]:_(<vscale x 4 x s16>) = G_VSLIDEDOWN_VL [[DEF]], [[DEF]], [[LSHR]](s32), [[VMSET_VL]](<vscale x 1 x s1>), [[C1]], 3
+    ; RV32-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 1 x s16>) = G_EXTRACT_SUBVECTOR [[VSLIDEDOWN_VL]](<vscale x 4 x s16>), 0
+    ; RV32-NEXT: $v8 = COPY [[EXTRACT_SUBVECTOR]](<vscale x 1 x s16>)
+    ; RV32-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64-LABEL: name: extract_subvector_nxv1i16_nxv4i16
+    ; RV64: [[DEF:%[0-9]+]]:_(<vscale x 4 x s16>) = G_IMPLICIT_DEF
+    ; RV64-NEXT: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; RV64-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
+    ; RV64-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; RV64-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL [[C1]](s64)
+    ; RV64-NEXT: [[VSLIDEDOWN_VL:%[0-9]+]]:_(<vscale x 4 x s16>) = G_VSLIDEDOWN_VL [[DEF]], [[DEF]], [[LSHR]](s64), [[VMSET_VL]](<vscale x 1 x s1>), [[C1]], 3
+    ; RV64-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 1 x s16>) = G_EXTRACT_SUBVECTOR [[VSLIDEDOWN_VL]](<vscale x 4 x s16>), 0
+    ; RV64-NEXT: $v8 = COPY [[EXTRACT_SUBVECTOR]](<vscale x 1 x s16>)
+    ; RV64-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 4 x s16>) = G_IMPLICIT_DEF
+    %1:_(<vscale x 1 x s16>) = G_EXTRACT_SUBVECTOR %0(<vscale x 4 x s16>), 2
+    $v8 = COPY %1(<vscale x 1 x s16>)
+    PseudoRET implicit $v8
+...
+---
+name:            extract_subvector_nxv4i32_nxv8i32
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: extract_subvector_nxv4i32_nxv8i32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 8 x s32>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 4 x s32>) = G_EXTRACT_SUBVECTOR [[DEF]](<vscale x 8 x s32>), 4
+    ; CHECK-NEXT: $v8 = COPY [[EXTRACT_SUBVECTOR]](<vscale x 4 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 8 x s32>) = G_IMPLICIT_DEF
+    %1:_(<vscale x 4 x s32>) = G_EXTRACT_SUBVECTOR %0(<vscale x 8 x s32>), 4
+    $v8 = COPY %1(<vscale x 4 x s32>)
+    PseudoRET implicit $v8
+...
+---
+name:            extract_subvector_nxv2i64_nxv8i64
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: extract_subvector_nxv2i64_nxv8i64
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 8 x s64>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 2 x s64>) = G_EXTRACT_SUBVECTOR [[DEF]](<vscale x 8 x s64>), 2
+    ; CHECK-NEXT: $v8 = COPY [[EXTRACT_SUBVECTOR]](<vscale x 2 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 8 x s64>) = G_IMPLICIT_DEF
+    %1:_(<vscale x 2 x s64>) = G_EXTRACT_SUBVECTOR %0(<vscale x 8 x s64>), 2
+    $v8 = COPY %1(<vscale x 2 x s64>)
+    PseudoRET implicit $v8
+...
+---
+name:            extract_subvector_subregidx_zero
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32-LABEL: name: extract_subvector_subregidx_zero
+    ; RV32: [[DEF:%[0-9]+]]:_(<vscale x 16 x s32>) = G_IMPLICIT_DEF
+    ; RV32-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 2 x s32>) = G_EXTRACT_SUBVECTOR [[DEF]](<vscale x 16 x s32>), 0
+    ; RV32-NEXT: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; RV32-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; RV32-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL [[C1]](s32)
+    ; RV32-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 2 x s32>) = G_IMPLICIT_DEF
+    ; RV32-NEXT: [[VSLIDEDOWN_VL:%[0-9]+]]:_(<vscale x 2 x s32>) = G_VSLIDEDOWN_VL [[DEF1]], [[EXTRACT_SUBVECTOR]], [[LSHR]](s32), [[VMSET_VL]](<vscale x 1 x s1>), [[C1]], 3
+    ; RV32-NEXT: [[EXTRACT_SUBVECTOR1:%[0-9]+]]:_(<vscale x 1 x s32>) = G_EXTRACT_SUBVECTOR [[VSLIDEDOWN_VL]](<vscale x 2 x s32>), 0
+    ; RV32-NEXT: $v8 = COPY [[EXTRACT_SUBVECTOR1]](<vscale x 1 x s32>)
+    ; RV32-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64-LABEL: name: extract_subvector_subregidx_zero
+    ; RV64: [[DEF:%[0-9]+]]:_(<vscale x 16 x s32>) = G_IMPLICIT_DEF
+    ; RV64-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 2 x s32>) = G_EXTRACT_SUBVECTOR [[DEF]](<vscale x 16 x s32>), 0
+    ; RV64-NEXT: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
+    ; RV64-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
+    ; RV64-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; RV64-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL [[C1]](s64)
+    ; RV64-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 2 x s32>) = G_IMPLICIT_DEF
+    ; RV64-NEXT: [[VSLIDEDOWN_VL:%[0-9]+]]:_(<vscale x 2 x s32>) = G_VSLIDEDOWN_VL [[DEF1]], [[EXTRACT_SUBVECTOR]], [[LSHR]](s64), [[VMSET_VL]](<vscale x 1 x s1>), [[C1]], 3
+    ; RV64-NEXT: [[EXTRACT_SUBVECTOR1:%[0-9]+]]:_(<vscale x 1 x s32>) = G_EXTRACT_SUBVECTOR [[VSLIDEDOWN_VL]](<vscale x 2 x s32>), 0
+    ; RV64-NEXT: $v8 = COPY [[EXTRACT_SUBVECTOR1]](<vscale x 1 x s32>)
+    ; RV64-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 16 x s32>) = G_IMPLICIT_DEF
+    %1:_(<vscale x 1 x s32>) = G_EXTRACT_SUBVECTOR %0(<vscale x 16 x s32>), 1
+    $v8 = COPY %1(<vscale x 1 x s32>)
+    PseudoRET implicit $v8
+...
+---
+name:            extract_subvector_subregidx
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32-LABEL: name: extract_subvector_subregidx
+    ; RV32: [[DEF:%[0-9]+]]:_(<vscale x 16 x s32>) = G_IMPLICIT_DEF
+    ; RV32-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 2 x s32>) = G_EXTRACT_SUBVECTOR [[DEF]](<vscale x 16 x s32>), 2
+    ; RV32-NEXT: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; RV32-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; RV32-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL [[C1]](s32)
+    ; RV32-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 2 x s32>) = G_IMPLICIT_DEF
+    ; RV32-NEXT: [[VSLIDEDOWN_VL:%[0-9]+]]:_(<vscale x 2 x s32>) = G_VSLIDEDOWN_VL [[DEF1]], [[EXTRACT_SUBVECTOR]], [[LSHR]](s32), [[VMSET_VL]](<vscale x 1 x s1>), [[C1]], 3
+    ; RV32-NEXT: [[EXTRACT_SUBVECTOR1:%[0-9]+]]:_(<vscale x 1 x s32>) = G_EXTRACT_SUBVECTOR [[VSLIDEDOWN_VL]](<vscale x 2 x s32>), 0
+    ; RV32-NEXT: $v8 = COPY [[EXTRACT_SUBVECTOR1]](<vscale x 1 x s32>)
+    ; RV32-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64-LABEL: name: extract_subvector_subregidx
+    ; RV64: [[DEF:%[0-9]+]]:_(<vscale x 16 x s32>) = G_IMPLICIT_DEF
+    ; RV64-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<vscale x 2 x s32>) = G_EXTRACT_SUBVECTOR [[DEF]](<vscale x 16 x s32>), 2
+    ; RV64-NEXT: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
+    ; RV64-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
+    ; RV64-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; RV64-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL [[C1]](s64)
+    ; RV64-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 2 x s32>) = G_IMPLICIT_DEF
+    ; RV64-NEXT: [[VSLIDEDOWN_VL:%[0-9]+]]:_(<vscale x 2 x s32>) = G_VSLIDEDOWN_VL [[DEF1]], [[EXTRACT_SUBVECTOR]], [[LSHR]](s64), [[VMSET_VL]](<vscale x 1 x s1>), [[C1]], 3
+    ; RV64-NEXT: [[EXTRACT_SUBVECTOR1:%[0-9]+]]:_(<vscale x 1 x s32>) = G_EXTRACT_SUBVECTOR [[VSLIDEDOWN_VL]](<vscale x 2 x s32>), 0
+    ; RV64-NEXT: $v8 = COPY [[EXTRACT_SUBVECTOR1]](<vscale x 1 x s32>)
+    ; RV64-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 16 x s32>) = G_IMPLICIT_DEF
+    %1:_(<vscale x 1 x s32>) = G_EXTRACT_SUBVECTOR %0(<vscale x 16 x s32>), 3
+    $v8 = COPY %1(<vscale x 1 x s32>)
+    PseudoRET implicit $v8
+...
+
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-icmp.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-icmp.mir
index d1df9540daaaa..fd92f123e886e 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-icmp.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-icmp.mir
@@ -15,12 +15,7 @@ body:             |
     ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
     ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 1 x s8>) = G_SPLAT_VECTOR [[C1]](s32)
     ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 1 x s8>) = G_SELECT [[DEF]](<vscale x 1 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
-    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[SPLAT_VECTOR2:%[0-9]+]]:_(<vscale x 1 x s8>) = G_SPLAT_VECTOR [[C2]](s32)
-    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV32-NEXT: [[SPLAT_VECTOR3:%[0-9]+]]:_(<vscale x 1 x s8>) = G_SPLAT_VECTOR [[C3]](s32)
-    ; RV32-NEXT: [[SELECT1:%[0-9]+]]:_(<vscale x 1 x s8>) = G_SELECT [[DEF]](<vscale x 1 x s1>), [[SPLAT_VECTOR3]], [[SPLAT_VECTOR2]]
-    ; RV32-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 1 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 1 x s8>), [[SELECT1]]
+    ; RV32-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 1 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 1 x s8>), [[SELECT]]
     ; RV32-NEXT: $v8 = COPY [[ICMP]](<vscale x 1 x s1>)
     ; RV32-NEXT: PseudoRET implicit $v8
     ;
@@ -33,14 +28,7 @@ body:             |
     ; RV64-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C1]](s32)
     ; RV64-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 1 x s8>) = G_SPLAT_VECTOR [[ANYEXT1]](s64)
     ; RV64-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 1 x s8>) = G_SELECT [[DEF]](<vscale x 1 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
-    ; RV64-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV64-NEXT: [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[C2]](s32)
-    ; RV64-NEXT: [[SPLAT_VECTOR2:%[0-9]+]]:_(<vscale x 1 x s8>) = G_SPLAT_VECTOR [[ANYEXT2]](s64)
-    ; RV64-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV64-NEXT: [[ANYEXT3:%[0-9]+]]:_(s64) = G_ANYEXT [[C3]](s32)
-    ; RV64-NEXT: [[SPLAT_VECTOR3:%[0-9]+]]:_(<vscale x 1 x s8>) = G_SPLAT_VECTOR [[ANYEXT3]](s64)
-    ; RV64-NEXT: [[SELECT1:%[0-9]+]]:_(<vscale x 1 x s8>) = G_SELECT [[DEF]](<vscale x 1 x s1>), [[SPLAT_VECTOR3]], [[SPLAT_VECTOR2]]
-    ; RV64-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 1 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 1 x s8>), [[SELECT1]]
+    ; RV64-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 1 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 1 x s8>), [[SELECT]]
     ; RV64-NEXT: $v8 = COPY [[ICMP]](<vscale x 1 x s1>)
     ; RV64-NEXT: PseudoRET implicit $v8
     %0:_(<vscale x 1 x s1>) = G_IMPLICIT_DEF
@@ -61,12 +49,7 @@ body:             |
     ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
     ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 2 x s8>) = G_SPLAT_VECTOR [[C1]](s32)
     ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 2 x s8>) = G_SELECT [[DEF]](<vscale x 2 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
-    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[SPLAT_VECTOR2:%[0-9]+]]:_(<vscale x 2 x s8>) = G_SPLAT_VECTOR [[C2]](s32)
-    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV32-NEXT: [[SPLAT_VECTOR3:%[0-9]+]]:_(<vscale x 2 x s8>) = G_SPLAT_VECTOR [[C3]](s32)
-    ; RV32-NEXT: [[SELECT1:%[0-9]+]]:_(<vscale x 2 x s8>) = G_SELECT [[DEF]](<vscale x 2 x s1>), [[SPLAT_VECTOR3]], [[SPLAT_VECTOR2]]
-    ; RV32-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 2 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 2 x s8>), [[SELECT1]]
+    ; RV32-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 2 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 2 x s8>), [[SELECT]]
     ; RV32-NEXT: $v8 = COPY [[ICMP]](<vscale x 2 x s1>)
     ; RV32-NEXT: PseudoRET implicit $v8
     ;
@@ -79,14 +62,7 @@ body:             |
     ; RV64-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C1]](s32)
     ; RV64-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 2 x s8>) = G_SPLAT_VECTOR [[ANYEXT1]](s64)
     ; RV64-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 2 x s8>) = G_SELECT [[DEF]](<vscale x 2 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
-    ; RV64-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV64-NEXT: [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[C2]](s32)
-    ; RV64-NEXT: [[SPLAT_VECTOR2:%[0-9]+]]:_(<vscale x 2 x s8>) = G_SPLAT_VECTOR [[ANYEXT2]](s64)
-    ; RV64-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV64-NEXT: [[ANYEXT3:%[0-9]+]]:_(s64) = G_ANYEXT [[C3]](s32)
-    ; RV64-NEXT: [[SPLAT_VECTOR3:%[0-9]+]]:_(<vscale x 2 x s8>) = G_SPLAT_VECTOR [[ANYEXT3]](s64)
-    ; RV64-NEXT: [[SELECT1:%[0-9]+]]:_(<vscale x 2 x s8>) = G_SELECT [[DEF]](<vscale x 2 x s1>), [[SPLAT_VECTOR3]], [[SPLAT_VECTOR2]]
-    ; RV64-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 2 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 2 x s8>), [[SELECT1]]
+    ; RV64-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 2 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 2 x s8>), [[SELECT]]
     ; RV64-NEXT: $v8 = COPY [[ICMP]](<vscale x 2 x s1>)
     ; RV64-NEXT: PseudoRET implicit $v8
     %0:_(<vscale x 2 x s1>) = G_IMPLICIT_DEF
@@ -107,12 +83,7 @@ body:             |
     ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
     ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SPLAT_VECTOR [[C1]](s32)
     ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SELECT [[DEF]](<vscale x 4 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
-    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[SPLAT_VECTOR2:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SPLAT_VECTOR [[C2]](s32)
-    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV32-NEXT: [[SPLAT_VECTOR3:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SPLAT_VECTOR [[C3]](s32)
-    ; RV32-NEXT: [[SELECT1:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SELECT [[DEF]](<vscale x 4 x s1>), [[SPLAT_VECTOR3]], [[SPLAT_VECTOR2]]
-    ; RV32-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 4 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 4 x s8>), [[SELECT1]]
+    ; RV32-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 4 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 4 x s8>), [[SELECT]]
     ; RV32-NEXT: $v8 = COPY [[ICMP]](<vscale x 4 x s1>)
     ; RV32-NEXT: PseudoRET implicit $v8
     ;
@@ -125,14 +96,7 @@ body:             |
     ; RV64-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C1]](s32)
     ; RV64-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SPLAT_VECTOR [[ANYEXT1]](s64)
     ; RV64-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SELECT [[DEF]](<vscale x 4 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
-    ; RV64-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV64-NEXT: [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[C2]](s32)
-    ; RV64-NEXT: [[SPLAT_VECTOR2:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SPLAT_VECTOR [[ANYEXT2]](s64)
-    ; RV64-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV64-NEXT: [[ANYEXT3:%[0-9]+]]:_(s64) = G_ANYEXT [[C3]](s32)
-    ; RV64-NEXT: [[SPLAT_VECTOR3:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SPLAT_VECTOR [[ANYEXT3]](s64)
-    ; RV64-NEXT: [[SELECT1:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SELECT [[DEF]](<vscale x 4 x s1>), [[SPLAT_VECTOR3]], [[SPLAT_VECTOR2]]
-    ; RV64-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 4 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 4 x s8>), [[SELECT1]]
+    ; RV64-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 4 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 4 x s8>), [[SELECT]]
     ; RV64-NEXT: $v8 = COPY [[ICMP]](<vscale x 4 x s1>)
     ; RV64-NEXT: PseudoRET implicit $v8
     %0:_(<vscale x 4 x s1>) = G_IMPLICIT_DEF
@@ -153,12 +117,7 @@ body:             |
     ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
     ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SPLAT_VECTOR [[C1]](s32)
     ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SELECT [[DEF]](<vscale x 8 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
-    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[SPLAT_VECTOR2:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SPLAT_VECTOR [[C2]](s32)
-    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV32-NEXT: [[SPLAT_VECTOR3:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SPLAT_VECTOR [[C3]](s32)
-    ; RV32-NEXT: [[SELECT1:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SELECT [[DEF]](<vscale x 8 x s1>), [[SPLAT_VECTOR3]], [[SPLAT_VECTOR2]]
-    ; RV32-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 8 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 8 x s8>), [[SELECT1]]
+    ; RV32-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 8 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 8 x s8>), [[SELECT]]
     ; RV32-NEXT: $v8 = COPY [[ICMP]](<vscale x 8 x s1>)
     ; RV32-NEXT: PseudoRET implicit $v8
     ;
@@ -171,14 +130,7 @@ body:             |
     ; RV64-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C1]](s32)
     ; RV64-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SPLAT_VECTOR [[ANYEXT1]](s64)
     ; RV64-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SELECT [[DEF]](<vscale x 8 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
-    ; RV64-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV64-NEXT: [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[C2]](s32)
-    ; RV64-NEXT: [[SPLAT_VECTOR2:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SPLAT_VECTOR [[ANYEXT2]](s64)
-    ; RV64-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV64-NEXT: [[ANYEXT3:%[0-9]+]]:_(s64) = G_ANYEXT [[C3]](s32)
-    ; RV64-NEXT: [[SPLAT_VECTOR3:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SPLAT_VECTOR [[ANYEXT3]](s64)
-    ; RV64-NEXT: [[SELECT1:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SELECT [[DEF]](<vscale x 8 x s1>), [[SPLAT_VECTOR3]], [[SPLAT_VECTOR2]]
-    ; RV64-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 8 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 8 x s8>), [[SELECT1]]
+    ; RV64-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 8 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 8 x s8>), [[SELECT]]
     ; RV64-NEXT: $v8 = COPY [[ICMP]](<vscale x 8 x s1>)
     ; RV64-NEXT: PseudoRET implicit $v8
     %0:_(<vscale x 8 x s1>) = G_IMPLICIT_DEF
@@ -199,12 +151,7 @@ body:             |
     ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
     ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 16 x s8>) = G_SPLAT_VECTOR [[C1]](s32)
     ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 16 x s8>) = G_SELECT [[DEF]](<vscale x 16 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
-    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[SPLAT_VECTOR2:%[0-9]+]]:_(<vscale x 16 x s8>) = G_SPLAT_VECTOR [[C2]](s32)
-    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV32-NEXT: [[SPLAT_VECTOR3:%[0-9]+]]:_(<vscale x 16 x s8>) = G_SPLAT_VECTOR [[C3]](s32)
-    ; RV32-NEXT: [[SELECT1:%[0-9]+]]:_(<vscale x 16 x s8>) = G_SELECT [[DEF]](<vscale x 16 x s1>), [[SPLAT_VECTOR3]], [[SPLAT_VECTOR2]]
-    ; RV32-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 16 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 16 x s8>), [[SELECT1]]
+    ; RV32-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 16 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 16 x s8>), [[SELECT]]
     ; RV32-NEXT: $v8 = COPY [[ICMP]](<vscale x 16 x s1>)
     ; RV32-NEXT: PseudoRET implicit $v8
     ;
@@ -217,14 +164,7 @@ body:             |
     ; RV64-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C1]](s32)
     ; RV64-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 16 x s8>) = G_SPLAT_VECTOR [[ANYEXT1]](s64)
     ; RV64-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 16 x s8>) = G_SELECT [[DEF]](<vscale x 16 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
-    ; RV64-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV64-NEXT: [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[C2]](s32)
-    ; RV64-NEXT: [[SPLAT_VECTOR2:%[0-9]+]]:_(<vscale x 16 x s8>) = G_SPLAT_VECTOR [[ANYEXT2]](s64)
-    ; RV64-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV64-NEXT: [[ANYEXT3:%[0-9]+]]:_(s64) = G_ANYEXT [[C3]](s32)
-    ; RV64-NEXT: [[SPLAT_VECTOR3:%[0-9]+]]:_(<vscale x 16 x s8>) = G_SPLAT_VECTOR [[ANYEXT3]](s64)
-    ; RV64-NEXT: [[SELECT1:%[0-9]+]]:_(<vscale x 16 x s8>) = G_SELECT [[DEF]](<vscale x 16 x s1>), [[SPLAT_VECTOR3]], [[SPLAT_VECTOR2]]
-    ; RV64-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 16 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 16 x s8>), [[SELECT1]]
+    ; RV64-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 16 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 16 x s8>), [[SELECT]]
     ; RV64-NEXT: $v8 = COPY [[ICMP]](<vscale x 16 x s1>)
     ; RV64-NEXT: PseudoRET implicit $v8
     %0:_(<vscale x 16 x s1>) = G_IMPLICIT_DEF
@@ -245,12 +185,7 @@ body:             |
     ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
     ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 32 x s8>) = G_SPLAT_VECTOR [[C1]](s32)
     ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 32 x s8>) = G_SELECT [[DEF]](<vscale x 32 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
-    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[SPLAT_VECTOR2:%[0-9]+]]:_(<vscale x 32 x s8>) = G_SPLAT_VECTOR [[C2]](s32)
-    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV32-NEXT: [[SPLAT_VECTOR3:%[0-9]+]]:_(<vscale x 32 x s8>) = G_SPLAT_VECTOR [[C3]](s32)
-    ; RV32-NEXT: [[SELECT1:%[0-9]+]]:_(<vscale x 32 x s8>) = G_SELECT [[DEF]](<vscale x 32 x s1>), [[SPLAT_VECTOR3]], [[SPLAT_VECTOR2]]
-    ; RV32-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 32 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 32 x s8>), [[SELECT1]]
+    ; RV32-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 32 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 32 x s8>), [[SELECT]]
     ; RV32-NEXT: $v8 = COPY [[ICMP]](<vscale x 32 x s1>)
     ; RV32-NEXT: PseudoRET implicit $v8
     ;
@@ -263,14 +198,7 @@ body:             |
     ; RV64-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C1]](s32)
     ; RV64-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 32 x s8>) = G_SPLAT_VECTOR [[ANYEXT1]](s64)
     ; RV64-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 32 x s8>) = G_SELECT [[DEF]](<vscale x 32 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
-    ; RV64-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV64-NEXT: [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[C2]](s32)
-    ; RV64-NEXT: [[SPLAT_VECTOR2:%[0-9]+]]:_(<vscale x 32 x s8>) = G_SPLAT_VECTOR [[ANYEXT2]](s64)
-    ; RV64-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV64-NEXT: [[ANYEXT3:%[0-9]+]]:_(s64) = G_ANYEXT [[C3]](s32)
-    ; RV64-NEXT: [[SPLAT_VECTOR3:%[0-9]+]]:_(<vscale x 32 x s8>) = G_SPLAT_VECTOR [[ANYEXT3]](s64)
-    ; RV64-NEXT: [[SELECT1:%[0-9]+]]:_(<vscale x 32 x s8>) = G_SELECT [[DEF]](<vscale x 32 x s1>), [[SPLAT_VECTOR3]], [[SPLAT_VECTOR2]]
-    ; RV64-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 32 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 32 x s8>), [[SELECT1]]
+    ; RV64-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 32 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 32 x s8>), [[SELECT]]
     ; RV64-NEXT: $v8 = COPY [[ICMP]](<vscale x 32 x s1>)
     ; RV64-NEXT: PseudoRET implicit $v8
     %0:_(<vscale x 32 x s1>) = G_IMPLICIT_DEF
@@ -291,12 +219,7 @@ body:             |
     ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
     ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SPLAT_VECTOR [[C1]](s32)
     ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SELECT [[DEF]](<vscale x 64 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
-    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[SPLAT_VECTOR2:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SPLAT_VECTOR [[C2]](s32)
-    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV32-NEXT: [[SPLAT_VECTOR3:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SPLAT_VECTOR [[C3]](s32)
-    ; RV32-NEXT: [[SELECT1:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SELECT [[DEF]](<vscale x 64 x s1>), [[SPLAT_VECTOR3]], [[SPLAT_VECTOR2]]
-    ; RV32-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 64 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 64 x s8>), [[SELECT1]]
+    ; RV32-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 64 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 64 x s8>), [[SELECT]]
     ; RV32-NEXT: $v8 = COPY [[ICMP]](<vscale x 64 x s1>)
     ; RV32-NEXT: PseudoRET implicit $v8
     ;
@@ -309,14 +232,7 @@ body:             |
     ; RV64-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C1]](s32)
     ; RV64-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SPLAT_VECTOR [[ANYEXT1]](s64)
     ; RV64-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SELECT [[DEF]](<vscale x 64 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
-    ; RV64-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV64-NEXT: [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[C2]](s32)
-    ; RV64-NEXT: [[SPLAT_VECTOR2:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SPLAT_VECTOR [[ANYEXT2]](s64)
-    ; RV64-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV64-NEXT: [[ANYEXT3:%[0-9]+]]:_(s64) = G_ANYEXT [[C3]](s32)
-    ; RV64-NEXT: [[SPLAT_VECTOR3:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SPLAT_VECTOR [[ANYEXT3]](s64)
-    ; RV64-NEXT: [[SELECT1:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SELECT [[DEF]](<vscale x 64 x s1>), [[SPLAT_VECTOR3]], [[SPLAT_VECTOR2]]
-    ; RV64-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 64 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 64 x s8>), [[SELECT1]]
+    ; RV64-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 64 x s1>) = G_ICMP intpred(sgt), [[SELECT]](<vscale x 64 x s8>), [[SELECT]]
     ; RV64-NEXT: $v8 = COPY [[ICMP]](<vscale x 64 x s1>)
     ; RV64-NEXT: PseudoRET implicit $v8
     %0:_(<vscale x 64 x s1>) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-sext.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-sext.mir
index 1571daf16b2d3..c7c8d347adfc9 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-sext.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-sext.mir
@@ -126,12 +126,10 @@ body:             |
     ; RV32-NEXT: {{  $}}
     ; RV32-NEXT: [[COPY:%[0-9]+]]:_(<vscale x 1 x s1>) = COPY $v0
     ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C1]](s32)
+    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 1 x s64>) = G_SPLAT_VECTOR [[MV]](s64)
-    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C2]](s32), [[C3]](s32)
+    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C1]](s32), [[C1]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 1 x s64>) = G_SPLAT_VECTOR [[MV1]](s64)
     ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 1 x s64>) = G_SELECT [[COPY]](<vscale x 1 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
     ; RV32-NEXT: $v8 = COPY [[SELECT]](<vscale x 1 x s64>)
@@ -276,12 +274,10 @@ body:             |
     ; RV32-NEXT: {{  $}}
     ; RV32-NEXT: [[COPY:%[0-9]+]]:_(<vscale x 2 x s1>) = COPY $v0
     ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C1]](s32)
+    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 2 x s64>) = G_SPLAT_VECTOR [[MV]](s64)
-    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C2]](s32), [[C3]](s32)
+    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C1]](s32), [[C1]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 2 x s64>) = G_SPLAT_VECTOR [[MV1]](s64)
     ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 2 x s64>) = G_SELECT [[COPY]](<vscale x 2 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
     ; RV32-NEXT: $v8m2 = COPY [[SELECT]](<vscale x 2 x s64>)
@@ -426,12 +422,10 @@ body:             |
     ; RV32-NEXT: {{  $}}
     ; RV32-NEXT: [[COPY:%[0-9]+]]:_(<vscale x 4 x s1>) = COPY $v0
     ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C1]](s32)
+    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 4 x s64>) = G_SPLAT_VECTOR [[MV]](s64)
-    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C2]](s32), [[C3]](s32)
+    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C1]](s32), [[C1]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 4 x s64>) = G_SPLAT_VECTOR [[MV1]](s64)
     ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 4 x s64>) = G_SELECT [[COPY]](<vscale x 4 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
     ; RV32-NEXT: $v8m4 = COPY [[SELECT]](<vscale x 4 x s64>)
@@ -576,12 +570,10 @@ body:             |
     ; RV32-NEXT: {{  $}}
     ; RV32-NEXT: [[COPY:%[0-9]+]]:_(<vscale x 8 x s1>) = COPY $v0
     ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C1]](s32)
+    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 8 x s64>) = G_SPLAT_VECTOR [[MV]](s64)
-    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C2]](s32), [[C3]](s32)
+    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C1]](s32), [[C1]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 8 x s64>) = G_SPLAT_VECTOR [[MV1]](s64)
     ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 8 x s64>) = G_SELECT [[COPY]](<vscale x 8 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
     ; RV32-NEXT: $v8m8 = COPY [[SELECT]](<vscale x 8 x s64>)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-splatvector-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-splatvector-rv32.mir
index 109536a07d5ec..5b14dcd0b6905 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-splatvector-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-splatvector-rv32.mir
@@ -8,8 +8,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv1i1_0
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMCLR_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL [[C]](s32)
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMCLR_VL [[C]](s32)
     ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 1 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 0
@@ -25,8 +26,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv1i1_1
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL [[C]](s32)
+    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL [[C]](s32)
     ; CHECK-NEXT: $v0 = COPY [[VMSET_VL1]](<vscale x 1 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 1
@@ -48,12 +50,11 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C1]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
     ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 1 x s8>) = G_SPLAT_VECTOR [[AND1]](s32)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 1 x s8>) = G_SPLAT_VECTOR [[C2]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 1 x s8>) = G_SPLAT_VECTOR [[C1]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 1 x s1>) = G_ICMP intpred(ne), [[SPLAT_VECTOR]](<vscale x 1 x s8>), [[SPLAT_VECTOR1]]
     ; CHECK-NEXT: $v0 = COPY [[ICMP]](<vscale x 1 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
@@ -70,8 +71,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv2i1_0
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMCLR_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL [[C]](s32)
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMCLR_VL [[C]](s32)
     ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 2 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 0
@@ -87,8 +89,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv2i1_1
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL [[C]](s32)
+    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL [[C]](s32)
     ; CHECK-NEXT: $v0 = COPY [[VMSET_VL1]](<vscale x 2 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 1
@@ -110,12 +113,11 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C1]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
     ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 2 x s8>) = G_SPLAT_VECTOR [[AND1]](s32)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 2 x s8>) = G_SPLAT_VECTOR [[C2]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 2 x s8>) = G_SPLAT_VECTOR [[C1]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 2 x s1>) = G_ICMP intpred(ne), [[SPLAT_VECTOR]](<vscale x 2 x s8>), [[SPLAT_VECTOR1]]
     ; CHECK-NEXT: $v0 = COPY [[ICMP]](<vscale x 2 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
@@ -132,8 +134,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv4i1_0
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 4 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 4 x s1>) = G_VMCLR_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 4 x s1>) = G_VMSET_VL [[C]](s32)
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 4 x s1>) = G_VMCLR_VL [[C]](s32)
     ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 4 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 0
@@ -149,8 +152,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv4i1_1
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 4 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 4 x s1>) = G_VMSET_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 4 x s1>) = G_VMSET_VL [[C]](s32)
+    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 4 x s1>) = G_VMSET_VL [[C]](s32)
     ; CHECK-NEXT: $v0 = COPY [[VMSET_VL1]](<vscale x 4 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 1
@@ -172,12 +176,11 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C1]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
     ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SPLAT_VECTOR [[AND1]](s32)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SPLAT_VECTOR [[C2]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SPLAT_VECTOR [[C1]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 4 x s1>) = G_ICMP intpred(ne), [[SPLAT_VECTOR]](<vscale x 4 x s8>), [[SPLAT_VECTOR1]]
     ; CHECK-NEXT: $v0 = COPY [[ICMP]](<vscale x 4 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
@@ -194,8 +197,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv8i1_0
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 8 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 8 x s1>) = G_VMCLR_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 8 x s1>) = G_VMSET_VL [[C]](s32)
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 8 x s1>) = G_VMCLR_VL [[C]](s32)
     ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 8 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 0
@@ -211,8 +215,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv8i1_1
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 8 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 8 x s1>) = G_VMSET_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 8 x s1>) = G_VMSET_VL [[C]](s32)
+    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 8 x s1>) = G_VMSET_VL [[C]](s32)
     ; CHECK-NEXT: $v0 = COPY [[VMSET_VL1]](<vscale x 8 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 1
@@ -234,12 +239,11 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C1]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
     ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SPLAT_VECTOR [[AND1]](s32)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SPLAT_VECTOR [[C2]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SPLAT_VECTOR [[C1]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 8 x s1>) = G_ICMP intpred(ne), [[SPLAT_VECTOR]](<vscale x 8 x s8>), [[SPLAT_VECTOR1]]
     ; CHECK-NEXT: $v0 = COPY [[ICMP]](<vscale x 8 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
@@ -256,8 +260,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv16i1_0
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 16 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 16 x s1>) = G_VMCLR_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 16 x s1>) = G_VMSET_VL [[C]](s32)
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 16 x s1>) = G_VMCLR_VL [[C]](s32)
     ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 16 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 0
@@ -273,8 +278,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv16i1_1
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 16 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 16 x s1>) = G_VMSET_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 16 x s1>) = G_VMSET_VL [[C]](s32)
+    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 16 x s1>) = G_VMSET_VL [[C]](s32)
     ; CHECK-NEXT: $v0 = COPY [[VMSET_VL1]](<vscale x 16 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 1
@@ -296,12 +302,11 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C1]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
     ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 16 x s8>) = G_SPLAT_VECTOR [[AND1]](s32)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 16 x s8>) = G_SPLAT_VECTOR [[C2]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 16 x s8>) = G_SPLAT_VECTOR [[C1]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 16 x s1>) = G_ICMP intpred(ne), [[SPLAT_VECTOR]](<vscale x 16 x s8>), [[SPLAT_VECTOR1]]
     ; CHECK-NEXT: $v0 = COPY [[ICMP]](<vscale x 16 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
@@ -318,8 +323,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv32i1_0
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 32 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 32 x s1>) = G_VMCLR_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 32 x s1>) = G_VMSET_VL [[C]](s32)
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 32 x s1>) = G_VMCLR_VL [[C]](s32)
     ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 32 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 0
@@ -335,8 +341,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv32i1_1
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 32 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 32 x s1>) = G_VMSET_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 32 x s1>) = G_VMSET_VL [[C]](s32)
+    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 32 x s1>) = G_VMSET_VL [[C]](s32)
     ; CHECK-NEXT: $v0 = COPY [[VMSET_VL1]](<vscale x 32 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 1
@@ -358,12 +365,11 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C1]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
     ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 32 x s8>) = G_SPLAT_VECTOR [[AND1]](s32)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 32 x s8>) = G_SPLAT_VECTOR [[C2]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 32 x s8>) = G_SPLAT_VECTOR [[C1]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 32 x s1>) = G_ICMP intpred(ne), [[SPLAT_VECTOR]](<vscale x 32 x s8>), [[SPLAT_VECTOR1]]
     ; CHECK-NEXT: $v0 = COPY [[ICMP]](<vscale x 32 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
@@ -380,8 +386,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv64i1_0
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 64 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 64 x s1>) = G_VMCLR_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 64 x s1>) = G_VMSET_VL [[C]](s32)
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 64 x s1>) = G_VMCLR_VL [[C]](s32)
     ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 64 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 0
@@ -397,8 +404,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv64i1_1
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 64 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 64 x s1>) = G_VMSET_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 64 x s1>) = G_VMSET_VL [[C]](s32)
+    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 64 x s1>) = G_VMSET_VL [[C]](s32)
     ; CHECK-NEXT: $v0 = COPY [[VMSET_VL1]](<vscale x 64 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 1
@@ -420,12 +428,11 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C1]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
     ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SPLAT_VECTOR [[AND1]](s32)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SPLAT_VECTOR [[C2]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SPLAT_VECTOR [[C1]](s32)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 64 x s1>) = G_ICMP intpred(ne), [[SPLAT_VECTOR]](<vscale x 64 x s8>), [[SPLAT_VECTOR1]]
     ; CHECK-NEXT: $v0 = COPY [[ICMP]](<vscale x 64 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-splatvector-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-splatvector-rv64.mir
index 7bf5f83dd9144..c95db3262471d 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-splatvector-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-splatvector-rv64.mir
@@ -8,8 +8,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv1i1_0
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMCLR_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL [[C]](s64)
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMCLR_VL [[C]](s64)
     ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 1 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 0
@@ -25,8 +26,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv1i1_1
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL [[C]](s64)
+    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL [[C]](s64)
     ; CHECK-NEXT: $v0 = COPY [[VMSET_VL1]](<vscale x 1 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 1
@@ -49,13 +51,12 @@ body:             |
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[AND1]](s32)
     ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 1 x s8>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C2]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C1]](s32)
     ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 1 x s8>) = G_SPLAT_VECTOR [[ANYEXT1]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 1 x s1>) = G_ICMP intpred(ne), [[SPLAT_VECTOR]](<vscale x 1 x s8>), [[SPLAT_VECTOR1]]
     ; CHECK-NEXT: $v0 = COPY [[ICMP]](<vscale x 1 x s1>)
@@ -73,8 +74,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv2i1_0
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMCLR_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL [[C]](s64)
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMCLR_VL [[C]](s64)
     ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 2 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 0
@@ -90,8 +92,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv2i1_1
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL [[C]](s64)
+    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL [[C]](s64)
     ; CHECK-NEXT: $v0 = COPY [[VMSET_VL1]](<vscale x 2 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 1
@@ -114,13 +117,12 @@ body:             |
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[AND1]](s32)
     ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 2 x s8>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C2]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C1]](s32)
     ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 2 x s8>) = G_SPLAT_VECTOR [[ANYEXT1]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 2 x s1>) = G_ICMP intpred(ne), [[SPLAT_VECTOR]](<vscale x 2 x s8>), [[SPLAT_VECTOR1]]
     ; CHECK-NEXT: $v0 = COPY [[ICMP]](<vscale x 2 x s1>)
@@ -138,8 +140,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv4i1_0
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 4 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 4 x s1>) = G_VMCLR_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 4 x s1>) = G_VMSET_VL [[C]](s64)
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 4 x s1>) = G_VMCLR_VL [[C]](s64)
     ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 4 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 0
@@ -155,8 +158,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv4i1_1
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 4 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 4 x s1>) = G_VMSET_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 4 x s1>) = G_VMSET_VL [[C]](s64)
+    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 4 x s1>) = G_VMSET_VL [[C]](s64)
     ; CHECK-NEXT: $v0 = COPY [[VMSET_VL1]](<vscale x 4 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 1
@@ -179,13 +183,12 @@ body:             |
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[AND1]](s32)
     ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C2]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C1]](s32)
     ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SPLAT_VECTOR [[ANYEXT1]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 4 x s1>) = G_ICMP intpred(ne), [[SPLAT_VECTOR]](<vscale x 4 x s8>), [[SPLAT_VECTOR1]]
     ; CHECK-NEXT: $v0 = COPY [[ICMP]](<vscale x 4 x s1>)
@@ -203,8 +206,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv8i1_0
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 8 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 8 x s1>) = G_VMCLR_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 8 x s1>) = G_VMSET_VL [[C]](s64)
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 8 x s1>) = G_VMCLR_VL [[C]](s64)
     ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 8 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 0
@@ -220,8 +224,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv8i1_1
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 8 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 8 x s1>) = G_VMSET_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 8 x s1>) = G_VMSET_VL [[C]](s64)
+    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 8 x s1>) = G_VMSET_VL [[C]](s64)
     ; CHECK-NEXT: $v0 = COPY [[VMSET_VL1]](<vscale x 8 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 1
@@ -244,13 +249,12 @@ body:             |
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[AND1]](s32)
     ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C2]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C1]](s32)
     ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SPLAT_VECTOR [[ANYEXT1]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 8 x s1>) = G_ICMP intpred(ne), [[SPLAT_VECTOR]](<vscale x 8 x s8>), [[SPLAT_VECTOR1]]
     ; CHECK-NEXT: $v0 = COPY [[ICMP]](<vscale x 8 x s1>)
@@ -268,8 +272,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv16i1_0
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 16 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 16 x s1>) = G_VMCLR_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 16 x s1>) = G_VMSET_VL [[C]](s64)
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 16 x s1>) = G_VMCLR_VL [[C]](s64)
     ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 16 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 0
@@ -285,8 +290,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv16i1_1
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 16 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 16 x s1>) = G_VMSET_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 16 x s1>) = G_VMSET_VL [[C]](s64)
+    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 16 x s1>) = G_VMSET_VL [[C]](s64)
     ; CHECK-NEXT: $v0 = COPY [[VMSET_VL1]](<vscale x 16 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 1
@@ -309,13 +315,12 @@ body:             |
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[AND1]](s32)
     ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 16 x s8>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C2]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C1]](s32)
     ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 16 x s8>) = G_SPLAT_VECTOR [[ANYEXT1]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 16 x s1>) = G_ICMP intpred(ne), [[SPLAT_VECTOR]](<vscale x 16 x s8>), [[SPLAT_VECTOR1]]
     ; CHECK-NEXT: $v0 = COPY [[ICMP]](<vscale x 16 x s1>)
@@ -333,8 +338,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv32i1_0
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 32 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 32 x s1>) = G_VMCLR_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 32 x s1>) = G_VMSET_VL [[C]](s64)
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 32 x s1>) = G_VMCLR_VL [[C]](s64)
     ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 32 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 0
@@ -350,8 +356,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv32i1_1
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 32 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 32 x s1>) = G_VMSET_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 32 x s1>) = G_VMSET_VL [[C]](s64)
+    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 32 x s1>) = G_VMSET_VL [[C]](s64)
     ; CHECK-NEXT: $v0 = COPY [[VMSET_VL1]](<vscale x 32 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 1
@@ -374,13 +381,12 @@ body:             |
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[AND1]](s32)
     ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 32 x s8>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C2]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C1]](s32)
     ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 32 x s8>) = G_SPLAT_VECTOR [[ANYEXT1]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 32 x s1>) = G_ICMP intpred(ne), [[SPLAT_VECTOR]](<vscale x 32 x s8>), [[SPLAT_VECTOR1]]
     ; CHECK-NEXT: $v0 = COPY [[ICMP]](<vscale x 32 x s1>)
@@ -398,8 +404,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv64i1_0
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 64 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 64 x s1>) = G_VMCLR_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 64 x s1>) = G_VMSET_VL [[C]](s64)
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:_(<vscale x 64 x s1>) = G_VMCLR_VL [[C]](s64)
     ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 64 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 0
@@ -415,8 +422,9 @@ tracksRegLiveness: true
 body:             |
   bb.1:
     ; CHECK-LABEL: name: splatvector_nxv64i1_1
-    ; CHECK: [[VMSET_VL:%[0-9]+]]:_(<vscale x 64 x s1>) = G_VMSET_VL $x0
-    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 64 x s1>) = G_VMSET_VL $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 64 x s1>) = G_VMSET_VL [[C]](s64)
+    ; CHECK-NEXT: [[VMSET_VL1:%[0-9]+]]:_(<vscale x 64 x s1>) = G_VMSET_VL [[C]](s64)
     ; CHECK-NEXT: $v0 = COPY [[VMSET_VL1]](<vscale x 64 x s1>)
     ; CHECK-NEXT: PseudoRET implicit $v0
     %0:_(s1) = G_CONSTANT i1 1
@@ -439,13 +447,12 @@ body:             |
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C1]]
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AND]], [[C]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[AND1]](s32)
     ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C2]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C1]](s32)
     ; CHECK-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SPLAT_VECTOR [[ANYEXT1]](s64)
     ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<vscale x 64 x s1>) = G_ICMP intpred(ne), [[SPLAT_VECTOR]](<vscale x 64 x s8>), [[SPLAT_VECTOR1]]
     ; CHECK-NEXT: $v0 = COPY [[ICMP]](<vscale x 64 x s1>)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-splatvector-s64-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-splatvector-s64-rv32.mir
index 806c9b97c91a9..a80ffb7087547 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-splatvector-s64-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-splatvector-s64-rv32.mir
@@ -10,18 +10,17 @@ body:             |
   bb.1:
     ; HasF64-LABEL: name: splatvector_nxv1i64
     ; HasF64: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; HasF64-NEXT: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; HasF64-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[DEF]](s32), [[DEF1]](s32)
+    ; HasF64-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[DEF]](s32), [[DEF]](s32)
     ; HasF64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 1 x s64>) = G_SPLAT_VECTOR [[MV]](s64)
     ; HasF64-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 1 x s64>)
     ; HasF64-NEXT: PseudoRET implicit $v8
     ;
     ; NoF64-LABEL: name: splatvector_nxv1i64
     ; NoF64: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; NoF64-NEXT: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; NoF64-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL $x0
-    ; NoF64-NEXT: [[DEF2:%[0-9]+]]:_(<vscale x 1 x s64>) = G_IMPLICIT_DEF
-    ; NoF64-NEXT: [[SPLAT_VECTOR_SPLIT_I64_VL:%[0-9]+]]:_(<vscale x 1 x s64>) = G_SPLAT_VECTOR_SPLIT_I64_VL [[DEF2]], [[DEF]](s32), [[DEF1]], $x0
+    ; NoF64-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; NoF64-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 1 x s1>) = G_VMSET_VL [[C]](s32)
+    ; NoF64-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 1 x s64>) = G_IMPLICIT_DEF
+    ; NoF64-NEXT: [[SPLAT_VECTOR_SPLIT_I64_VL:%[0-9]+]]:_(<vscale x 1 x s64>) = G_SPLAT_VECTOR_SPLIT_I64_VL [[DEF1]], [[DEF]](s32), [[DEF]], [[C]](s32)
     ; NoF64-NEXT: $v8 = COPY [[SPLAT_VECTOR_SPLIT_I64_VL]](<vscale x 1 x s64>)
     ; NoF64-NEXT: PseudoRET implicit $v8
     %0:_(s64) = G_IMPLICIT_DEF
@@ -38,18 +37,17 @@ body:             |
   bb.1:
     ; HasF64-LABEL: name: splatvector_nxv2i64
     ; HasF64: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; HasF64-NEXT: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; HasF64-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[DEF]](s32), [[DEF1]](s32)
+    ; HasF64-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[DEF]](s32), [[DEF]](s32)
     ; HasF64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 2 x s64>) = G_SPLAT_VECTOR [[MV]](s64)
     ; HasF64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]](<vscale x 2 x s64>)
     ; HasF64-NEXT: PseudoRET implicit $v8m2
     ;
     ; NoF64-LABEL: name: splatvector_nxv2i64
     ; NoF64: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; NoF64-NEXT: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; NoF64-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL $x0
-    ; NoF64-NEXT: [[DEF2:%[0-9]+]]:_(<vscale x 2 x s64>) = G_IMPLICIT_DEF
-    ; NoF64-NEXT: [[SPLAT_VECTOR_SPLIT_I64_VL:%[0-9]+]]:_(<vscale x 2 x s64>) = G_SPLAT_VECTOR_SPLIT_I64_VL [[DEF2]], [[DEF]](s32), [[DEF1]], $x0
+    ; NoF64-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; NoF64-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 2 x s1>) = G_VMSET_VL [[C]](s32)
+    ; NoF64-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 2 x s64>) = G_IMPLICIT_DEF
+    ; NoF64-NEXT: [[SPLAT_VECTOR_SPLIT_I64_VL:%[0-9]+]]:_(<vscale x 2 x s64>) = G_SPLAT_VECTOR_SPLIT_I64_VL [[DEF1]], [[DEF]](s32), [[DEF]], [[C]](s32)
     ; NoF64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR_SPLIT_I64_VL]](<vscale x 2 x s64>)
     ; NoF64-NEXT: PseudoRET implicit $v8m2
     %0:_(s64) = G_IMPLICIT_DEF
@@ -66,18 +64,17 @@ body:             |
   bb.1:
     ; HasF64-LABEL: name: splatvector_nxv4i64
     ; HasF64: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; HasF64-NEXT: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; HasF64-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[DEF]](s32), [[DEF1]](s32)
+    ; HasF64-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[DEF]](s32), [[DEF]](s32)
     ; HasF64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 4 x s64>) = G_SPLAT_VECTOR [[MV]](s64)
     ; HasF64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]](<vscale x 4 x s64>)
     ; HasF64-NEXT: PseudoRET implicit $v8m4
     ;
     ; NoF64-LABEL: name: splatvector_nxv4i64
     ; NoF64: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; NoF64-NEXT: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; NoF64-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 4 x s1>) = G_VMSET_VL $x0
-    ; NoF64-NEXT: [[DEF2:%[0-9]+]]:_(<vscale x 4 x s64>) = G_IMPLICIT_DEF
-    ; NoF64-NEXT: [[SPLAT_VECTOR_SPLIT_I64_VL:%[0-9]+]]:_(<vscale x 4 x s64>) = G_SPLAT_VECTOR_SPLIT_I64_VL [[DEF2]], [[DEF]](s32), [[DEF1]], $x0
+    ; NoF64-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; NoF64-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 4 x s1>) = G_VMSET_VL [[C]](s32)
+    ; NoF64-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 4 x s64>) = G_IMPLICIT_DEF
+    ; NoF64-NEXT: [[SPLAT_VECTOR_SPLIT_I64_VL:%[0-9]+]]:_(<vscale x 4 x s64>) = G_SPLAT_VECTOR_SPLIT_I64_VL [[DEF1]], [[DEF]](s32), [[DEF]], [[C]](s32)
     ; NoF64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR_SPLIT_I64_VL]](<vscale x 4 x s64>)
     ; NoF64-NEXT: PseudoRET implicit $v8m4
     %0:_(s64) = G_IMPLICIT_DEF
@@ -94,18 +91,17 @@ body:             |
   bb.1:
     ; HasF64-LABEL: name: splatvector_nxv8i64
     ; HasF64: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; HasF64-NEXT: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; HasF64-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[DEF]](s32), [[DEF1]](s32)
+    ; HasF64-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[DEF]](s32), [[DEF]](s32)
     ; HasF64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 8 x s64>) = G_SPLAT_VECTOR [[MV]](s64)
     ; HasF64-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]](<vscale x 8 x s64>)
     ; HasF64-NEXT: PseudoRET implicit $v8m8
     ;
     ; NoF64-LABEL: name: splatvector_nxv8i64
     ; NoF64: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; NoF64-NEXT: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; NoF64-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 8 x s1>) = G_VMSET_VL $x0
-    ; NoF64-NEXT: [[DEF2:%[0-9]+]]:_(<vscale x 8 x s64>) = G_IMPLICIT_DEF
-    ; NoF64-NEXT: [[SPLAT_VECTOR_SPLIT_I64_VL:%[0-9]+]]:_(<vscale x 8 x s64>) = G_SPLAT_VECTOR_SPLIT_I64_VL [[DEF2]], [[DEF]](s32), [[DEF1]], $x0
+    ; NoF64-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; NoF64-NEXT: [[VMSET_VL:%[0-9]+]]:_(<vscale x 8 x s1>) = G_VMSET_VL [[C]](s32)
+    ; NoF64-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 8 x s64>) = G_IMPLICIT_DEF
+    ; NoF64-NEXT: [[SPLAT_VECTOR_SPLIT_I64_VL:%[0-9]+]]:_(<vscale x 8 x s64>) = G_SPLAT_VECTOR_SPLIT_I64_VL [[DEF1]], [[DEF]](s32), [[DEF]], [[C]](s32)
     ; NoF64-NEXT: $v8m8 = COPY [[SPLAT_VECTOR_SPLIT_I64_VL]](<vscale x 8 x s64>)
     ; NoF64-NEXT: PseudoRET implicit $v8m8
     %0:_(s64) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv32.mir
index 899f7955a2730..e7f391d5a9ee9 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv32.mir
@@ -40,8 +40,7 @@ body:             |
     ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
-    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[LSHR]], [[C1]]
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[LSHR]], [[C]]
     ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %0:_(s32) = G_VSCALE i32 3
@@ -149,8 +148,7 @@ body:             |
     ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
-    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[LSHR]], [[C1]]
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[LSHR]], [[C]]
     ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %0:_(s64) = G_VSCALE i64 3
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv64.mir
index c0453a04a1815..fea094d73736e 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv64.mir
@@ -40,8 +40,7 @@ body:             |
     ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
     ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
-    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[LSHR]], [[C1]]
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[LSHR]], [[C]]
     ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
     ; CHECK-NEXT: PseudoRET implicit $x10
     %0:_(s64) = G_VSCALE i64 3
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-zext.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-zext.mir
index fe4ddfa847dd2..2a6e7a6fa8e52 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-zext.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-zext.mir
@@ -126,12 +126,10 @@ body:             |
     ; RV32-NEXT: {{  $}}
     ; RV32-NEXT: [[COPY:%[0-9]+]]:_(<vscale x 1 x s1>) = COPY $v8
     ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C1]](s32)
+    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 1 x s64>) = G_SPLAT_VECTOR [[MV]](s64)
-    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C2]](s32), [[C3]](s32)
+    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C1]](s32), [[C]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 1 x s64>) = G_SPLAT_VECTOR [[MV1]](s64)
     ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 1 x s64>) = G_SELECT [[COPY]](<vscale x 1 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
     ; RV32-NEXT: $v8 = COPY [[SELECT]](<vscale x 1 x s64>)
@@ -276,12 +274,10 @@ body:             |
     ; RV32-NEXT: {{  $}}
     ; RV32-NEXT: [[COPY:%[0-9]+]]:_(<vscale x 2 x s1>) = COPY $v8
     ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C1]](s32)
+    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 2 x s64>) = G_SPLAT_VECTOR [[MV]](s64)
-    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C2]](s32), [[C3]](s32)
+    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C1]](s32), [[C]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 2 x s64>) = G_SPLAT_VECTOR [[MV1]](s64)
     ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 2 x s64>) = G_SELECT [[COPY]](<vscale x 2 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
     ; RV32-NEXT: $v8m2 = COPY [[SELECT]](<vscale x 2 x s64>)
@@ -426,12 +422,10 @@ body:             |
     ; RV32-NEXT: {{  $}}
     ; RV32-NEXT: [[COPY:%[0-9]+]]:_(<vscale x 4 x s1>) = COPY $v8
     ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C1]](s32)
+    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 4 x s64>) = G_SPLAT_VECTOR [[MV]](s64)
-    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C2]](s32), [[C3]](s32)
+    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C1]](s32), [[C]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 4 x s64>) = G_SPLAT_VECTOR [[MV1]](s64)
     ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 4 x s64>) = G_SELECT [[COPY]](<vscale x 4 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
     ; RV32-NEXT: $v8m4 = COPY [[SELECT]](<vscale x 4 x s64>)
@@ -576,12 +570,10 @@ body:             |
     ; RV32-NEXT: {{  $}}
     ; RV32-NEXT: [[COPY:%[0-9]+]]:_(<vscale x 8 x s1>) = COPY $v8
     ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C1]](s32)
+    ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_(<vscale x 8 x s64>) = G_SPLAT_VECTOR [[MV]](s64)
-    ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C2]](s32), [[C3]](s32)
+    ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; RV32-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C1]](s32), [[C]](s32)
     ; RV32-NEXT: [[SPLAT_VECTOR1:%[0-9]+]]:_(<vscale x 8 x s64>) = G_SPLAT_VECTOR [[MV1]](s64)
     ; RV32-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 8 x s64>) = G_SELECT [[COPY]](<vscale x 8 x s1>), [[SPLAT_VECTOR1]], [[SPLAT_VECTOR]]
     ; RV32-NEXT: $v8m8 = COPY [[SELECT]](<vscale x 8 x s64>)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/splatvector-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/splatvector-rv32.mir
new file mode 100644
index 0000000000000..b44bece8eadf8
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/splatvector-rv32.mir
@@ -0,0 +1,549 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=riscv32 -mattr=+m,+v -run-pass=regbankselect \
+# RUN:   -disable-gisel-legality-check -simplify-mir -verify-machineinstrs %s \
+# RUN:   -o - | FileCheck %s
+
+---
+name:            splat_zero_nxv1i8
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv1i8
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 1 x s8>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 1 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %3:_(s32) = G_CONSTANT i32 0
+    %0:_(<vscale x 1 x s8>) = G_SPLAT_VECTOR %3(s32)
+    $v8 = COPY %0(<vscale x 1 x s8>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv2i8
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv2i8
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 2 x s8>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 2 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %3:_(s32) = G_CONSTANT i32 0
+    %0:_(<vscale x 2 x s8>) = G_SPLAT_VECTOR %3(s32)
+    $v8 = COPY %0(<vscale x 2 x s8>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv4i8
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv4i8
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 4 x s8>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 4 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %3:_(s32) = G_CONSTANT i32 0
+    %0:_(<vscale x 4 x s8>) = G_SPLAT_VECTOR %3(s32)
+    $v8 = COPY %0(<vscale x 4 x s8>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv8i8
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv8i8
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 8 x s8>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 8 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %3:_(s32) = G_CONSTANT i32 0
+    %0:_(<vscale x 8 x s8>) = G_SPLAT_VECTOR %3(s32)
+    $v8 = COPY %0(<vscale x 8 x s8>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv16i8
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv16i8
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 16 x s8>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]](<vscale x 16 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %3:_(s32) = G_CONSTANT i32 0
+    %0:_(<vscale x 16 x s8>) = G_SPLAT_VECTOR %3(s32)
+    $v8m2 = COPY %0(<vscale x 16 x s8>)
+    PseudoRET implicit $v8m2
+
+...
+---
+name:            splat_zero_nxv32i8
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv32i8
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 32 x s8>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]](<vscale x 32 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %3:_(s32) = G_CONSTANT i32 0
+    %0:_(<vscale x 32 x s8>) = G_SPLAT_VECTOR %3(s32)
+    $v8m4 = COPY %0(<vscale x 32 x s8>)
+    PseudoRET implicit $v8m4
+
+...
+---
+name:            splat_zero_nxv64i8
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv64i8
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 64 x s8>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]](<vscale x 64 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %3:_(s32) = G_CONSTANT i32 0
+    %0:_(<vscale x 64 x s8>) = G_SPLAT_VECTOR %3(s32)
+    $v8m8 = COPY %0(<vscale x 64 x s8>)
+    PseudoRET implicit $v8m8
+
+...
+---
+name:            splat_zero_nxv1i16
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv1i16
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 1 x s16>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 1 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %3:_(s32) = G_CONSTANT i32 0
+    %0:_(<vscale x 1 x s16>) = G_SPLAT_VECTOR %3(s32)
+    $v8 = COPY %0(<vscale x 1 x s16>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv2i16
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv2i16
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 2 x s16>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 2 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %3:_(s32) = G_CONSTANT i32 0
+    %0:_(<vscale x 2 x s16>) = G_SPLAT_VECTOR %3(s32)
+    $v8 = COPY %0(<vscale x 2 x s16>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv4i16
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv4i16
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 4 x s16>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 4 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %3:_(s32) = G_CONSTANT i32 0
+    %0:_(<vscale x 4 x s16>) = G_SPLAT_VECTOR %3(s32)
+    $v8 = COPY %0(<vscale x 4 x s16>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv8i16
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv8i16
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 8 x s16>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]](<vscale x 8 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %3:_(s32) = G_CONSTANT i32 0
+    %0:_(<vscale x 8 x s16>) = G_SPLAT_VECTOR %3(s32)
+    $v8m2 = COPY %0(<vscale x 8 x s16>)
+    PseudoRET implicit $v8m2
+
+...
+---
+name:            splat_zero_nxv16i16
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv16i16
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 16 x s16>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]](<vscale x 16 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %3:_(s32) = G_CONSTANT i32 0
+    %0:_(<vscale x 16 x s16>) = G_SPLAT_VECTOR %3(s32)
+    $v8m4 = COPY %0(<vscale x 16 x s16>)
+    PseudoRET implicit $v8m4
+
+...
+---
+name:            splat_zero_nxv32i16
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv32i16
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 32 x s16>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]](<vscale x 32 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %3:_(s32) = G_CONSTANT i32 0
+    %0:_(<vscale x 32 x s16>) = G_SPLAT_VECTOR %3(s32)
+    $v8m8 = COPY %0(<vscale x 32 x s16>)
+    PseudoRET implicit $v8m8
+
+...
+---
+name:            splat_zero_nxv1i32
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv1i32
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 1 x s32>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 1 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(s32) = G_CONSTANT i32 0
+    %0:_(<vscale x 1 x s32>) = G_SPLAT_VECTOR %1(s32)
+    $v8 = COPY %0(<vscale x 1 x s32>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv2i32
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv2i32
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 2 x s32>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 2 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(s32) = G_CONSTANT i32 0
+    %0:_(<vscale x 2 x s32>) = G_SPLAT_VECTOR %1(s32)
+    $v8 = COPY %0(<vscale x 2 x s32>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv4i32
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv4i32
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 4 x s32>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]](<vscale x 4 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %1:_(s32) = G_CONSTANT i32 0
+    %0:_(<vscale x 4 x s32>) = G_SPLAT_VECTOR %1(s32)
+    $v8m2 = COPY %0(<vscale x 4 x s32>)
+    PseudoRET implicit $v8m2
+
+...
+---
+name:            splat_zero_nxv8i32
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv8i32
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 8 x s32>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]](<vscale x 8 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %1:_(s32) = G_CONSTANT i32 0
+    %0:_(<vscale x 8 x s32>) = G_SPLAT_VECTOR %1(s32)
+    $v8m4 = COPY %0(<vscale x 8 x s32>)
+    PseudoRET implicit $v8m4
+
+...
+---
+name:            splat_zero_nxv16i32
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv16i32
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 16 x s32>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]](<vscale x 16 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %1:_(s32) = G_CONSTANT i32 0
+    %0:_(<vscale x 16 x s32>) = G_SPLAT_VECTOR %1(s32)
+    $v8m8 = COPY %0(<vscale x 16 x s32>)
+    PseudoRET implicit $v8m8
+
+...
+---
+name:            splat_zero_nxv1i64
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv1i64
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:fprb(s64) = G_MERGE_VALUES [[C]](s32), [[C1]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 1 x s64>) = G_SPLAT_VECTOR [[MV]](s64)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 1 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %2:_(s32) = G_CONSTANT i32 0
+    %3:_(s32) = G_CONSTANT i32 0
+    %1:_(s64) = G_MERGE_VALUES %2(s32), %3(s32)
+    %0:_(<vscale x 1 x s64>) = G_SPLAT_VECTOR %1(s64)
+    $v8 = COPY %0(<vscale x 1 x s64>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv2i64
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv2i64
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:fprb(s64) = G_MERGE_VALUES [[C]](s32), [[C1]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 2 x s64>) = G_SPLAT_VECTOR [[MV]](s64)
+    ; CHECK-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]](<vscale x 2 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %2:_(s32) = G_CONSTANT i32 0
+    %3:_(s32) = G_CONSTANT i32 0
+    %1:_(s64) = G_MERGE_VALUES %2(s32), %3(s32)
+    %0:_(<vscale x 2 x s64>) = G_SPLAT_VECTOR %1(s64)
+    $v8m2 = COPY %0(<vscale x 2 x s64>)
+    PseudoRET implicit $v8m2
+
+...
+---
+name:            splat_zero_nxv4i64
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv4i64
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:fprb(s64) = G_MERGE_VALUES [[C]](s32), [[C1]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 4 x s64>) = G_SPLAT_VECTOR [[MV]](s64)
+    ; CHECK-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]](<vscale x 4 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %2:_(s32) = G_CONSTANT i32 0
+    %3:_(s32) = G_CONSTANT i32 0
+    %1:_(s64) = G_MERGE_VALUES %2(s32), %3(s32)
+    %0:_(<vscale x 4 x s64>) = G_SPLAT_VECTOR %1(s64)
+    $v8m4 = COPY %0(<vscale x 4 x s64>)
+    PseudoRET implicit $v8m4
+
+...
+---
+name:            splat_zero_nxv8i64
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv8i64
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:fprb(s64) = G_MERGE_VALUES [[C]](s32), [[C1]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 8 x s64>) = G_SPLAT_VECTOR [[MV]](s64)
+    ; CHECK-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]](<vscale x 8 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %2:_(s32) = G_CONSTANT i32 0
+    %3:_(s32) = G_CONSTANT i32 0
+    %1:_(s64) = G_MERGE_VALUES %2(s32), %3(s32)
+    %0:_(<vscale x 8 x s64>) = G_SPLAT_VECTOR %1(s64)
+    $v8m8 = COPY %0(<vscale x 8 x s64>)
+    PseudoRET implicit $v8m8
+
+...
+
+---
+name:            splat_zero_nxv1f32
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv1f32
+    ; CHECK: [[C:%[0-9]+]]:fprb(s32) = G_FCONSTANT float 0.000000e+00
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 1 x s32>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 1 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(s32) = G_FCONSTANT float 0.000000e+00
+    %0:_(<vscale x 1 x s32>) = G_SPLAT_VECTOR %1(s32)
+    $v8 = COPY %0(<vscale x 1 x s32>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv2f32
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv2f32
+    ; CHECK: [[C:%[0-9]+]]:fprb(s32) = G_FCONSTANT float 0.000000e+00
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 2 x s32>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 2 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(s32) = G_FCONSTANT float 0.000000e+00
+    %0:_(<vscale x 2 x s32>) = G_SPLAT_VECTOR %1(s32)
+    $v8 = COPY %0(<vscale x 2 x s32>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv4f32
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv4f32
+    ; CHECK: [[C:%[0-9]+]]:fprb(s32) = G_FCONSTANT float 0.000000e+00
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 4 x s32>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]](<vscale x 4 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %1:_(s32) = G_FCONSTANT float 0.000000e+00
+    %0:_(<vscale x 4 x s32>) = G_SPLAT_VECTOR %1(s32)
+    $v8m2 = COPY %0(<vscale x 4 x s32>)
+    PseudoRET implicit $v8m2
+
+...
+---
+name:            splat_zero_nxv8f32
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv8f32
+    ; CHECK: [[C:%[0-9]+]]:fprb(s32) = G_FCONSTANT float 0.000000e+00
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 8 x s32>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]](<vscale x 8 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %1:_(s32) = G_FCONSTANT float 0.000000e+00
+    %0:_(<vscale x 8 x s32>) = G_SPLAT_VECTOR %1(s32)
+    $v8m4 = COPY %0(<vscale x 8 x s32>)
+    PseudoRET implicit $v8m4
+
+...
+---
+name:            splat_zero_nxv16f32
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv16f32
+    ; CHECK: [[C:%[0-9]+]]:fprb(s32) = G_FCONSTANT float 0.000000e+00
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 16 x s32>) = G_SPLAT_VECTOR [[C]](s32)
+    ; CHECK-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]](<vscale x 16 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %1:_(s32) = G_FCONSTANT float 0.000000e+00
+    %0:_(<vscale x 16 x s32>) = G_SPLAT_VECTOR %1(s32)
+    $v8m8 = COPY %0(<vscale x 16 x s32>)
+    PseudoRET implicit $v8m8
+
+...
+---
+name:            splat_zero_nxv1f64
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv1f64
+    ; CHECK: [[C:%[0-9]+]]:fprb(s64) = G_FCONSTANT double 0.000000e+00
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 1 x s64>) = G_SPLAT_VECTOR [[C]](s64)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 1 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(s64) = G_FCONSTANT double 0.000000e+00
+    %0:_(<vscale x 1 x s64>) = G_SPLAT_VECTOR %1(s64)
+    $v8 = COPY %0(<vscale x 1 x s64>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv2f64
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv2f64
+    ; CHECK: [[C:%[0-9]+]]:fprb(s64) = G_FCONSTANT double 0.000000e+00
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 2 x s64>) = G_SPLAT_VECTOR [[C]](s64)
+    ; CHECK-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]](<vscale x 2 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %1:_(s64) = G_FCONSTANT double 0.000000e+00
+    %0:_(<vscale x 2 x s64>) = G_SPLAT_VECTOR %1(s64)
+    $v8m2 = COPY %0(<vscale x 2 x s64>)
+    PseudoRET implicit $v8m2
+
+...
+---
+name:            splat_zero_nxv4f64
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv4f64
+    ; CHECK: [[C:%[0-9]+]]:fprb(s64) = G_FCONSTANT double 0.000000e+00
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 4 x s64>) = G_SPLAT_VECTOR [[C]](s64)
+    ; CHECK-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]](<vscale x 4 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %1:_(s64) = G_FCONSTANT double 0.000000e+00
+    %0:_(<vscale x 4 x s64>) = G_SPLAT_VECTOR %1(s64)
+    $v8m4 = COPY %0(<vscale x 4 x s64>)
+    PseudoRET implicit $v8m4
+
+...
+---
+name:            splat_zero_nxv8f64
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv8f64
+    ; CHECK: [[C:%[0-9]+]]:fprb(s64) = G_FCONSTANT double 0.000000e+00
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 8 x s64>) = G_SPLAT_VECTOR [[C]](s64)
+    ; CHECK-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]](<vscale x 8 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %1:_(s64) = G_FCONSTANT double 0.000000e+00
+    %0:_(<vscale x 8 x s64>) = G_SPLAT_VECTOR %1(s64)
+    $v8m8 = COPY %0(<vscale x 8 x s64>)
+    PseudoRET implicit $v8m8
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/splatvector-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/splatvector-rv64.mir
new file mode 100644
index 0000000000000..efd5237253449
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/splatvector-rv64.mir
@@ -0,0 +1,583 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=riscv64 -mattr=+m,+v -run-pass=regbankselect \
+# RUN:   -disable-gisel-legality-check -simplify-mir -verify-machineinstrs %s \
+# RUN:   -o - | FileCheck %s
+
+---
+name:            splat_zero_nxv1i8
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv1i8
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[C]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 1 x s8>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 1 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %3:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_ANYEXT %3(s32)
+    %0:_(<vscale x 1 x s8>) = G_SPLAT_VECTOR %2(s64)
+    $v8 = COPY %0(<vscale x 1 x s8>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv2i8
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv2i8
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[C]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 2 x s8>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 2 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %3:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_ANYEXT %3(s32)
+    %0:_(<vscale x 2 x s8>) = G_SPLAT_VECTOR %2(s64)
+    $v8 = COPY %0(<vscale x 2 x s8>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv4i8
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv4i8
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[C]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 4 x s8>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 4 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %3:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_ANYEXT %3(s32)
+    %0:_(<vscale x 4 x s8>) = G_SPLAT_VECTOR %2(s64)
+    $v8 = COPY %0(<vscale x 4 x s8>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv8i8
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv8i8
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[C]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 8 x s8>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 8 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %3:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_ANYEXT %3(s32)
+    %0:_(<vscale x 8 x s8>) = G_SPLAT_VECTOR %2(s64)
+    $v8 = COPY %0(<vscale x 8 x s8>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv16i8
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv16i8
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[C]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 16 x s8>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]](<vscale x 16 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %3:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_ANYEXT %3(s32)
+    %0:_(<vscale x 16 x s8>) = G_SPLAT_VECTOR %2(s64)
+    $v8m2 = COPY %0(<vscale x 16 x s8>)
+    PseudoRET implicit $v8m2
+
+...
+---
+name:            splat_zero_nxv32i8
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv32i8
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[C]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 32 x s8>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]](<vscale x 32 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %3:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_ANYEXT %3(s32)
+    %0:_(<vscale x 32 x s8>) = G_SPLAT_VECTOR %2(s64)
+    $v8m4 = COPY %0(<vscale x 32 x s8>)
+    PseudoRET implicit $v8m4
+
+...
+---
+name:            splat_zero_nxv64i8
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv64i8
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[C]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 64 x s8>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]](<vscale x 64 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %3:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_ANYEXT %3(s32)
+    %0:_(<vscale x 64 x s8>) = G_SPLAT_VECTOR %2(s64)
+    $v8m8 = COPY %0(<vscale x 64 x s8>)
+    PseudoRET implicit $v8m8
+
+...
+---
+name:            splat_zero_nxv1i16
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv1i16
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[C]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 1 x s16>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 1 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %3:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_ANYEXT %3(s32)
+    %0:_(<vscale x 1 x s16>) = G_SPLAT_VECTOR %2(s64)
+    $v8 = COPY %0(<vscale x 1 x s16>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv2i16
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv2i16
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[C]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 2 x s16>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 2 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %3:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_ANYEXT %3(s32)
+    %0:_(<vscale x 2 x s16>) = G_SPLAT_VECTOR %2(s64)
+    $v8 = COPY %0(<vscale x 2 x s16>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv4i16
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv4i16
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[C]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 4 x s16>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 4 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %3:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_ANYEXT %3(s32)
+    %0:_(<vscale x 4 x s16>) = G_SPLAT_VECTOR %2(s64)
+    $v8 = COPY %0(<vscale x 4 x s16>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv8i16
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv8i16
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[C]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 8 x s16>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]](<vscale x 8 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %3:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_ANYEXT %3(s32)
+    %0:_(<vscale x 8 x s16>) = G_SPLAT_VECTOR %2(s64)
+    $v8m2 = COPY %0(<vscale x 8 x s16>)
+    PseudoRET implicit $v8m2
+
+...
+---
+name:            splat_zero_nxv16i16
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv16i16
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[C]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 16 x s16>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]](<vscale x 16 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %3:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_ANYEXT %3(s32)
+    %0:_(<vscale x 16 x s16>) = G_SPLAT_VECTOR %2(s64)
+    $v8m4 = COPY %0(<vscale x 16 x s16>)
+    PseudoRET implicit $v8m4
+
+...
+---
+name:            splat_zero_nxv32i16
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv32i16
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[C]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 32 x s16>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]](<vscale x 32 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %3:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_ANYEXT %3(s32)
+    %0:_(<vscale x 32 x s16>) = G_SPLAT_VECTOR %2(s64)
+    $v8m8 = COPY %0(<vscale x 32 x s16>)
+    PseudoRET implicit $v8m8
+
+...
+---
+name:            splat_zero_nxv1i32
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv1i32
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[C]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 1 x s32>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 1 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_ANYEXT %1(s32)
+    %0:_(<vscale x 1 x s32>) = G_SPLAT_VECTOR %2(s64)
+    $v8 = COPY %0(<vscale x 1 x s32>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv2i32
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv2i32
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[C]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 2 x s32>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 2 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_ANYEXT %1(s32)
+    %0:_(<vscale x 2 x s32>) = G_SPLAT_VECTOR %2(s64)
+    $v8 = COPY %0(<vscale x 2 x s32>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv4i32
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv4i32
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[C]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 4 x s32>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]](<vscale x 4 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %1:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_ANYEXT %1(s32)
+    %0:_(<vscale x 4 x s32>) = G_SPLAT_VECTOR %2(s64)
+    $v8m2 = COPY %0(<vscale x 4 x s32>)
+    PseudoRET implicit $v8m2
+
+...
+---
+name:            splat_zero_nxv8i32
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv8i32
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[C]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 8 x s32>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]](<vscale x 8 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %1:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_ANYEXT %1(s32)
+    %0:_(<vscale x 8 x s32>) = G_SPLAT_VECTOR %2(s64)
+    $v8m4 = COPY %0(<vscale x 8 x s32>)
+    PseudoRET implicit $v8m4
+
+...
+---
+name:            splat_zero_nxv16i32
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv16i32
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[C]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 16 x s32>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]](<vscale x 16 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %1:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_ANYEXT %1(s32)
+    %0:_(<vscale x 16 x s32>) = G_SPLAT_VECTOR %2(s64)
+    $v8m8 = COPY %0(<vscale x 16 x s32>)
+    PseudoRET implicit $v8m8
+
+...
+---
+name:            splat_zero_nxv1i64
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv1i64
+    ; CHECK: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 1 x s64>) = G_SPLAT_VECTOR [[C]](s64)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 1 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(s64) = G_CONSTANT i64 0
+    %0:_(<vscale x 1 x s64>) = G_SPLAT_VECTOR %1(s64)
+    $v8 = COPY %0(<vscale x 1 x s64>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv2i64
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv2i64
+    ; CHECK: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 2 x s64>) = G_SPLAT_VECTOR [[C]](s64)
+    ; CHECK-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]](<vscale x 2 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %1:_(s64) = G_CONSTANT i64 0
+    %0:_(<vscale x 2 x s64>) = G_SPLAT_VECTOR %1(s64)
+    $v8m2 = COPY %0(<vscale x 2 x s64>)
+    PseudoRET implicit $v8m2
+
+...
+---
+name:            splat_zero_nxv4i64
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv4i64
+    ; CHECK: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 4 x s64>) = G_SPLAT_VECTOR [[C]](s64)
+    ; CHECK-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]](<vscale x 4 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %1:_(s64) = G_CONSTANT i64 0
+    %0:_(<vscale x 4 x s64>) = G_SPLAT_VECTOR %1(s64)
+    $v8m4 = COPY %0(<vscale x 4 x s64>)
+    PseudoRET implicit $v8m4
+
+...
+---
+name:            splat_zero_nxv8i64
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv8i64
+    ; CHECK: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 8 x s64>) = G_SPLAT_VECTOR [[C]](s64)
+    ; CHECK-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]](<vscale x 8 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %1:_(s64) = G_CONSTANT i64 0
+    %0:_(<vscale x 8 x s64>) = G_SPLAT_VECTOR %1(s64)
+    $v8m8 = COPY %0(<vscale x 8 x s64>)
+    PseudoRET implicit $v8m8
+
+...
+---
+name:            splat_zero_nxv1f32
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv1f32
+    ; CHECK: [[C:%[0-9]+]]:fprb(s32) = G_FCONSTANT float 0.000000e+00
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(s32) = COPY [[C]](s32)
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[COPY]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 1 x s32>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 1 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(s32) = G_FCONSTANT float 0.000000e+00
+    %2:_(s64) = G_ANYEXT %1(s32)
+    %0:_(<vscale x 1 x s32>) = G_SPLAT_VECTOR %2(s64)
+    $v8 = COPY %0(<vscale x 1 x s32>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv2f32
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv2f32
+    ; CHECK: [[C:%[0-9]+]]:fprb(s32) = G_FCONSTANT float 0.000000e+00
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(s32) = COPY [[C]](s32)
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[COPY]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 2 x s32>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 2 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(s32) = G_FCONSTANT float 0.000000e+00
+    %2:_(s64) = G_ANYEXT %1(s32)
+    %0:_(<vscale x 2 x s32>) = G_SPLAT_VECTOR %2(s64)
+    $v8 = COPY %0(<vscale x 2 x s32>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv4f32
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv4f32
+    ; CHECK: [[C:%[0-9]+]]:fprb(s32) = G_FCONSTANT float 0.000000e+00
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(s32) = COPY [[C]](s32)
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[COPY]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 4 x s32>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]](<vscale x 4 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %1:_(s32) = G_FCONSTANT float 0.000000e+00
+    %2:_(s64) = G_ANYEXT %1(s32)
+    %0:_(<vscale x 4 x s32>) = G_SPLAT_VECTOR %2(s64)
+    $v8m2 = COPY %0(<vscale x 4 x s32>)
+    PseudoRET implicit $v8m2
+
+...
+---
+name:            splat_zero_nxv8f32
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv8f32
+    ; CHECK: [[C:%[0-9]+]]:fprb(s32) = G_FCONSTANT float 0.000000e+00
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(s32) = COPY [[C]](s32)
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[COPY]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 8 x s32>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]](<vscale x 8 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %1:_(s32) = G_FCONSTANT float 0.000000e+00
+    %2:_(s64) = G_ANYEXT %1(s32)
+    %0:_(<vscale x 8 x s32>) = G_SPLAT_VECTOR %2(s64)
+    $v8m4 = COPY %0(<vscale x 8 x s32>)
+    PseudoRET implicit $v8m4
+
+...
+---
+name:            splat_zero_nxv16f32
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv16f32
+    ; CHECK: [[C:%[0-9]+]]:fprb(s32) = G_FCONSTANT float 0.000000e+00
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(s32) = COPY [[C]](s32)
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:gprb(s64) = G_ANYEXT [[COPY]](s32)
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 16 x s32>) = G_SPLAT_VECTOR [[ANYEXT]](s64)
+    ; CHECK-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]](<vscale x 16 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %1:_(s32) = G_FCONSTANT float 0.000000e+00
+    %2:_(s64) = G_ANYEXT %1(s32)
+    %0:_(<vscale x 16 x s32>) = G_SPLAT_VECTOR %2(s64)
+    $v8m8 = COPY %0(<vscale x 16 x s32>)
+    PseudoRET implicit $v8m8
+
+...
+---
+name:            splat_zero_nxv1f64
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv1f64
+    ; CHECK: [[C:%[0-9]+]]:fprb(s64) = G_FCONSTANT double 0.000000e+00
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 1 x s64>) = G_SPLAT_VECTOR [[C]](s64)
+    ; CHECK-NEXT: $v8 = COPY [[SPLAT_VECTOR]](<vscale x 1 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(s64) = G_FCONSTANT double 0.000000e+00
+    %0:_(<vscale x 1 x s64>) = G_SPLAT_VECTOR %1(s64)
+    $v8 = COPY %0(<vscale x 1 x s64>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            splat_zero_nxv2f64
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv2f64
+    ; CHECK: [[C:%[0-9]+]]:fprb(s64) = G_FCONSTANT double 0.000000e+00
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 2 x s64>) = G_SPLAT_VECTOR [[C]](s64)
+    ; CHECK-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]](<vscale x 2 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %1:_(s64) = G_FCONSTANT double 0.000000e+00
+    %0:_(<vscale x 2 x s64>) = G_SPLAT_VECTOR %1(s64)
+    $v8m2 = COPY %0(<vscale x 2 x s64>)
+    PseudoRET implicit $v8m2
+
+...
+---
+name:            splat_zero_nxv4f64
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv4f64
+    ; CHECK: [[C:%[0-9]+]]:fprb(s64) = G_FCONSTANT double 0.000000e+00
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 4 x s64>) = G_SPLAT_VECTOR [[C]](s64)
+    ; CHECK-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]](<vscale x 4 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %1:_(s64) = G_FCONSTANT double 0.000000e+00
+    %0:_(<vscale x 4 x s64>) = G_SPLAT_VECTOR %1(s64)
+    $v8m4 = COPY %0(<vscale x 4 x s64>)
+    PseudoRET implicit $v8m4
+
+...
+---
+name:            splat_zero_nxv8f64
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv8f64
+    ; CHECK: [[C:%[0-9]+]]:fprb(s64) = G_FCONSTANT double 0.000000e+00
+    ; CHECK-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:vrb(<vscale x 8 x s64>) = G_SPLAT_VECTOR [[C]](s64)
+    ; CHECK-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]](<vscale x 8 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %1:_(s64) = G_FCONSTANT double 0.000000e+00
+    %0:_(<vscale x 8 x s64>) = G_SPLAT_VECTOR %1(s64)
+    $v8m8 = COPY %0(<vscale x 8 x s64>)
+    PseudoRET implicit $v8m8
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vmclr-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vmclr-rv32.mir
new file mode 100644
index 0000000000000..b2ad2dd777f6c
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vmclr-rv32.mir
@@ -0,0 +1,125 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=riscv32 -mattr=+m,+v -run-pass=regbankselect \
+# RUN:   -disable-gisel-legality-check -simplify-mir -verify-machineinstrs %s \
+# RUN:   -o - | FileCheck %s
+
+---
+name:            splat_zero_nxv1i1
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv1i1
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:vrb(<vscale x 1 x s1>) = G_VMCLR_VL [[C]](s32)
+    ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 1 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:_(s32) = G_CONSTANT i32 -1
+    %1:_(<vscale x 1 x s1>) = G_VMCLR_VL %0(s32)
+    $v0 = COPY %1(<vscale x 1 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv2i1
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv2i1
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:vrb(<vscale x 2 x s1>) = G_VMCLR_VL [[C]](s32)
+    ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 2 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:_(s32) = G_CONSTANT i32 -1
+    %1:_(<vscale x 2 x s1>) = G_VMCLR_VL %0(s32)
+    $v0 = COPY %1(<vscale x 2 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv4i1
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv4i1
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:vrb(<vscale x 4 x s1>) = G_VMCLR_VL [[C]](s32)
+    ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 4 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:_(s32) = G_CONSTANT i32 -1
+    %1:_(<vscale x 4 x s1>) = G_VMCLR_VL %0(s32)
+    $v0 = COPY %1(<vscale x 4 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv8i1
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv8i1
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:vrb(<vscale x 8 x s1>) = G_VMCLR_VL [[C]](s32)
+    ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 8 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:_(s32) = G_CONSTANT i32 -1
+    %1:_(<vscale x 8 x s1>) = G_VMCLR_VL %0(s32)
+    $v0 = COPY %1(<vscale x 8 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv16i1
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv16i1
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:vrb(<vscale x 16 x s1>) = G_VMCLR_VL [[C]](s32)
+    ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 16 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:_(s32) = G_CONSTANT i32 -1
+    %1:_(<vscale x 16 x s1>) = G_VMCLR_VL %0(s32)
+    $v0 = COPY %1(<vscale x 16 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv32i1
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv32i1
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:vrb(<vscale x 32 x s1>) = G_VMCLR_VL [[C]](s32)
+    ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 32 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:_(s32) = G_CONSTANT i32 -1
+    %1:_(<vscale x 32 x s1>) = G_VMCLR_VL %0(s32)
+    $v0 = COPY %1(<vscale x 32 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv64i1
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv64i1
+    ; CHECK: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:vrb(<vscale x 64 x s1>) = G_VMCLR_VL [[C]](s32)
+    ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 64 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:_(s32) = G_CONSTANT i32 -1
+    %1:_(<vscale x 64 x s1>) = G_VMCLR_VL %0(s32)
+    $v0 = COPY %1(<vscale x 64 x s1>)
+    PseudoRET implicit $v0
+
+...
+
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vmclr-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vmclr-rv64.mir
new file mode 100644
index 0000000000000..9997805eb5f76
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vmclr-rv64.mir
@@ -0,0 +1,125 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=riscv64 -mattr=+m,+v -run-pass=regbankselect \
+# RUN:   -disable-gisel-legality-check -simplify-mir -verify-machineinstrs %s \
+# RUN:   -o - | FileCheck %s
+
+---
+name:            splat_zero_nxv1i1
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv1i1
+    ; CHECK: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:vrb(<vscale x 1 x s1>) = G_VMCLR_VL [[C]](s64)
+    ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 1 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:_(s64) = G_CONSTANT i64 -1
+    %1:_(<vscale x 1 x s1>) = G_VMCLR_VL %0(s64)
+    $v0 = COPY %1(<vscale x 1 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv2i1
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv2i1
+    ; CHECK: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:vrb(<vscale x 2 x s1>) = G_VMCLR_VL [[C]](s64)
+    ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 2 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:_(s64) = G_CONSTANT i64 -1
+    %1:_(<vscale x 2 x s1>) = G_VMCLR_VL %0(s64)
+    $v0 = COPY %1(<vscale x 2 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv4i1
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv4i1
+    ; CHECK: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:vrb(<vscale x 4 x s1>) = G_VMCLR_VL [[C]](s64)
+    ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 4 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:_(s64) = G_CONSTANT i64 -1
+    %1:_(<vscale x 4 x s1>) = G_VMCLR_VL %0(s64)
+    $v0 = COPY %1(<vscale x 4 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv8i1
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv8i1
+    ; CHECK: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:vrb(<vscale x 8 x s1>) = G_VMCLR_VL [[C]](s64)
+    ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 8 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:_(s64) = G_CONSTANT i64 -1
+    %1:_(<vscale x 8 x s1>) = G_VMCLR_VL %0(s64)
+    $v0 = COPY %1(<vscale x 8 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv16i1
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv16i1
+    ; CHECK: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:vrb(<vscale x 16 x s1>) = G_VMCLR_VL [[C]](s64)
+    ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 16 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:_(s64) = G_CONSTANT i64 -1
+    %1:_(<vscale x 16 x s1>) = G_VMCLR_VL %0(s64)
+    $v0 = COPY %1(<vscale x 16 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv32i1
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv32i1
+    ; CHECK: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:vrb(<vscale x 32 x s1>) = G_VMCLR_VL [[C]](s64)
+    ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 32 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:_(s64) = G_CONSTANT i64 -1
+    %1:_(<vscale x 32 x s1>) = G_VMCLR_VL %0(s64)
+    $v0 = COPY %1(<vscale x 32 x s1>)
+    PseudoRET implicit $v0
+
+...
+---
+name:            splat_zero_nxv64i1
+legalized:       true
+regBankSelected: false
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: splat_zero_nxv64i1
+    ; CHECK: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[VMCLR_VL:%[0-9]+]]:vrb(<vscale x 64 x s1>) = G_VMCLR_VL [[C]](s64)
+    ; CHECK-NEXT: $v0 = COPY [[VMCLR_VL]](<vscale x 64 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v0
+    %0:_(s64) = G_CONSTANT i64 -1
+    %1:_(<vscale x 64 x s1>) = G_VMCLR_VL %0(s64)
+    $v0 = COPY %1(<vscale x 64 x s1>)
+    PseudoRET implicit $v0
+
+...
+
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/rvv/splat-vector.ll b/llvm/test/CodeGen/RISCV/GlobalISel/rvv/splat-vector.ll
new file mode 100644
index 0000000000000..4e58c4dcec2d8
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/rvv/splat-vector.ll
@@ -0,0 +1,73 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv32 -mattr=+v,+zvfh -global-isel \
+; RUN:   -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV32 %s
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zvfh -global-isel \
+; RUN:   -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV64 %s
+
+define <vscale x 1 x i1> @splat_zero_nxv1i1() {
+; CHECK-LABEL: splat_zero_nxv1i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
+; CHECK-NEXT:    vmclr.m v0
+; CHECK-NEXT:    ret
+  ret <vscale x 1 x i1> zeroinitializer
+}
+
+define <vscale x 2 x i1> @splat_zero_nxv2i1() {
+; CHECK-LABEL: splat_zero_nxv2i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf4, ta, ma
+; CHECK-NEXT:    vmclr.m v0
+; CHECK-NEXT:    ret
+  ret <vscale x 2 x i1> zeroinitializer
+}
+
+define <vscale x 4 x i1> @splat_zero_nxv4i1() {
+; CHECK-LABEL: splat_zero_nxv4i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vmclr.m v0
+; CHECK-NEXT:    ret
+  ret <vscale x 4 x i1> zeroinitializer
+}
+
+define <vscale x 8 x i1> @splat_zero_nxv8i1() {
+; CHECK-LABEL: splat_zero_nxv8i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vmclr.m v0
+; CHECK-NEXT:    ret
+  ret <vscale x 8 x i1> zeroinitializer
+}
+
+define <vscale x 16 x i1> @splat_zero_nxv16i1() {
+; CHECK-LABEL: splat_zero_nxv16i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmclr.m v0
+; CHECK-NEXT:    ret
+  ret <vscale x 16 x i1> zeroinitializer
+}
+
+define <vscale x 32 x i1> @splat_zero_nxv32i1() {
+; CHECK-LABEL: splat_zero_nxv32i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
+; CHECK-NEXT:    vmclr.m v0
+; CHECK-NEXT:    ret
+  ret <vscale x 32 x i1> zeroinitializer
+}
+
+define <vscale x 64 x i1> @splat_zero_nxv64i1() {
+; CHECK-LABEL: splat_zero_nxv64i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmclr.m v0
+; CHECK-NEXT:    ret
+  ret <vscale x 64 x i1> zeroinitializer
+}
+
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; RV32: {{.*}}
+; RV64: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/shift.ll b/llvm/test/CodeGen/RISCV/GlobalISel/shift.ll
index b75cbf8e871a1..3e090507ad642 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/shift.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/shift.ll
@@ -46,3 +46,57 @@ define i16 @test_shl_i48(i48 %x) {
   %trunc = trunc i48 %shl to i16
   ret i16 %trunc
 }
+
+define i16 @test_lshr_i48_2(i48 %x, i48 %y) {
+; RV32-LABEL: test_lshr_i48_2:
+; RV32:       # %bb.0:
+; RV32-NEXT:    andi a2, a2, 15
+; RV32-NEXT:    srl a0, a0, a2
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: test_lshr_i48_2:
+; RV64:       # %bb.0:
+; RV64-NEXT:    andi a1, a1, 15
+; RV64-NEXT:    srlw a0, a0, a1
+; RV64-NEXT:    ret
+  %and = and i48 %y, 15
+  %lshr = lshr i48 %x, %and
+  %trunc = trunc i48 %lshr to i16
+  ret i16 %trunc
+}
+
+define i16 @test_ashr_i48_2(i48 %x, i48 %y) {
+; RV32-LABEL: test_ashr_i48_2:
+; RV32:       # %bb.0:
+; RV32-NEXT:    andi a2, a2, 15
+; RV32-NEXT:    sra a0, a0, a2
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: test_ashr_i48_2:
+; RV64:       # %bb.0:
+; RV64-NEXT:    andi a1, a1, 15
+; RV64-NEXT:    sraw a0, a0, a1
+; RV64-NEXT:    ret
+  %and = and i48 %y, 15
+  %ashr = ashr i48 %x, %and
+  %trunc = trunc i48 %ashr to i16
+  ret i16 %trunc
+}
+
+define i16 @test_shl_i48_2(i48 %x, i48 %y) {
+; RV32-LABEL: test_shl_i48_2:
+; RV32:       # %bb.0:
+; RV32-NEXT:    andi a2, a2, 15
+; RV32-NEXT:    sll a0, a0, a2
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: test_shl_i48_2:
+; RV64:       # %bb.0:
+; RV64-NEXT:    andi a1, a1, 15
+; RV64-NEXT:    sllw a0, a0, a1
+; RV64-NEXT:    ret
+  %and = and i48 %y, 15
+  %shl = shl i48 %x, %and
+  %trunc = trunc i48 %shl to i16
+  ret i16 %trunc
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll b/llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll
index 01cab0d0e157b..3df4aca40ec94 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll
@@ -69,15 +69,16 @@ define i32 @va1(ptr %fmt, ...) {
 ; RV64-NEXT:    sd a2, 32(sp)
 ; RV64-NEXT:    sd a3, 40(sp)
 ; RV64-NEXT:    sd a4, 48(sp)
-; RV64-NEXT:    addi a0, sp, 24
-; RV64-NEXT:    sd a0, 8(sp)
-; RV64-NEXT:    lwu a0, 8(sp)
-; RV64-NEXT:    lw a1, 12(sp)
 ; RV64-NEXT:    sd a5, 56(sp)
+; RV64-NEXT:    addi a0, sp, 8
+; RV64-NEXT:    addi a1, sp, 24
+; RV64-NEXT:    sd a1, 8(sp)
+; RV64-NEXT:    lw a0, 4(a0)
+; RV64-NEXT:    lwu a1, 8(sp)
 ; RV64-NEXT:    sd a6, 64(sp)
 ; RV64-NEXT:    sd a7, 72(sp)
-; RV64-NEXT:    slli a1, a1, 32
-; RV64-NEXT:    or a0, a1, a0
+; RV64-NEXT:    slli a0, a0, 32
+; RV64-NEXT:    or a0, a0, a1
 ; RV64-NEXT:    addi a1, a0, 4
 ; RV64-NEXT:    srli a2, a1, 32
 ; RV64-NEXT:    sw a1, 8(sp)
@@ -128,15 +129,16 @@ define i32 @va1(ptr %fmt, ...) {
 ; RV64-WITHFP-NEXT:    sd a2, 16(s0)
 ; RV64-WITHFP-NEXT:    sd a3, 24(s0)
 ; RV64-WITHFP-NEXT:    sd a4, 32(s0)
-; RV64-WITHFP-NEXT:    addi a0, s0, 8
-; RV64-WITHFP-NEXT:    sd a0, -24(s0)
-; RV64-WITHFP-NEXT:    lwu a0, -24(s0)
-; RV64-WITHFP-NEXT:    lw a1, -20(s0)
 ; RV64-WITHFP-NEXT:    sd a5, 40(s0)
+; RV64-WITHFP-NEXT:    addi a0, s0, -24
+; RV64-WITHFP-NEXT:    addi a1, s0, 8
+; RV64-WITHFP-NEXT:    sd a1, -24(s0)
+; RV64-WITHFP-NEXT:    lw a0, 4(a0)
+; RV64-WITHFP-NEXT:    lwu a1, -24(s0)
 ; RV64-WITHFP-NEXT:    sd a6, 48(s0)
 ; RV64-WITHFP-NEXT:    sd a7, 56(s0)
-; RV64-WITHFP-NEXT:    slli a1, a1, 32
-; RV64-WITHFP-NEXT:    or a0, a1, a0
+; RV64-WITHFP-NEXT:    slli a0, a0, 32
+; RV64-WITHFP-NEXT:    or a0, a0, a1
 ; RV64-WITHFP-NEXT:    addi a1, a0, 4
 ; RV64-WITHFP-NEXT:    srli a2, a1, 32
 ; RV64-WITHFP-NEXT:    sw a1, -24(s0)
@@ -1207,105 +1209,111 @@ declare void @llvm.va_copy(ptr, ptr)
 define iXLen @va4_va_copy(i32 %argno, ...) nounwind {
 ; RV32-LABEL: va4_va_copy:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
-; RV32-NEXT:    sw a1, 20(sp)
-; RV32-NEXT:    sw a2, 24(sp)
-; RV32-NEXT:    sw a3, 28(sp)
-; RV32-NEXT:    sw a4, 32(sp)
-; RV32-NEXT:    sw a5, 36(sp)
-; RV32-NEXT:    sw a6, 40(sp)
-; RV32-NEXT:    sw a7, 44(sp)
-; RV32-NEXT:    addi a0, sp, 20
-; RV32-NEXT:    sw a0, 4(sp)
-; RV32-NEXT:    lw a0, 4(sp)
+; RV32-NEXT:    addi sp, sp, -64
+; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw a1, 36(sp)
+; RV32-NEXT:    sw a2, 40(sp)
+; RV32-NEXT:    sw a3, 44(sp)
+; RV32-NEXT:    sw a4, 48(sp)
+; RV32-NEXT:    sw a5, 52(sp)
+; RV32-NEXT:    sw a6, 56(sp)
+; RV32-NEXT:    sw a7, 60(sp)
+; RV32-NEXT:    addi a0, sp, 36
+; RV32-NEXT:    sw a0, 16(sp)
+; RV32-NEXT:    lw a0, 16(sp)
 ; RV32-NEXT:    addi a0, a0, 3
-; RV32-NEXT:    andi a0, a0, -4
+; RV32-NEXT:    li s0, -4
+; RV32-NEXT:    and a0, a0, s0
 ; RV32-NEXT:    addi a1, a0, 4
-; RV32-NEXT:    sw a1, 4(sp)
-; RV32-NEXT:    lw a1, 4(sp)
-; RV32-NEXT:    lw s0, 0(a0)
-; RV32-NEXT:    sw a1, 0(sp)
-; RV32-NEXT:    lw a0, 0(sp)
+; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    lw a1, 16(sp)
+; RV32-NEXT:    lw s1, 0(a0)
+; RV32-NEXT:    sw a1, 12(sp)
+; RV32-NEXT:    lw a0, 12(sp)
 ; RV32-NEXT:    call notdead
-; RV32-NEXT:    lw a0, 4(sp)
+; RV32-NEXT:    lw a0, 16(sp)
 ; RV32-NEXT:    addi a0, a0, 3
-; RV32-NEXT:    andi a0, a0, -4
+; RV32-NEXT:    and a0, a0, s0
 ; RV32-NEXT:    addi a1, a0, 4
-; RV32-NEXT:    sw a1, 4(sp)
-; RV32-NEXT:    lw a1, 4(sp)
+; RV32-NEXT:    sw a1, 16(sp)
+; RV32-NEXT:    lw a1, 16(sp)
 ; RV32-NEXT:    lw a0, 0(a0)
 ; RV32-NEXT:    addi a1, a1, 3
-; RV32-NEXT:    andi a1, a1, -4
+; RV32-NEXT:    and a1, a1, s0
 ; RV32-NEXT:    addi a2, a1, 4
-; RV32-NEXT:    sw a2, 4(sp)
-; RV32-NEXT:    lw a2, 4(sp)
+; RV32-NEXT:    sw a2, 16(sp)
+; RV32-NEXT:    lw a2, 16(sp)
 ; RV32-NEXT:    lw a1, 0(a1)
 ; RV32-NEXT:    addi a2, a2, 3
 ; RV32-NEXT:    andi a2, a2, -4
 ; RV32-NEXT:    addi a3, a2, 4
-; RV32-NEXT:    sw a3, 4(sp)
+; RV32-NEXT:    sw a3, 16(sp)
 ; RV32-NEXT:    lw a2, 0(a2)
-; RV32-NEXT:    add a0, a0, s0
+; RV32-NEXT:    add a0, a0, s1
 ; RV32-NEXT:    add a1, a1, a2
 ; RV32-NEXT:    add a0, a0, a1
-; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
-; RV32-NEXT:    addi sp, sp, 48
+; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 64
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: va4_va_copy:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    addi sp, sp, -96
-; RV64-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; RV64-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; RV64-NEXT:    sd a1, 40(sp)
-; RV64-NEXT:    sd a2, 48(sp)
-; RV64-NEXT:    sd a3, 56(sp)
-; RV64-NEXT:    sd a4, 64(sp)
-; RV64-NEXT:    sd a5, 72(sp)
-; RV64-NEXT:    sd a6, 80(sp)
-; RV64-NEXT:    sd a7, 88(sp)
-; RV64-NEXT:    addi a0, sp, 40
-; RV64-NEXT:    sd a0, 8(sp)
-; RV64-NEXT:    ld a0, 8(sp)
+; RV64-NEXT:    addi sp, sp, -112
+; RV64-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd a1, 56(sp)
+; RV64-NEXT:    sd a2, 64(sp)
+; RV64-NEXT:    sd a3, 72(sp)
+; RV64-NEXT:    sd a4, 80(sp)
+; RV64-NEXT:    sd a5, 88(sp)
+; RV64-NEXT:    sd a6, 96(sp)
+; RV64-NEXT:    sd a7, 104(sp)
+; RV64-NEXT:    addi a0, sp, 56
+; RV64-NEXT:    sd a0, 16(sp)
+; RV64-NEXT:    ld a0, 16(sp)
 ; RV64-NEXT:    addi a0, a0, 7
-; RV64-NEXT:    andi a0, a0, -8
+; RV64-NEXT:    li s0, -8
+; RV64-NEXT:    and a0, a0, s0
 ; RV64-NEXT:    addi a1, a0, 8
+; RV64-NEXT:    sd a1, 16(sp)
+; RV64-NEXT:    ld a1, 16(sp)
+; RV64-NEXT:    ld s1, 0(a0)
 ; RV64-NEXT:    sd a1, 8(sp)
-; RV64-NEXT:    ld a1, 8(sp)
-; RV64-NEXT:    ld s0, 0(a0)
-; RV64-NEXT:    sd a1, 0(sp)
-; RV64-NEXT:    lw a0, 4(sp)
-; RV64-NEXT:    lwu a1, 0(sp)
+; RV64-NEXT:    lw a0, 12(sp)
+; RV64-NEXT:    lwu a1, 8(sp)
 ; RV64-NEXT:    slli a0, a0, 32
 ; RV64-NEXT:    or a0, a0, a1
 ; RV64-NEXT:    call notdead
-; RV64-NEXT:    ld a0, 8(sp)
+; RV64-NEXT:    ld a0, 16(sp)
 ; RV64-NEXT:    addi a0, a0, 7
-; RV64-NEXT:    andi a0, a0, -8
+; RV64-NEXT:    and a0, a0, s0
 ; RV64-NEXT:    addi a1, a0, 8
-; RV64-NEXT:    sd a1, 8(sp)
-; RV64-NEXT:    ld a1, 8(sp)
+; RV64-NEXT:    sd a1, 16(sp)
+; RV64-NEXT:    ld a1, 16(sp)
 ; RV64-NEXT:    ld a0, 0(a0)
 ; RV64-NEXT:    addi a1, a1, 7
-; RV64-NEXT:    andi a1, a1, -8
+; RV64-NEXT:    and a1, a1, s0
 ; RV64-NEXT:    addi a2, a1, 8
-; RV64-NEXT:    sd a2, 8(sp)
-; RV64-NEXT:    ld a2, 8(sp)
+; RV64-NEXT:    sd a2, 16(sp)
+; RV64-NEXT:    ld a2, 16(sp)
 ; RV64-NEXT:    ld a1, 0(a1)
 ; RV64-NEXT:    addi a2, a2, 7
 ; RV64-NEXT:    andi a2, a2, -8
 ; RV64-NEXT:    addi a3, a2, 8
-; RV64-NEXT:    sd a3, 8(sp)
+; RV64-NEXT:    sd a3, 16(sp)
 ; RV64-NEXT:    ld a2, 0(a2)
-; RV64-NEXT:    add a0, a0, s0
+; RV64-NEXT:    add a0, a0, s1
 ; RV64-NEXT:    add a1, a1, a2
 ; RV64-NEXT:    add a0, a0, a1
-; RV64-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
-; RV64-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
-; RV64-NEXT:    addi sp, sp, 96
+; RV64-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64-NEXT:    addi sp, sp, 112
 ; RV64-NEXT:    ret
 ;
 ; RV32-WITHFP-LABEL: va4_va_copy:
@@ -1314,6 +1322,7 @@ define iXLen @va4_va_copy(i32 %argno, ...) nounwind {
 ; RV32-WITHFP-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
 ; RV32-WITHFP-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
 ; RV32-WITHFP-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32-WITHFP-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32-WITHFP-NEXT:    addi s0, sp, 32
 ; RV32-WITHFP-NEXT:    sw a1, 4(s0)
 ; RV32-WITHFP-NEXT:    sw a2, 8(s0)
@@ -1323,41 +1332,43 @@ define iXLen @va4_va_copy(i32 %argno, ...) nounwind {
 ; RV32-WITHFP-NEXT:    sw a6, 24(s0)
 ; RV32-WITHFP-NEXT:    sw a7, 28(s0)
 ; RV32-WITHFP-NEXT:    addi a0, s0, 4
-; RV32-WITHFP-NEXT:    sw a0, -16(s0)
-; RV32-WITHFP-NEXT:    lw a0, -16(s0)
+; RV32-WITHFP-NEXT:    sw a0, -20(s0)
+; RV32-WITHFP-NEXT:    lw a0, -20(s0)
 ; RV32-WITHFP-NEXT:    addi a0, a0, 3
-; RV32-WITHFP-NEXT:    andi a0, a0, -4
+; RV32-WITHFP-NEXT:    li s1, -4
+; RV32-WITHFP-NEXT:    and a0, a0, s1
 ; RV32-WITHFP-NEXT:    addi a1, a0, 4
-; RV32-WITHFP-NEXT:    sw a1, -16(s0)
-; RV32-WITHFP-NEXT:    lw a1, -16(s0)
-; RV32-WITHFP-NEXT:    lw s1, 0(a0)
 ; RV32-WITHFP-NEXT:    sw a1, -20(s0)
-; RV32-WITHFP-NEXT:    lw a0, -20(s0)
+; RV32-WITHFP-NEXT:    lw a1, -20(s0)
+; RV32-WITHFP-NEXT:    lw s2, 0(a0)
+; RV32-WITHFP-NEXT:    sw a1, -24(s0)
+; RV32-WITHFP-NEXT:    lw a0, -24(s0)
 ; RV32-WITHFP-NEXT:    call notdead
-; RV32-WITHFP-NEXT:    lw a0, -16(s0)
+; RV32-WITHFP-NEXT:    lw a0, -20(s0)
 ; RV32-WITHFP-NEXT:    addi a0, a0, 3
-; RV32-WITHFP-NEXT:    andi a0, a0, -4
+; RV32-WITHFP-NEXT:    and a0, a0, s1
 ; RV32-WITHFP-NEXT:    addi a1, a0, 4
-; RV32-WITHFP-NEXT:    sw a1, -16(s0)
-; RV32-WITHFP-NEXT:    lw a1, -16(s0)
+; RV32-WITHFP-NEXT:    sw a1, -20(s0)
+; RV32-WITHFP-NEXT:    lw a1, -20(s0)
 ; RV32-WITHFP-NEXT:    lw a0, 0(a0)
 ; RV32-WITHFP-NEXT:    addi a1, a1, 3
-; RV32-WITHFP-NEXT:    andi a1, a1, -4
+; RV32-WITHFP-NEXT:    and a1, a1, s1
 ; RV32-WITHFP-NEXT:    addi a2, a1, 4
-; RV32-WITHFP-NEXT:    sw a2, -16(s0)
-; RV32-WITHFP-NEXT:    lw a2, -16(s0)
+; RV32-WITHFP-NEXT:    sw a2, -20(s0)
+; RV32-WITHFP-NEXT:    lw a2, -20(s0)
 ; RV32-WITHFP-NEXT:    lw a1, 0(a1)
 ; RV32-WITHFP-NEXT:    addi a2, a2, 3
 ; RV32-WITHFP-NEXT:    andi a2, a2, -4
 ; RV32-WITHFP-NEXT:    addi a3, a2, 4
-; RV32-WITHFP-NEXT:    sw a3, -16(s0)
+; RV32-WITHFP-NEXT:    sw a3, -20(s0)
 ; RV32-WITHFP-NEXT:    lw a2, 0(a2)
-; RV32-WITHFP-NEXT:    add a0, a0, s1
+; RV32-WITHFP-NEXT:    add a0, a0, s2
 ; RV32-WITHFP-NEXT:    add a1, a1, a2
 ; RV32-WITHFP-NEXT:    add a0, a0, a1
 ; RV32-WITHFP-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32-WITHFP-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32-WITHFP-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32-WITHFP-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
 ; RV32-WITHFP-NEXT:    addi sp, sp, 64
 ; RV32-WITHFP-NEXT:    ret
 ;
@@ -1367,6 +1378,7 @@ define iXLen @va4_va_copy(i32 %argno, ...) nounwind {
 ; RV64-WITHFP-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
 ; RV64-WITHFP-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; RV64-WITHFP-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64-WITHFP-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64-WITHFP-NEXT:    addi s0, sp, 48
 ; RV64-WITHFP-NEXT:    sd a1, 8(s0)
 ; RV64-WITHFP-NEXT:    sd a2, 16(s0)
@@ -1376,44 +1388,46 @@ define iXLen @va4_va_copy(i32 %argno, ...) nounwind {
 ; RV64-WITHFP-NEXT:    sd a6, 48(s0)
 ; RV64-WITHFP-NEXT:    sd a7, 56(s0)
 ; RV64-WITHFP-NEXT:    addi a0, s0, 8
-; RV64-WITHFP-NEXT:    sd a0, -32(s0)
-; RV64-WITHFP-NEXT:    ld a0, -32(s0)
+; RV64-WITHFP-NEXT:    sd a0, -40(s0)
+; RV64-WITHFP-NEXT:    ld a0, -40(s0)
 ; RV64-WITHFP-NEXT:    addi a0, a0, 7
-; RV64-WITHFP-NEXT:    andi a0, a0, -8
+; RV64-WITHFP-NEXT:    li s1, -8
+; RV64-WITHFP-NEXT:    and a0, a0, s1
 ; RV64-WITHFP-NEXT:    addi a1, a0, 8
-; RV64-WITHFP-NEXT:    sd a1, -32(s0)
-; RV64-WITHFP-NEXT:    ld a1, -32(s0)
-; RV64-WITHFP-NEXT:    ld s1, 0(a0)
 ; RV64-WITHFP-NEXT:    sd a1, -40(s0)
-; RV64-WITHFP-NEXT:    lw a0, -36(s0)
-; RV64-WITHFP-NEXT:    lwu a1, -40(s0)
+; RV64-WITHFP-NEXT:    ld a1, -40(s0)
+; RV64-WITHFP-NEXT:    ld s2, 0(a0)
+; RV64-WITHFP-NEXT:    sd a1, -48(s0)
+; RV64-WITHFP-NEXT:    lw a0, -44(s0)
+; RV64-WITHFP-NEXT:    lwu a1, -48(s0)
 ; RV64-WITHFP-NEXT:    slli a0, a0, 32
 ; RV64-WITHFP-NEXT:    or a0, a0, a1
 ; RV64-WITHFP-NEXT:    call notdead
-; RV64-WITHFP-NEXT:    ld a0, -32(s0)
+; RV64-WITHFP-NEXT:    ld a0, -40(s0)
 ; RV64-WITHFP-NEXT:    addi a0, a0, 7
-; RV64-WITHFP-NEXT:    andi a0, a0, -8
+; RV64-WITHFP-NEXT:    and a0, a0, s1
 ; RV64-WITHFP-NEXT:    addi a1, a0, 8
-; RV64-WITHFP-NEXT:    sd a1, -32(s0)
-; RV64-WITHFP-NEXT:    ld a1, -32(s0)
+; RV64-WITHFP-NEXT:    sd a1, -40(s0)
+; RV64-WITHFP-NEXT:    ld a1, -40(s0)
 ; RV64-WITHFP-NEXT:    ld a0, 0(a0)
 ; RV64-WITHFP-NEXT:    addi a1, a1, 7
-; RV64-WITHFP-NEXT:    andi a1, a1, -8
+; RV64-WITHFP-NEXT:    and a1, a1, s1
 ; RV64-WITHFP-NEXT:    addi a2, a1, 8
-; RV64-WITHFP-NEXT:    sd a2, -32(s0)
-; RV64-WITHFP-NEXT:    ld a2, -32(s0)
+; RV64-WITHFP-NEXT:    sd a2, -40(s0)
+; RV64-WITHFP-NEXT:    ld a2, -40(s0)
 ; RV64-WITHFP-NEXT:    ld a1, 0(a1)
 ; RV64-WITHFP-NEXT:    addi a2, a2, 7
 ; RV64-WITHFP-NEXT:    andi a2, a2, -8
 ; RV64-WITHFP-NEXT:    addi a3, a2, 8
-; RV64-WITHFP-NEXT:    sd a3, -32(s0)
+; RV64-WITHFP-NEXT:    sd a3, -40(s0)
 ; RV64-WITHFP-NEXT:    ld a2, 0(a2)
-; RV64-WITHFP-NEXT:    add a0, a0, s1
+; RV64-WITHFP-NEXT:    add a0, a0, s2
 ; RV64-WITHFP-NEXT:    add a1, a1, a2
 ; RV64-WITHFP-NEXT:    add a0, a0, a1
 ; RV64-WITHFP-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64-WITHFP-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
 ; RV64-WITHFP-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64-WITHFP-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
 ; RV64-WITHFP-NEXT:    addi sp, sp, 112
 ; RV64-WITHFP-NEXT:    ret
   %vargs = alloca ptr
@@ -1609,22 +1623,23 @@ define i32 @va_large_stack(ptr %fmt, ...) {
 ; RV64-NEXT:    add a0, sp, a0
 ; RV64-NEXT:    sd a4, 304(a0)
 ; RV64-NEXT:    lui a0, 24414
-; RV64-NEXT:    addiw a0, a0, 280
 ; RV64-NEXT:    add a0, sp, a0
-; RV64-NEXT:    sd a0, 8(sp)
-; RV64-NEXT:    lwu a0, 8(sp)
-; RV64-NEXT:    lw a1, 12(sp)
-; RV64-NEXT:    lui a2, 24414
-; RV64-NEXT:    add a2, sp, a2
-; RV64-NEXT:    sd a5, 312(a2)
+; RV64-NEXT:    sd a5, 312(a0)
+; RV64-NEXT:    addi a0, sp, 8
+; RV64-NEXT:    lui a1, 24414
+; RV64-NEXT:    addiw a1, a1, 280
+; RV64-NEXT:    add a1, sp, a1
+; RV64-NEXT:    sd a1, 8(sp)
+; RV64-NEXT:    lw a0, 4(a0)
+; RV64-NEXT:    lwu a1, 8(sp)
 ; RV64-NEXT:    lui a2, 24414
 ; RV64-NEXT:    add a2, sp, a2
 ; RV64-NEXT:    sd a6, 320(a2)
 ; RV64-NEXT:    lui a2, 24414
 ; RV64-NEXT:    add a2, sp, a2
 ; RV64-NEXT:    sd a7, 328(a2)
-; RV64-NEXT:    slli a1, a1, 32
-; RV64-NEXT:    or a0, a1, a0
+; RV64-NEXT:    slli a0, a0, 32
+; RV64-NEXT:    or a0, a0, a1
 ; RV64-NEXT:    addi a1, a0, 4
 ; RV64-NEXT:    srli a2, a1, 32
 ; RV64-NEXT:    sw a1, 8(sp)
diff --git a/llvm/test/CodeGen/RISCV/branch-relaxation.ll b/llvm/test/CodeGen/RISCV/branch-relaxation.ll
index 3d48dc9637eae..ec77d54da116d 100644
--- a/llvm/test/CodeGen/RISCV/branch-relaxation.ll
+++ b/llvm/test/CodeGen/RISCV/branch-relaxation.ll
@@ -824,10 +824,7 @@ define void @relax_jal_spill_32_adjust_spill_slot() {
 ; CHECK-RV32-NEXT:    #APP
 ; CHECK-RV32-NEXT:    # reg use t6
 ; CHECK-RV32-NEXT:    #NO_APP
-; CHECK-RV32-NEXT:    lui a0, 2
-; CHECK-RV32-NEXT:    sub sp, s0, a0
-; CHECK-RV32-NEXT:    addi a0, a0, -2032
-; CHECK-RV32-NEXT:    add sp, sp, a0
+; CHECK-RV32-NEXT:    addi sp, s0, -2032
 ; CHECK-RV32-NEXT:    lw ra, 2028(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    lw s0, 2024(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    lw s1, 2020(sp) # 4-byte Folded Reload
@@ -1073,10 +1070,7 @@ define void @relax_jal_spill_32_adjust_spill_slot() {
 ; CHECK-RV64-NEXT:    #APP
 ; CHECK-RV64-NEXT:    # reg use t6
 ; CHECK-RV64-NEXT:    #NO_APP
-; CHECK-RV64-NEXT:    lui a0, 2
-; CHECK-RV64-NEXT:    sub sp, s0, a0
-; CHECK-RV64-NEXT:    addiw a0, a0, -2032
-; CHECK-RV64-NEXT:    add sp, sp, a0
+; CHECK-RV64-NEXT:    addi sp, s0, -2032
 ; CHECK-RV64-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
 ; CHECK-RV64-NEXT:    ld s0, 2016(sp) # 8-byte Folded Reload
 ; CHECK-RV64-NEXT:    ld s1, 2008(sp) # 8-byte Folded Reload
@@ -2323,10 +2317,7 @@ define void @relax_jal_spill_64_adjust_spill_slot() {
 ; CHECK-RV32-NEXT:    #APP
 ; CHECK-RV32-NEXT:    # reg use t6
 ; CHECK-RV32-NEXT:    #NO_APP
-; CHECK-RV32-NEXT:    lui a0, 2
-; CHECK-RV32-NEXT:    sub sp, s0, a0
-; CHECK-RV32-NEXT:    addi a0, a0, -2032
-; CHECK-RV32-NEXT:    add sp, sp, a0
+; CHECK-RV32-NEXT:    addi sp, s0, -2032
 ; CHECK-RV32-NEXT:    lw ra, 2028(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    lw s0, 2024(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    lw s1, 2020(sp) # 4-byte Folded Reload
@@ -2560,10 +2551,7 @@ define void @relax_jal_spill_64_adjust_spill_slot() {
 ; CHECK-RV64-NEXT:    #APP
 ; CHECK-RV64-NEXT:    # reg use t6
 ; CHECK-RV64-NEXT:    #NO_APP
-; CHECK-RV64-NEXT:    lui a0, 2
-; CHECK-RV64-NEXT:    sub sp, s0, a0
-; CHECK-RV64-NEXT:    addiw a0, a0, -2032
-; CHECK-RV64-NEXT:    add sp, sp, a0
+; CHECK-RV64-NEXT:    addi sp, s0, -2032
 ; CHECK-RV64-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
 ; CHECK-RV64-NEXT:    ld s0, 2016(sp) # 8-byte Folded Reload
 ; CHECK-RV64-NEXT:    ld s1, 2008(sp) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/fastcc-without-f-reg.ll b/llvm/test/CodeGen/RISCV/fastcc-without-f-reg.ll
index de5bb8a30db16..0eefc34ad552a 100644
--- a/llvm/test/CodeGen/RISCV/fastcc-without-f-reg.ll
+++ b/llvm/test/CodeGen/RISCV/fastcc-without-f-reg.ll
@@ -23,26 +23,34 @@ define half @caller_half(half %x) nounwind {
 ;
 ; ZFINX32-LABEL: caller_half:
 ; ZFINX32:       # %bb.0: # %entry
+; ZFINX32-NEXT:    # kill: def $x10_w killed $x10_w def $x10
 ; ZFINX32-NEXT:    lui a1, 1048560
 ; ZFINX32-NEXT:    or a0, a0, a1
+; ZFINX32-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; ZFINX32-NEXT:    tail h
 ;
 ; ZFINX64-LABEL: caller_half:
 ; ZFINX64:       # %bb.0: # %entry
+; ZFINX64-NEXT:    # kill: def $x10_w killed $x10_w def $x10
 ; ZFINX64-NEXT:    lui a1, 1048560
 ; ZFINX64-NEXT:    or a0, a0, a1
+; ZFINX64-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; ZFINX64-NEXT:    tail h
 ;
 ; ZDINX32-LABEL: caller_half:
 ; ZDINX32:       # %bb.0: # %entry
+; ZDINX32-NEXT:    # kill: def $x10_w killed $x10_w def $x10
 ; ZDINX32-NEXT:    lui a1, 1048560
 ; ZDINX32-NEXT:    or a0, a0, a1
+; ZDINX32-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; ZDINX32-NEXT:    tail h
 ;
 ; ZDINX64-LABEL: caller_half:
 ; ZDINX64:       # %bb.0: # %entry
+; ZDINX64-NEXT:    # kill: def $x10_w killed $x10_w def $x10
 ; ZDINX64-NEXT:    lui a1, 1048560
 ; ZDINX64-NEXT:    or a0, a0, a1
+; ZDINX64-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; ZDINX64-NEXT:    tail h
 entry:
   %0 = tail call fastcc half @h(half %x)
@@ -60,26 +68,34 @@ define internal fastcc half @h(half %x) nounwind {
 ;
 ; ZFINX32-LABEL: h:
 ; ZFINX32:       # %bb.0:
+; ZFINX32-NEXT:    # kill: def $x10_w killed $x10_w def $x10
 ; ZFINX32-NEXT:    lui a1, 1048560
 ; ZFINX32-NEXT:    or a0, a0, a1
+; ZFINX32-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; ZFINX32-NEXT:    ret
 ;
 ; ZFINX64-LABEL: h:
 ; ZFINX64:       # %bb.0:
+; ZFINX64-NEXT:    # kill: def $x10_w killed $x10_w def $x10
 ; ZFINX64-NEXT:    lui a1, 1048560
 ; ZFINX64-NEXT:    or a0, a0, a1
+; ZFINX64-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; ZFINX64-NEXT:    ret
 ;
 ; ZDINX32-LABEL: h:
 ; ZDINX32:       # %bb.0:
+; ZDINX32-NEXT:    # kill: def $x10_w killed $x10_w def $x10
 ; ZDINX32-NEXT:    lui a1, 1048560
 ; ZDINX32-NEXT:    or a0, a0, a1
+; ZDINX32-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; ZDINX32-NEXT:    ret
 ;
 ; ZDINX64-LABEL: h:
 ; ZDINX64:       # %bb.0:
+; ZDINX64-NEXT:    # kill: def $x10_w killed $x10_w def $x10
 ; ZDINX64-NEXT:    lui a1, 1048560
 ; ZDINX64-NEXT:    or a0, a0, a1
+; ZDINX64-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; ZDINX64-NEXT:    ret
   ret half %x
 }
@@ -220,24 +236,28 @@ define fastcc half @callee_half_32(<32 x half> %A) nounwind {
 ; ZFINX32:       # %bb.0:
 ; ZFINX32-NEXT:    lui a1, 1048560
 ; ZFINX32-NEXT:    or a0, a0, a1
+; ZFINX32-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; ZFINX32-NEXT:    ret
 ;
 ; ZFINX64-LABEL: callee_half_32:
 ; ZFINX64:       # %bb.0:
 ; ZFINX64-NEXT:    lui a1, 1048560
 ; ZFINX64-NEXT:    or a0, a0, a1
+; ZFINX64-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; ZFINX64-NEXT:    ret
 ;
 ; ZDINX32-LABEL: callee_half_32:
 ; ZDINX32:       # %bb.0:
 ; ZDINX32-NEXT:    lui a1, 1048560
 ; ZDINX32-NEXT:    or a0, a0, a1
+; ZDINX32-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; ZDINX32-NEXT:    ret
 ;
 ; ZDINX64-LABEL: callee_half_32:
 ; ZDINX64:       # %bb.0:
 ; ZDINX64-NEXT:    lui a1, 1048560
 ; ZDINX64-NEXT:    or a0, a0, a1
+; ZDINX64-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; ZDINX64-NEXT:    ret
 	%B = extractelement <32 x half> %A, i32 0
 	ret half %B
@@ -492,8 +512,10 @@ define half @caller_half_32(<32 x half> %A) nounwind {
 ; ZFINX32-NEXT:    lw a3, 96(sp) # 4-byte Folded Reload
 ; ZFINX32-NEXT:    lw a4, 92(sp) # 4-byte Folded Reload
 ; ZFINX32-NEXT:    call callee_half_32
+; ZFINX32-NEXT:    # kill: def $x10_w killed $x10_w def $x10
 ; ZFINX32-NEXT:    lui a1, 1048560
 ; ZFINX32-NEXT:    or a0, a0, a1
+; ZFINX32-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; ZFINX32-NEXT:    lw ra, 156(sp) # 4-byte Folded Reload
 ; ZFINX32-NEXT:    lw s0, 152(sp) # 4-byte Folded Reload
 ; ZFINX32-NEXT:    lw s1, 148(sp) # 4-byte Folded Reload
@@ -588,8 +610,10 @@ define half @caller_half_32(<32 x half> %A) nounwind {
 ; ZFINX64-NEXT:    ld a3, 176(sp) # 8-byte Folded Reload
 ; ZFINX64-NEXT:    ld a4, 168(sp) # 8-byte Folded Reload
 ; ZFINX64-NEXT:    call callee_half_32
+; ZFINX64-NEXT:    # kill: def $x10_w killed $x10_w def $x10
 ; ZFINX64-NEXT:    lui a1, 1048560
 ; ZFINX64-NEXT:    or a0, a0, a1
+; ZFINX64-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; ZFINX64-NEXT:    ld ra, 296(sp) # 8-byte Folded Reload
 ; ZFINX64-NEXT:    ld s0, 288(sp) # 8-byte Folded Reload
 ; ZFINX64-NEXT:    ld s1, 280(sp) # 8-byte Folded Reload
@@ -684,8 +708,10 @@ define half @caller_half_32(<32 x half> %A) nounwind {
 ; ZDINX32-NEXT:    lw a3, 96(sp) # 4-byte Folded Reload
 ; ZDINX32-NEXT:    lw a4, 92(sp) # 4-byte Folded Reload
 ; ZDINX32-NEXT:    call callee_half_32
+; ZDINX32-NEXT:    # kill: def $x10_w killed $x10_w def $x10
 ; ZDINX32-NEXT:    lui a1, 1048560
 ; ZDINX32-NEXT:    or a0, a0, a1
+; ZDINX32-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; ZDINX32-NEXT:    lw ra, 156(sp) # 4-byte Folded Reload
 ; ZDINX32-NEXT:    lw s0, 152(sp) # 4-byte Folded Reload
 ; ZDINX32-NEXT:    lw s1, 148(sp) # 4-byte Folded Reload
@@ -780,8 +806,10 @@ define half @caller_half_32(<32 x half> %A) nounwind {
 ; ZDINX64-NEXT:    ld a3, 176(sp) # 8-byte Folded Reload
 ; ZDINX64-NEXT:    ld a4, 168(sp) # 8-byte Folded Reload
 ; ZDINX64-NEXT:    call callee_half_32
+; ZDINX64-NEXT:    # kill: def $x10_w killed $x10_w def $x10
 ; ZDINX64-NEXT:    lui a1, 1048560
 ; ZDINX64-NEXT:    or a0, a0, a1
+; ZDINX64-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; ZDINX64-NEXT:    ld ra, 296(sp) # 8-byte Folded Reload
 ; ZDINX64-NEXT:    ld s0, 288(sp) # 8-byte Folded Reload
 ; ZDINX64-NEXT:    ld s1, 280(sp) # 8-byte Folded Reload
@@ -917,79 +945,87 @@ define float @caller_float_32(<32 x float> %A) nounwind {
 ;
 ; ZHINX64-LABEL: caller_float_32:
 ; ZHINX64:       # %bb.0:
-; ZHINX64-NEXT:    addi sp, sp, -192
-; ZHINX64-NEXT:    sd ra, 184(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s0, 176(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s1, 168(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s2, 160(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s3, 152(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s4, 144(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s5, 136(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s6, 128(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s7, 120(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s8, 112(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s9, 104(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s10, 96(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s11, 88(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    addi sp, sp, -208
+; ZHINX64-NEXT:    sd ra, 200(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s0, 192(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s1, 184(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s2, 176(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s3, 168(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s4, 160(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s5, 152(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s6, 144(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s7, 136(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s8, 128(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s9, 120(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s10, 112(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s11, 104(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    lw t0, 208(sp)
+; ZHINX64-NEXT:    sw t0, 100(sp) # 4-byte Folded Spill
+; ZHINX64-NEXT:    lw t0, 216(sp)
+; ZHINX64-NEXT:    sw t0, 96(sp) # 4-byte Folded Spill
 ; ZHINX64-NEXT:    lw t0, 224(sp)
-; ZHINX64-NEXT:    lw t1, 232(sp)
-; ZHINX64-NEXT:    lw t2, 240(sp)
-; ZHINX64-NEXT:    lw s0, 248(sp)
-; ZHINX64-NEXT:    lw t3, 256(sp)
-; ZHINX64-NEXT:    lw t4, 264(sp)
-; ZHINX64-NEXT:    lw t5, 272(sp)
-; ZHINX64-NEXT:    lw t6, 280(sp)
-; ZHINX64-NEXT:    lw s1, 288(sp)
-; ZHINX64-NEXT:    lw s2, 296(sp)
-; ZHINX64-NEXT:    lw s3, 304(sp)
-; ZHINX64-NEXT:    lw s4, 312(sp)
-; ZHINX64-NEXT:    lw s5, 320(sp)
-; ZHINX64-NEXT:    lw s6, 328(sp)
-; ZHINX64-NEXT:    lw s7, 336(sp)
-; ZHINX64-NEXT:    lw s8, 344(sp)
-; ZHINX64-NEXT:    lw s9, 352(sp)
-; ZHINX64-NEXT:    lw s10, 360(sp)
-; ZHINX64-NEXT:    lw s11, 368(sp)
-; ZHINX64-NEXT:    lw ra, 376(sp)
-; ZHINX64-NEXT:    sw ra, 76(sp)
-; ZHINX64-NEXT:    sw s11, 72(sp)
-; ZHINX64-NEXT:    sw s10, 68(sp)
-; ZHINX64-NEXT:    sw s9, 64(sp)
-; ZHINX64-NEXT:    sw s8, 60(sp)
-; ZHINX64-NEXT:    sw s7, 56(sp)
-; ZHINX64-NEXT:    sw s6, 52(sp)
-; ZHINX64-NEXT:    sw s5, 48(sp)
-; ZHINX64-NEXT:    sw s4, 44(sp)
-; ZHINX64-NEXT:    sw s3, 40(sp)
-; ZHINX64-NEXT:    sw s2, 36(sp)
-; ZHINX64-NEXT:    sw s1, 32(sp)
-; ZHINX64-NEXT:    sw t6, 28(sp)
-; ZHINX64-NEXT:    sw t5, 24(sp)
-; ZHINX64-NEXT:    sw t4, 20(sp)
-; ZHINX64-NEXT:    sw t3, 16(sp)
-; ZHINX64-NEXT:    lw t3, 192(sp)
-; ZHINX64-NEXT:    lw t4, 200(sp)
-; ZHINX64-NEXT:    lw t5, 208(sp)
-; ZHINX64-NEXT:    lw t6, 216(sp)
+; ZHINX64-NEXT:    sw t0, 92(sp) # 4-byte Folded Spill
+; ZHINX64-NEXT:    lw t0, 232(sp)
+; ZHINX64-NEXT:    sw t0, 88(sp) # 4-byte Folded Spill
+; ZHINX64-NEXT:    lw t6, 240(sp)
+; ZHINX64-NEXT:    lw t5, 248(sp)
+; ZHINX64-NEXT:    lw t4, 256(sp)
+; ZHINX64-NEXT:    lw s0, 264(sp)
+; ZHINX64-NEXT:    lw s1, 272(sp)
+; ZHINX64-NEXT:    lw s2, 280(sp)
+; ZHINX64-NEXT:    lw s3, 288(sp)
+; ZHINX64-NEXT:    lw s4, 296(sp)
+; ZHINX64-NEXT:    lw s5, 304(sp)
+; ZHINX64-NEXT:    lw s6, 312(sp)
+; ZHINX64-NEXT:    lw s7, 320(sp)
+; ZHINX64-NEXT:    lw s8, 328(sp)
+; ZHINX64-NEXT:    lw s9, 336(sp)
+; ZHINX64-NEXT:    lw s10, 344(sp)
+; ZHINX64-NEXT:    lw s11, 352(sp)
+; ZHINX64-NEXT:    lw ra, 360(sp)
+; ZHINX64-NEXT:    lw t3, 368(sp)
+; ZHINX64-NEXT:    lw t2, 376(sp)
+; ZHINX64-NEXT:    lw t1, 384(sp)
+; ZHINX64-NEXT:    lw t0, 392(sp)
+; ZHINX64-NEXT:    sw t0, 76(sp)
+; ZHINX64-NEXT:    sw t1, 72(sp)
+; ZHINX64-NEXT:    sw t2, 68(sp)
+; ZHINX64-NEXT:    sw t3, 64(sp)
+; ZHINX64-NEXT:    sw ra, 60(sp)
+; ZHINX64-NEXT:    sw s11, 56(sp)
+; ZHINX64-NEXT:    sw s10, 52(sp)
+; ZHINX64-NEXT:    sw s9, 48(sp)
+; ZHINX64-NEXT:    sw s8, 44(sp)
+; ZHINX64-NEXT:    sw s7, 40(sp)
+; ZHINX64-NEXT:    sw s6, 36(sp)
+; ZHINX64-NEXT:    sw s5, 32(sp)
+; ZHINX64-NEXT:    sw s4, 28(sp)
+; ZHINX64-NEXT:    sw s3, 24(sp)
+; ZHINX64-NEXT:    sw s2, 20(sp)
+; ZHINX64-NEXT:    sw s1, 16(sp)
 ; ZHINX64-NEXT:    sw s0, 12(sp)
-; ZHINX64-NEXT:    sw t2, 8(sp)
-; ZHINX64-NEXT:    sw t1, 4(sp)
-; ZHINX64-NEXT:    sw t0, 0(sp)
+; ZHINX64-NEXT:    sw t4, 8(sp)
+; ZHINX64-NEXT:    sw t5, 4(sp)
+; ZHINX64-NEXT:    sw t6, 0(sp)
+; ZHINX64-NEXT:    lw t3, 100(sp) # 4-byte Folded Reload
+; ZHINX64-NEXT:    lw t4, 96(sp) # 4-byte Folded Reload
+; ZHINX64-NEXT:    lw t5, 92(sp) # 4-byte Folded Reload
+; ZHINX64-NEXT:    lw t6, 88(sp) # 4-byte Folded Reload
 ; ZHINX64-NEXT:    call callee_float_32
-; ZHINX64-NEXT:    ld ra, 184(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s0, 176(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s1, 168(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s2, 160(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s3, 152(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s4, 144(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s5, 136(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s6, 128(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s7, 120(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s8, 112(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s9, 104(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s10, 96(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s11, 88(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    addi sp, sp, 192
+; ZHINX64-NEXT:    ld ra, 200(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s0, 192(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s1, 184(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s2, 176(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s3, 168(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s4, 160(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s5, 152(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s6, 144(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s7, 136(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s8, 128(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s9, 120(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s10, 112(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s11, 104(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    addi sp, sp, 208
 ; ZHINX64-NEXT:    ret
 ;
 ; ZFINX32-LABEL: caller_float_32:
@@ -1079,79 +1115,87 @@ define float @caller_float_32(<32 x float> %A) nounwind {
 ;
 ; ZFINX64-LABEL: caller_float_32:
 ; ZFINX64:       # %bb.0:
-; ZFINX64-NEXT:    addi sp, sp, -192
-; ZFINX64-NEXT:    sd ra, 184(sp) # 8-byte Folded Spill
-; ZFINX64-NEXT:    sd s0, 176(sp) # 8-byte Folded Spill
-; ZFINX64-NEXT:    sd s1, 168(sp) # 8-byte Folded Spill
-; ZFINX64-NEXT:    sd s2, 160(sp) # 8-byte Folded Spill
-; ZFINX64-NEXT:    sd s3, 152(sp) # 8-byte Folded Spill
-; ZFINX64-NEXT:    sd s4, 144(sp) # 8-byte Folded Spill
-; ZFINX64-NEXT:    sd s5, 136(sp) # 8-byte Folded Spill
-; ZFINX64-NEXT:    sd s6, 128(sp) # 8-byte Folded Spill
-; ZFINX64-NEXT:    sd s7, 120(sp) # 8-byte Folded Spill
-; ZFINX64-NEXT:    sd s8, 112(sp) # 8-byte Folded Spill
-; ZFINX64-NEXT:    sd s9, 104(sp) # 8-byte Folded Spill
-; ZFINX64-NEXT:    sd s10, 96(sp) # 8-byte Folded Spill
-; ZFINX64-NEXT:    sd s11, 88(sp) # 8-byte Folded Spill
+; ZFINX64-NEXT:    addi sp, sp, -208
+; ZFINX64-NEXT:    sd ra, 200(sp) # 8-byte Folded Spill
+; ZFINX64-NEXT:    sd s0, 192(sp) # 8-byte Folded Spill
+; ZFINX64-NEXT:    sd s1, 184(sp) # 8-byte Folded Spill
+; ZFINX64-NEXT:    sd s2, 176(sp) # 8-byte Folded Spill
+; ZFINX64-NEXT:    sd s3, 168(sp) # 8-byte Folded Spill
+; ZFINX64-NEXT:    sd s4, 160(sp) # 8-byte Folded Spill
+; ZFINX64-NEXT:    sd s5, 152(sp) # 8-byte Folded Spill
+; ZFINX64-NEXT:    sd s6, 144(sp) # 8-byte Folded Spill
+; ZFINX64-NEXT:    sd s7, 136(sp) # 8-byte Folded Spill
+; ZFINX64-NEXT:    sd s8, 128(sp) # 8-byte Folded Spill
+; ZFINX64-NEXT:    sd s9, 120(sp) # 8-byte Folded Spill
+; ZFINX64-NEXT:    sd s10, 112(sp) # 8-byte Folded Spill
+; ZFINX64-NEXT:    sd s11, 104(sp) # 8-byte Folded Spill
+; ZFINX64-NEXT:    lw t0, 208(sp)
+; ZFINX64-NEXT:    sw t0, 100(sp) # 4-byte Folded Spill
+; ZFINX64-NEXT:    lw t0, 216(sp)
+; ZFINX64-NEXT:    sw t0, 96(sp) # 4-byte Folded Spill
 ; ZFINX64-NEXT:    lw t0, 224(sp)
-; ZFINX64-NEXT:    lw t1, 232(sp)
-; ZFINX64-NEXT:    lw t2, 240(sp)
-; ZFINX64-NEXT:    lw s0, 248(sp)
-; ZFINX64-NEXT:    lw t3, 256(sp)
-; ZFINX64-NEXT:    lw t4, 264(sp)
-; ZFINX64-NEXT:    lw t5, 272(sp)
-; ZFINX64-NEXT:    lw t6, 280(sp)
-; ZFINX64-NEXT:    lw s1, 288(sp)
-; ZFINX64-NEXT:    lw s2, 296(sp)
-; ZFINX64-NEXT:    lw s3, 304(sp)
-; ZFINX64-NEXT:    lw s4, 312(sp)
-; ZFINX64-NEXT:    lw s5, 320(sp)
-; ZFINX64-NEXT:    lw s6, 328(sp)
-; ZFINX64-NEXT:    lw s7, 336(sp)
-; ZFINX64-NEXT:    lw s8, 344(sp)
-; ZFINX64-NEXT:    lw s9, 352(sp)
-; ZFINX64-NEXT:    lw s10, 360(sp)
-; ZFINX64-NEXT:    lw s11, 368(sp)
-; ZFINX64-NEXT:    lw ra, 376(sp)
-; ZFINX64-NEXT:    sw ra, 76(sp)
-; ZFINX64-NEXT:    sw s11, 72(sp)
-; ZFINX64-NEXT:    sw s10, 68(sp)
-; ZFINX64-NEXT:    sw s9, 64(sp)
-; ZFINX64-NEXT:    sw s8, 60(sp)
-; ZFINX64-NEXT:    sw s7, 56(sp)
-; ZFINX64-NEXT:    sw s6, 52(sp)
-; ZFINX64-NEXT:    sw s5, 48(sp)
-; ZFINX64-NEXT:    sw s4, 44(sp)
-; ZFINX64-NEXT:    sw s3, 40(sp)
-; ZFINX64-NEXT:    sw s2, 36(sp)
-; ZFINX64-NEXT:    sw s1, 32(sp)
-; ZFINX64-NEXT:    sw t6, 28(sp)
-; ZFINX64-NEXT:    sw t5, 24(sp)
-; ZFINX64-NEXT:    sw t4, 20(sp)
-; ZFINX64-NEXT:    sw t3, 16(sp)
-; ZFINX64-NEXT:    lw t3, 192(sp)
-; ZFINX64-NEXT:    lw t4, 200(sp)
-; ZFINX64-NEXT:    lw t5, 208(sp)
-; ZFINX64-NEXT:    lw t6, 216(sp)
+; ZFINX64-NEXT:    sw t0, 92(sp) # 4-byte Folded Spill
+; ZFINX64-NEXT:    lw t0, 232(sp)
+; ZFINX64-NEXT:    sw t0, 88(sp) # 4-byte Folded Spill
+; ZFINX64-NEXT:    lw t6, 240(sp)
+; ZFINX64-NEXT:    lw t5, 248(sp)
+; ZFINX64-NEXT:    lw t4, 256(sp)
+; ZFINX64-NEXT:    lw s0, 264(sp)
+; ZFINX64-NEXT:    lw s1, 272(sp)
+; ZFINX64-NEXT:    lw s2, 280(sp)
+; ZFINX64-NEXT:    lw s3, 288(sp)
+; ZFINX64-NEXT:    lw s4, 296(sp)
+; ZFINX64-NEXT:    lw s5, 304(sp)
+; ZFINX64-NEXT:    lw s6, 312(sp)
+; ZFINX64-NEXT:    lw s7, 320(sp)
+; ZFINX64-NEXT:    lw s8, 328(sp)
+; ZFINX64-NEXT:    lw s9, 336(sp)
+; ZFINX64-NEXT:    lw s10, 344(sp)
+; ZFINX64-NEXT:    lw s11, 352(sp)
+; ZFINX64-NEXT:    lw ra, 360(sp)
+; ZFINX64-NEXT:    lw t3, 368(sp)
+; ZFINX64-NEXT:    lw t2, 376(sp)
+; ZFINX64-NEXT:    lw t1, 384(sp)
+; ZFINX64-NEXT:    lw t0, 392(sp)
+; ZFINX64-NEXT:    sw t0, 76(sp)
+; ZFINX64-NEXT:    sw t1, 72(sp)
+; ZFINX64-NEXT:    sw t2, 68(sp)
+; ZFINX64-NEXT:    sw t3, 64(sp)
+; ZFINX64-NEXT:    sw ra, 60(sp)
+; ZFINX64-NEXT:    sw s11, 56(sp)
+; ZFINX64-NEXT:    sw s10, 52(sp)
+; ZFINX64-NEXT:    sw s9, 48(sp)
+; ZFINX64-NEXT:    sw s8, 44(sp)
+; ZFINX64-NEXT:    sw s7, 40(sp)
+; ZFINX64-NEXT:    sw s6, 36(sp)
+; ZFINX64-NEXT:    sw s5, 32(sp)
+; ZFINX64-NEXT:    sw s4, 28(sp)
+; ZFINX64-NEXT:    sw s3, 24(sp)
+; ZFINX64-NEXT:    sw s2, 20(sp)
+; ZFINX64-NEXT:    sw s1, 16(sp)
 ; ZFINX64-NEXT:    sw s0, 12(sp)
-; ZFINX64-NEXT:    sw t2, 8(sp)
-; ZFINX64-NEXT:    sw t1, 4(sp)
-; ZFINX64-NEXT:    sw t0, 0(sp)
+; ZFINX64-NEXT:    sw t4, 8(sp)
+; ZFINX64-NEXT:    sw t5, 4(sp)
+; ZFINX64-NEXT:    sw t6, 0(sp)
+; ZFINX64-NEXT:    lw t3, 100(sp) # 4-byte Folded Reload
+; ZFINX64-NEXT:    lw t4, 96(sp) # 4-byte Folded Reload
+; ZFINX64-NEXT:    lw t5, 92(sp) # 4-byte Folded Reload
+; ZFINX64-NEXT:    lw t6, 88(sp) # 4-byte Folded Reload
 ; ZFINX64-NEXT:    call callee_float_32
-; ZFINX64-NEXT:    ld ra, 184(sp) # 8-byte Folded Reload
-; ZFINX64-NEXT:    ld s0, 176(sp) # 8-byte Folded Reload
-; ZFINX64-NEXT:    ld s1, 168(sp) # 8-byte Folded Reload
-; ZFINX64-NEXT:    ld s2, 160(sp) # 8-byte Folded Reload
-; ZFINX64-NEXT:    ld s3, 152(sp) # 8-byte Folded Reload
-; ZFINX64-NEXT:    ld s4, 144(sp) # 8-byte Folded Reload
-; ZFINX64-NEXT:    ld s5, 136(sp) # 8-byte Folded Reload
-; ZFINX64-NEXT:    ld s6, 128(sp) # 8-byte Folded Reload
-; ZFINX64-NEXT:    ld s7, 120(sp) # 8-byte Folded Reload
-; ZFINX64-NEXT:    ld s8, 112(sp) # 8-byte Folded Reload
-; ZFINX64-NEXT:    ld s9, 104(sp) # 8-byte Folded Reload
-; ZFINX64-NEXT:    ld s10, 96(sp) # 8-byte Folded Reload
-; ZFINX64-NEXT:    ld s11, 88(sp) # 8-byte Folded Reload
-; ZFINX64-NEXT:    addi sp, sp, 192
+; ZFINX64-NEXT:    ld ra, 200(sp) # 8-byte Folded Reload
+; ZFINX64-NEXT:    ld s0, 192(sp) # 8-byte Folded Reload
+; ZFINX64-NEXT:    ld s1, 184(sp) # 8-byte Folded Reload
+; ZFINX64-NEXT:    ld s2, 176(sp) # 8-byte Folded Reload
+; ZFINX64-NEXT:    ld s3, 168(sp) # 8-byte Folded Reload
+; ZFINX64-NEXT:    ld s4, 160(sp) # 8-byte Folded Reload
+; ZFINX64-NEXT:    ld s5, 152(sp) # 8-byte Folded Reload
+; ZFINX64-NEXT:    ld s6, 144(sp) # 8-byte Folded Reload
+; ZFINX64-NEXT:    ld s7, 136(sp) # 8-byte Folded Reload
+; ZFINX64-NEXT:    ld s8, 128(sp) # 8-byte Folded Reload
+; ZFINX64-NEXT:    ld s9, 120(sp) # 8-byte Folded Reload
+; ZFINX64-NEXT:    ld s10, 112(sp) # 8-byte Folded Reload
+; ZFINX64-NEXT:    ld s11, 104(sp) # 8-byte Folded Reload
+; ZFINX64-NEXT:    addi sp, sp, 208
 ; ZFINX64-NEXT:    ret
 ;
 ; ZDINX32-LABEL: caller_float_32:
@@ -1241,79 +1285,87 @@ define float @caller_float_32(<32 x float> %A) nounwind {
 ;
 ; ZDINX64-LABEL: caller_float_32:
 ; ZDINX64:       # %bb.0:
-; ZDINX64-NEXT:    addi sp, sp, -192
-; ZDINX64-NEXT:    sd ra, 184(sp) # 8-byte Folded Spill
-; ZDINX64-NEXT:    sd s0, 176(sp) # 8-byte Folded Spill
-; ZDINX64-NEXT:    sd s1, 168(sp) # 8-byte Folded Spill
-; ZDINX64-NEXT:    sd s2, 160(sp) # 8-byte Folded Spill
-; ZDINX64-NEXT:    sd s3, 152(sp) # 8-byte Folded Spill
-; ZDINX64-NEXT:    sd s4, 144(sp) # 8-byte Folded Spill
-; ZDINX64-NEXT:    sd s5, 136(sp) # 8-byte Folded Spill
-; ZDINX64-NEXT:    sd s6, 128(sp) # 8-byte Folded Spill
-; ZDINX64-NEXT:    sd s7, 120(sp) # 8-byte Folded Spill
-; ZDINX64-NEXT:    sd s8, 112(sp) # 8-byte Folded Spill
-; ZDINX64-NEXT:    sd s9, 104(sp) # 8-byte Folded Spill
-; ZDINX64-NEXT:    sd s10, 96(sp) # 8-byte Folded Spill
-; ZDINX64-NEXT:    sd s11, 88(sp) # 8-byte Folded Spill
+; ZDINX64-NEXT:    addi sp, sp, -208
+; ZDINX64-NEXT:    sd ra, 200(sp) # 8-byte Folded Spill
+; ZDINX64-NEXT:    sd s0, 192(sp) # 8-byte Folded Spill
+; ZDINX64-NEXT:    sd s1, 184(sp) # 8-byte Folded Spill
+; ZDINX64-NEXT:    sd s2, 176(sp) # 8-byte Folded Spill
+; ZDINX64-NEXT:    sd s3, 168(sp) # 8-byte Folded Spill
+; ZDINX64-NEXT:    sd s4, 160(sp) # 8-byte Folded Spill
+; ZDINX64-NEXT:    sd s5, 152(sp) # 8-byte Folded Spill
+; ZDINX64-NEXT:    sd s6, 144(sp) # 8-byte Folded Spill
+; ZDINX64-NEXT:    sd s7, 136(sp) # 8-byte Folded Spill
+; ZDINX64-NEXT:    sd s8, 128(sp) # 8-byte Folded Spill
+; ZDINX64-NEXT:    sd s9, 120(sp) # 8-byte Folded Spill
+; ZDINX64-NEXT:    sd s10, 112(sp) # 8-byte Folded Spill
+; ZDINX64-NEXT:    sd s11, 104(sp) # 8-byte Folded Spill
+; ZDINX64-NEXT:    lw t0, 208(sp)
+; ZDINX64-NEXT:    sw t0, 100(sp) # 4-byte Folded Spill
+; ZDINX64-NEXT:    lw t0, 216(sp)
+; ZDINX64-NEXT:    sw t0, 96(sp) # 4-byte Folded Spill
 ; ZDINX64-NEXT:    lw t0, 224(sp)
-; ZDINX64-NEXT:    lw t1, 232(sp)
-; ZDINX64-NEXT:    lw t2, 240(sp)
-; ZDINX64-NEXT:    lw s0, 248(sp)
-; ZDINX64-NEXT:    lw t3, 256(sp)
-; ZDINX64-NEXT:    lw t4, 264(sp)
-; ZDINX64-NEXT:    lw t5, 272(sp)
-; ZDINX64-NEXT:    lw t6, 280(sp)
-; ZDINX64-NEXT:    lw s1, 288(sp)
-; ZDINX64-NEXT:    lw s2, 296(sp)
-; ZDINX64-NEXT:    lw s3, 304(sp)
-; ZDINX64-NEXT:    lw s4, 312(sp)
-; ZDINX64-NEXT:    lw s5, 320(sp)
-; ZDINX64-NEXT:    lw s6, 328(sp)
-; ZDINX64-NEXT:    lw s7, 336(sp)
-; ZDINX64-NEXT:    lw s8, 344(sp)
-; ZDINX64-NEXT:    lw s9, 352(sp)
-; ZDINX64-NEXT:    lw s10, 360(sp)
-; ZDINX64-NEXT:    lw s11, 368(sp)
-; ZDINX64-NEXT:    lw ra, 376(sp)
-; ZDINX64-NEXT:    sw ra, 76(sp)
-; ZDINX64-NEXT:    sw s11, 72(sp)
-; ZDINX64-NEXT:    sw s10, 68(sp)
-; ZDINX64-NEXT:    sw s9, 64(sp)
-; ZDINX64-NEXT:    sw s8, 60(sp)
-; ZDINX64-NEXT:    sw s7, 56(sp)
-; ZDINX64-NEXT:    sw s6, 52(sp)
-; ZDINX64-NEXT:    sw s5, 48(sp)
-; ZDINX64-NEXT:    sw s4, 44(sp)
-; ZDINX64-NEXT:    sw s3, 40(sp)
-; ZDINX64-NEXT:    sw s2, 36(sp)
-; ZDINX64-NEXT:    sw s1, 32(sp)
-; ZDINX64-NEXT:    sw t6, 28(sp)
-; ZDINX64-NEXT:    sw t5, 24(sp)
-; ZDINX64-NEXT:    sw t4, 20(sp)
-; ZDINX64-NEXT:    sw t3, 16(sp)
-; ZDINX64-NEXT:    lw t3, 192(sp)
-; ZDINX64-NEXT:    lw t4, 200(sp)
-; ZDINX64-NEXT:    lw t5, 208(sp)
-; ZDINX64-NEXT:    lw t6, 216(sp)
+; ZDINX64-NEXT:    sw t0, 92(sp) # 4-byte Folded Spill
+; ZDINX64-NEXT:    lw t0, 232(sp)
+; ZDINX64-NEXT:    sw t0, 88(sp) # 4-byte Folded Spill
+; ZDINX64-NEXT:    lw t6, 240(sp)
+; ZDINX64-NEXT:    lw t5, 248(sp)
+; ZDINX64-NEXT:    lw t4, 256(sp)
+; ZDINX64-NEXT:    lw s0, 264(sp)
+; ZDINX64-NEXT:    lw s1, 272(sp)
+; ZDINX64-NEXT:    lw s2, 280(sp)
+; ZDINX64-NEXT:    lw s3, 288(sp)
+; ZDINX64-NEXT:    lw s4, 296(sp)
+; ZDINX64-NEXT:    lw s5, 304(sp)
+; ZDINX64-NEXT:    lw s6, 312(sp)
+; ZDINX64-NEXT:    lw s7, 320(sp)
+; ZDINX64-NEXT:    lw s8, 328(sp)
+; ZDINX64-NEXT:    lw s9, 336(sp)
+; ZDINX64-NEXT:    lw s10, 344(sp)
+; ZDINX64-NEXT:    lw s11, 352(sp)
+; ZDINX64-NEXT:    lw ra, 360(sp)
+; ZDINX64-NEXT:    lw t3, 368(sp)
+; ZDINX64-NEXT:    lw t2, 376(sp)
+; ZDINX64-NEXT:    lw t1, 384(sp)
+; ZDINX64-NEXT:    lw t0, 392(sp)
+; ZDINX64-NEXT:    sw t0, 76(sp)
+; ZDINX64-NEXT:    sw t1, 72(sp)
+; ZDINX64-NEXT:    sw t2, 68(sp)
+; ZDINX64-NEXT:    sw t3, 64(sp)
+; ZDINX64-NEXT:    sw ra, 60(sp)
+; ZDINX64-NEXT:    sw s11, 56(sp)
+; ZDINX64-NEXT:    sw s10, 52(sp)
+; ZDINX64-NEXT:    sw s9, 48(sp)
+; ZDINX64-NEXT:    sw s8, 44(sp)
+; ZDINX64-NEXT:    sw s7, 40(sp)
+; ZDINX64-NEXT:    sw s6, 36(sp)
+; ZDINX64-NEXT:    sw s5, 32(sp)
+; ZDINX64-NEXT:    sw s4, 28(sp)
+; ZDINX64-NEXT:    sw s3, 24(sp)
+; ZDINX64-NEXT:    sw s2, 20(sp)
+; ZDINX64-NEXT:    sw s1, 16(sp)
 ; ZDINX64-NEXT:    sw s0, 12(sp)
-; ZDINX64-NEXT:    sw t2, 8(sp)
-; ZDINX64-NEXT:    sw t1, 4(sp)
-; ZDINX64-NEXT:    sw t0, 0(sp)
+; ZDINX64-NEXT:    sw t4, 8(sp)
+; ZDINX64-NEXT:    sw t5, 4(sp)
+; ZDINX64-NEXT:    sw t6, 0(sp)
+; ZDINX64-NEXT:    lw t3, 100(sp) # 4-byte Folded Reload
+; ZDINX64-NEXT:    lw t4, 96(sp) # 4-byte Folded Reload
+; ZDINX64-NEXT:    lw t5, 92(sp) # 4-byte Folded Reload
+; ZDINX64-NEXT:    lw t6, 88(sp) # 4-byte Folded Reload
 ; ZDINX64-NEXT:    call callee_float_32
-; ZDINX64-NEXT:    ld ra, 184(sp) # 8-byte Folded Reload
-; ZDINX64-NEXT:    ld s0, 176(sp) # 8-byte Folded Reload
-; ZDINX64-NEXT:    ld s1, 168(sp) # 8-byte Folded Reload
-; ZDINX64-NEXT:    ld s2, 160(sp) # 8-byte Folded Reload
-; ZDINX64-NEXT:    ld s3, 152(sp) # 8-byte Folded Reload
-; ZDINX64-NEXT:    ld s4, 144(sp) # 8-byte Folded Reload
-; ZDINX64-NEXT:    ld s5, 136(sp) # 8-byte Folded Reload
-; ZDINX64-NEXT:    ld s6, 128(sp) # 8-byte Folded Reload
-; ZDINX64-NEXT:    ld s7, 120(sp) # 8-byte Folded Reload
-; ZDINX64-NEXT:    ld s8, 112(sp) # 8-byte Folded Reload
-; ZDINX64-NEXT:    ld s9, 104(sp) # 8-byte Folded Reload
-; ZDINX64-NEXT:    ld s10, 96(sp) # 8-byte Folded Reload
-; ZDINX64-NEXT:    ld s11, 88(sp) # 8-byte Folded Reload
-; ZDINX64-NEXT:    addi sp, sp, 192
+; ZDINX64-NEXT:    ld ra, 200(sp) # 8-byte Folded Reload
+; ZDINX64-NEXT:    ld s0, 192(sp) # 8-byte Folded Reload
+; ZDINX64-NEXT:    ld s1, 184(sp) # 8-byte Folded Reload
+; ZDINX64-NEXT:    ld s2, 176(sp) # 8-byte Folded Reload
+; ZDINX64-NEXT:    ld s3, 168(sp) # 8-byte Folded Reload
+; ZDINX64-NEXT:    ld s4, 160(sp) # 8-byte Folded Reload
+; ZDINX64-NEXT:    ld s5, 152(sp) # 8-byte Folded Reload
+; ZDINX64-NEXT:    ld s6, 144(sp) # 8-byte Folded Reload
+; ZDINX64-NEXT:    ld s7, 136(sp) # 8-byte Folded Reload
+; ZDINX64-NEXT:    ld s8, 128(sp) # 8-byte Folded Reload
+; ZDINX64-NEXT:    ld s9, 120(sp) # 8-byte Folded Reload
+; ZDINX64-NEXT:    ld s10, 112(sp) # 8-byte Folded Reload
+; ZDINX64-NEXT:    ld s11, 104(sp) # 8-byte Folded Reload
+; ZDINX64-NEXT:    addi sp, sp, 208
 ; ZDINX64-NEXT:    ret
 	%C = call fastcc float @callee_float_32(<32 x float> %A)
 	ret float %C
diff --git a/llvm/test/CodeGen/RISCV/float-arith.ll b/llvm/test/CodeGen/RISCV/float-arith.ll
index 3f32734db0ba7..bf500d1a2adb3 100644
--- a/llvm/test/CodeGen/RISCV/float-arith.ll
+++ b/llvm/test/CodeGen/RISCV/float-arith.ll
@@ -706,18 +706,11 @@ define float @fnmadd_s_3(float %a, float %b, float %c) nounwind {
 ; CHECKIF-NEXT:    fneg.s fa0, fa5
 ; CHECKIF-NEXT:    ret
 ;
-; RV32IZFINX-LABEL: fnmadd_s_3:
-; RV32IZFINX:       # %bb.0:
-; RV32IZFINX-NEXT:    fmadd.s a0, a0, a1, a2
-; RV32IZFINX-NEXT:    fneg.s a0, a0
-; RV32IZFINX-NEXT:    ret
-;
-; RV64IZFINX-LABEL: fnmadd_s_3:
-; RV64IZFINX:       # %bb.0:
-; RV64IZFINX-NEXT:    fmadd.s a0, a0, a1, a2
-; RV64IZFINX-NEXT:    lui a1, 524288
-; RV64IZFINX-NEXT:    xor a0, a0, a1
-; RV64IZFINX-NEXT:    ret
+; CHECKIZFINX-LABEL: fnmadd_s_3:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fmadd.s a0, a0, a1, a2
+; CHECKIZFINX-NEXT:    fneg.s a0, a0
+; CHECKIZFINX-NEXT:    ret
 ;
 ; RV32I-LABEL: fnmadd_s_3:
 ; RV32I:       # %bb.0:
@@ -761,17 +754,10 @@ define float @fnmadd_nsz(float %a, float %b, float %c) nounwind {
 ; CHECKIF-NEXT:    fnmadd.s fa0, fa0, fa1, fa2
 ; CHECKIF-NEXT:    ret
 ;
-; RV32IZFINX-LABEL: fnmadd_nsz:
-; RV32IZFINX:       # %bb.0:
-; RV32IZFINX-NEXT:    fnmadd.s a0, a0, a1, a2
-; RV32IZFINX-NEXT:    ret
-;
-; RV64IZFINX-LABEL: fnmadd_nsz:
-; RV64IZFINX:       # %bb.0:
-; RV64IZFINX-NEXT:    fmadd.s a0, a0, a1, a2
-; RV64IZFINX-NEXT:    lui a1, 524288
-; RV64IZFINX-NEXT:    xor a0, a0, a1
-; RV64IZFINX-NEXT:    ret
+; CHECKIZFINX-LABEL: fnmadd_nsz:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fnmadd.s a0, a0, a1, a2
+; CHECKIZFINX-NEXT:    ret
 ;
 ; RV32I-LABEL: fnmadd_nsz:
 ; RV32I:       # %bb.0:
@@ -1247,3 +1233,6 @@ define float @fsgnjx_f32(float %x, float %y) nounwind {
   %mul = fmul float %z, %y
   ret float %mul
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; RV32IZFINX: {{.*}}
+; RV64IZFINX: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/float-bitmanip-dagcombines.ll b/llvm/test/CodeGen/RISCV/float-bitmanip-dagcombines.ll
index 2338219687ef7..86f6f079243c2 100644
--- a/llvm/test/CodeGen/RISCV/float-bitmanip-dagcombines.ll
+++ b/llvm/test/CodeGen/RISCV/float-bitmanip-dagcombines.ll
@@ -50,8 +50,7 @@ define float @fneg(float %a) nounwind {
 ;
 ; RV64IZFINX-LABEL: fneg:
 ; RV64IZFINX:       # %bb.0:
-; RV64IZFINX-NEXT:    lui a1, 524288
-; RV64IZFINX-NEXT:    xor a0, a0, a1
+; RV64IZFINX-NEXT:    fneg.s a0, a0
 ; RV64IZFINX-NEXT:    ret
   %1 = fneg float %a
   ret float %1
@@ -91,8 +90,7 @@ define float @fabs(float %a) nounwind {
 ;
 ; RV64IZFINX-LABEL: fabs:
 ; RV64IZFINX:       # %bb.0:
-; RV64IZFINX-NEXT:    slli a0, a0, 33
-; RV64IZFINX-NEXT:    srli a0, a0, 33
+; RV64IZFINX-NEXT:    fabs.s a0, a0
 ; RV64IZFINX-NEXT:    ret
   %1 = call float @llvm.fabs.f32(float %a)
   ret float %1
diff --git a/llvm/test/CodeGen/RISCV/float-frem.ll b/llvm/test/CodeGen/RISCV/float-frem.ll
index 651b1b116adc7..31d39a5ab6d6e 100644
--- a/llvm/test/CodeGen/RISCV/float-frem.ll
+++ b/llvm/test/CodeGen/RISCV/float-frem.ll
@@ -27,12 +27,7 @@ define float @frem_f32(float %a, float %b) nounwind {
 ;
 ; RV64IZFINX-LABEL: frem_f32:
 ; RV64IZFINX:       # %bb.0:
-; RV64IZFINX-NEXT:    addi sp, sp, -16
-; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZFINX-NEXT:    call fmodf
-; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IZFINX-NEXT:    addi sp, sp, 16
-; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:    tail fmodf
 ;
 ; RV32I-LABEL: frem_f32:
 ; RV32I:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/float-imm.ll b/llvm/test/CodeGen/RISCV/float-imm.ll
index 69a506cd850f2..58cbc72e2197c 100644
--- a/llvm/test/CodeGen/RISCV/float-imm.ll
+++ b/llvm/test/CodeGen/RISCV/float-imm.ll
@@ -20,12 +20,14 @@ define float @float_imm() nounwind {
 ; RV32ZFINX:       # %bb.0:
 ; RV32ZFINX-NEXT:    lui a0, 263313
 ; RV32ZFINX-NEXT:    addi a0, a0, -37
+; RV32ZFINX-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; RV32ZFINX-NEXT:    ret
 ;
 ; RV64ZFINX-LABEL: float_imm:
 ; RV64ZFINX:       # %bb.0:
 ; RV64ZFINX-NEXT:    lui a0, 263313
 ; RV64ZFINX-NEXT:    addiw a0, a0, -37
+; RV64ZFINX-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; RV64ZFINX-NEXT:    ret
   ret float 3.14159274101257324218750
 }
diff --git a/llvm/test/CodeGen/RISCV/float-intrinsics.ll b/llvm/test/CodeGen/RISCV/float-intrinsics.ll
index 52442026dab50..b05eac9c9dee2 100644
--- a/llvm/test/CodeGen/RISCV/float-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/float-intrinsics.ll
@@ -136,12 +136,7 @@ define float @sin_f32(float %a) nounwind {
 ;
 ; RV64IZFINX-LABEL: sin_f32:
 ; RV64IZFINX:       # %bb.0:
-; RV64IZFINX-NEXT:    addi sp, sp, -16
-; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZFINX-NEXT:    call sinf
-; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IZFINX-NEXT:    addi sp, sp, 16
-; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:    tail sinf
 ;
 ; RV32I-LABEL: sin_f32:
 ; RV32I:       # %bb.0:
@@ -181,12 +176,7 @@ define float @cos_f32(float %a) nounwind {
 ;
 ; RV64IZFINX-LABEL: cos_f32:
 ; RV64IZFINX:       # %bb.0:
-; RV64IZFINX-NEXT:    addi sp, sp, -16
-; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZFINX-NEXT:    call cosf
-; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IZFINX-NEXT:    addi sp, sp, 16
-; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:    tail cosf
 ;
 ; RV32I-LABEL: cos_f32:
 ; RV32I:       # %bb.0:
@@ -327,12 +317,7 @@ define float @pow_f32(float %a, float %b) nounwind {
 ;
 ; RV64IZFINX-LABEL: pow_f32:
 ; RV64IZFINX:       # %bb.0:
-; RV64IZFINX-NEXT:    addi sp, sp, -16
-; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZFINX-NEXT:    call powf
-; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IZFINX-NEXT:    addi sp, sp, 16
-; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:    tail powf
 ;
 ; RV32I-LABEL: pow_f32:
 ; RV32I:       # %bb.0:
@@ -372,12 +357,7 @@ define float @exp_f32(float %a) nounwind {
 ;
 ; RV64IZFINX-LABEL: exp_f32:
 ; RV64IZFINX:       # %bb.0:
-; RV64IZFINX-NEXT:    addi sp, sp, -16
-; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZFINX-NEXT:    call expf
-; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IZFINX-NEXT:    addi sp, sp, 16
-; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:    tail expf
 ;
 ; RV32I-LABEL: exp_f32:
 ; RV32I:       # %bb.0:
@@ -417,12 +397,7 @@ define float @exp2_f32(float %a) nounwind {
 ;
 ; RV64IZFINX-LABEL: exp2_f32:
 ; RV64IZFINX:       # %bb.0:
-; RV64IZFINX-NEXT:    addi sp, sp, -16
-; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZFINX-NEXT:    call exp2f
-; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IZFINX-NEXT:    addi sp, sp, 16
-; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:    tail exp2f
 ;
 ; RV32I-LABEL: exp2_f32:
 ; RV32I:       # %bb.0:
@@ -462,12 +437,7 @@ define float @log_f32(float %a) nounwind {
 ;
 ; RV64IZFINX-LABEL: log_f32:
 ; RV64IZFINX:       # %bb.0:
-; RV64IZFINX-NEXT:    addi sp, sp, -16
-; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZFINX-NEXT:    call logf
-; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IZFINX-NEXT:    addi sp, sp, 16
-; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:    tail logf
 ;
 ; RV32I-LABEL: log_f32:
 ; RV32I:       # %bb.0:
@@ -507,12 +477,7 @@ define float @log10_f32(float %a) nounwind {
 ;
 ; RV64IZFINX-LABEL: log10_f32:
 ; RV64IZFINX:       # %bb.0:
-; RV64IZFINX-NEXT:    addi sp, sp, -16
-; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZFINX-NEXT:    call log10f
-; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IZFINX-NEXT:    addi sp, sp, 16
-; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:    tail log10f
 ;
 ; RV32I-LABEL: log10_f32:
 ; RV32I:       # %bb.0:
@@ -552,12 +517,7 @@ define float @log2_f32(float %a) nounwind {
 ;
 ; RV64IZFINX-LABEL: log2_f32:
 ; RV64IZFINX:       # %bb.0:
-; RV64IZFINX-NEXT:    addi sp, sp, -16
-; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZFINX-NEXT:    call log2f
-; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IZFINX-NEXT:    addi sp, sp, 16
-; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:    tail log2f
 ;
 ; RV32I-LABEL: log2_f32:
 ; RV32I:       # %bb.0:
@@ -698,8 +658,7 @@ define float @fabs_f32(float %a) nounwind {
 ;
 ; RV64IZFINX-LABEL: fabs_f32:
 ; RV64IZFINX:       # %bb.0:
-; RV64IZFINX-NEXT:    slli a0, a0, 33
-; RV64IZFINX-NEXT:    srli a0, a0, 33
+; RV64IZFINX-NEXT:    fabs.s a0, a0
 ; RV64IZFINX-NEXT:    ret
 ;
 ; RV32I-LABEL: fabs_f32:
@@ -1195,12 +1154,7 @@ define float @nearbyint_f32(float %a) nounwind {
 ;
 ; RV64IZFINX-LABEL: nearbyint_f32:
 ; RV64IZFINX:       # %bb.0:
-; RV64IZFINX-NEXT:    addi sp, sp, -16
-; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZFINX-NEXT:    call nearbyintf
-; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IZFINX-NEXT:    addi sp, sp, 16
-; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:    tail nearbyintf
 ;
 ; RV32I-LABEL: nearbyint_f32:
 ; RV32I:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/half-intrinsics.ll b/llvm/test/CodeGen/RISCV/half-intrinsics.ll
index 81e29329e7181..18cdb18106f34 100644
--- a/llvm/test/CodeGen/RISCV/half-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/half-intrinsics.ll
@@ -153,8 +153,8 @@ define half @powi_f16(half %a, i32 %b) nounwind {
 ; RV64IZHINX:       # %bb.0:
 ; RV64IZHINX-NEXT:    addi sp, sp, -16
 ; RV64IZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZHINX-NEXT:    sext.w a1, a1
 ; RV64IZHINX-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINX-NEXT:    sext.w a1, a1
 ; RV64IZHINX-NEXT:    call __powisf2
 ; RV64IZHINX-NEXT:    fcvt.h.s a0, a0
 ; RV64IZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
@@ -233,8 +233,8 @@ define half @powi_f16(half %a, i32 %b) nounwind {
 ; RV64IZHINXMIN:       # %bb.0:
 ; RV64IZHINXMIN-NEXT:    addi sp, sp, -16
 ; RV64IZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZHINXMIN-NEXT:    sext.w a1, a1
 ; RV64IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINXMIN-NEXT:    sext.w a1, a1
 ; RV64IZHINXMIN-NEXT:    call __powisf2
 ; RV64IZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; RV64IZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/instruction-count-remark.mir b/llvm/test/CodeGen/RISCV/instruction-count-remark.mir
new file mode 100644
index 0000000000000..4f429ab5274e8
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/instruction-count-remark.mir
@@ -0,0 +1,75 @@
+# RUN: llc -mtriple=riscv32 -verify-machineinstrs -start-before=riscv-expand-pseudo -simplify-mir -o /dev/null -pass-remarks-analysis=asm-printer %s 2>&1 | FileCheck %s
+---
+name: instrs
+tracksRegLiveness: true
+body: |
+  bb.0:
+    $x0 = ADDI $x0, 0
+    $x0 = ADDI $x0, 0
+    $x0 = ADDI $x0, 0
+    $x0 = LW $x0, 0
+    $x0 = LW $x0, 0
+    $x0 = XORI $x0, 0
+    ; CHECK: addi : 3
+    ; CHECK-NEXT: lw : 2
+    ; CHECK-NEXT: xori : 1
+    ; CHECK: 6 instructions in function
+...
+---
+name: bundles
+tracksRegLiveness: true
+body: |
+  bb.0:
+    $x0 = ADDI $x0, 0
+    BUNDLE {
+    $x0 = ADDI $x0, 0
+    $x0 = ADDI $x0, 0
+    $x0 = LW $x0, 0
+    }
+    $x0 = LW $x0, 0
+    $x0 = XORI $x0, 0
+    ; CHECK: addi : 3
+    ; CHECK-NEXT: lw : 2
+    ; CHECK-NEXT: xori : 1
+    ; CHECK: 6 instructions in function
+...
+---
+name: metainstrs
+tracksRegLiveness: true
+body: |
+  bb.0:
+    $x0 = ADDI $x0, 0
+    $x0 = ADDI $x0, 0
+    $x0 = ADDI $x0, 0
+    $x0 = IMPLICIT_DEF
+    $x0 = LW $x0, 0
+    $x0 = LW $x0, 0
+    CFI_INSTRUCTION adjust_cfa_offset 4
+    $x0 = XORI $x0, 0
+    DBG_VALUE $x0, 0
+    ; CHECK: addi : 3
+    ; CHECK-NEXT: lw : 2
+    ; CHECK-NEXT: xori : 1
+    ; CHECK: 6 instructions in function
+...
+---
+name: metabundles
+tracksRegLiveness: true
+body: |
+  bb.0:
+    $x0 = ADDI $x0, 0
+    BUNDLE {
+    CFI_INSTRUCTION adjust_cfa_offset 4
+    $x0 = ADDI $x0, 0
+    $x0 = ADDI $x0, 0
+    DBG_VALUE $x0, 0
+    $x0 = LW $x0, 0
+    }
+    $x0 = LW $x0, 0
+    $x0 = IMPLICIT_DEF
+    $x0 = XORI $x0, 0
+    ; CHECK: addi : 3
+    ; CHECK-NEXT: lw : 2
+    ; CHECK-NEXT: xori : 1
+    ; CHECK: 6 instructions in function
+...
diff --git a/llvm/test/CodeGen/RISCV/kcfi-mir.ll b/llvm/test/CodeGen/RISCV/kcfi-mir.ll
index e478930d59abc..f35be0564cb25 100644
--- a/llvm/test/CodeGen/RISCV/kcfi-mir.ll
+++ b/llvm/test/CodeGen/RISCV/kcfi-mir.ll
@@ -10,7 +10,7 @@ define void @f1(ptr noundef %x) !kcfi_type !1 {
   ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION def_cfa_offset 16
   ; CHECK-NEXT:   SD killed $x1, $x2, 8 :: (store (s64) into %stack.0)
   ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x1, -8
-  ; CHECK-NEXT:   BUNDLE implicit-def $x6, implicit-def $x6_h, implicit-def $x7, implicit-def $x7_h, implicit-def $x28, implicit-def $x28_h, implicit-def $x29, implicit-def $x29_h, implicit-def $x30, implicit-def $x30_h, implicit-def $x31, implicit-def $x31_h, implicit-def dead $x1, implicit-def $x2, implicit-def $x2_h, implicit killed $x10 {
+  ; CHECK-NEXT:   BUNDLE implicit-def $x6, implicit-def $x6_w, implicit-def $x6_h, implicit-def $x7, implicit-def $x7_w, implicit-def $x7_h, implicit-def $x28, implicit-def $x28_w, implicit-def $x28_h, implicit-def $x29, implicit-def $x29_w, implicit-def $x29_h, implicit-def $x30, implicit-def $x30_w, implicit-def $x30_h, implicit-def $x31, implicit-def $x31_w, implicit-def $x31_h, implicit-def dead $x1, implicit-def $x2, implicit-def $x2_w, implicit-def $x2_h, implicit killed $x10 {
   ; CHECK-NEXT:     KCFI_CHECK $x10, 12345678, implicit-def $x6, implicit-def $x7, implicit-def $x28, implicit-def $x29, implicit-def $x30, implicit-def $x31
   ; CHECK-NEXT:     PseudoCALLIndirect killed $x10, csr_ilp32_lp64, implicit-def dead $x1, implicit-def $x2
   ; CHECK-NEXT:   }
@@ -26,7 +26,7 @@ define void @f2(ptr noundef %x) #0 {
   ; CHECK: bb.0 (%ir-block.0):
   ; CHECK-NEXT:   liveins: $x10
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   BUNDLE implicit-def $x6, implicit-def $x6_h, implicit-def $x7, implicit-def $x7_h, implicit-def $x28, implicit-def $x28_h, implicit-def $x29, implicit-def $x29_h, implicit-def $x30, implicit-def $x30_h, implicit-def $x31, implicit-def $x31_h, implicit killed $x10, implicit $x2 {
+  ; CHECK-NEXT:   BUNDLE implicit-def $x6, implicit-def $x6_w, implicit-def $x6_h, implicit-def $x7, implicit-def $x7_w, implicit-def $x7_h, implicit-def $x28, implicit-def $x28_w, implicit-def $x28_h, implicit-def $x29, implicit-def $x29_w, implicit-def $x29_h, implicit-def $x30, implicit-def $x30_w, implicit-def $x30_h, implicit-def $x31, implicit-def $x31_w, implicit-def $x31_h, implicit killed $x10, implicit $x2 {
   ; CHECK-NEXT:     KCFI_CHECK $x10, 12345678, implicit-def $x6, implicit-def $x7, implicit-def $x28, implicit-def $x29, implicit-def $x30, implicit-def $x31
   ; CHECK-NEXT:     PseudoTAILIndirect killed $x10, implicit $x2
   ; CHECK-NEXT:   }
diff --git a/llvm/test/CodeGen/RISCV/llvm.frexp.ll b/llvm/test/CodeGen/RISCV/llvm.frexp.ll
index 442b0cf5b4a85..2c9d640e03a63 100644
--- a/llvm/test/CodeGen/RISCV/llvm.frexp.ll
+++ b/llvm/test/CodeGen/RISCV/llvm.frexp.ll
@@ -62,8 +62,10 @@ define { half, i32 } @test_frexp_f16_i32(half %a) nounwind {
 ; RV32IZFINXZDINX-NEXT:    call frexpf
 ; RV32IZFINXZDINX-NEXT:    call __truncsfhf2
 ; RV32IZFINXZDINX-NEXT:    lw a1, 8(sp)
+; RV32IZFINXZDINX-NEXT:    # kill: def $x10_w killed $x10_w def $x10
 ; RV32IZFINXZDINX-NEXT:    lui a2, 1048560
 ; RV32IZFINXZDINX-NEXT:    or a0, a0, a2
+; RV32IZFINXZDINX-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
 ; RV32IZFINXZDINX-NEXT:    ret
@@ -77,8 +79,10 @@ define { half, i32 } @test_frexp_f16_i32(half %a) nounwind {
 ; RV64IZFINXZDINX-NEXT:    call frexpf
 ; RV64IZFINXZDINX-NEXT:    call __truncsfhf2
 ; RV64IZFINXZDINX-NEXT:    ld a1, 0(sp)
+; RV64IZFINXZDINX-NEXT:    # kill: def $x10_w killed $x10_w def $x10
 ; RV64IZFINXZDINX-NEXT:    lui a2, 1048560
 ; RV64IZFINXZDINX-NEXT:    or a0, a0, a2
+; RV64IZFINXZDINX-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
 ; RV64IZFINXZDINX-NEXT:    ret
@@ -157,8 +161,10 @@ define half @test_frexp_f16_i32_only_use_fract(half %a) nounwind {
 ; RV32IZFINXZDINX-NEXT:    addi a1, sp, 8
 ; RV32IZFINXZDINX-NEXT:    call frexpf
 ; RV32IZFINXZDINX-NEXT:    call __truncsfhf2
+; RV32IZFINXZDINX-NEXT:    # kill: def $x10_w killed $x10_w def $x10
 ; RV32IZFINXZDINX-NEXT:    lui a1, 1048560
 ; RV32IZFINXZDINX-NEXT:    or a0, a0, a1
+; RV32IZFINXZDINX-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
 ; RV32IZFINXZDINX-NEXT:    ret
@@ -171,8 +177,10 @@ define half @test_frexp_f16_i32_only_use_fract(half %a) nounwind {
 ; RV64IZFINXZDINX-NEXT:    mv a1, sp
 ; RV64IZFINXZDINX-NEXT:    call frexpf
 ; RV64IZFINXZDINX-NEXT:    call __truncsfhf2
+; RV64IZFINXZDINX-NEXT:    # kill: def $x10_w killed $x10_w def $x10
 ; RV64IZFINXZDINX-NEXT:    lui a1, 1048560
 ; RV64IZFINXZDINX-NEXT:    or a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    # kill: def $x10_w killed $x10_w killed $x10
 ; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
 ; RV64IZFINXZDINX-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/make-compressible-zfinx.mir b/llvm/test/CodeGen/RISCV/make-compressible-zfinx.mir
new file mode 100644
index 0000000000000..d0223dc5911ad
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/make-compressible-zfinx.mir
@@ -0,0 +1,296 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -o - %s -mtriple=riscv32 -mattr=+c,+zfinx -simplify-mir \
+# RUN:   -run-pass=riscv-make-compressible | FileCheck --check-prefixes=CHECK %s
+# RUN: llc -o - %s -mtriple=riscv64 -mattr=+c,+zfinx -simplify-mir \
+# RUN:   -run-pass=riscv-make-compressible | FileCheck --check-prefixes=CHECK %s
+
+--- |
+
+  define void @store_common_value_float(ptr %a, ptr %b, ptr %c, float %d, float %e, float %f, float %g, float %h, float %i, float %j) #0 {
+  entry:
+    store float %j, ptr %a, align 4
+    store float %j, ptr %b, align 4
+    store float %j, ptr %c, align 4
+    ret void
+  }
+
+  define void @store_common_ptr_float(float %a, float %b, float %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, ptr %p) #0 {
+  entry:
+    store volatile float %a, ptr %p, align 4
+    store volatile float %b, ptr %p, align 4
+    store volatile float %c, ptr %p, align 4
+    ret void
+  }
+
+  define void @load_common_ptr_float(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, ptr %g) #0 {
+  entry:
+    %0 = load float, ptr %g, align 4
+    %arrayidx1 = getelementptr inbounds float, ptr %g, i32 1
+    %1 = load float, ptr %arrayidx1, align 4
+    %arrayidx2 = getelementptr inbounds float, ptr %g, i32 2
+    %2 = load float, ptr %arrayidx2, align 4
+    tail call void @load_common_ptr_float_1(float %0, float %1, float %2)
+    ret void
+  }
+
+  declare void @load_common_ptr_float_1(float, float, float) #0
+
+  define void @store_large_offset_float(ptr %p, float %a, float %b, float %c, float %d) #0 {
+  entry:
+    %0 = getelementptr inbounds float, ptr %p, i32 100
+    store volatile float %a, ptr %0, align 4
+    %1 = getelementptr inbounds float, ptr %p, i32 101
+    store volatile float %b, ptr %1, align 4
+    %2 = getelementptr inbounds float, ptr %p, i32 102
+    store volatile float %c, ptr %2, align 4
+    %3 = getelementptr inbounds float, ptr %p, i32 103
+    store volatile float %d, ptr %3, align 4
+    ret void
+  }
+
+  define void @load_large_offset_float(ptr %p) #0 {
+  entry:
+    %arrayidx = getelementptr inbounds float, ptr %p, i32 100
+    %0 = load float, ptr %arrayidx, align 4
+    %arrayidx1 = getelementptr inbounds float, ptr %p, i32 101
+    %1 = load float, ptr %arrayidx1, align 4
+    %arrayidx2 = getelementptr inbounds float, ptr %p, i32 102
+    %2 = load float, ptr %arrayidx2, align 4
+    tail call void @load_large_offset_float_1(float %0, float %1, float %2)
+    ret void
+  }
+
+  declare void @load_large_offset_float_1(float, float, float) #0
+
+  define void @store_common_value_float_no_opt(ptr %a, float %b, float %c, float %d, float %e, float %f, float %g, float %h) #0 {
+  entry:
+    store float %h, ptr %a, align 4
+    ret void
+  }
+
+  define void @store_common_ptr_float_no_opt(float %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, ptr %p) #0 {
+  entry:
+    store volatile float %a, ptr %p, align 4
+    ret void
+  }
+
+  define float @load_common_ptr_float_no_opt(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, ptr %g) #0 {
+  entry:
+    %0 = load float, ptr %g, align 4
+    ret float %0
+  }
+
+  define void @store_large_offset_float_no_opt(ptr %p, float %a, float %b) #0 {
+  entry:
+    %0 = getelementptr inbounds float, ptr %p, i32 100
+    store volatile float %a, ptr %0, align 4
+    %1 = getelementptr inbounds float, ptr %p, i32 101
+    store volatile float %b, ptr %1, align 4
+    ret void
+  }
+
+  define { float, float } @load_large_offset_float_no_opt(ptr %p) #0 {
+  entry:
+    %arrayidx = getelementptr inbounds float, ptr %p, i32 100
+    %0 = load float, ptr %arrayidx, align 4
+    %arrayidx1 = getelementptr inbounds float, ptr %p, i32 101
+    %1 = load float, ptr %arrayidx1, align 4
+    %2 = insertvalue { float, float } undef, float %0, 0
+    %3 = insertvalue { float, float } %2, float %1, 1
+    ret { float, float } %3
+  }
+
+  attributes #0 = { minsize }
+
+...
+---
+name:            store_common_value_float
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11, $x12
+
+    ; CHECK-LABEL: name: store_common_value_float
+    ; CHECK: liveins: $x10, $x11, $x12
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x13_w = PseudoMV_FPR32INX $x0_w
+    ; CHECK-NEXT: SW_INX $x13_w, killed renamable $x10, 0 :: (store (s32) into %ir.a)
+    ; CHECK-NEXT: SW_INX $x13_w, killed renamable $x11, 0 :: (store (s32) into %ir.b)
+    ; CHECK-NEXT: SW_INX killed $x13_w, killed renamable $x12, 0 :: (store (s32) into %ir.c)
+    ; CHECK-NEXT: PseudoRET
+    SW_INX $x0_w, killed renamable $x10, 0 :: (store (s32) into %ir.a)
+    SW_INX $x0_w, killed renamable $x11, 0 :: (store (s32) into %ir.b)
+    SW_INX killed $x0_w, killed renamable $x12, 0 :: (store (s32) into %ir.c)
+    PseudoRET
+
+...
+---
+name:            store_common_ptr_float
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10_w, $x11_w, $x12_w, $x16
+
+    ; CHECK-LABEL: name: store_common_ptr_float
+    ; CHECK: liveins: $x10_w, $x11_w, $x12_w, $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x13 = ADDI $x16, 0
+    ; CHECK-NEXT: SW_INX killed renamable $x10_w, $x13, 0 :: (volatile store (s32) into %ir.p)
+    ; CHECK-NEXT: SW_INX killed renamable $x11_w, $x13, 0 :: (volatile store (s32) into %ir.p)
+    ; CHECK-NEXT: SW_INX killed renamable $x12_w, killed $x13, 0 :: (volatile store (s32) into %ir.p)
+    ; CHECK-NEXT: PseudoRET
+    SW_INX killed renamable $x10_w, renamable $x16, 0 :: (volatile store (s32) into %ir.p)
+    SW_INX killed renamable $x11_w, renamable $x16, 0 :: (volatile store (s32) into %ir.p)
+    SW_INX killed renamable $x12_w, killed renamable $x16, 0 :: (volatile store (s32) into %ir.p)
+    PseudoRET
+
+...
+---
+name:            load_common_ptr_float
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x16
+
+    ; CHECK-LABEL: name: load_common_ptr_float
+    ; CHECK: liveins: $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x13 = ADDI $x16, 0
+    ; CHECK-NEXT: renamable $x10_w = LW_INX $x13, 0 :: (load (s32) from %ir.g)
+    ; CHECK-NEXT: renamable $x11_w = LW_INX $x13, 4 :: (load (s32) from %ir.arrayidx1)
+    ; CHECK-NEXT: renamable $x12_w = LW_INX killed $x13, 8 :: (load (s32) from %ir.arrayidx2)
+    ; CHECK-NEXT: PseudoTAIL target-flags(riscv-call) @load_common_ptr_float_1, implicit $x2, implicit $x10_w, implicit $x11_w, implicit $x12_w
+    renamable $x10_w = LW_INX renamable $x16, 0 :: (load (s32) from %ir.g)
+    renamable $x11_w = LW_INX renamable $x16, 4 :: (load (s32) from %ir.arrayidx1)
+    renamable $x12_w = LW_INX killed renamable $x16, 8 :: (load (s32) from %ir.arrayidx2)
+    PseudoTAIL target-flags(riscv-call) @load_common_ptr_float_1, implicit $x2, implicit $x10_w, implicit $x11_w, implicit $x12_w
+
+...
+---
+name:            store_large_offset_float
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11_w, $x11_w, $x12_w, $x13_w
+
+    ; CHECK-LABEL: name: store_large_offset_float
+    ; CHECK: liveins: $x10, $x11_w, $x11_w, $x12_w, $x13_w
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x14 = ADDI $x10, 384
+    ; CHECK-NEXT: SW_INX killed renamable $x10_w, $x14, 16 :: (volatile store (s32) into %ir.0)
+    ; CHECK-NEXT: SW_INX killed renamable $x11_w, $x14, 20 :: (volatile store (s32) into %ir.1)
+    ; CHECK-NEXT: SW_INX killed renamable $x12_w, $x14, 24 :: (volatile store (s32) into %ir.2)
+    ; CHECK-NEXT: SW_INX killed renamable $x13_w, killed $x14, 28 :: (volatile store (s32) into %ir.3)
+    ; CHECK-NEXT: PseudoRET
+    SW_INX killed renamable $x10_w, renamable $x10, 400 :: (volatile store (s32) into %ir.0)
+    SW_INX killed renamable $x11_w, renamable $x10, 404 :: (volatile store (s32) into %ir.1)
+    SW_INX killed renamable $x12_w, renamable $x10, 408 :: (volatile store (s32) into %ir.2)
+    SW_INX killed renamable $x13_w, killed renamable $x10, 412 :: (volatile store (s32) into %ir.3)
+    PseudoRET
+
+...
+---
+name:            load_large_offset_float
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: load_large_offset_float
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x14 = ADDI $x10, 384
+    ; CHECK-NEXT: renamable $x11_w = LW_INX $x14, 16 :: (load (s32) from %ir.arrayidx)
+    ; CHECK-NEXT: renamable $x12_w = LW_INX $x14, 20 :: (load (s32) from %ir.arrayidx1)
+    ; CHECK-NEXT: renamable $x13_w = LW_INX killed $x14, 24 :: (load (s32) from %ir.arrayidx2)
+    ; CHECK-NEXT: PseudoTAIL target-flags(riscv-call) @load_large_offset_float_1, implicit $x2, implicit $x11_w, implicit $x12_w, implicit $x12_w
+    renamable $x11_w = LW_INX renamable $x10, 400 :: (load (s32) from %ir.arrayidx)
+    renamable $x12_w = LW_INX renamable $x10, 404 :: (load (s32) from %ir.arrayidx1)
+    renamable $x13_w = LW_INX killed renamable $x10, 408 :: (load (s32) from %ir.arrayidx2)
+    PseudoTAIL target-flags(riscv-call) @load_large_offset_float_1, implicit $x2, implicit $x11_w, implicit $x12_w, implicit $x12_w
+
+...
+---
+name:            store_common_value_float_no_opt
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x16_w
+
+    ; CHECK-LABEL: name: store_common_value_float_no_opt
+    ; CHECK: liveins: $x10, $x16_w
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: SW_INX killed renamable $x16_w, killed renamable $x10, 0 :: (store (s32) into %ir.a)
+    ; CHECK-NEXT: PseudoRET
+    SW_INX killed renamable $x16_w, killed renamable $x10, 0 :: (store (s32) into %ir.a)
+    PseudoRET
+
+...
+---
+name:            store_common_ptr_float_no_opt
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x16, $x10_w
+
+    ; CHECK-LABEL: name: store_common_ptr_float_no_opt
+    ; CHECK: liveins: $x16, $x10_w
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: SW_INX killed renamable $x10_w, killed renamable $x16, 0 :: (volatile store (s32) into %ir.p)
+    ; CHECK-NEXT: PseudoRET
+    SW_INX killed renamable $x10_w, killed renamable $x16, 0 :: (volatile store (s32) into %ir.p)
+    PseudoRET
+
+...
+---
+name:            load_common_ptr_float_no_opt
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x16
+
+    ; CHECK-LABEL: name: load_common_ptr_float_no_opt
+    ; CHECK: liveins: $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $x10_w = LW_INX killed renamable $x16, 0 :: (load (s32) from %ir.g)
+    ; CHECK-NEXT: PseudoRET implicit $x10_w
+    renamable $x10_w = LW_INX killed renamable $x16, 0 :: (load (s32) from %ir.g)
+    PseudoRET implicit $x10_w
+
+...
+---
+name:            store_large_offset_float_no_opt
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11_w, $x12_w
+
+    ; CHECK-LABEL: name: store_large_offset_float_no_opt
+    ; CHECK: liveins: $x10, $x11_w, $x12_w
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: SW_INX killed renamable $x11_w, renamable $x10, 400 :: (volatile store (s32) into %ir.0)
+    ; CHECK-NEXT: SW_INX killed renamable $x12_w, killed renamable $x10, 404 :: (volatile store (s32) into %ir.1)
+    ; CHECK-NEXT: PseudoRET
+    SW_INX killed renamable $x11_w, renamable $x10, 400 :: (volatile store (s32) into %ir.0)
+    SW_INX killed renamable $x12_w, killed renamable $x10, 404 :: (volatile store (s32) into %ir.1)
+    PseudoRET
+
+...
+---
+name:            load_large_offset_float_no_opt
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: load_large_offset_float_no_opt
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $x11_w = LW_INX renamable $x10, 400 :: (load (s32) from %ir.arrayidx)
+    ; CHECK-NEXT: renamable $x12_w = LW_INX killed renamable $x10, 404 :: (load (s32) from %ir.arrayidx1)
+    ; CHECK-NEXT: PseudoRET implicit $x11_w, implicit $x12_w
+    renamable $x11_w = LW_INX renamable $x10, 400 :: (load (s32) from %ir.arrayidx)
+    renamable $x12_w = LW_INX killed renamable $x10, 404 :: (load (s32) from %ir.arrayidx1)
+    PseudoRET implicit $x11_w, implicit $x12_w
+
+...
diff --git a/llvm/test/CodeGen/RISCV/out-of-reach-emergency-slot.mir b/llvm/test/CodeGen/RISCV/out-of-reach-emergency-slot.mir
index 08716be713b0f..a3a1818993f0b 100644
--- a/llvm/test/CodeGen/RISCV/out-of-reach-emergency-slot.mir
+++ b/llvm/test/CodeGen/RISCV/out-of-reach-emergency-slot.mir
@@ -41,10 +41,7 @@
   ; CHECK-NEXT:    sd a0, -8(a1)
   ; CHECK-NEXT:    ld a1, 0(sp)
   ; CHECK-NEXT:    call foo
-  ; CHECK-NEXT:    lui a0, 2
-  ; CHECK-NEXT:    sub sp, s0, a0
-  ; CHECK-NEXT:    addiw a0, a0, -2032
-  ; CHECK-NEXT:    add sp, sp, a0
+  ; CHECK-NEXT:    addi sp, s0, -2032
   ; CHECK-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
   ; CHECK-NEXT:    ld s0, 2016(sp) # 8-byte Folded Reload
   ; CHECK-NEXT:    addi sp, sp, 2032
diff --git a/llvm/test/CodeGen/RISCV/rv32zbs.ll b/llvm/test/CodeGen/RISCV/rv32zbs.ll
index 30aba61ba4746..c0b9e0b3c7748 100644
--- a/llvm/test/CodeGen/RISCV/rv32zbs.ll
+++ b/llvm/test/CodeGen/RISCV/rv32zbs.ll
@@ -837,3 +837,49 @@ define i64 @bset_trailing_ones_i64_no_mask(i64 %a) nounwind {
   %not = xor i64 %shift, -1
   ret i64 %not
 }
+
+define i1 @icmp_eq_pow2(i32 %x) nounwind {
+; RV32I-LABEL: icmp_eq_pow2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 8
+; RV32I-NEXT:    xor a0, a0, a1
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    ret
+;
+; RV32ZBS-LABEL: icmp_eq_pow2:
+; RV32ZBS:       # %bb.0:
+; RV32ZBS-NEXT:    binvi a0, a0, 15
+; RV32ZBS-NEXT:    seqz a0, a0
+; RV32ZBS-NEXT:    ret
+  %cmp = icmp eq i32 %x, 32768
+  ret i1 %cmp
+}
+
+define i1 @icmp_ne_pow2(i32 %x) nounwind {
+; RV32I-LABEL: icmp_ne_pow2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 8
+; RV32I-NEXT:    xor a0, a0, a1
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    ret
+;
+; RV32ZBS-LABEL: icmp_ne_pow2:
+; RV32ZBS:       # %bb.0:
+; RV32ZBS-NEXT:    binvi a0, a0, 15
+; RV32ZBS-NEXT:    seqz a0, a0
+; RV32ZBS-NEXT:    ret
+  %cmp = icmp eq i32 %x, 32768
+  ret i1 %cmp
+}
+
+define i1 @icmp_eq_nonpow2(i32 %x) nounwind {
+; CHECK-LABEL: icmp_eq_nonpow2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lui a1, 8
+; CHECK-NEXT:    addi a1, a1, -1
+; CHECK-NEXT:    xor a0, a0, a1
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %cmp = icmp eq i32 %x, 32767
+  ret i1 %cmp
+}
diff --git a/llvm/test/CodeGen/RISCV/rv64zbs.ll b/llvm/test/CodeGen/RISCV/rv64zbs.ll
index d370b1877470c..c1e1e16d0d4ae 100644
--- a/llvm/test/CodeGen/RISCV/rv64zbs.ll
+++ b/llvm/test/CodeGen/RISCV/rv64zbs.ll
@@ -1146,3 +1146,100 @@ define signext i64 @bset_trailing_ones_i64_no_mask(i64 signext %a) nounwind {
   %not = xor i64 %shift, -1
   ret i64 %not
 }
+
+define i1 @icmp_eq_pow2(i32 signext %x) nounwind {
+; RV64I-LABEL: icmp_eq_pow2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 8
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    ret
+;
+; RV64ZBS-LABEL: icmp_eq_pow2:
+; RV64ZBS:       # %bb.0:
+; RV64ZBS-NEXT:    binvi a0, a0, 15
+; RV64ZBS-NEXT:    seqz a0, a0
+; RV64ZBS-NEXT:    ret
+  %cmp = icmp eq i32 %x, 32768
+  ret i1 %cmp
+}
+
+define i1 @icmp_eq_pow2_64(i64 %x) nounwind {
+; RV64I-LABEL: icmp_eq_pow2_64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a1, 1
+; RV64I-NEXT:    slli a1, a1, 40
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    ret
+;
+; RV64ZBS-LABEL: icmp_eq_pow2_64:
+; RV64ZBS:       # %bb.0:
+; RV64ZBS-NEXT:    binvi a0, a0, 40
+; RV64ZBS-NEXT:    seqz a0, a0
+; RV64ZBS-NEXT:    ret
+  %cmp = icmp eq i64 %x, 1099511627776
+  ret i1 %cmp
+}
+
+define i1 @icmp_ne_pow2(i32 signext %x) nounwind {
+; RV64I-LABEL: icmp_ne_pow2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 8
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    ret
+;
+; RV64ZBS-LABEL: icmp_ne_pow2:
+; RV64ZBS:       # %bb.0:
+; RV64ZBS-NEXT:    binvi a0, a0, 15
+; RV64ZBS-NEXT:    seqz a0, a0
+; RV64ZBS-NEXT:    ret
+  %cmp = icmp eq i32 %x, 32768
+  ret i1 %cmp
+}
+
+define i1 @icmp_eq_nonpow2(i32 signext %x) nounwind {
+; CHECK-LABEL: icmp_eq_nonpow2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lui a1, 8
+; CHECK-NEXT:    addiw a1, a1, -1
+; CHECK-NEXT:    xor a0, a0, a1
+; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    ret
+  %cmp = icmp eq i32 %x, 32767
+  ret i1 %cmp
+}
+
+define signext i32 @fold_sextinreg_shl_to_sllw(i64 %x) nounwind {
+; CHECK-LABEL: fold_sextinreg_shl_to_sllw:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li a1, 1
+; CHECK-NEXT:    sllw a0, a1, a0
+; CHECK-NEXT:    ret
+entry:
+  %mask = and i64 %x, 31
+  %shl = shl i64 1, %mask
+  %trunc = trunc i64 %shl to i32
+  ret i32 %trunc
+}
+
+define signext i32 @fold_sextinreg_shl_to_sllw_large_shamt(i64 %x) nounwind {
+; RV64I-LABEL: fold_sextinreg_shl_to_sllw_large_shamt:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    li a1, 1
+; RV64I-NEXT:    sll a0, a1, a0
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    ret
+;
+; RV64ZBS-LABEL: fold_sextinreg_shl_to_sllw_large_shamt:
+; RV64ZBS:       # %bb.0: # %entry
+; RV64ZBS-NEXT:    bset a0, zero, a0
+; RV64ZBS-NEXT:    sext.w a0, a0
+; RV64ZBS-NEXT:    ret
+entry:
+  %mask = and i64 %x, 63
+  %shl = shl i64 1, %mask
+  %trunc = trunc i64 %shl to i32
+  ret i32 %trunc
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/addi-scalable-offset.mir b/llvm/test/CodeGen/RISCV/rvv/addi-scalable-offset.mir
index 5f0e1a9b9aa24..43fb0c10ca46f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/addi-scalable-offset.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/addi-scalable-offset.mir
@@ -46,9 +46,7 @@ body: |
     ; CHECK-NEXT: $x10 = ADDI killed $x10, -2048
     ; CHECK-NEXT: $x10 = ADDI killed $x10, -224
     ; CHECK-NEXT: VS1R_V killed renamable $v8, killed renamable $x10
-    ; CHECK-NEXT: $x2 = frame-destroy ADDI $x8, -2048
-    ; CHECK-NEXT: $x2 = frame-destroy ADDI killed $x2, -224
-    ; CHECK-NEXT: $x2 = frame-destroy ADDI $x2, 240
+    ; CHECK-NEXT: $x2 = frame-destroy ADDI $x8, -2032
     ; CHECK-NEXT: $x1 = LD $x2, 2024 :: (load (s64) from %stack.3)
     ; CHECK-NEXT: $x8 = LD $x2, 2016 :: (load (s64) from %stack.4)
     ; CHECK-NEXT: $x2 = frame-destroy ADDI $x2, 2032
diff --git a/llvm/test/CodeGen/RISCV/rvv/callee-saved-regs.ll b/llvm/test/CodeGen/RISCV/rvv/callee-saved-regs.ll
index c1ce2e988fc51..c0b10be847d1f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/callee-saved-regs.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/callee-saved-regs.ll
@@ -109,9 +109,7 @@ define riscv_vector_cc void @local_stack_allocation_frame_pointer() "frame-point
 ; SPILL-O2-NEXT:    addi sp, sp, -480
 ; SPILL-O2-NEXT:    lbu a0, -1912(s0)
 ; SPILL-O2-NEXT:    sb a0, -1912(s0)
-; SPILL-O2-NEXT:    addi sp, s0, -2048
-; SPILL-O2-NEXT:    addi sp, sp, -464
-; SPILL-O2-NEXT:    addi sp, sp, 480
+; SPILL-O2-NEXT:    addi sp, s0, -2032
 ; SPILL-O2-NEXT:    lw ra, 2028(sp) # 4-byte Folded Reload
 ; SPILL-O2-NEXT:    lw s0, 2024(sp) # 4-byte Folded Reload
 ; SPILL-O2-NEXT:    addi sp, sp, 2032
diff --git a/llvm/test/CodeGen/RISCV/rvv/emergency-slot.mir b/llvm/test/CodeGen/RISCV/rvv/emergency-slot.mir
index c4bc794b8aeb3..1b9ce12af01f9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/emergency-slot.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/emergency-slot.mir
@@ -150,9 +150,7 @@ body:             |
   ; CHECK-NEXT:   PseudoBR %bb.2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
-  ; CHECK-NEXT:   $x2 = frame-destroy ADDI $x8, -2048
-  ; CHECK-NEXT:   $x2 = frame-destroy ADDI killed $x2, -256
-  ; CHECK-NEXT:   $x2 = frame-destroy ADDI $x2, 272
+  ; CHECK-NEXT:   $x2 = frame-destroy ADDI $x8, -2032
   ; CHECK-NEXT:   $x1 = LD $x2, 2024 :: (load (s64) from %stack.3)
   ; CHECK-NEXT:   $x8 = LD $x2, 2016 :: (load (s64) from %stack.4)
   ; CHECK-NEXT:   $x18 = LD $x2, 2008 :: (load (s64) from %stack.5)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-fp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-fp-vp.ll
index 793e8eb5aee6a..6d5be7f14bf70 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-fp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-fp-vp.ll
@@ -1,63 +1,33 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
-; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN:   -verify-machineinstrs < %s | FileCheck %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
-; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
-; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
-; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN:   -verify-machineinstrs < %s | FileCheck %s
 
 declare half @llvm.vp.reduce.fadd.v2f16(half, <2 x half>, <2 x i1>, i32)
 
 define half @vpreduce_fadd_v2f16(half %s, <2 x half> %v, <2 x i1> %m, i32 zeroext %evl) {
-; ZVFH-LABEL: vpreduce_fadd_v2f16:
-; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; ZVFH-NEXT:    vfmv.s.f v9, fa0
-; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFH-NEXT:    vfredusum.vs v9, v8, v9, v0.t
-; ZVFH-NEXT:    vfmv.f.s fa0, v9
-; ZVFH-NEXT:    ret
-;
-; ZVFHMIN-LABEL: vpreduce_fadd_v2f16:
-; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfmv.s.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfredusum.vs v8, v9, v8, v0.t
-; ZVFHMIN-NEXT:    vfmv.f.s fa5, v8
-; ZVFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; ZVFHMIN-NEXT:    ret
+; CHECK-LABEL: vpreduce_fadd_v2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vfredusum.vs v9, v8, v9, v0.t
+; CHECK-NEXT:    vfmv.f.s fa0, v9
+; CHECK-NEXT:    ret
   %r = call reassoc half @llvm.vp.reduce.fadd.v2f16(half %s, <2 x half> %v, <2 x i1> %m, i32 %evl)
   ret half %r
 }
 
 define half @vpreduce_ord_fadd_v2f16(half %s, <2 x half> %v, <2 x i1> %m, i32 zeroext %evl) {
-; ZVFH-LABEL: vpreduce_ord_fadd_v2f16:
-; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; ZVFH-NEXT:    vfmv.s.f v9, fa0
-; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFH-NEXT:    vfredosum.vs v9, v8, v9, v0.t
-; ZVFH-NEXT:    vfmv.f.s fa0, v9
-; ZVFH-NEXT:    ret
-;
-; ZVFHMIN-LABEL: vpreduce_ord_fadd_v2f16:
-; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfmv.s.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfredosum.vs v8, v9, v8, v0.t
-; ZVFHMIN-NEXT:    vfmv.f.s fa5, v8
-; ZVFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; ZVFHMIN-NEXT:    ret
+; CHECK-LABEL: vpreduce_ord_fadd_v2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vfredosum.vs v9, v8, v9, v0.t
+; CHECK-NEXT:    vfmv.f.s fa0, v9
+; CHECK-NEXT:    ret
   %r = call half @llvm.vp.reduce.fadd.v2f16(half %s, <2 x half> %v, <2 x i1> %m, i32 %evl)
   ret half %r
 }
@@ -65,53 +35,27 @@ define half @vpreduce_ord_fadd_v2f16(half %s, <2 x half> %v, <2 x i1> %m, i32 ze
 declare half @llvm.vp.reduce.fadd.v4f16(half, <4 x half>, <4 x i1>, i32)
 
 define half @vpreduce_fadd_v4f16(half %s, <4 x half> %v, <4 x i1> %m, i32 zeroext %evl) {
-; ZVFH-LABEL: vpreduce_fadd_v4f16:
-; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; ZVFH-NEXT:    vfmv.s.f v9, fa0
-; ZVFH-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; ZVFH-NEXT:    vfredusum.vs v9, v8, v9, v0.t
-; ZVFH-NEXT:    vfmv.f.s fa0, v9
-; ZVFH-NEXT:    ret
-;
-; ZVFHMIN-LABEL: vpreduce_fadd_v4f16:
-; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; ZVFHMIN-NEXT:    vfmv.s.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; ZVFHMIN-NEXT:    vfredusum.vs v8, v9, v8, v0.t
-; ZVFHMIN-NEXT:    vfmv.f.s fa5, v8
-; ZVFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; ZVFHMIN-NEXT:    ret
+; CHECK-LABEL: vpreduce_fadd_v4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vfredusum.vs v9, v8, v9, v0.t
+; CHECK-NEXT:    vfmv.f.s fa0, v9
+; CHECK-NEXT:    ret
   %r = call reassoc half @llvm.vp.reduce.fadd.v4f16(half %s, <4 x half> %v, <4 x i1> %m, i32 %evl)
   ret half %r
 }
 
 define half @vpreduce_ord_fadd_v4f16(half %s, <4 x half> %v, <4 x i1> %m, i32 zeroext %evl) {
-; ZVFH-LABEL: vpreduce_ord_fadd_v4f16:
-; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; ZVFH-NEXT:    vfmv.s.f v9, fa0
-; ZVFH-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; ZVFH-NEXT:    vfredosum.vs v9, v8, v9, v0.t
-; ZVFH-NEXT:    vfmv.f.s fa0, v9
-; ZVFH-NEXT:    ret
-;
-; ZVFHMIN-LABEL: vpreduce_ord_fadd_v4f16:
-; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; ZVFHMIN-NEXT:    vfmv.s.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; ZVFHMIN-NEXT:    vfredosum.vs v8, v9, v8, v0.t
-; ZVFHMIN-NEXT:    vfmv.f.s fa5, v8
-; ZVFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; ZVFHMIN-NEXT:    ret
+; CHECK-LABEL: vpreduce_ord_fadd_v4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vfredosum.vs v9, v8, v9, v0.t
+; CHECK-NEXT:    vfmv.f.s fa0, v9
+; CHECK-NEXT:    ret
   %r = call half @llvm.vp.reduce.fadd.v4f16(half %s, <4 x half> %v, <4 x i1> %m, i32 %evl)
   ret half %r
 }
diff --git a/llvm/test/CodeGen/RISCV/rvv/insertelt-fp.ll b/llvm/test/CodeGen/RISCV/rvv/insertelt-fp.ll
index 8cfa88e6f9569..607e0085c3f46 100644
--- a/llvm/test/CodeGen/RISCV/rvv/insertelt-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/insertelt-fp.ll
@@ -1,209 +1,585 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
-; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
-; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zfbfmin,+zvfh,+zvfbfmin,+v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zfbfmin,+zvfh,+zvfbfmin,+v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zfbfmin,+zvfhmin,+zvfbfmin,+v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zfbfmin,+zvfhmin,+zvfbfmin,+v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
-define <vscale x 1 x half> @insertelt_nxv1f16_0(<vscale x 1 x half> %v, half %elt) {
-; CHECK-LABEL: insertelt_nxv1f16_0:
+define <vscale x 1 x bfloat> @insertelt_nxv1bf16_0(<vscale x 1 x bfloat> %v, bfloat %elt) {
+; CHECK-LABEL: insertelt_nxv1bf16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, tu, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa0
+; CHECK-NEXT:    fmv.x.h a0, fa0
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, tu, ma
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
-  %r = insertelement <vscale x 1 x half> %v, half %elt, i32 0
-  ret <vscale x 1 x half> %r
+  %r = insertelement <vscale x 1 x bfloat> %v, bfloat %elt, i32 0
+  ret <vscale x 1 x bfloat> %r
 }
 
-define <vscale x 1 x half> @insertelt_nxv1f16_imm(<vscale x 1 x half> %v, half %elt) {
-; CHECK-LABEL: insertelt_nxv1f16_imm:
+define <vscale x 1 x bfloat> @insertelt_nxv1bf16_imm(<vscale x 1 x bfloat> %v, bfloat %elt) {
+; CHECK-LABEL: insertelt_nxv1bf16_imm:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmv.x.h a0, fa0
 ; CHECK-NEXT:    vsetivli zero, 4, e16, mf4, tu, ma
-; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    vmv.s.x v9, a0
 ; CHECK-NEXT:    vslideup.vi v8, v9, 3
 ; CHECK-NEXT:    ret
-  %r = insertelement <vscale x 1 x half> %v, half %elt, i32 3
-  ret <vscale x 1 x half> %r
+  %r = insertelement <vscale x 1 x bfloat> %v, bfloat %elt, i32 3
+  ret <vscale x 1 x bfloat> %r
 }
 
-define <vscale x 1 x half> @insertelt_nxv1f16_idx(<vscale x 1 x half> %v, half %elt, i32 zeroext %idx) {
-; CHECK-LABEL: insertelt_nxv1f16_idx:
+define <vscale x 1 x bfloat> @insertelt_nxv1bf16_idx(<vscale x 1 x bfloat> %v, bfloat %elt, i32 zeroext %idx) {
+; CHECK-LABEL: insertelt_nxv1bf16_idx:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi a1, a0, 1
-; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    fmv.x.h a2, fa0
+; CHECK-NEXT:    vsetvli a3, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmv.s.x v9, a2
 ; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, tu, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    ret
-  %r = insertelement <vscale x 1 x half> %v, half %elt, i32 %idx
-  ret <vscale x 1 x half> %r
+  %r = insertelement <vscale x 1 x bfloat> %v, bfloat %elt, i32 %idx
+  ret <vscale x 1 x bfloat> %r
 }
 
-define <vscale x 2 x half> @insertelt_nxv2f16_0(<vscale x 2 x half> %v, half %elt) {
-; CHECK-LABEL: insertelt_nxv2f16_0:
+define <vscale x 2 x bfloat> @insertelt_nxv2bf16_0(<vscale x 2 x bfloat> %v, bfloat %elt) {
+; CHECK-LABEL: insertelt_nxv2bf16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, tu, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa0
+; CHECK-NEXT:    fmv.x.h a0, fa0
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, tu, ma
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
-  %r = insertelement <vscale x 2 x half> %v, half %elt, i32 0
-  ret <vscale x 2 x half> %r
+  %r = insertelement <vscale x 2 x bfloat> %v, bfloat %elt, i32 0
+  ret <vscale x 2 x bfloat> %r
 }
 
-define <vscale x 2 x half> @insertelt_nxv2f16_imm(<vscale x 2 x half> %v, half %elt) {
-; CHECK-LABEL: insertelt_nxv2f16_imm:
+define <vscale x 2 x bfloat> @insertelt_nxv2bf16_imm(<vscale x 2 x bfloat> %v, bfloat %elt) {
+; CHECK-LABEL: insertelt_nxv2bf16_imm:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmv.x.h a0, fa0
 ; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, tu, ma
-; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    vmv.s.x v9, a0
 ; CHECK-NEXT:    vslideup.vi v8, v9, 3
 ; CHECK-NEXT:    ret
-  %r = insertelement <vscale x 2 x half> %v, half %elt, i32 3
-  ret <vscale x 2 x half> %r
+  %r = insertelement <vscale x 2 x bfloat> %v, bfloat %elt, i32 3
+  ret <vscale x 2 x bfloat> %r
 }
 
-define <vscale x 2 x half> @insertelt_nxv2f16_idx(<vscale x 2 x half> %v, half %elt, i32 zeroext %idx) {
-; CHECK-LABEL: insertelt_nxv2f16_idx:
+define <vscale x 2 x bfloat> @insertelt_nxv2bf16_idx(<vscale x 2 x bfloat> %v, bfloat %elt, i32 zeroext %idx) {
+; CHECK-LABEL: insertelt_nxv2bf16_idx:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi a1, a0, 1
-; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    fmv.x.h a2, fa0
+; CHECK-NEXT:    vsetvli a3, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmv.s.x v9, a2
 ; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, tu, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    ret
-  %r = insertelement <vscale x 2 x half> %v, half %elt, i32 %idx
-  ret <vscale x 2 x half> %r
+  %r = insertelement <vscale x 2 x bfloat> %v, bfloat %elt, i32 %idx
+  ret <vscale x 2 x bfloat> %r
 }
 
-define <vscale x 4 x half> @insertelt_nxv4f16_0(<vscale x 4 x half> %v, half %elt) {
-; CHECK-LABEL: insertelt_nxv4f16_0:
+define <vscale x 4 x bfloat> @insertelt_nxv4bf16_0(<vscale x 4 x bfloat> %v, bfloat %elt) {
+; CHECK-LABEL: insertelt_nxv4bf16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, tu, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa0
+; CHECK-NEXT:    fmv.x.h a0, fa0
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, tu, ma
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
-  %r = insertelement <vscale x 4 x half> %v, half %elt, i32 0
-  ret <vscale x 4 x half> %r
+  %r = insertelement <vscale x 4 x bfloat> %v, bfloat %elt, i32 0
+  ret <vscale x 4 x bfloat> %r
 }
 
-define <vscale x 4 x half> @insertelt_nxv4f16_imm(<vscale x 4 x half> %v, half %elt) {
-; CHECK-LABEL: insertelt_nxv4f16_imm:
+define <vscale x 4 x bfloat> @insertelt_nxv4bf16_imm(<vscale x 4 x bfloat> %v, bfloat %elt) {
+; CHECK-LABEL: insertelt_nxv4bf16_imm:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmv.x.h a0, fa0
 ; CHECK-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
-; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    vmv.s.x v9, a0
 ; CHECK-NEXT:    vslideup.vi v8, v9, 3
 ; CHECK-NEXT:    ret
-  %r = insertelement <vscale x 4 x half> %v, half %elt, i32 3
-  ret <vscale x 4 x half> %r
+  %r = insertelement <vscale x 4 x bfloat> %v, bfloat %elt, i32 3
+  ret <vscale x 4 x bfloat> %r
 }
 
-define <vscale x 4 x half> @insertelt_nxv4f16_idx(<vscale x 4 x half> %v, half %elt, i32 zeroext %idx) {
-; CHECK-LABEL: insertelt_nxv4f16_idx:
+define <vscale x 4 x bfloat> @insertelt_nxv4bf16_idx(<vscale x 4 x bfloat> %v, bfloat %elt, i32 zeroext %idx) {
+; CHECK-LABEL: insertelt_nxv4bf16_idx:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi a1, a0, 1
-; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    fmv.x.h a2, fa0
+; CHECK-NEXT:    vsetvli a3, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmv.s.x v9, a2
 ; CHECK-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    ret
-  %r = insertelement <vscale x 4 x half> %v, half %elt, i32 %idx
-  ret <vscale x 4 x half> %r
+  %r = insertelement <vscale x 4 x bfloat> %v, bfloat %elt, i32 %idx
+  ret <vscale x 4 x bfloat> %r
 }
 
-define <vscale x 8 x half> @insertelt_nxv8f16_0(<vscale x 8 x half> %v, half %elt) {
-; CHECK-LABEL: insertelt_nxv8f16_0:
+define <vscale x 8 x bfloat> @insertelt_nxv8bf16_0(<vscale x 8 x bfloat> %v, bfloat %elt) {
+; CHECK-LABEL: insertelt_nxv8bf16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, tu, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa0
+; CHECK-NEXT:    fmv.x.h a0, fa0
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, tu, ma
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
-  %r = insertelement <vscale x 8 x half> %v, half %elt, i32 0
-  ret <vscale x 8 x half> %r
+  %r = insertelement <vscale x 8 x bfloat> %v, bfloat %elt, i32 0
+  ret <vscale x 8 x bfloat> %r
 }
 
-define <vscale x 8 x half> @insertelt_nxv8f16_imm(<vscale x 8 x half> %v, half %elt) {
-; CHECK-LABEL: insertelt_nxv8f16_imm:
+define <vscale x 8 x bfloat> @insertelt_nxv8bf16_imm(<vscale x 8 x bfloat> %v, bfloat %elt) {
+; CHECK-LABEL: insertelt_nxv8bf16_imm:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmv.x.h a0, fa0
 ; CHECK-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
-; CHECK-NEXT:    vfmv.s.f v10, fa0
+; CHECK-NEXT:    vmv.s.x v10, a0
 ; CHECK-NEXT:    vslideup.vi v8, v10, 3
 ; CHECK-NEXT:    ret
-  %r = insertelement <vscale x 8 x half> %v, half %elt, i32 3
-  ret <vscale x 8 x half> %r
+  %r = insertelement <vscale x 8 x bfloat> %v, bfloat %elt, i32 3
+  ret <vscale x 8 x bfloat> %r
 }
 
-define <vscale x 8 x half> @insertelt_nxv8f16_idx(<vscale x 8 x half> %v, half %elt, i32 zeroext %idx) {
-; CHECK-LABEL: insertelt_nxv8f16_idx:
+define <vscale x 8 x bfloat> @insertelt_nxv8bf16_idx(<vscale x 8 x bfloat> %v, bfloat %elt, i32 zeroext %idx) {
+; CHECK-LABEL: insertelt_nxv8bf16_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vfmv.s.f v10, fa0
+; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmv.s.x v10, a1
 ; CHECK-NEXT:    addi a1, a0, 1
 ; CHECK-NEXT:    vsetvli zero, a1, e16, m2, tu, ma
 ; CHECK-NEXT:    vslideup.vx v8, v10, a0
 ; CHECK-NEXT:    ret
-  %r = insertelement <vscale x 8 x half> %v, half %elt, i32 %idx
-  ret <vscale x 8 x half> %r
+  %r = insertelement <vscale x 8 x bfloat> %v, bfloat %elt, i32 %idx
+  ret <vscale x 8 x bfloat> %r
 }
 
-define <vscale x 16 x half> @insertelt_nxv16f16_0(<vscale x 16 x half> %v, half %elt) {
-; CHECK-LABEL: insertelt_nxv16f16_0:
+define <vscale x 16 x bfloat> @insertelt_nxv16bf16_0(<vscale x 16 x bfloat> %v, bfloat %elt) {
+; CHECK-LABEL: insertelt_nxv16bf16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, tu, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa0
+; CHECK-NEXT:    fmv.x.h a0, fa0
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, tu, ma
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
-  %r = insertelement <vscale x 16 x half> %v, half %elt, i32 0
-  ret <vscale x 16 x half> %r
+  %r = insertelement <vscale x 16 x bfloat> %v, bfloat %elt, i32 0
+  ret <vscale x 16 x bfloat> %r
 }
 
-define <vscale x 16 x half> @insertelt_nxv16f16_imm(<vscale x 16 x half> %v, half %elt) {
-; CHECK-LABEL: insertelt_nxv16f16_imm:
+define <vscale x 16 x bfloat> @insertelt_nxv16bf16_imm(<vscale x 16 x bfloat> %v, bfloat %elt) {
+; CHECK-LABEL: insertelt_nxv16bf16_imm:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmv.x.h a0, fa0
 ; CHECK-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
-; CHECK-NEXT:    vfmv.s.f v12, fa0
+; CHECK-NEXT:    vmv.s.x v12, a0
 ; CHECK-NEXT:    vslideup.vi v8, v12, 3
 ; CHECK-NEXT:    ret
-  %r = insertelement <vscale x 16 x half> %v, half %elt, i32 3
-  ret <vscale x 16 x half> %r
+  %r = insertelement <vscale x 16 x bfloat> %v, bfloat %elt, i32 3
+  ret <vscale x 16 x bfloat> %r
 }
 
-define <vscale x 16 x half> @insertelt_nxv16f16_idx(<vscale x 16 x half> %v, half %elt, i32 zeroext %idx) {
-; CHECK-LABEL: insertelt_nxv16f16_idx:
+define <vscale x 16 x bfloat> @insertelt_nxv16bf16_idx(<vscale x 16 x bfloat> %v, bfloat %elt, i32 zeroext %idx) {
+; CHECK-LABEL: insertelt_nxv16bf16_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vfmv.s.f v12, fa0
+; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmv.s.x v12, a1
 ; CHECK-NEXT:    addi a1, a0, 1
 ; CHECK-NEXT:    vsetvli zero, a1, e16, m4, tu, ma
 ; CHECK-NEXT:    vslideup.vx v8, v12, a0
 ; CHECK-NEXT:    ret
-  %r = insertelement <vscale x 16 x half> %v, half %elt, i32 %idx
-  ret <vscale x 16 x half> %r
+  %r = insertelement <vscale x 16 x bfloat> %v, bfloat %elt, i32 %idx
+  ret <vscale x 16 x bfloat> %r
 }
 
-define <vscale x 32 x half> @insertelt_nxv32f16_0(<vscale x 32 x half> %v, half %elt) {
-; CHECK-LABEL: insertelt_nxv32f16_0:
+define <vscale x 32 x bfloat> @insertelt_nxv32bf16_0(<vscale x 32 x bfloat> %v, bfloat %elt) {
+; CHECK-LABEL: insertelt_nxv32bf16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, tu, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa0
+; CHECK-NEXT:    fmv.x.h a0, fa0
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, tu, ma
+; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
-  %r = insertelement <vscale x 32 x half> %v, half %elt, i32 0
-  ret <vscale x 32 x half> %r
+  %r = insertelement <vscale x 32 x bfloat> %v, bfloat %elt, i32 0
+  ret <vscale x 32 x bfloat> %r
 }
 
-define <vscale x 32 x half> @insertelt_nxv32f16_imm(<vscale x 32 x half> %v, half %elt) {
-; CHECK-LABEL: insertelt_nxv32f16_imm:
+define <vscale x 32 x bfloat> @insertelt_nxv32bf16_imm(<vscale x 32 x bfloat> %v, bfloat %elt) {
+; CHECK-LABEL: insertelt_nxv32bf16_imm:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmv.x.h a0, fa0
 ; CHECK-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
-; CHECK-NEXT:    vfmv.s.f v16, fa0
+; CHECK-NEXT:    vmv.s.x v16, a0
 ; CHECK-NEXT:    vslideup.vi v8, v16, 3
 ; CHECK-NEXT:    ret
-  %r = insertelement <vscale x 32 x half> %v, half %elt, i32 3
-  ret <vscale x 32 x half> %r
+  %r = insertelement <vscale x 32 x bfloat> %v, bfloat %elt, i32 3
+  ret <vscale x 32 x bfloat> %r
 }
 
-define <vscale x 32 x half> @insertelt_nxv32f16_idx(<vscale x 32 x half> %v, half %elt, i32 zeroext %idx) {
-; CHECK-LABEL: insertelt_nxv32f16_idx:
+define <vscale x 32 x bfloat> @insertelt_nxv32bf16_idx(<vscale x 32 x bfloat> %v, bfloat %elt, i32 zeroext %idx) {
+; CHECK-LABEL: insertelt_nxv32bf16_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vfmv.s.f v16, fa0
+; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmv.s.x v16, a1
 ; CHECK-NEXT:    addi a1, a0, 1
 ; CHECK-NEXT:    vsetvli zero, a1, e16, m8, tu, ma
 ; CHECK-NEXT:    vslideup.vx v8, v16, a0
 ; CHECK-NEXT:    ret
+  %r = insertelement <vscale x 32 x bfloat> %v, bfloat %elt, i32 %idx
+  ret <vscale x 32 x bfloat> %r
+}
+
+define <vscale x 1 x half> @insertelt_nxv1f16_0(<vscale x 1 x half> %v, half %elt) {
+; ZVFH-LABEL: insertelt_nxv1f16_0:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetvli a0, zero, e16, m1, tu, ma
+; ZVFH-NEXT:    vfmv.s.f v8, fa0
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: insertelt_nxv1f16_0:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, tu, ma
+; ZVFHMIN-NEXT:    vmv.s.x v8, a0
+; ZVFHMIN-NEXT:    ret
+  %r = insertelement <vscale x 1 x half> %v, half %elt, i32 0
+  ret <vscale x 1 x half> %r
+}
+
+define <vscale x 1 x half> @insertelt_nxv1f16_imm(<vscale x 1 x half> %v, half %elt) {
+; ZVFH-LABEL: insertelt_nxv1f16_imm:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 4, e16, mf4, tu, ma
+; ZVFH-NEXT:    vfmv.s.f v9, fa0
+; ZVFH-NEXT:    vslideup.vi v8, v9, 3
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: insertelt_nxv1f16_imm:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
+; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, mf4, tu, ma
+; ZVFHMIN-NEXT:    vmv.s.x v9, a0
+; ZVFHMIN-NEXT:    vslideup.vi v8, v9, 3
+; ZVFHMIN-NEXT:    ret
+  %r = insertelement <vscale x 1 x half> %v, half %elt, i32 3
+  ret <vscale x 1 x half> %r
+}
+
+define <vscale x 1 x half> @insertelt_nxv1f16_idx(<vscale x 1 x half> %v, half %elt, i32 zeroext %idx) {
+; ZVFH-LABEL: insertelt_nxv1f16_idx:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    addi a1, a0, 1
+; ZVFH-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
+; ZVFH-NEXT:    vfmv.s.f v9, fa0
+; ZVFH-NEXT:    vsetvli zero, a1, e16, mf4, tu, ma
+; ZVFH-NEXT:    vslideup.vx v8, v9, a0
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: insertelt_nxv1f16_idx:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    addi a1, a0, 1
+; ZVFHMIN-NEXT:    fmv.x.h a2, fa0
+; ZVFHMIN-NEXT:    vsetvli a3, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv.s.x v9, a2
+; ZVFHMIN-NEXT:    vsetvli zero, a1, e16, mf4, tu, ma
+; ZVFHMIN-NEXT:    vslideup.vx v8, v9, a0
+; ZVFHMIN-NEXT:    ret
+  %r = insertelement <vscale x 1 x half> %v, half %elt, i32 %idx
+  ret <vscale x 1 x half> %r
+}
+
+define <vscale x 2 x half> @insertelt_nxv2f16_0(<vscale x 2 x half> %v, half %elt) {
+; ZVFH-LABEL: insertelt_nxv2f16_0:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetvli a0, zero, e16, m1, tu, ma
+; ZVFH-NEXT:    vfmv.s.f v8, fa0
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: insertelt_nxv2f16_0:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, tu, ma
+; ZVFHMIN-NEXT:    vmv.s.x v8, a0
+; ZVFHMIN-NEXT:    ret
+  %r = insertelement <vscale x 2 x half> %v, half %elt, i32 0
+  ret <vscale x 2 x half> %r
+}
+
+define <vscale x 2 x half> @insertelt_nxv2f16_imm(<vscale x 2 x half> %v, half %elt) {
+; ZVFH-LABEL: insertelt_nxv2f16_imm:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 4, e16, mf2, tu, ma
+; ZVFH-NEXT:    vfmv.s.f v9, fa0
+; ZVFH-NEXT:    vslideup.vi v8, v9, 3
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: insertelt_nxv2f16_imm:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
+; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, mf2, tu, ma
+; ZVFHMIN-NEXT:    vmv.s.x v9, a0
+; ZVFHMIN-NEXT:    vslideup.vi v8, v9, 3
+; ZVFHMIN-NEXT:    ret
+  %r = insertelement <vscale x 2 x half> %v, half %elt, i32 3
+  ret <vscale x 2 x half> %r
+}
+
+define <vscale x 2 x half> @insertelt_nxv2f16_idx(<vscale x 2 x half> %v, half %elt, i32 zeroext %idx) {
+; ZVFH-LABEL: insertelt_nxv2f16_idx:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    addi a1, a0, 1
+; ZVFH-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
+; ZVFH-NEXT:    vfmv.s.f v9, fa0
+; ZVFH-NEXT:    vsetvli zero, a1, e16, mf2, tu, ma
+; ZVFH-NEXT:    vslideup.vx v8, v9, a0
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: insertelt_nxv2f16_idx:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    addi a1, a0, 1
+; ZVFHMIN-NEXT:    fmv.x.h a2, fa0
+; ZVFHMIN-NEXT:    vsetvli a3, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv.s.x v9, a2
+; ZVFHMIN-NEXT:    vsetvli zero, a1, e16, mf2, tu, ma
+; ZVFHMIN-NEXT:    vslideup.vx v8, v9, a0
+; ZVFHMIN-NEXT:    ret
+  %r = insertelement <vscale x 2 x half> %v, half %elt, i32 %idx
+  ret <vscale x 2 x half> %r
+}
+
+define <vscale x 4 x half> @insertelt_nxv4f16_0(<vscale x 4 x half> %v, half %elt) {
+; ZVFH-LABEL: insertelt_nxv4f16_0:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetvli a0, zero, e16, m1, tu, ma
+; ZVFH-NEXT:    vfmv.s.f v8, fa0
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: insertelt_nxv4f16_0:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, tu, ma
+; ZVFHMIN-NEXT:    vmv.s.x v8, a0
+; ZVFHMIN-NEXT:    ret
+  %r = insertelement <vscale x 4 x half> %v, half %elt, i32 0
+  ret <vscale x 4 x half> %r
+}
+
+define <vscale x 4 x half> @insertelt_nxv4f16_imm(<vscale x 4 x half> %v, half %elt) {
+; ZVFH-LABEL: insertelt_nxv4f16_imm:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
+; ZVFH-NEXT:    vfmv.s.f v9, fa0
+; ZVFH-NEXT:    vslideup.vi v8, v9, 3
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: insertelt_nxv4f16_imm:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
+; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
+; ZVFHMIN-NEXT:    vmv.s.x v9, a0
+; ZVFHMIN-NEXT:    vslideup.vi v8, v9, 3
+; ZVFHMIN-NEXT:    ret
+  %r = insertelement <vscale x 4 x half> %v, half %elt, i32 3
+  ret <vscale x 4 x half> %r
+}
+
+define <vscale x 4 x half> @insertelt_nxv4f16_idx(<vscale x 4 x half> %v, half %elt, i32 zeroext %idx) {
+; ZVFH-LABEL: insertelt_nxv4f16_idx:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    addi a1, a0, 1
+; ZVFH-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
+; ZVFH-NEXT:    vfmv.s.f v9, fa0
+; ZVFH-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
+; ZVFH-NEXT:    vslideup.vx v8, v9, a0
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: insertelt_nxv4f16_idx:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    addi a1, a0, 1
+; ZVFHMIN-NEXT:    fmv.x.h a2, fa0
+; ZVFHMIN-NEXT:    vsetvli a3, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv.s.x v9, a2
+; ZVFHMIN-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
+; ZVFHMIN-NEXT:    vslideup.vx v8, v9, a0
+; ZVFHMIN-NEXT:    ret
+  %r = insertelement <vscale x 4 x half> %v, half %elt, i32 %idx
+  ret <vscale x 4 x half> %r
+}
+
+define <vscale x 8 x half> @insertelt_nxv8f16_0(<vscale x 8 x half> %v, half %elt) {
+; ZVFH-LABEL: insertelt_nxv8f16_0:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetvli a0, zero, e16, m1, tu, ma
+; ZVFH-NEXT:    vfmv.s.f v8, fa0
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: insertelt_nxv8f16_0:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, tu, ma
+; ZVFHMIN-NEXT:    vmv.s.x v8, a0
+; ZVFHMIN-NEXT:    ret
+  %r = insertelement <vscale x 8 x half> %v, half %elt, i32 0
+  ret <vscale x 8 x half> %r
+}
+
+define <vscale x 8 x half> @insertelt_nxv8f16_imm(<vscale x 8 x half> %v, half %elt) {
+; ZVFH-LABEL: insertelt_nxv8f16_imm:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
+; ZVFH-NEXT:    vfmv.s.f v10, fa0
+; ZVFH-NEXT:    vslideup.vi v8, v10, 3
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: insertelt_nxv8f16_imm:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
+; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
+; ZVFHMIN-NEXT:    vmv.s.x v10, a0
+; ZVFHMIN-NEXT:    vslideup.vi v8, v10, 3
+; ZVFHMIN-NEXT:    ret
+  %r = insertelement <vscale x 8 x half> %v, half %elt, i32 3
+  ret <vscale x 8 x half> %r
+}
+
+define <vscale x 8 x half> @insertelt_nxv8f16_idx(<vscale x 8 x half> %v, half %elt, i32 zeroext %idx) {
+; ZVFH-LABEL: insertelt_nxv8f16_idx:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; ZVFH-NEXT:    vfmv.s.f v10, fa0
+; ZVFH-NEXT:    addi a1, a0, 1
+; ZVFH-NEXT:    vsetvli zero, a1, e16, m2, tu, ma
+; ZVFH-NEXT:    vslideup.vx v8, v10, a0
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: insertelt_nxv8f16_idx:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv.s.x v10, a1
+; ZVFHMIN-NEXT:    addi a1, a0, 1
+; ZVFHMIN-NEXT:    vsetvli zero, a1, e16, m2, tu, ma
+; ZVFHMIN-NEXT:    vslideup.vx v8, v10, a0
+; ZVFHMIN-NEXT:    ret
+  %r = insertelement <vscale x 8 x half> %v, half %elt, i32 %idx
+  ret <vscale x 8 x half> %r
+}
+
+define <vscale x 16 x half> @insertelt_nxv16f16_0(<vscale x 16 x half> %v, half %elt) {
+; ZVFH-LABEL: insertelt_nxv16f16_0:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetvli a0, zero, e16, m1, tu, ma
+; ZVFH-NEXT:    vfmv.s.f v8, fa0
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: insertelt_nxv16f16_0:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, tu, ma
+; ZVFHMIN-NEXT:    vmv.s.x v8, a0
+; ZVFHMIN-NEXT:    ret
+  %r = insertelement <vscale x 16 x half> %v, half %elt, i32 0
+  ret <vscale x 16 x half> %r
+}
+
+define <vscale x 16 x half> @insertelt_nxv16f16_imm(<vscale x 16 x half> %v, half %elt) {
+; ZVFH-LABEL: insertelt_nxv16f16_imm:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
+; ZVFH-NEXT:    vfmv.s.f v12, fa0
+; ZVFH-NEXT:    vslideup.vi v8, v12, 3
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: insertelt_nxv16f16_imm:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
+; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
+; ZVFHMIN-NEXT:    vmv.s.x v12, a0
+; ZVFHMIN-NEXT:    vslideup.vi v8, v12, 3
+; ZVFHMIN-NEXT:    ret
+  %r = insertelement <vscale x 16 x half> %v, half %elt, i32 3
+  ret <vscale x 16 x half> %r
+}
+
+define <vscale x 16 x half> @insertelt_nxv16f16_idx(<vscale x 16 x half> %v, half %elt, i32 zeroext %idx) {
+; ZVFH-LABEL: insertelt_nxv16f16_idx:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; ZVFH-NEXT:    vfmv.s.f v12, fa0
+; ZVFH-NEXT:    addi a1, a0, 1
+; ZVFH-NEXT:    vsetvli zero, a1, e16, m4, tu, ma
+; ZVFH-NEXT:    vslideup.vx v8, v12, a0
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: insertelt_nxv16f16_idx:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv.s.x v12, a1
+; ZVFHMIN-NEXT:    addi a1, a0, 1
+; ZVFHMIN-NEXT:    vsetvli zero, a1, e16, m4, tu, ma
+; ZVFHMIN-NEXT:    vslideup.vx v8, v12, a0
+; ZVFHMIN-NEXT:    ret
+  %r = insertelement <vscale x 16 x half> %v, half %elt, i32 %idx
+  ret <vscale x 16 x half> %r
+}
+
+define <vscale x 32 x half> @insertelt_nxv32f16_0(<vscale x 32 x half> %v, half %elt) {
+; ZVFH-LABEL: insertelt_nxv32f16_0:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetvli a0, zero, e16, m1, tu, ma
+; ZVFH-NEXT:    vfmv.s.f v8, fa0
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: insertelt_nxv32f16_0:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, tu, ma
+; ZVFHMIN-NEXT:    vmv.s.x v8, a0
+; ZVFHMIN-NEXT:    ret
+  %r = insertelement <vscale x 32 x half> %v, half %elt, i32 0
+  ret <vscale x 32 x half> %r
+}
+
+define <vscale x 32 x half> @insertelt_nxv32f16_imm(<vscale x 32 x half> %v, half %elt) {
+; ZVFH-LABEL: insertelt_nxv32f16_imm:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
+; ZVFH-NEXT:    vfmv.s.f v16, fa0
+; ZVFH-NEXT:    vslideup.vi v8, v16, 3
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: insertelt_nxv32f16_imm:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fmv.x.h a0, fa0
+; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
+; ZVFHMIN-NEXT:    vmv.s.x v16, a0
+; ZVFHMIN-NEXT:    vslideup.vi v8, v16, 3
+; ZVFHMIN-NEXT:    ret
+  %r = insertelement <vscale x 32 x half> %v, half %elt, i32 3
+  ret <vscale x 32 x half> %r
+}
+
+define <vscale x 32 x half> @insertelt_nxv32f16_idx(<vscale x 32 x half> %v, half %elt, i32 zeroext %idx) {
+; ZVFH-LABEL: insertelt_nxv32f16_idx:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; ZVFH-NEXT:    vfmv.s.f v16, fa0
+; ZVFH-NEXT:    addi a1, a0, 1
+; ZVFH-NEXT:    vsetvli zero, a1, e16, m8, tu, ma
+; ZVFH-NEXT:    vslideup.vx v8, v16, a0
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: insertelt_nxv32f16_idx:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv.s.x v16, a1
+; ZVFHMIN-NEXT:    addi a1, a0, 1
+; ZVFHMIN-NEXT:    vsetvli zero, a1, e16, m8, tu, ma
+; ZVFHMIN-NEXT:    vslideup.vx v8, v16, a0
+; ZVFHMIN-NEXT:    ret
   %r = insertelement <vscale x 32 x half> %v, half %elt, i32 %idx
   ret <vscale x 32 x half> %r
 }
diff --git a/llvm/test/CodeGen/RISCV/rvv/large-rvv-stack-size.mir b/llvm/test/CodeGen/RISCV/rvv/large-rvv-stack-size.mir
index b4d8805b65bd8..22a7425bf98b8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/large-rvv-stack-size.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/large-rvv-stack-size.mir
@@ -33,9 +33,7 @@
   ; CHECK-NEXT:    vs1r.v v25, (a0) # Unknown-size Folded Spill
   ; CHECK-NEXT:    ld a0, 8(sp)
   ; CHECK-NEXT:    call spillslot
-  ; CHECK-NEXT:    addi sp, s0, -2048
-  ; CHECK-NEXT:    addi sp, sp, -256
-  ; CHECK-NEXT:    addi sp, sp, 272
+  ; CHECK-NEXT:    addi sp, s0, -2032
   ; CHECK-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
   ; CHECK-NEXT:    ld s0, 2016(sp) # 8-byte Folded Reload
   ; CHECK-NEXT:    addi sp, sp, 2032
diff --git a/llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll
index 9d0cb22eb5f47..a6c6db345032e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll
@@ -1,10 +1,16 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-UNKNOWN
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-256
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-512
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-UNKNOWN
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-256
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-512
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-UNKNOWN
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvfbfmin -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-256
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvfbfmin -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-512
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-UNKNOWN
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvfbfmin -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-256
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvfbfmin -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-512
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfhmin,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-UNKNOWN
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfhmin,+zvfbfmin -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-256
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfhmin,+zvfbfmin -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,RV32-BITS-512
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfhmin,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-UNKNOWN
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfhmin,+zvfbfmin -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-256
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfhmin,+zvfbfmin -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,RV64-BITS-512
 
 ;
 ; VECTOR_REVERSE - masks
@@ -1515,6 +1521,113 @@ define <vscale x 8 x i64> @reverse_nxv8i64(<vscale x 8 x i64> %a) {
 ; VECTOR_REVERSE - floating point
 ;
 
+define <vscale x 1 x bfloat> @reverse_nxv1bf16(<vscale x 1 x bfloat> %a) {
+; CHECK-LABEL: reverse_nxv1bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    srli a0, a0, 3
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vid.v v9
+; CHECK-NEXT:    vrsub.vx v10, v9, a0
+; CHECK-NEXT:    vrgather.vv v9, v8, v10
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %res = call <vscale x 1 x bfloat> @llvm.vector.reverse.nxv1bf16(<vscale x 1 x bfloat> %a)
+  ret <vscale x 1 x bfloat> %res
+}
+
+define <vscale x 2 x bfloat> @reverse_nxv2bf16(<vscale x 2 x bfloat> %a) {
+; CHECK-LABEL: reverse_nxv2bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    srli a0, a0, 2
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vid.v v9
+; CHECK-NEXT:    vrsub.vx v10, v9, a0
+; CHECK-NEXT:    vrgather.vv v9, v8, v10
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %res = call <vscale x 2 x bfloat> @llvm.vector.reverse.nxv2bf16(<vscale x 2 x bfloat> %a)
+  ret <vscale x 2 x bfloat> %res
+}
+
+define <vscale x 4 x bfloat> @reverse_nxv4bf16(<vscale x 4 x bfloat> %a) {
+; CHECK-LABEL: reverse_nxv4bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    srli a0, a0, 1
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vid.v v9
+; CHECK-NEXT:    vrsub.vx v10, v9, a0
+; CHECK-NEXT:    vrgather.vv v9, v8, v10
+; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    ret
+  %res = call <vscale x 4 x bfloat> @llvm.vector.reverse.nxv4bf16(<vscale x 4 x bfloat> %a)
+  ret <vscale x 4 x bfloat> %res
+}
+
+define <vscale x 8 x bfloat> @reverse_nxv8bf16(<vscale x 8 x bfloat> %a) {
+; CHECK-LABEL: reverse_nxv8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    srli a0, a0, 1
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vx v12, v10, a0
+; CHECK-NEXT:    vrgather.vv v11, v8, v12
+; CHECK-NEXT:    vrgather.vv v10, v9, v12
+; CHECK-NEXT:    vmv2r.v v8, v10
+; CHECK-NEXT:    ret
+  %res = call <vscale x 8 x bfloat> @llvm.vector.reverse.nxv8bf16(<vscale x 8 x bfloat> %a)
+  ret <vscale x 8 x bfloat> %res
+}
+
+define <vscale x 16 x bfloat> @reverse_nxv16bf16(<vscale x 16 x bfloat> %a) {
+; CHECK-LABEL: reverse_nxv16bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    srli a0, a0, 1
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vid.v v12
+; CHECK-NEXT:    vrsub.vx v16, v12, a0
+; CHECK-NEXT:    vrgather.vv v15, v8, v16
+; CHECK-NEXT:    vrgather.vv v14, v9, v16
+; CHECK-NEXT:    vrgather.vv v13, v10, v16
+; CHECK-NEXT:    vrgather.vv v12, v11, v16
+; CHECK-NEXT:    vmv4r.v v8, v12
+; CHECK-NEXT:    ret
+  %res = call <vscale x 16 x bfloat> @llvm.vector.reverse.nxv16bf16(<vscale x 16 x bfloat> %a)
+  ret <vscale x 16 x bfloat> %res
+}
+
+define <vscale x 32 x bfloat> @reverse_nxv32bf16(<vscale x 32 x bfloat> %a) {
+; CHECK-LABEL: reverse_nxv32bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv8r.v v16, v8
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    srli a0, a0, 1
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    vrsub.vx v24, v8, a0
+; CHECK-NEXT:    vrgather.vv v15, v16, v24
+; CHECK-NEXT:    vrgather.vv v14, v17, v24
+; CHECK-NEXT:    vrgather.vv v13, v18, v24
+; CHECK-NEXT:    vrgather.vv v12, v19, v24
+; CHECK-NEXT:    vrgather.vv v11, v20, v24
+; CHECK-NEXT:    vrgather.vv v10, v21, v24
+; CHECK-NEXT:    vrgather.vv v9, v22, v24
+; CHECK-NEXT:    vrgather.vv v8, v23, v24
+; CHECK-NEXT:    ret
+  %res = call <vscale x 32 x bfloat> @llvm.vector.reverse.nxv32bf16(<vscale x 32 x bfloat> %a)
+  ret <vscale x 32 x bfloat> %res
+}
+
 define <vscale x 1 x half> @reverse_nxv1f16(<vscale x 1 x half> %a) {
 ; CHECK-LABEL: reverse_nxv1f16:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-to-vmv.mir b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-to-vmv.mir
index e2f1fe4094b5c..a5622fd466217 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-to-vmv.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-to-vmv.mir
@@ -159,3 +159,29 @@ body: |
     $v0 = COPY %mask
     %x:vrnov0 = PseudoVMERGE_VVM_M1 $noreg, %false, %true, $v0, 4, 5 /* e32 */
 ...
+---
+# Shouldn't be converted because true is in a different block
+name: same_mask_diff_blocks
+body: |
+  ; CHECK-LABEL: name: same_mask_diff_blocks
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $v8, $v0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   %false:vr = COPY $v8
+  ; CHECK-NEXT:   %mask:vr = COPY $v0
+  ; CHECK-NEXT:   $v0 = COPY %mask
+  ; CHECK-NEXT:   %true:vrnov0 = PseudoVADD_VV_M1_MASK $noreg, $noreg, $noreg, $v0, 4, 5 /* e32 */, 0 /* tu, mu */
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   $v0 = COPY %mask
+  ; CHECK-NEXT:   [[PseudoVMERGE_VVM_M1_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_M1 $noreg, %false, %true, $v0, 4, 5 /* e32 */
+  bb.0:
+    liveins: $v8, $v0
+    %false:vr = COPY $v8
+    %mask:vr = COPY $v0
+    $v0 = COPY %mask
+    %true:vrnov0 = PseudoVADD_VV_M1_MASK $noreg, $noreg, $noreg, $v0, 4, 5 /* e32 */, 0 /* tu, mu */
+  bb.1:
+    $v0 = COPY %mask
+    %5:vrnov0 = PseudoVMERGE_VVM_M1 $noreg, %false, %true, $v0, 4, 5 /* e32 */
diff --git a/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-vp.ll
index 0f8e74942d58d..f3ccf74019bb5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-vp.ll
@@ -1,339 +1,33 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
-; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
-; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+zfbfmin,+zvfbfmin,+v \
-; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
-; RUN:     --check-prefixes=CHECK,ZVFH
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
-; RUN:     -target-abi=ilp32d -verify-machineinstrs < %s | FileCheck %s \
-; RUN:     --check-prefixes=CHECK,ZVFHMIN
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfhmin,+zvfhmin,+zfbfmin,+zvfbfmin,+v \
-; RUN:     -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s \
-; RUN:     --check-prefixes=CHECK,ZVFHMIN
-
-declare bfloat @llvm.vp.reduce.fadd.nxv1bf16(bfloat, <vscale x 1 x bfloat>, <vscale x 1 x i1>, i32)
-
-define bfloat @vpreduce_fadd_nxv1bf16(bfloat %s, <vscale x 1 x bfloat> %v, <vscale x 1 x i1> %m, i32 zeroext %evl) {
-; CHECK-LABEL: vpreduce_fadd_nxv1bf16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
-; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa5
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    vfredusum.vs v8, v9, v8, v0.t
-; CHECK-NEXT:    vfmv.f.s fa5, v8
-; CHECK-NEXT:    fcvt.bf16.s fa0, fa5
-; CHECK-NEXT:    ret
-  %r = call reassoc bfloat @llvm.vp.reduce.fadd.nxv1bf16(bfloat %s, <vscale x 1 x bfloat> %v, <vscale x 1 x i1> %m, i32 %evl)
-  ret bfloat %r
-}
-
-define bfloat @vpreduce_ord_fadd_nxv1bf16(bfloat %s, <vscale x 1 x bfloat> %v, <vscale x 1 x i1> %m, i32 zeroext %evl) {
-; CHECK-LABEL: vpreduce_ord_fadd_nxv1bf16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
-; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa5
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    vfredosum.vs v8, v9, v8, v0.t
-; CHECK-NEXT:    vfmv.f.s fa5, v8
-; CHECK-NEXT:    fcvt.bf16.s fa0, fa5
-; CHECK-NEXT:    ret
-  %r = call bfloat @llvm.vp.reduce.fadd.nxv1bf16(bfloat %s, <vscale x 1 x bfloat> %v, <vscale x 1 x i1> %m, i32 %evl)
-  ret bfloat %r
-}
-
-declare bfloat @llvm.vp.reduce.fadd.nxv2bf16(bfloat, <vscale x 2 x bfloat>, <vscale x 2 x i1>, i32)
-
-define bfloat @vpreduce_fadd_nxv2bf16(bfloat %s, <vscale x 2 x bfloat> %v, <vscale x 2 x i1> %m, i32 zeroext %evl) {
-; CHECK-LABEL: vpreduce_fadd_nxv2bf16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
-; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa5
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    vfredusum.vs v8, v9, v8, v0.t
-; CHECK-NEXT:    vfmv.f.s fa5, v8
-; CHECK-NEXT:    fcvt.bf16.s fa0, fa5
-; CHECK-NEXT:    ret
-  %r = call reassoc bfloat @llvm.vp.reduce.fadd.nxv2bf16(bfloat %s, <vscale x 2 x bfloat> %v, <vscale x 2 x i1> %m, i32 %evl)
-  ret bfloat %r
-}
-
-define bfloat @vpreduce_ord_fadd_nxv2bf16(bfloat %s, <vscale x 2 x bfloat> %v, <vscale x 2 x i1> %m, i32 zeroext %evl) {
-; CHECK-LABEL: vpreduce_ord_fadd_nxv2bf16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
-; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa5
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    vfredosum.vs v8, v9, v8, v0.t
-; CHECK-NEXT:    vfmv.f.s fa5, v8
-; CHECK-NEXT:    fcvt.bf16.s fa0, fa5
-; CHECK-NEXT:    ret
-  %r = call bfloat @llvm.vp.reduce.fadd.nxv2bf16(bfloat %s, <vscale x 2 x bfloat> %v, <vscale x 2 x i1> %m, i32 %evl)
-  ret bfloat %r
-}
-
-declare bfloat @llvm.vp.reduce.fadd.nxv4bf16(bfloat, <vscale x 4 x bfloat>, <vscale x 4 x i1>, i32)
-
-define bfloat @vpreduce_fadd_nxv4bf16(bfloat %s, <vscale x 4 x bfloat> %v, <vscale x 4 x i1> %m, i32 zeroext %evl) {
-; CHECK-LABEL: vpreduce_fadd_nxv4bf16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
-; CHECK-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa5
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    vfredusum.vs v8, v10, v8, v0.t
-; CHECK-NEXT:    vfmv.f.s fa5, v8
-; CHECK-NEXT:    fcvt.bf16.s fa0, fa5
-; CHECK-NEXT:    ret
-  %r = call reassoc bfloat @llvm.vp.reduce.fadd.nxv4bf16(bfloat %s, <vscale x 4 x bfloat> %v, <vscale x 4 x i1> %m, i32 %evl)
-  ret bfloat %r
-}
-
-define bfloat @vpreduce_ord_fadd_nxv4bf16(bfloat %s, <vscale x 4 x bfloat> %v, <vscale x 4 x i1> %m, i32 zeroext %evl) {
-; CHECK-LABEL: vpreduce_ord_fadd_nxv4bf16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
-; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
-; CHECK-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa5
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    vfredosum.vs v8, v10, v8, v0.t
-; CHECK-NEXT:    vfmv.f.s fa5, v8
-; CHECK-NEXT:    fcvt.bf16.s fa0, fa5
-; CHECK-NEXT:    ret
-  %r = call bfloat @llvm.vp.reduce.fadd.nxv4bf16(bfloat %s, <vscale x 4 x bfloat> %v, <vscale x 4 x i1> %m, i32 %evl)
-  ret bfloat %r
-}
-
-declare bfloat @llvm.vp.reduce.fadd.nxv64bf16(bfloat, <vscale x 64 x bfloat>, <vscale x 64 x i1>, i32)
-
-define bfloat @vpreduce_fadd_nxv64bf16(bfloat %s, <vscale x 64 x bfloat> %v, <vscale x 64 x i1> %m, i32 zeroext %evl) {
-; CHECK-LABEL: vpreduce_fadd_nxv64bf16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    csrr a3, vlenb
-; CHECK-NEXT:    srli a1, a3, 1
-; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vx v7, v0, a1
-; CHECK-NEXT:    slli a5, a3, 2
-; CHECK-NEXT:    sub a1, a0, a5
-; CHECK-NEXT:    sltu a2, a0, a1
-; CHECK-NEXT:    addi a2, a2, -1
-; CHECK-NEXT:    and a1, a2, a1
-; CHECK-NEXT:    slli a4, a3, 1
-; CHECK-NEXT:    sub a2, a1, a4
-; CHECK-NEXT:    sltu a6, a1, a2
-; CHECK-NEXT:    bltu a1, a4, .LBB6_2
-; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    mv a1, a4
-; CHECK-NEXT:  .LBB6_2:
-; CHECK-NEXT:    addi a6, a6, -1
-; CHECK-NEXT:    bltu a0, a5, .LBB6_4
-; CHECK-NEXT:  # %bb.3:
-; CHECK-NEXT:    mv a0, a5
-; CHECK-NEXT:  .LBB6_4:
-; CHECK-NEXT:    and a2, a6, a2
-; CHECK-NEXT:    sub a5, a0, a4
-; CHECK-NEXT:    sltu a6, a0, a5
-; CHECK-NEXT:    addi a6, a6, -1
-; CHECK-NEXT:    and a5, a6, a5
-; CHECK-NEXT:    srli a3, a3, 2
-; CHECK-NEXT:    vsetvli a6, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vx v6, v0, a3
-; CHECK-NEXT:    bltu a0, a4, .LBB6_6
-; CHECK-NEXT:  # %bb.5:
-; CHECK-NEXT:    mv a0, a4
-; CHECK-NEXT:  .LBB6_6:
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v8
-; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
-; CHECK-NEXT:    vsetivli zero, 1, e32, m8, ta, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa5
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    vfredusum.vs v8, v24, v8, v0.t
-; CHECK-NEXT:    vfmv.f.s fa5, v8
-; CHECK-NEXT:    fcvt.bf16.s fa5, fa5
-; CHECK-NEXT:    fcvt.s.bf16 fa5, fa5
-; CHECK-NEXT:    vsetivli zero, 1, e32, m8, ta, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa5
-; CHECK-NEXT:    vsetvli zero, a5, e16, m4, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v12
-; CHECK-NEXT:    vmv1r.v v0, v6
-; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vfredusum.vs v8, v24, v8, v0.t
-; CHECK-NEXT:    vfmv.f.s fa5, v8
-; CHECK-NEXT:    fcvt.bf16.s fa5, fa5
-; CHECK-NEXT:    fcvt.s.bf16 fa5, fa5
-; CHECK-NEXT:    vsetivli zero, 1, e32, m8, ta, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa5
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v16
-; CHECK-NEXT:    vmv1r.v v0, v7
-; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vfredusum.vs v8, v24, v8, v0.t
-; CHECK-NEXT:    vfmv.f.s fa5, v8
-; CHECK-NEXT:    fcvt.bf16.s fa5, fa5
-; CHECK-NEXT:    fcvt.s.bf16 fa5, fa5
-; CHECK-NEXT:    vsetivli zero, 1, e32, m8, ta, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa5
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vx v0, v7, a3
-; CHECK-NEXT:    vsetvli zero, a2, e16, m4, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v20
-; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vfredusum.vs v8, v24, v8, v0.t
-; CHECK-NEXT:    vfmv.f.s fa5, v8
-; CHECK-NEXT:    fcvt.bf16.s fa0, fa5
-; CHECK-NEXT:    ret
-  %r = call reassoc bfloat @llvm.vp.reduce.fadd.nxv64bf16(bfloat %s, <vscale x 64 x bfloat> %v, <vscale x 64 x i1> %m, i32 %evl)
-  ret bfloat %r
-}
-
-define bfloat @vpreduce_ord_fadd_nxv64bf16(bfloat %s, <vscale x 64 x bfloat> %v, <vscale x 64 x i1> %m, i32 zeroext %evl) {
-; CHECK-LABEL: vpreduce_ord_fadd_nxv64bf16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    csrr a3, vlenb
-; CHECK-NEXT:    srli a1, a3, 1
-; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vx v7, v0, a1
-; CHECK-NEXT:    slli a5, a3, 2
-; CHECK-NEXT:    sub a1, a0, a5
-; CHECK-NEXT:    sltu a2, a0, a1
-; CHECK-NEXT:    addi a2, a2, -1
-; CHECK-NEXT:    and a1, a2, a1
-; CHECK-NEXT:    slli a4, a3, 1
-; CHECK-NEXT:    sub a2, a1, a4
-; CHECK-NEXT:    sltu a6, a1, a2
-; CHECK-NEXT:    bltu a1, a4, .LBB7_2
-; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    mv a1, a4
-; CHECK-NEXT:  .LBB7_2:
-; CHECK-NEXT:    addi a6, a6, -1
-; CHECK-NEXT:    bltu a0, a5, .LBB7_4
-; CHECK-NEXT:  # %bb.3:
-; CHECK-NEXT:    mv a0, a5
-; CHECK-NEXT:  .LBB7_4:
-; CHECK-NEXT:    and a2, a6, a2
-; CHECK-NEXT:    sub a5, a0, a4
-; CHECK-NEXT:    sltu a6, a0, a5
-; CHECK-NEXT:    addi a6, a6, -1
-; CHECK-NEXT:    and a5, a6, a5
-; CHECK-NEXT:    srli a3, a3, 2
-; CHECK-NEXT:    vsetvli a6, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vx v6, v0, a3
-; CHECK-NEXT:    bltu a0, a4, .LBB7_6
-; CHECK-NEXT:  # %bb.5:
-; CHECK-NEXT:    mv a0, a4
-; CHECK-NEXT:  .LBB7_6:
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v8
-; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
-; CHECK-NEXT:    vsetivli zero, 1, e32, m8, ta, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa5
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    vfredosum.vs v8, v24, v8, v0.t
-; CHECK-NEXT:    vfmv.f.s fa5, v8
-; CHECK-NEXT:    fcvt.bf16.s fa5, fa5
-; CHECK-NEXT:    fcvt.s.bf16 fa5, fa5
-; CHECK-NEXT:    vsetivli zero, 1, e32, m8, ta, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa5
-; CHECK-NEXT:    vsetvli zero, a5, e16, m4, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v12
-; CHECK-NEXT:    vmv1r.v v0, v6
-; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vfredosum.vs v8, v24, v8, v0.t
-; CHECK-NEXT:    vfmv.f.s fa5, v8
-; CHECK-NEXT:    fcvt.bf16.s fa5, fa5
-; CHECK-NEXT:    fcvt.s.bf16 fa5, fa5
-; CHECK-NEXT:    vsetivli zero, 1, e32, m8, ta, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa5
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v16
-; CHECK-NEXT:    vmv1r.v v0, v7
-; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vfredosum.vs v8, v24, v8, v0.t
-; CHECK-NEXT:    vfmv.f.s fa5, v8
-; CHECK-NEXT:    fcvt.bf16.s fa5, fa5
-; CHECK-NEXT:    fcvt.s.bf16 fa5, fa5
-; CHECK-NEXT:    vsetivli zero, 1, e32, m8, ta, ma
-; CHECK-NEXT:    vfmv.s.f v8, fa5
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vx v0, v7, a3
-; CHECK-NEXT:    vsetvli zero, a2, e16, m4, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v20
-; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vfredosum.vs v8, v24, v8, v0.t
-; CHECK-NEXT:    vfmv.f.s fa5, v8
-; CHECK-NEXT:    fcvt.bf16.s fa0, fa5
-; CHECK-NEXT:    ret
-  %r = call bfloat @llvm.vp.reduce.fadd.nxv64bf16(bfloat %s, <vscale x 64 x bfloat> %v, <vscale x 64 x i1> %m, i32 %evl)
-  ret bfloat %r
-}
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare half @llvm.vp.reduce.fadd.nxv1f16(half, <vscale x 1 x half>, <vscale x 1 x i1>, i32)
 
 define half @vpreduce_fadd_nxv1f16(half %s, <vscale x 1 x half> %v, <vscale x 1 x i1> %m, i32 zeroext %evl) {
-; ZVFH-LABEL: vpreduce_fadd_nxv1f16:
-; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; ZVFH-NEXT:    vfmv.s.f v9, fa0
-; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFH-NEXT:    vfredusum.vs v9, v8, v9, v0.t
-; ZVFH-NEXT:    vfmv.f.s fa0, v9
-; ZVFH-NEXT:    ret
-;
-; ZVFHMIN-LABEL: vpreduce_fadd_nxv1f16:
-; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfmv.s.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfredusum.vs v8, v9, v8, v0.t
-; ZVFHMIN-NEXT:    vfmv.f.s fa5, v8
-; ZVFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; ZVFHMIN-NEXT:    ret
+; CHECK-LABEL: vpreduce_fadd_nxv1f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vfredusum.vs v9, v8, v9, v0.t
+; CHECK-NEXT:    vfmv.f.s fa0, v9
+; CHECK-NEXT:    ret
   %r = call reassoc half @llvm.vp.reduce.fadd.nxv1f16(half %s, <vscale x 1 x half> %v, <vscale x 1 x i1> %m, i32 %evl)
   ret half %r
 }
 
 define half @vpreduce_ord_fadd_nxv1f16(half %s, <vscale x 1 x half> %v, <vscale x 1 x i1> %m, i32 zeroext %evl) {
-; ZVFH-LABEL: vpreduce_ord_fadd_nxv1f16:
-; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; ZVFH-NEXT:    vfmv.s.f v9, fa0
-; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFH-NEXT:    vfredosum.vs v9, v8, v9, v0.t
-; ZVFH-NEXT:    vfmv.f.s fa0, v9
-; ZVFH-NEXT:    ret
-;
-; ZVFHMIN-LABEL: vpreduce_ord_fadd_nxv1f16:
-; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfmv.s.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfredosum.vs v8, v9, v8, v0.t
-; ZVFHMIN-NEXT:    vfmv.f.s fa5, v8
-; ZVFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; ZVFHMIN-NEXT:    ret
+; CHECK-LABEL: vpreduce_ord_fadd_nxv1f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vfredosum.vs v9, v8, v9, v0.t
+; CHECK-NEXT:    vfmv.f.s fa0, v9
+; CHECK-NEXT:    ret
   %r = call half @llvm.vp.reduce.fadd.nxv1f16(half %s, <vscale x 1 x half> %v, <vscale x 1 x i1> %m, i32 %evl)
   ret half %r
 }
@@ -341,53 +35,27 @@ define half @vpreduce_ord_fadd_nxv1f16(half %s, <vscale x 1 x half> %v, <vscale
 declare half @llvm.vp.reduce.fadd.nxv2f16(half, <vscale x 2 x half>, <vscale x 2 x i1>, i32)
 
 define half @vpreduce_fadd_nxv2f16(half %s, <vscale x 2 x half> %v, <vscale x 2 x i1> %m, i32 zeroext %evl) {
-; ZVFH-LABEL: vpreduce_fadd_nxv2f16:
-; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; ZVFH-NEXT:    vfmv.s.f v9, fa0
-; ZVFH-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; ZVFH-NEXT:    vfredusum.vs v9, v8, v9, v0.t
-; ZVFH-NEXT:    vfmv.f.s fa0, v9
-; ZVFH-NEXT:    ret
-;
-; ZVFHMIN-LABEL: vpreduce_fadd_nxv2f16:
-; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; ZVFHMIN-NEXT:    vfmv.s.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; ZVFHMIN-NEXT:    vfredusum.vs v8, v9, v8, v0.t
-; ZVFHMIN-NEXT:    vfmv.f.s fa5, v8
-; ZVFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; ZVFHMIN-NEXT:    ret
+; CHECK-LABEL: vpreduce_fadd_nxv2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vfredusum.vs v9, v8, v9, v0.t
+; CHECK-NEXT:    vfmv.f.s fa0, v9
+; CHECK-NEXT:    ret
   %r = call reassoc half @llvm.vp.reduce.fadd.nxv2f16(half %s, <vscale x 2 x half> %v, <vscale x 2 x i1> %m, i32 %evl)
   ret half %r
 }
 
 define half @vpreduce_ord_fadd_nxv2f16(half %s, <vscale x 2 x half> %v, <vscale x 2 x i1> %m, i32 zeroext %evl) {
-; ZVFH-LABEL: vpreduce_ord_fadd_nxv2f16:
-; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; ZVFH-NEXT:    vfmv.s.f v9, fa0
-; ZVFH-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; ZVFH-NEXT:    vfredosum.vs v9, v8, v9, v0.t
-; ZVFH-NEXT:    vfmv.f.s fa0, v9
-; ZVFH-NEXT:    ret
-;
-; ZVFHMIN-LABEL: vpreduce_ord_fadd_nxv2f16:
-; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; ZVFHMIN-NEXT:    vfmv.s.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; ZVFHMIN-NEXT:    vfredosum.vs v8, v9, v8, v0.t
-; ZVFHMIN-NEXT:    vfmv.f.s fa5, v8
-; ZVFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; ZVFHMIN-NEXT:    ret
+; CHECK-LABEL: vpreduce_ord_fadd_nxv2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vfredosum.vs v9, v8, v9, v0.t
+; CHECK-NEXT:    vfmv.f.s fa0, v9
+; CHECK-NEXT:    ret
   %r = call half @llvm.vp.reduce.fadd.nxv2f16(half %s, <vscale x 2 x half> %v, <vscale x 2 x i1> %m, i32 %evl)
   ret half %r
 }
@@ -395,53 +63,27 @@ define half @vpreduce_ord_fadd_nxv2f16(half %s, <vscale x 2 x half> %v, <vscale
 declare half @llvm.vp.reduce.fadd.nxv4f16(half, <vscale x 4 x half>, <vscale x 4 x i1>, i32)
 
 define half @vpreduce_fadd_nxv4f16(half %s, <vscale x 4 x half> %v, <vscale x 4 x i1> %m, i32 zeroext %evl) {
-; ZVFH-LABEL: vpreduce_fadd_nxv4f16:
-; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; ZVFH-NEXT:    vfmv.s.f v9, fa0
-; ZVFH-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; ZVFH-NEXT:    vfredusum.vs v9, v8, v9, v0.t
-; ZVFH-NEXT:    vfmv.f.s fa0, v9
-; ZVFH-NEXT:    ret
-;
-; ZVFHMIN-LABEL: vpreduce_fadd_nxv4f16:
-; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; ZVFHMIN-NEXT:    vfmv.s.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; ZVFHMIN-NEXT:    vfredusum.vs v8, v10, v8, v0.t
-; ZVFHMIN-NEXT:    vfmv.f.s fa5, v8
-; ZVFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; ZVFHMIN-NEXT:    ret
+; CHECK-LABEL: vpreduce_fadd_nxv4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vfredusum.vs v9, v8, v9, v0.t
+; CHECK-NEXT:    vfmv.f.s fa0, v9
+; CHECK-NEXT:    ret
   %r = call reassoc half @llvm.vp.reduce.fadd.nxv4f16(half %s, <vscale x 4 x half> %v, <vscale x 4 x i1> %m, i32 %evl)
   ret half %r
 }
 
 define half @vpreduce_ord_fadd_nxv4f16(half %s, <vscale x 4 x half> %v, <vscale x 4 x i1> %m, i32 zeroext %evl) {
-; ZVFH-LABEL: vpreduce_ord_fadd_nxv4f16:
-; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; ZVFH-NEXT:    vfmv.s.f v9, fa0
-; ZVFH-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; ZVFH-NEXT:    vfredosum.vs v9, v8, v9, v0.t
-; ZVFH-NEXT:    vfmv.f.s fa0, v9
-; ZVFH-NEXT:    ret
-;
-; ZVFHMIN-LABEL: vpreduce_ord_fadd_nxv4f16:
-; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; ZVFHMIN-NEXT:    vfmv.s.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; ZVFHMIN-NEXT:    vfredosum.vs v8, v10, v8, v0.t
-; ZVFHMIN-NEXT:    vfmv.f.s fa5, v8
-; ZVFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; ZVFHMIN-NEXT:    ret
+; CHECK-LABEL: vpreduce_ord_fadd_nxv4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vfredosum.vs v9, v8, v9, v0.t
+; CHECK-NEXT:    vfmv.f.s fa0, v9
+; CHECK-NEXT:    ret
   %r = call half @llvm.vp.reduce.fadd.nxv4f16(half %s, <vscale x 4 x half> %v, <vscale x 4 x i1> %m, i32 %evl)
   ret half %r
 }
@@ -449,213 +91,59 @@ define half @vpreduce_ord_fadd_nxv4f16(half %s, <vscale x 4 x half> %v, <vscale
 declare half @llvm.vp.reduce.fadd.nxv64f16(half, <vscale x 64 x half>, <vscale x 64 x i1>, i32)
 
 define half @vpreduce_fadd_nxv64f16(half %s, <vscale x 64 x half> %v, <vscale x 64 x i1> %m, i32 zeroext %evl) {
-; ZVFH-LABEL: vpreduce_fadd_nxv64f16:
-; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    csrr a2, vlenb
-; ZVFH-NEXT:    srli a1, a2, 1
-; ZVFH-NEXT:    vsetvli a3, zero, e8, m1, ta, ma
-; ZVFH-NEXT:    vslidedown.vx v24, v0, a1
-; ZVFH-NEXT:    slli a2, a2, 2
-; ZVFH-NEXT:    sub a1, a0, a2
-; ZVFH-NEXT:    sltu a3, a0, a1
-; ZVFH-NEXT:    addi a3, a3, -1
-; ZVFH-NEXT:    and a1, a3, a1
-; ZVFH-NEXT:    bltu a0, a2, .LBB14_2
-; ZVFH-NEXT:  # %bb.1:
-; ZVFH-NEXT:    mv a0, a2
-; ZVFH-NEXT:  .LBB14_2:
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; ZVFH-NEXT:    vfmv.s.f v25, fa0
-; ZVFH-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; ZVFH-NEXT:    vfredusum.vs v25, v8, v25, v0.t
-; ZVFH-NEXT:    vmv1r.v v0, v24
-; ZVFH-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
-; ZVFH-NEXT:    vfredusum.vs v25, v16, v25, v0.t
-; ZVFH-NEXT:    vfmv.f.s fa0, v25
-; ZVFH-NEXT:    ret
-;
-; ZVFHMIN-LABEL: vpreduce_fadd_nxv64f16:
-; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    csrr a3, vlenb
-; ZVFHMIN-NEXT:    srli a1, a3, 1
-; ZVFHMIN-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
-; ZVFHMIN-NEXT:    vslidedown.vx v7, v0, a1
-; ZVFHMIN-NEXT:    slli a5, a3, 2
-; ZVFHMIN-NEXT:    sub a1, a0, a5
-; ZVFHMIN-NEXT:    sltu a2, a0, a1
-; ZVFHMIN-NEXT:    addi a2, a2, -1
-; ZVFHMIN-NEXT:    and a1, a2, a1
-; ZVFHMIN-NEXT:    slli a4, a3, 1
-; ZVFHMIN-NEXT:    sub a2, a1, a4
-; ZVFHMIN-NEXT:    sltu a6, a1, a2
-; ZVFHMIN-NEXT:    bltu a1, a4, .LBB14_2
-; ZVFHMIN-NEXT:  # %bb.1:
-; ZVFHMIN-NEXT:    mv a1, a4
-; ZVFHMIN-NEXT:  .LBB14_2:
-; ZVFHMIN-NEXT:    addi a6, a6, -1
-; ZVFHMIN-NEXT:    bltu a0, a5, .LBB14_4
-; ZVFHMIN-NEXT:  # %bb.3:
-; ZVFHMIN-NEXT:    mv a0, a5
-; ZVFHMIN-NEXT:  .LBB14_4:
-; ZVFHMIN-NEXT:    and a2, a6, a2
-; ZVFHMIN-NEXT:    sub a5, a0, a4
-; ZVFHMIN-NEXT:    sltu a6, a0, a5
-; ZVFHMIN-NEXT:    addi a6, a6, -1
-; ZVFHMIN-NEXT:    and a5, a6, a5
-; ZVFHMIN-NEXT:    srli a3, a3, 2
-; ZVFHMIN-NEXT:    vsetvli a6, zero, e8, mf2, ta, ma
-; ZVFHMIN-NEXT:    vslidedown.vx v6, v0, a3
-; ZVFHMIN-NEXT:    bltu a0, a4, .LBB14_6
-; ZVFHMIN-NEXT:  # %bb.5:
-; ZVFHMIN-NEXT:    mv a0, a4
-; ZVFHMIN-NEXT:  .LBB14_6:
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v8
-; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT:    vsetivli zero, 1, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfmv.s.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfredusum.vs v8, v24, v8, v0.t
-; ZVFHMIN-NEXT:    vfmv.f.s fa5, v8
-; ZVFHMIN-NEXT:    fcvt.h.s fa5, fa5
-; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa5
-; ZVFHMIN-NEXT:    vsetivli zero, 1, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfmv.s.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a5, e16, m4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v12
-; ZVFHMIN-NEXT:    vmv1r.v v0, v6
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfredusum.vs v8, v24, v8, v0.t
-; ZVFHMIN-NEXT:    vfmv.f.s fa5, v8
-; ZVFHMIN-NEXT:    fcvt.h.s fa5, fa5
-; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa5
-; ZVFHMIN-NEXT:    vsetivli zero, 1, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfmv.s.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v16
-; ZVFHMIN-NEXT:    vmv1r.v v0, v7
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfredusum.vs v8, v24, v8, v0.t
-; ZVFHMIN-NEXT:    vfmv.f.s fa5, v8
-; ZVFHMIN-NEXT:    fcvt.h.s fa5, fa5
-; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa5
-; ZVFHMIN-NEXT:    vsetivli zero, 1, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfmv.s.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
-; ZVFHMIN-NEXT:    vslidedown.vx v0, v7, a3
-; ZVFHMIN-NEXT:    vsetvli zero, a2, e16, m4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v20
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfredusum.vs v8, v24, v8, v0.t
-; ZVFHMIN-NEXT:    vfmv.f.s fa5, v8
-; ZVFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; ZVFHMIN-NEXT:    ret
+; CHECK-LABEL: vpreduce_fadd_nxv64f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a2, vlenb
+; CHECK-NEXT:    srli a1, a2, 1
+; CHECK-NEXT:    vsetvli a3, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vx v24, v0, a1
+; CHECK-NEXT:    slli a2, a2, 2
+; CHECK-NEXT:    sub a1, a0, a2
+; CHECK-NEXT:    sltu a3, a0, a1
+; CHECK-NEXT:    addi a3, a3, -1
+; CHECK-NEXT:    and a1, a3, a1
+; CHECK-NEXT:    bltu a0, a2, .LBB6_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    mv a0, a2
+; CHECK-NEXT:  .LBB6_2:
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vfmv.s.f v25, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; CHECK-NEXT:    vfredusum.vs v25, v8, v25, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v24
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
+; CHECK-NEXT:    vfredusum.vs v25, v16, v25, v0.t
+; CHECK-NEXT:    vfmv.f.s fa0, v25
+; CHECK-NEXT:    ret
   %r = call reassoc half @llvm.vp.reduce.fadd.nxv64f16(half %s, <vscale x 64 x half> %v, <vscale x 64 x i1> %m, i32 %evl)
   ret half %r
 }
 
 define half @vpreduce_ord_fadd_nxv64f16(half %s, <vscale x 64 x half> %v, <vscale x 64 x i1> %m, i32 zeroext %evl) {
-; ZVFH-LABEL: vpreduce_ord_fadd_nxv64f16:
-; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    csrr a2, vlenb
-; ZVFH-NEXT:    srli a1, a2, 1
-; ZVFH-NEXT:    vsetvli a3, zero, e8, m1, ta, ma
-; ZVFH-NEXT:    vslidedown.vx v24, v0, a1
-; ZVFH-NEXT:    slli a2, a2, 2
-; ZVFH-NEXT:    sub a1, a0, a2
-; ZVFH-NEXT:    sltu a3, a0, a1
-; ZVFH-NEXT:    addi a3, a3, -1
-; ZVFH-NEXT:    and a1, a3, a1
-; ZVFH-NEXT:    bltu a0, a2, .LBB15_2
-; ZVFH-NEXT:  # %bb.1:
-; ZVFH-NEXT:    mv a0, a2
-; ZVFH-NEXT:  .LBB15_2:
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; ZVFH-NEXT:    vfmv.s.f v25, fa0
-; ZVFH-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; ZVFH-NEXT:    vfredosum.vs v25, v8, v25, v0.t
-; ZVFH-NEXT:    vmv1r.v v0, v24
-; ZVFH-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
-; ZVFH-NEXT:    vfredosum.vs v25, v16, v25, v0.t
-; ZVFH-NEXT:    vfmv.f.s fa0, v25
-; ZVFH-NEXT:    ret
-;
-; ZVFHMIN-LABEL: vpreduce_ord_fadd_nxv64f16:
-; ZVFHMIN:       # %bb.0:
-; ZVFHMIN-NEXT:    csrr a3, vlenb
-; ZVFHMIN-NEXT:    srli a1, a3, 1
-; ZVFHMIN-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
-; ZVFHMIN-NEXT:    vslidedown.vx v7, v0, a1
-; ZVFHMIN-NEXT:    slli a5, a3, 2
-; ZVFHMIN-NEXT:    sub a1, a0, a5
-; ZVFHMIN-NEXT:    sltu a2, a0, a1
-; ZVFHMIN-NEXT:    addi a2, a2, -1
-; ZVFHMIN-NEXT:    and a1, a2, a1
-; ZVFHMIN-NEXT:    slli a4, a3, 1
-; ZVFHMIN-NEXT:    sub a2, a1, a4
-; ZVFHMIN-NEXT:    sltu a6, a1, a2
-; ZVFHMIN-NEXT:    bltu a1, a4, .LBB15_2
-; ZVFHMIN-NEXT:  # %bb.1:
-; ZVFHMIN-NEXT:    mv a1, a4
-; ZVFHMIN-NEXT:  .LBB15_2:
-; ZVFHMIN-NEXT:    addi a6, a6, -1
-; ZVFHMIN-NEXT:    bltu a0, a5, .LBB15_4
-; ZVFHMIN-NEXT:  # %bb.3:
-; ZVFHMIN-NEXT:    mv a0, a5
-; ZVFHMIN-NEXT:  .LBB15_4:
-; ZVFHMIN-NEXT:    and a2, a6, a2
-; ZVFHMIN-NEXT:    sub a5, a0, a4
-; ZVFHMIN-NEXT:    sltu a6, a0, a5
-; ZVFHMIN-NEXT:    addi a6, a6, -1
-; ZVFHMIN-NEXT:    and a5, a6, a5
-; ZVFHMIN-NEXT:    srli a3, a3, 2
-; ZVFHMIN-NEXT:    vsetvli a6, zero, e8, mf2, ta, ma
-; ZVFHMIN-NEXT:    vslidedown.vx v6, v0, a3
-; ZVFHMIN-NEXT:    bltu a0, a4, .LBB15_6
-; ZVFHMIN-NEXT:  # %bb.5:
-; ZVFHMIN-NEXT:    mv a0, a4
-; ZVFHMIN-NEXT:  .LBB15_6:
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v8
-; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT:    vsetivli zero, 1, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfmv.s.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfredosum.vs v8, v24, v8, v0.t
-; ZVFHMIN-NEXT:    vfmv.f.s fa5, v8
-; ZVFHMIN-NEXT:    fcvt.h.s fa5, fa5
-; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa5
-; ZVFHMIN-NEXT:    vsetivli zero, 1, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfmv.s.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a5, e16, m4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v12
-; ZVFHMIN-NEXT:    vmv1r.v v0, v6
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfredosum.vs v8, v24, v8, v0.t
-; ZVFHMIN-NEXT:    vfmv.f.s fa5, v8
-; ZVFHMIN-NEXT:    fcvt.h.s fa5, fa5
-; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa5
-; ZVFHMIN-NEXT:    vsetivli zero, 1, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfmv.s.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v16
-; ZVFHMIN-NEXT:    vmv1r.v v0, v7
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfredosum.vs v8, v24, v8, v0.t
-; ZVFHMIN-NEXT:    vfmv.f.s fa5, v8
-; ZVFHMIN-NEXT:    fcvt.h.s fa5, fa5
-; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa5
-; ZVFHMIN-NEXT:    vsetivli zero, 1, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfmv.s.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
-; ZVFHMIN-NEXT:    vslidedown.vx v0, v7, a3
-; ZVFHMIN-NEXT:    vsetvli zero, a2, e16, m4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v20
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vfredosum.vs v8, v24, v8, v0.t
-; ZVFHMIN-NEXT:    vfmv.f.s fa5, v8
-; ZVFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; ZVFHMIN-NEXT:    ret
+; CHECK-LABEL: vpreduce_ord_fadd_nxv64f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a2, vlenb
+; CHECK-NEXT:    srli a1, a2, 1
+; CHECK-NEXT:    vsetvli a3, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vx v24, v0, a1
+; CHECK-NEXT:    slli a2, a2, 2
+; CHECK-NEXT:    sub a1, a0, a2
+; CHECK-NEXT:    sltu a3, a0, a1
+; CHECK-NEXT:    addi a3, a3, -1
+; CHECK-NEXT:    and a1, a3, a1
+; CHECK-NEXT:    bltu a0, a2, .LBB7_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    mv a0, a2
+; CHECK-NEXT:  .LBB7_2:
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vfmv.s.f v25, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; CHECK-NEXT:    vfredosum.vs v25, v8, v25, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v24
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
+; CHECK-NEXT:    vfredosum.vs v25, v16, v25, v0.t
+; CHECK-NEXT:    vfmv.f.s fa0, v25
+; CHECK-NEXT:    ret
   %r = call half @llvm.vp.reduce.fadd.nxv64f16(half %s, <vscale x 64 x half> %v, <vscale x 64 x i1> %m, i32 %evl)
   ret half %r
 }
@@ -868,12 +356,12 @@ define float @vreduce_fminimum_nxv4f32(float %start, <vscale x 4 x float> %val,
 ; CHECK-NEXT:    feq.s a1, fa0, fa0
 ; CHECK-NEXT:    xori a1, a1, 1
 ; CHECK-NEXT:    or a0, a0, a1
-; CHECK-NEXT:    beqz a0, .LBB30_2
+; CHECK-NEXT:    beqz a0, .LBB22_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    lui a0, 523264
 ; CHECK-NEXT:    fmv.w.x fa0, a0
 ; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB30_2:
+; CHECK-NEXT:  .LBB22_2:
 ; CHECK-NEXT:    vfmv.f.s fa0, v10
 ; CHECK-NEXT:    ret
   %s = call float @llvm.vp.reduce.fminimum.nxv4f32(float %start, <vscale x 4 x float> %val, <vscale x 4 x i1> %m, i32 %evl)
@@ -892,12 +380,12 @@ define float @vreduce_fmaximum_nxv4f32(float %start, <vscale x 4 x float> %val,
 ; CHECK-NEXT:    feq.s a1, fa0, fa0
 ; CHECK-NEXT:    xori a1, a1, 1
 ; CHECK-NEXT:    or a0, a0, a1
-; CHECK-NEXT:    beqz a0, .LBB31_2
+; CHECK-NEXT:    beqz a0, .LBB23_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    lui a0, 523264
 ; CHECK-NEXT:    fmv.w.x fa0, a0
 ; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB31_2:
+; CHECK-NEXT:  .LBB23_2:
 ; CHECK-NEXT:    vfmv.f.s fa0, v10
 ; CHECK-NEXT:    ret
   %s = call float @llvm.vp.reduce.fmaximum.nxv4f32(float %start, <vscale x 4 x float> %val, <vscale x 4 x i1> %m, i32 %evl)
@@ -942,12 +430,12 @@ define float @vreduce_fminimum_v4f32(float %start, <4 x float> %val, <4 x i1> %m
 ; CHECK-NEXT:    feq.s a1, fa0, fa0
 ; CHECK-NEXT:    xori a1, a1, 1
 ; CHECK-NEXT:    or a0, a0, a1
-; CHECK-NEXT:    beqz a0, .LBB34_2
+; CHECK-NEXT:    beqz a0, .LBB26_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    lui a0, 523264
 ; CHECK-NEXT:    fmv.w.x fa0, a0
 ; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB34_2:
+; CHECK-NEXT:  .LBB26_2:
 ; CHECK-NEXT:    vfmv.f.s fa0, v9
 ; CHECK-NEXT:    ret
   %s = call float @llvm.vp.reduce.fminimum.v4f32(float %start, <4 x float> %val, <4 x i1> %m, i32 %evl)
@@ -966,12 +454,12 @@ define float @vreduce_fmaximum_v4f32(float %start, <4 x float> %val, <4 x i1> %m
 ; CHECK-NEXT:    feq.s a1, fa0, fa0
 ; CHECK-NEXT:    xori a1, a1, 1
 ; CHECK-NEXT:    or a0, a0, a1
-; CHECK-NEXT:    beqz a0, .LBB35_2
+; CHECK-NEXT:    beqz a0, .LBB27_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    lui a0, 523264
 ; CHECK-NEXT:    fmv.w.x fa0, a0
 ; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB35_2:
+; CHECK-NEXT:  .LBB27_2:
 ; CHECK-NEXT:    vfmv.f.s fa0, v9
 ; CHECK-NEXT:    ret
   %s = call float @llvm.vp.reduce.fmaximum.v4f32(float %start, <4 x float> %val, <4 x i1> %m, i32 %evl)
diff --git a/llvm/test/CodeGen/RISCV/stack-guard-global.ll b/llvm/test/CodeGen/RISCV/stack-guard-global.ll
new file mode 100644
index 0000000000000..da26ceb5e9c01
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/stack-guard-global.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:     | FileCheck %s
+
+define void @foo(i64 %t) sspstrong nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -32
+; CHECK-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    addi s0, sp, 32
+; CHECK-NEXT:    lui s1, %hi(__stack_chk_guard)
+; CHECK-NEXT:    ld a1, %lo(__stack_chk_guard)(s1)
+; CHECK-NEXT:    sd a1, -32(s0)
+; CHECK-NEXT:    slli a0, a0, 2
+; CHECK-NEXT:    addi a0, a0, 15
+; CHECK-NEXT:    andi a0, a0, -16
+; CHECK-NEXT:    sub a0, sp, a0
+; CHECK-NEXT:    mv sp, a0
+; CHECK-NEXT:    call baz
+; CHECK-NEXT:    ld a0, %lo(__stack_chk_guard)(s1)
+; CHECK-NEXT:    ld a1, -32(s0)
+; CHECK-NEXT:    bne a0, a1, .LBB0_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    addi sp, s0, -32
+; CHECK-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 32
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:    call __stack_chk_fail
+  %vla = alloca i32, i64 %t, align 4
+  call void @baz(ptr %vla)
+  ret void
+}
+
+declare void @baz(ptr)
+
+!llvm.module.flags = !{!1}
+!1 = !{i32 2, !"stack-protector-guard", !"global"}
diff --git a/llvm/test/CodeGen/RISCV/stack-guard-tls.ll b/llvm/test/CodeGen/RISCV/stack-guard-tls.ll
new file mode 100644
index 0000000000000..2992097937feb
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/stack-guard-tls.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:     | FileCheck %s
+
+define void @foo(i64 %t) sspstrong nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -32
+; CHECK-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    addi s0, sp, 32
+; CHECK-NEXT:    ld a1, 500(tp)
+; CHECK-NEXT:    sd a1, -24(s0)
+; CHECK-NEXT:    slli a0, a0, 2
+; CHECK-NEXT:    addi a0, a0, 15
+; CHECK-NEXT:    andi a0, a0, -16
+; CHECK-NEXT:    sub a0, sp, a0
+; CHECK-NEXT:    mv sp, a0
+; CHECK-NEXT:    call baz
+; CHECK-NEXT:    ld a0, 500(tp)
+; CHECK-NEXT:    ld a1, -24(s0)
+; CHECK-NEXT:    bne a0, a1, .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %SP_return
+; CHECK-NEXT:    addi sp, s0, -32
+; CHECK-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 32
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB0_2: # %CallStackCheckFailBlk
+; CHECK-NEXT:    call __stack_chk_fail
+  %vla = alloca i32, i64 %t, align 4
+  call void @baz(ptr %vla)
+  ret void
+}
+
+declare void @baz(ptr)
+
+!llvm.module.flags = !{!1, !2, !3}
+!1 = !{i32 2, !"stack-protector-guard", !"tls"}
+!2 = !{i32 2, !"stack-protector-guard-reg", !"tp"}
+!3 = !{i32 2, !"stack-protector-guard-offset", i32 500}
diff --git a/llvm/test/CodeGen/RISCV/stack-realignment.ll b/llvm/test/CodeGen/RISCV/stack-realignment.ll
index 034ebadc76af2..58876e4888a91 100644
--- a/llvm/test/CodeGen/RISCV/stack-realignment.ll
+++ b/llvm/test/CodeGen/RISCV/stack-realignment.ll
@@ -815,8 +815,7 @@ define void @caller1024() {
 ; RV32I-NEXT:    andi sp, sp, -1024
 ; RV32I-NEXT:    addi a0, sp, 1024
 ; RV32I-NEXT:    call callee
-; RV32I-NEXT:    addi sp, s0, -2048
-; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    addi sp, s0, -2032
 ; RV32I-NEXT:    lw ra, 2028(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 2024(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 2032
@@ -836,8 +835,7 @@ define void @caller1024() {
 ; RV32I-ILP32E-NEXT:    andi sp, sp, -1024
 ; RV32I-ILP32E-NEXT:    addi a0, sp, 1024
 ; RV32I-ILP32E-NEXT:    call callee
-; RV32I-ILP32E-NEXT:    addi sp, s0, -2048
-; RV32I-ILP32E-NEXT:    addi sp, sp, 4
+; RV32I-ILP32E-NEXT:    addi sp, s0, -2044
 ; RV32I-ILP32E-NEXT:    lw ra, 2040(sp) # 4-byte Folded Reload
 ; RV32I-ILP32E-NEXT:    lw s0, 2036(sp) # 4-byte Folded Reload
 ; RV32I-ILP32E-NEXT:    addi sp, sp, 2044
@@ -857,8 +855,7 @@ define void @caller1024() {
 ; RV64I-NEXT:    andi sp, sp, -1024
 ; RV64I-NEXT:    addi a0, sp, 1024
 ; RV64I-NEXT:    call callee
-; RV64I-NEXT:    addi sp, s0, -2048
-; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    addi sp, s0, -2032
 ; RV64I-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 2016(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 2032
@@ -878,8 +875,7 @@ define void @caller1024() {
 ; RV64I-LP64E-NEXT:    andi sp, sp, -1024
 ; RV64I-LP64E-NEXT:    addi a0, sp, 1024
 ; RV64I-LP64E-NEXT:    call callee
-; RV64I-LP64E-NEXT:    addi sp, s0, -2048
-; RV64I-LP64E-NEXT:    addi sp, sp, 8
+; RV64I-LP64E-NEXT:    addi sp, s0, -2040
 ; RV64I-LP64E-NEXT:    ld ra, 2032(sp) # 8-byte Folded Reload
 ; RV64I-LP64E-NEXT:    ld s0, 2024(sp) # 8-byte Folded Reload
 ; RV64I-LP64E-NEXT:    addi sp, sp, 2040
@@ -959,10 +955,7 @@ define void @caller2048() {
 ; RV32I-NEXT:    addi a0, sp, 2047
 ; RV32I-NEXT:    addi a0, a0, 1
 ; RV32I-NEXT:    call callee
-; RV32I-NEXT:    lui a0, 1
-; RV32I-NEXT:    sub sp, s0, a0
-; RV32I-NEXT:    addi sp, sp, 2032
-; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    addi sp, s0, -2032
 ; RV32I-NEXT:    lw ra, 2028(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 2024(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 2032
@@ -984,10 +977,7 @@ define void @caller2048() {
 ; RV32I-ILP32E-NEXT:    addi a0, sp, 2047
 ; RV32I-ILP32E-NEXT:    addi a0, a0, 1
 ; RV32I-ILP32E-NEXT:    call callee
-; RV32I-ILP32E-NEXT:    lui a0, 1
-; RV32I-ILP32E-NEXT:    sub sp, s0, a0
-; RV32I-ILP32E-NEXT:    addi sp, sp, 2044
-; RV32I-ILP32E-NEXT:    addi sp, sp, 8
+; RV32I-ILP32E-NEXT:    addi sp, s0, -2044
 ; RV32I-ILP32E-NEXT:    lw ra, 2040(sp) # 4-byte Folded Reload
 ; RV32I-ILP32E-NEXT:    lw s0, 2036(sp) # 4-byte Folded Reload
 ; RV32I-ILP32E-NEXT:    addi sp, sp, 2044
@@ -1009,10 +999,7 @@ define void @caller2048() {
 ; RV64I-NEXT:    addi a0, sp, 2047
 ; RV64I-NEXT:    addi a0, a0, 1
 ; RV64I-NEXT:    call callee
-; RV64I-NEXT:    lui a0, 1
-; RV64I-NEXT:    sub sp, s0, a0
-; RV64I-NEXT:    addi sp, sp, 2032
-; RV64I-NEXT:    addi sp, sp, 32
+; RV64I-NEXT:    addi sp, s0, -2032
 ; RV64I-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 2016(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 2032
@@ -1034,10 +1021,7 @@ define void @caller2048() {
 ; RV64I-LP64E-NEXT:    addi a0, sp, 2047
 ; RV64I-LP64E-NEXT:    addi a0, a0, 1
 ; RV64I-LP64E-NEXT:    call callee
-; RV64I-LP64E-NEXT:    lui a0, 1
-; RV64I-LP64E-NEXT:    sub sp, s0, a0
-; RV64I-LP64E-NEXT:    addi sp, sp, 2040
-; RV64I-LP64E-NEXT:    addi sp, sp, 16
+; RV64I-LP64E-NEXT:    addi sp, s0, -2040
 ; RV64I-LP64E-NEXT:    ld ra, 2032(sp) # 8-byte Folded Reload
 ; RV64I-LP64E-NEXT:    ld s0, 2024(sp) # 8-byte Folded Reload
 ; RV64I-LP64E-NEXT:    addi sp, sp, 2040
@@ -1119,10 +1103,7 @@ define void @caller4096() {
 ; RV32I-NEXT:    lui a0, 1
 ; RV32I-NEXT:    add a0, sp, a0
 ; RV32I-NEXT:    call callee
-; RV32I-NEXT:    lui a0, 2
-; RV32I-NEXT:    sub sp, s0, a0
-; RV32I-NEXT:    addi a0, a0, -2032
-; RV32I-NEXT:    add sp, sp, a0
+; RV32I-NEXT:    addi sp, s0, -2032
 ; RV32I-NEXT:    lw ra, 2028(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 2024(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 2032
@@ -1146,10 +1127,7 @@ define void @caller4096() {
 ; RV32I-ILP32E-NEXT:    lui a0, 1
 ; RV32I-ILP32E-NEXT:    add a0, sp, a0
 ; RV32I-ILP32E-NEXT:    call callee
-; RV32I-ILP32E-NEXT:    lui a0, 2
-; RV32I-ILP32E-NEXT:    sub sp, s0, a0
-; RV32I-ILP32E-NEXT:    addi a0, a0, -2044
-; RV32I-ILP32E-NEXT:    add sp, sp, a0
+; RV32I-ILP32E-NEXT:    addi sp, s0, -2044
 ; RV32I-ILP32E-NEXT:    lw ra, 2040(sp) # 4-byte Folded Reload
 ; RV32I-ILP32E-NEXT:    lw s0, 2036(sp) # 4-byte Folded Reload
 ; RV32I-ILP32E-NEXT:    addi sp, sp, 2044
@@ -1173,10 +1151,7 @@ define void @caller4096() {
 ; RV64I-NEXT:    lui a0, 1
 ; RV64I-NEXT:    add a0, sp, a0
 ; RV64I-NEXT:    call callee
-; RV64I-NEXT:    lui a0, 2
-; RV64I-NEXT:    sub sp, s0, a0
-; RV64I-NEXT:    addiw a0, a0, -2032
-; RV64I-NEXT:    add sp, sp, a0
+; RV64I-NEXT:    addi sp, s0, -2032
 ; RV64I-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 2016(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 2032
@@ -1200,10 +1175,7 @@ define void @caller4096() {
 ; RV64I-LP64E-NEXT:    lui a0, 1
 ; RV64I-LP64E-NEXT:    add a0, sp, a0
 ; RV64I-LP64E-NEXT:    call callee
-; RV64I-LP64E-NEXT:    lui a0, 2
-; RV64I-LP64E-NEXT:    sub sp, s0, a0
-; RV64I-LP64E-NEXT:    addiw a0, a0, -2040
-; RV64I-LP64E-NEXT:    add sp, sp, a0
+; RV64I-LP64E-NEXT:    addi sp, s0, -2040
 ; RV64I-LP64E-NEXT:    ld ra, 2032(sp) # 8-byte Folded Reload
 ; RV64I-LP64E-NEXT:    ld s0, 2024(sp) # 8-byte Folded Reload
 ; RV64I-LP64E-NEXT:    addi sp, sp, 2040
diff --git a/llvm/test/CodeGen/RISCV/unaligned-load-store.ll b/llvm/test/CodeGen/RISCV/unaligned-load-store.ll
index 74d34b2b64d41..0ca9fc065ff2d 100644
--- a/llvm/test/CodeGen/RISCV/unaligned-load-store.ll
+++ b/llvm/test/CodeGen/RISCV/unaligned-load-store.ll
@@ -97,18 +97,29 @@ define i32 @load_i32(ptr %p) {
 ; SLOWBASE-NEXT:    or a0, a0, a1
 ; SLOWBASE-NEXT:    ret
 ;
-; SLOWZBKB-LABEL: load_i32:
-; SLOWZBKB:       # %bb.0:
-; SLOWZBKB-NEXT:    lbu a1, 0(a0)
-; SLOWZBKB-NEXT:    lbu a2, 1(a0)
-; SLOWZBKB-NEXT:    lbu a3, 2(a0)
-; SLOWZBKB-NEXT:    lbu a0, 3(a0)
-; SLOWZBKB-NEXT:    packh a1, a1, a2
-; SLOWZBKB-NEXT:    slli a3, a3, 16
-; SLOWZBKB-NEXT:    slli a0, a0, 24
-; SLOWZBKB-NEXT:    or a0, a0, a3
-; SLOWZBKB-NEXT:    or a0, a0, a1
-; SLOWZBKB-NEXT:    ret
+; RV32IZBKB-LABEL: load_i32:
+; RV32IZBKB:       # %bb.0:
+; RV32IZBKB-NEXT:    lbu a1, 1(a0)
+; RV32IZBKB-NEXT:    lbu a2, 2(a0)
+; RV32IZBKB-NEXT:    lbu a3, 3(a0)
+; RV32IZBKB-NEXT:    lbu a0, 0(a0)
+; RV32IZBKB-NEXT:    packh a2, a2, a3
+; RV32IZBKB-NEXT:    packh a0, a0, a1
+; RV32IZBKB-NEXT:    pack a0, a0, a2
+; RV32IZBKB-NEXT:    ret
+;
+; RV64IZBKB-LABEL: load_i32:
+; RV64IZBKB:       # %bb.0:
+; RV64IZBKB-NEXT:    lbu a1, 0(a0)
+; RV64IZBKB-NEXT:    lbu a2, 1(a0)
+; RV64IZBKB-NEXT:    lbu a3, 2(a0)
+; RV64IZBKB-NEXT:    lbu a0, 3(a0)
+; RV64IZBKB-NEXT:    packh a1, a1, a2
+; RV64IZBKB-NEXT:    slli a3, a3, 16
+; RV64IZBKB-NEXT:    slli a0, a0, 24
+; RV64IZBKB-NEXT:    or a0, a0, a3
+; RV64IZBKB-NEXT:    or a0, a0, a1
+; RV64IZBKB-NEXT:    ret
 ;
 ; FAST-LABEL: load_i32:
 ; FAST:       # %bb.0:
@@ -176,20 +187,16 @@ define i64 @load_i64(ptr %p) {
 ; RV32IZBKB-NEXT:    lbu a2, 1(a0)
 ; RV32IZBKB-NEXT:    lbu a3, 2(a0)
 ; RV32IZBKB-NEXT:    lbu a4, 3(a0)
+; RV32IZBKB-NEXT:    lbu a5, 4(a0)
+; RV32IZBKB-NEXT:    lbu a6, 5(a0)
+; RV32IZBKB-NEXT:    lbu a7, 6(a0)
+; RV32IZBKB-NEXT:    lbu t0, 7(a0)
+; RV32IZBKB-NEXT:    packh a0, a3, a4
 ; RV32IZBKB-NEXT:    packh a1, a1, a2
-; RV32IZBKB-NEXT:    slli a3, a3, 16
-; RV32IZBKB-NEXT:    slli a4, a4, 24
-; RV32IZBKB-NEXT:    or a3, a4, a3
-; RV32IZBKB-NEXT:    lbu a2, 4(a0)
-; RV32IZBKB-NEXT:    lbu a4, 5(a0)
-; RV32IZBKB-NEXT:    lbu a5, 6(a0)
-; RV32IZBKB-NEXT:    lbu a6, 7(a0)
-; RV32IZBKB-NEXT:    or a0, a3, a1
-; RV32IZBKB-NEXT:    packh a1, a2, a4
-; RV32IZBKB-NEXT:    slli a5, a5, 16
-; RV32IZBKB-NEXT:    slli a6, a6, 24
-; RV32IZBKB-NEXT:    or a2, a6, a5
-; RV32IZBKB-NEXT:    or a1, a2, a1
+; RV32IZBKB-NEXT:    pack a0, a1, a0
+; RV32IZBKB-NEXT:    packh a1, a7, t0
+; RV32IZBKB-NEXT:    packh a2, a5, a6
+; RV32IZBKB-NEXT:    pack a1, a2, a1
 ; RV32IZBKB-NEXT:    ret
 ;
 ; RV64IZBKB-LABEL: load_i64:
@@ -199,18 +206,16 @@ define i64 @load_i64(ptr %p) {
 ; RV64IZBKB-NEXT:    lbu a3, 6(a0)
 ; RV64IZBKB-NEXT:    lbu a4, 7(a0)
 ; RV64IZBKB-NEXT:    packh a1, a1, a2
-; RV64IZBKB-NEXT:    slli a3, a3, 16
-; RV64IZBKB-NEXT:    slli a4, a4, 24
-; RV64IZBKB-NEXT:    or a3, a4, a3
-; RV64IZBKB-NEXT:    lbu a2, 0(a0)
+; RV64IZBKB-NEXT:    packh a2, a3, a4
+; RV64IZBKB-NEXT:    lbu a3, 0(a0)
 ; RV64IZBKB-NEXT:    lbu a4, 1(a0)
 ; RV64IZBKB-NEXT:    lbu a5, 2(a0)
 ; RV64IZBKB-NEXT:    lbu a0, 3(a0)
-; RV64IZBKB-NEXT:    or a1, a3, a1
-; RV64IZBKB-NEXT:    packh a2, a2, a4
-; RV64IZBKB-NEXT:    slli a5, a5, 16
-; RV64IZBKB-NEXT:    slli a0, a0, 24
-; RV64IZBKB-NEXT:    or a0, a0, a5
+; RV64IZBKB-NEXT:    slli a2, a2, 16
+; RV64IZBKB-NEXT:    or a1, a2, a1
+; RV64IZBKB-NEXT:    packh a2, a3, a4
+; RV64IZBKB-NEXT:    packh a0, a5, a0
+; RV64IZBKB-NEXT:    slli a0, a0, 16
 ; RV64IZBKB-NEXT:    or a0, a0, a2
 ; RV64IZBKB-NEXT:    pack a0, a0, a1
 ; RV64IZBKB-NEXT:    ret
@@ -574,3 +579,5 @@ define void @store_large_constant(ptr %x) {
   store i64 18364758544493064720, ptr %x, align 1
   ret void
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; SLOWZBKB: {{.*}}
diff --git a/llvm/test/CodeGen/SPARC/fp128-split.ll b/llvm/test/CodeGen/SPARC/fp128-split.ll
index 8a127c9c28cc7..ef3e9deb98d4d 100644
--- a/llvm/test/CodeGen/SPARC/fp128-split.ll
+++ b/llvm/test/CodeGen/SPARC/fp128-split.ll
@@ -13,16 +13,16 @@ define fp128 @testcase(fp128 %0) {
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:qfpregs = COPY $q0
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:dfpregs = COPY [[COPY]].sub_odd64
   ; CHECK-NEXT:   [[ADDri:%[0-9]+]]:i64regs = ADDri %stack.0, 0
-  ; CHECK-NEXT:   [[ORri:%[0-9]+]]:i64regs = ORri killed [[ADDri]], 8
-  ; CHECK-NEXT:   STDFrr [[ORri]], $g0, killed [[COPY1]] :: (store (s64) into %stack.0 + 8)
+  ; CHECK-NEXT:   %3:i64regs = disjoint ORri killed [[ADDri]], 8
+  ; CHECK-NEXT:   STDFrr %3, $g0, killed [[COPY1]] :: (store (s64) into %stack.0 + 8)
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:dfpregs = COPY [[COPY]].sub_even64
   ; CHECK-NEXT:   STDFri %stack.0, 0, killed [[COPY2]] :: (store (s64) into %stack.0, align 16)
-  ; CHECK-NEXT:   [[LDXrr:%[0-9]+]]:i64regs = LDXrr [[ORri]], $g0 :: (load (s64) from %stack.0 + 8)
+  ; CHECK-NEXT:   [[LDXrr:%[0-9]+]]:i64regs = LDXrr %3, $g0 :: (load (s64) from %stack.0 + 8)
   ; CHECK-NEXT:   [[LDXri:%[0-9]+]]:i64regs = LDXri %stack.0, 0 :: (load (s64) from %stack.0, align 16)
   ; CHECK-NEXT:   [[ADDri1:%[0-9]+]]:i64regs = ADDri %stack.1, 0
-  ; CHECK-NEXT:   [[ORri1:%[0-9]+]]:i64regs = ORri killed [[ADDri1]], 8
+  ; CHECK-NEXT:   %8:i64regs = disjoint ORri killed [[ADDri1]], 8
   ; CHECK-NEXT:   [[ADDri2:%[0-9]+]]:i64regs = ADDri [[LDXrr]], -1
-  ; CHECK-NEXT:   STXrr [[ORri1]], $g0, killed [[ADDri2]] :: (store (s64) into %stack.1 + 8, basealign 16)
+  ; CHECK-NEXT:   STXrr %8, $g0, killed [[ADDri2]] :: (store (s64) into %stack.1 + 8, basealign 16)
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:intregs = COPY $g0
   ; CHECK-NEXT:   [[MOVRri:%[0-9]+]]:intregs = MOVRri [[LDXrr]], 1, [[COPY3]], 49
   ; CHECK-NEXT:   [[SRLri:%[0-9]+]]:i64regs = SRLri killed [[MOVRri]], 0
@@ -31,7 +31,7 @@ define fp128 @testcase(fp128 %0) {
   ; CHECK-NEXT:   [[LDDFri:%[0-9]+]]:dfpregs = LDDFri %stack.1, 0 :: (load (s64) from %stack.1, align 16)
   ; CHECK-NEXT:   [[DEF:%[0-9]+]]:qfpregs = IMPLICIT_DEF
   ; CHECK-NEXT:   [[INSERT_SUBREG:%[0-9]+]]:qfpregs = INSERT_SUBREG [[DEF]], killed [[LDDFri]], %subreg.sub_even64
-  ; CHECK-NEXT:   [[LDDFrr:%[0-9]+]]:dfpregs = LDDFrr [[ORri1]], $g0 :: (load (s64) from %stack.1 + 8)
+  ; CHECK-NEXT:   [[LDDFrr:%[0-9]+]]:dfpregs = LDDFrr %8, $g0 :: (load (s64) from %stack.1 + 8)
   ; CHECK-NEXT:   [[INSERT_SUBREG1:%[0-9]+]]:qfpregs = INSERT_SUBREG [[INSERT_SUBREG]], killed [[LDDFrr]], %subreg.sub_odd64
   ; CHECK-NEXT:   $q0 = COPY [[INSERT_SUBREG1]]
   ; CHECK-NEXT:   RETL 8, implicit $q0
diff --git a/llvm/test/CodeGen/SPIRV/ShaderBufferImage.ll b/llvm/test/CodeGen/SPIRV/ShaderBufferImage.ll
new file mode 100644
index 0000000000000..3c002e1849b8d
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/ShaderBufferImage.ll
@@ -0,0 +1,20 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-vulkan-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-library %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-NOT: OpCapability ImageBasic
+; CHECK-NOT: OpCapability ImageReadWrite
+; CHECK: OpCapability ImageBuffer
+; CHECK-NOT: OpCapability ImageBasic
+; CHECK-NOT: OpCapability ImageReadWrite
+
+; CHECK-DAG: [[Float:%[0-9]+]] = OpTypeFloat 32
+; CHECK-DAG: [[Void:%[0-9]+]] = OpTypeVoid
+; CHECK-DAG: [[ImageType:%[0-9]+]] = OpTypeImage [[Float]] Buffer 2 0 0 2 R32i {{$}}
+; CHECK-DAG: [[ImageFuncType:%[0-9]+]] = OpTypeFunction [[Void]] [[ImageType]]
+
+; CHECK: {{%[0-9]+}} = OpFunction [[Void]] DontInline [[ImageFuncType]]
+define void @ImageWithNoAccessQualifier(target("spirv.Image", float, 5, 2, 0, 0, 2, 24) %img) #0 {
+  ret void
+}
+
+attributes #0 = { convergent noinline norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
diff --git a/llvm/test/CodeGen/SPIRV/ShaderImage.ll b/llvm/test/CodeGen/SPIRV/ShaderImage.ll
new file mode 100644
index 0000000000000..6ac58ce42f950
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/ShaderImage.ll
@@ -0,0 +1,21 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-vulkan-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-library %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: [[Float:%[0-9]+]] = OpTypeFloat 32
+; CHECK-DAG: [[Void:%[0-9]+]] = OpTypeVoid
+; CHECK-DAG: [[ImageType:%[0-9]+]] = OpTypeImage [[Float]] Buffer 2 0 0 1 R32i {{$}}
+; CHECK-DAG: [[ImageFuncType:%[0-9]+]] = OpTypeFunction [[Void]] [[ImageType]]
+; CHECK-DAG: [[SampledImageType:%[0-9]+]] = OpTypeSampledImage [[ImageType]]
+; CHECK-DAG: [[SampledImageFuncType:%[0-9]+]] = OpTypeFunction [[Void]] [[SampledImageType]]
+
+; CHECK: {{%[0-9]+}} = OpFunction [[Void]] DontInline [[ImageFuncType]]
+define void @ImageWithNoAccessQualifier(target("spirv.Image", float, 5, 2, 0, 0, 1, 24) %img) #0 {
+  ret void
+}
+
+; CHECK: {{%[0-9]+}} = OpFunction [[Void]] DontInline [[SampledImageFuncType]]
+define void @SampledImageWithNoAccessQualifier(target("spirv.SampledImage", float, 5, 2, 0, 0, 1, 24) %img) #0 {
+  ret void
+}
+
+attributes #0 = { convergent noinline norecurse "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cross.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cross.ll
new file mode 100644
index 0000000000000..2e0eb8c429ac2
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/cross.ll
@@ -0,0 +1,33 @@
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; Make sure SPIRV operation function calls for cross are lowered correctly.
+
+; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
+; CHECK-DAG: %[[#float_32:]] = OpTypeFloat 32
+; CHECK-DAG: %[[#float_16:]] = OpTypeFloat 16
+; CHECK-DAG: %[[#vec3_float_16:]] = OpTypeVector %[[#float_16]] 3
+; CHECK-DAG: %[[#vec3_float_32:]] = OpTypeVector %[[#float_32]] 3
+
+define noundef <3 x half> @cross_half4(<3 x half> noundef %a, <3 x half> noundef %b) {
+entry:
+  ; CHECK: %[[#]] = OpFunction %[[#vec3_float_16]] None %[[#]]
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#vec3_float_16]]
+  ; CHECK: %[[#arg1:]] = OpFunctionParameter %[[#vec3_float_16]]
+  ; CHECK: %[[#]] = OpExtInst %[[#vec3_float_16]] %[[#op_ext_glsl]] Cross %[[#arg0]] %[[#arg1]]
+  %hlsl.cross = call <3 x half> @llvm.spv.cross.v4f16(<3 x half> %a, <3 x half> %b)
+  ret <3 x half> %hlsl.cross
+}
+
+define noundef <3 x float> @cross_float4(<3 x float> noundef %a, <3 x float> noundef %b) {
+entry:
+  ; CHECK: %[[#]] = OpFunction %[[#vec3_float_32]] None %[[#]]
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#vec3_float_32]]
+  ; CHECK: %[[#arg1:]] = OpFunctionParameter %[[#vec3_float_32]]
+  ; CHECK: %[[#]] = OpExtInst %[[#vec3_float_32]] %[[#op_ext_glsl]] Cross %[[#arg0]] %[[#arg1]]
+  %hlsl.cross = call <3 x float> @llvm.spv.cross.v4f32(<3 x float> %a, <3 x float> %b)
+  ret <3 x float> %hlsl.cross
+}
+
+declare <3 x half> @llvm.spv.cross.v4f16(<3 x half>, <3 x half>)
+declare <3 x float> @llvm.spv.cross.v4f32(<3 x float>, <3 x float>)
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/radians.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/radians.ll
new file mode 100644
index 0000000000000..1fe8ab30ed953
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/radians.ll
@@ -0,0 +1,48 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
+
+; CHECK-DAG: %[[#float_32:]] = OpTypeFloat 32
+; CHECK-DAG: %[[#float_16:]] = OpTypeFloat 16
+
+; CHECK-DAG: %[[#vec4_float_32:]] = OpTypeVector %[[#float_32]] 4
+; CHECK-DAG: %[[#vec4_float_16:]] = OpTypeVector %[[#float_16]] 4
+
+declare half @llvm.spv.radians.f16(half)
+declare float @llvm.spv.radians.f32(float)
+
+declare <4 x float> @llvm.spv.radians.v4f32(<4 x float>)
+declare <4 x half> @llvm.spv.radians.v4f16(<4 x half>)
+
+define noundef float @radians_float(float noundef %a) {
+entry:
+; CHECK: %[[#float_32_arg:]] = OpFunctionParameter %[[#float_32]]
+; CHECK: %[[#]] = OpExtInst %[[#float_32]] %[[#op_ext_glsl]] Radians %[[#float_32_arg]]
+  %elt.radians = call float @llvm.spv.radians.f32(float %a)
+  ret float %elt.radians
+}
+
+define noundef half @radians_half(half noundef %a) {
+entry:
+; CHECK: %[[#float_16_arg:]] = OpFunctionParameter %[[#float_16]]
+; CHECK: %[[#]] = OpExtInst %[[#float_16]] %[[#op_ext_glsl]] Radians %[[#float_16_arg]]
+  %elt.radians = call half @llvm.spv.radians.f16(half %a)
+  ret half %elt.radians
+}
+
+define noundef <4 x float> @radians_float_vector(<4 x float> noundef %a) {
+entry:
+; CHECK: %[[#vec4_float_32_arg:]] = OpFunctionParameter %[[#vec4_float_32]]
+; CHECK: %[[#]] = OpExtInst %[[#vec4_float_32]] %[[#op_ext_glsl]] Radians %[[#vec4_float_32_arg]]
+  %elt.radians = call <4 x float> @llvm.spv.radians.v4f32(<4 x float> %a)
+  ret <4 x float> %elt.radians
+}
+
+define noundef <4 x half> @radians_half_vector(<4 x half> noundef %a) {
+entry:
+; CHECK: %[[#vec4_float_16_arg:]] = OpFunctionParameter %[[#vec4_float_16]]
+; CHECK: %[[#]] = OpExtInst %[[#vec4_float_16]] %[[#op_ext_glsl]] Radians %[[#vec4_float_16_arg]]
+  %elt.radians = call <4 x half> @llvm.spv.radians.v4f16(<4 x half> %a)
+  ret <4 x half> %elt.radians
+}
diff --git a/llvm/test/CodeGen/SPIRV/opencl/radians.ll b/llvm/test/CodeGen/SPIRV/opencl/radians.ll
new file mode 100644
index 0000000000000..f7bb8d5226cd1
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/opencl/radians.ll
@@ -0,0 +1,51 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "OpenCL.std"
+
+; CHECK-DAG: %[[#float_32:]] = OpTypeFloat 32
+; CHECK-DAG: %[[#float_16:]] = OpTypeFloat 16
+
+; CHECK-DAG: %[[#vec4_float_32:]] = OpTypeVector %[[#float_32]] 4
+; CHECK-DAG: %[[#vec4_float_16:]] = OpTypeVector %[[#float_16]] 4
+
+declare half @llvm.spv.radians.f16(half)
+declare float @llvm.spv.radians.f32(float)
+
+declare <4 x float> @llvm.spv.radians.v4f32(<4 x float>)
+declare <4 x half> @llvm.spv.radians.v4f16(<4 x half>)
+
+define noundef float @radians_float(float noundef %a) {
+entry:
+; CHECK: %[[#float_32_arg:]] = OpFunctionParameter %[[#float_32]]
+; CHECK: %[[#]] = OpExtInst %[[#float_32]] %[[#op_ext_glsl]] radians %[[#float_32_arg]]
+  %elt.radians = call float @llvm.spv.radians.f32(float %a)
+  ret float %elt.radians
+}
+
+define noundef half @radians_half(half noundef %a) {
+entry:
+; CHECK: %[[#float_16_arg:]] = OpFunctionParameter %[[#float_16]]
+; CHECK: %[[#]] = OpExtInst %[[#float_16]] %[[#op_ext_glsl]] radians %[[#float_16_arg]]
+  %elt.radians = call half @llvm.spv.radians.f16(half %a)
+  ret half %elt.radians
+}
+
+define noundef <4 x float> @radians_float_vector(<4 x float> noundef %a) {
+entry:
+; CHECK: %[[#vec4_float_32_arg:]] = OpFunctionParameter %[[#vec4_float_32]]
+; CHECK: %[[#]] = OpExtInst %[[#vec4_float_32]] %[[#op_ext_glsl]] radians %[[#vec4_float_32_arg]]
+  %elt.radians = call <4 x float> @llvm.spv.radians.v4f32(<4 x float> %a)
+  ret <4 x float> %elt.radians
+}
+
+define noundef <4 x half> @radians_half_vector(<4 x half> noundef %a) {
+entry:
+; CHECK: %[[#vec4_float_16_arg:]] = OpFunctionParameter %[[#vec4_float_16]]
+; CHECK: %[[#]] = OpExtInst %[[#vec4_float_16]] %[[#op_ext_glsl]] radians %[[#vec4_float_16_arg]]
+  %elt.radians = call <4 x half> @llvm.spv.radians.v4f16(<4 x half> %a)
+  ret <4 x half> %elt.radians
+}
+
diff --git a/llvm/test/CodeGen/X86/2006-08-21-ExtraMovInst.ll b/llvm/test/CodeGen/X86/2006-08-21-ExtraMovInst.ll
index ac749bccb3c55..f3bdf561a9456 100644
--- a/llvm/test/CodeGen/X86/2006-08-21-ExtraMovInst.ll
+++ b/llvm/test/CodeGen/X86/2006-08-21-ExtraMovInst.ll
@@ -7,7 +7,7 @@ define i32 @foo(i32 %t, i32 %C) {
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    decl %eax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %cond_true
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    incl %eax
diff --git a/llvm/test/CodeGen/X86/2007-01-13-StackPtrIndex.ll b/llvm/test/CodeGen/X86/2007-01-13-StackPtrIndex.ll
index 46dddd8fcd851..1e5ee2f71d9b4 100644
--- a/llvm/test/CodeGen/X86/2007-01-13-StackPtrIndex.ll
+++ b/llvm/test/CodeGen/X86/2007-01-13-StackPtrIndex.ll
@@ -27,7 +27,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
 ; CHECK-NEXT:    js .LBB0_14
 ; CHECK-NEXT:  # %bb.12:
 ; CHECK-NEXT:    xorl %r8d, %r8d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_13: # %a25b
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    testb %r8b, %r8b
@@ -38,7 +38,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
 ; CHECK-NEXT:    jne .LBB0_1
 ; CHECK-NEXT:  # %bb.15:
 ; CHECK-NEXT:    xorl %r8d, %r8d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_16: # %a25b140
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    testb %r8b, %r8b
@@ -56,7 +56,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    movb $1, %r10b
 ; CHECK-NEXT:    jmp .LBB0_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_9: # %b1606
 ; CHECK-NEXT:    # in Loop: Header=BB0_3 Depth=1
 ; CHECK-NEXT:    testb %r9b, %r9b
@@ -83,7 +83,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
 ; CHECK-NEXT:    # in Loop: Header=BB0_3 Depth=1
 ; CHECK-NEXT:    testq %rdx, %rdx
 ; CHECK-NEXT:    js .LBB0_18
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_36: # %a30b
 ; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
 ; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
@@ -93,7 +93,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
 ; CHECK-NEXT:    # in Loop: Header=BB0_3 Depth=1
 ; CHECK-NEXT:    testb %r10b, %r10b
 ; CHECK-NEXT:    jne .LBB0_4
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_19: # %a30b294
 ; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
 ; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
@@ -115,7 +115,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
 ; CHECK-NEXT:    # in Loop: Header=BB0_3 Depth=1
 ; CHECK-NEXT:    testb %r8b, %r8b
 ; CHECK-NEXT:    jne .LBB0_8
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_33: # %a74b
 ; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
 ; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
@@ -128,7 +128,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
 ; CHECK-NEXT:    # in Loop: Header=BB0_3 Depth=1
 ; CHECK-NEXT:    testl %eax, %eax
 ; CHECK-NEXT:    js .LBB0_9
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_35: # %a97b
 ; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
 ; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
@@ -142,7 +142,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
 ; CHECK-NEXT:    testb %r9b, %r9b
 ; CHECK-NEXT:    jne .LBB0_35
 ; CHECK-NEXT:    jmp .LBB0_9
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_21: # %b377
 ; CHECK-NEXT:    # in Loop: Header=BB0_20 Depth=2
 ; CHECK-NEXT:    testb %r9b, %r9b
@@ -153,7 +153,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
 ; CHECK-NEXT:    # Child Loop BB0_37 Depth 3
 ; CHECK-NEXT:    testq %rsi, %rsi
 ; CHECK-NEXT:    js .LBB0_21
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_37: # %a35b
 ; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
 ; CHECK-NEXT:    # Parent Loop BB0_20 Depth=2
@@ -161,7 +161,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
 ; CHECK-NEXT:    testb %r9b, %r9b
 ; CHECK-NEXT:    je .LBB0_37
 ; CHECK-NEXT:    jmp .LBB0_21
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_27: # %b1016
 ; CHECK-NEXT:    # in Loop: Header=BB0_25 Depth=2
 ; CHECK-NEXT:    testq %rsi, %rsi
@@ -173,7 +173,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
 ; CHECK-NEXT:    # Child Loop BB0_28 Depth 3
 ; CHECK-NEXT:    testq %rdx, %rdx
 ; CHECK-NEXT:    js .LBB0_26
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_38: # %a53b
 ; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
 ; CHECK-NEXT:    # Parent Loop BB0_25 Depth=2
@@ -184,7 +184,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
 ; CHECK-NEXT:    # in Loop: Header=BB0_25 Depth=2
 ; CHECK-NEXT:    testb %r10b, %r10b
 ; CHECK-NEXT:    jne .LBB0_27
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_28: # %a53b1019
 ; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
 ; CHECK-NEXT:    # Parent Loop BB0_25 Depth=2
@@ -192,7 +192,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
 ; CHECK-NEXT:    testq %rdx, %rdx
 ; CHECK-NEXT:    jle .LBB0_28
 ; CHECK-NEXT:    jmp .LBB0_27
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_31: # %b1263
 ; CHECK-NEXT:    # in Loop: Header=BB0_29 Depth=2
 ; CHECK-NEXT:    testq %rdx, %rdx
@@ -204,7 +204,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
 ; CHECK-NEXT:    # Child Loop BB0_32 Depth 3
 ; CHECK-NEXT:    testq %rsi, %rsi
 ; CHECK-NEXT:    js .LBB0_30
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_39: # %a63b
 ; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
 ; CHECK-NEXT:    # Parent Loop BB0_29 Depth=2
@@ -215,7 +215,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
 ; CHECK-NEXT:    # in Loop: Header=BB0_29 Depth=2
 ; CHECK-NEXT:    testq %rsi, %rsi
 ; CHECK-NEXT:    jle .LBB0_31
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_32: # %a63b1266
 ; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
 ; CHECK-NEXT:    # Parent Loop BB0_29 Depth=2
@@ -223,7 +223,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
 ; CHECK-NEXT:    testq %rsi, %rsi
 ; CHECK-NEXT:    jle .LBB0_32
 ; CHECK-NEXT:    jmp .LBB0_31
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_24: # %b712
 ; CHECK-NEXT:    # in Loop: Header=BB0_22 Depth=2
 ; CHECK-NEXT:    testb %r9b, %r9b
@@ -234,7 +234,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
 ; CHECK-NEXT:    # Child Loop BB0_23 Depth 3
 ; CHECK-NEXT:    testq %rdx, %rdx
 ; CHECK-NEXT:    js .LBB0_24
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_23: # %a45b
 ; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
 ; CHECK-NEXT:    # Parent Loop BB0_22 Depth=2
diff --git a/llvm/test/CodeGen/X86/2007-03-15-GEP-Idx-Sink.ll b/llvm/test/CodeGen/X86/2007-03-15-GEP-Idx-Sink.ll
index f21aaca7ca5f1..49e2bf207e52a 100644
--- a/llvm/test/CodeGen/X86/2007-03-15-GEP-Idx-Sink.ll
+++ b/llvm/test/CodeGen/X86/2007-03-15-GEP-Idx-Sink.ll
@@ -15,7 +15,7 @@ define void @foo(ptr %buf, i32 %size, i32 %col, ptr %p) nounwind {
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; CHECK-NEXT:    addl $8, %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_2: ## %bb
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl (%esi), %edi
diff --git a/llvm/test/CodeGen/X86/2007-10-12-CoalesceExtSubReg.ll b/llvm/test/CodeGen/X86/2007-10-12-CoalesceExtSubReg.ll
index 2f75ab29e708f..cfb3e508576dd 100644
--- a/llvm/test/CodeGen/X86/2007-10-12-CoalesceExtSubReg.ll
+++ b/llvm/test/CodeGen/X86/2007-10-12-CoalesceExtSubReg.ll
@@ -9,7 +9,7 @@ define signext i16 @f(ptr %bp, ptr %ss)   {
 ; CHECK-NEXT:    .cfi_offset %esi, -8
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %cond_next127
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl (%eax), %edx
diff --git a/llvm/test/CodeGen/X86/2007-10-12-SpillerUnfold2.ll b/llvm/test/CodeGen/X86/2007-10-12-SpillerUnfold2.ll
index f9996e2df50e0..6ebb97d63e7c6 100644
--- a/llvm/test/CodeGen/X86/2007-10-12-SpillerUnfold2.ll
+++ b/llvm/test/CodeGen/X86/2007-10-12-SpillerUnfold2.ll
@@ -6,7 +6,7 @@ define signext   i16 @t(ptr %qmatrix, ptr %dct, ptr %acBaseTable, ptr %acExtTabl
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %cond_next127
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %eax, %edx
diff --git a/llvm/test/CodeGen/X86/2007-11-06-InstrSched.ll b/llvm/test/CodeGen/X86/2007-11-06-InstrSched.ll
index 750d06d9e6031..bbce246a5d394 100644
--- a/llvm/test/CodeGen/X86/2007-11-06-InstrSched.ll
+++ b/llvm/test/CodeGen/X86/2007-11-06-InstrSched.ll
@@ -14,7 +14,7 @@ define float @foo(ptr %x, ptr %y, i32 %c) nounwind {
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    xorl %esi, %esi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_3: # %bb18
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    xorps %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/2007-11-30-LoadFolding-Bug.ll b/llvm/test/CodeGen/X86/2007-11-30-LoadFolding-Bug.ll
index 68566c7b37097..8d690ba06e3bd 100644
--- a/llvm/test/CodeGen/X86/2007-11-30-LoadFolding-Bug.ll
+++ b/llvm/test/CodeGen/X86/2007-11-30-LoadFolding-Bug.ll
@@ -16,7 +16,7 @@ define fastcc void @mp_sqrt(i32 %n, i32 %radix, ptr %in, ptr %out, ptr %tmp1, pt
 ; CHECK-NEXT:    movb $1, %cl
 ; CHECK-NEXT:    movl $1, %ebx
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %bb.i5
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %ecx, %eax
@@ -37,7 +37,7 @@ define fastcc void @mp_sqrt(i32 %n, i32 %radix, ptr %in, ptr %out, ptr %tmp1, pt
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    xorl %ecx, %ecx
 ; CHECK-NEXT:    xorpd %xmm1, %xmm1
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_7: # %bb.i28.i
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cvttsd2si %xmm1, %edi
@@ -85,7 +85,7 @@ define fastcc void @mp_sqrt(i32 %n, i32 %radix, ptr %in, ptr %out, ptr %tmp1, pt
 ; CHECK-NEXT:    popl %ebx
 ; CHECK-NEXT:    popl %ebp
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_9: # %bb.i.i
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp .LBB0_9
diff --git a/llvm/test/CodeGen/X86/2008-04-28-CoalescerBug.ll b/llvm/test/CodeGen/X86/2008-04-28-CoalescerBug.ll
index 8e6d2c11b7b3d..c95fc00b3ee6d 100644
--- a/llvm/test/CodeGen/X86/2008-04-28-CoalescerBug.ll
+++ b/llvm/test/CodeGen/X86/2008-04-28-CoalescerBug.ll
@@ -16,13 +16,13 @@ define void @t(ptr %depth, ptr %bop, i32 %mode) nounwind  {
 ; CHECK-NEXT:  ## %bb.1: ## %entry
 ; CHECK-NEXT:    cmpl $1, %edx
 ; CHECK-NEXT:    jne LBB0_10
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_2: ## %bb2898.us
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp LBB0_2
 ; CHECK-NEXT:  LBB0_3: ## %bb13086.preheader
 ; CHECK-NEXT:    movb $1, %al
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_4: ## %bb13088
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    testb %al, %al
@@ -31,7 +31,7 @@ define void @t(ptr %depth, ptr %bop, i32 %mode) nounwind  {
 ; CHECK-NEXT:    ## in Loop: Header=BB0_4 Depth=1
 ; CHECK-NEXT:    xorl %ecx, %ecx
 ; CHECK-NEXT:    jmp LBB0_7
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_5: ## in Loop: Header=BB0_4 Depth=1
 ; CHECK-NEXT:    movl $65535, %ecx ## imm = 0xFFFF
 ; CHECK-NEXT:  LBB0_7: ## %bb13107
diff --git a/llvm/test/CodeGen/X86/2008-08-06-CmpStride.ll b/llvm/test/CodeGen/X86/2008-08-06-CmpStride.ll
index ca92c555058ab..5086ed40a43a2 100644
--- a/llvm/test/CodeGen/X86/2008-08-06-CmpStride.ll
+++ b/llvm/test/CodeGen/X86/2008-08-06-CmpStride.ll
@@ -10,7 +10,7 @@ define i32 @main() nounwind {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    movl $10271, %ebx # imm = 0x281F
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %forbody
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl $.str, %edi
diff --git a/llvm/test/CodeGen/X86/2008-12-01-loop-iv-used-outside-loop.ll b/llvm/test/CodeGen/X86/2008-12-01-loop-iv-used-outside-loop.ll
index 18d3cec442c6c..c2a7d6be8baa0 100644
--- a/llvm/test/CodeGen/X86/2008-12-01-loop-iv-used-outside-loop.ll
+++ b/llvm/test/CodeGen/X86/2008-12-01-loop-iv-used-outside-loop.ll
@@ -11,7 +11,7 @@ define ptr @test(ptr %Q, ptr %L) nounwind {
 ; CHECK:       ## %bb.0: ## %entry
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    jmp LBB0_2
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_1: ## %bb
 ; CHECK-NEXT:    ## in Loop: Header=BB0_2 Depth=1
 ; CHECK-NEXT:    incl %eax
diff --git a/llvm/test/CodeGen/X86/2009-02-26-MachineLICMBug.ll b/llvm/test/CodeGen/X86/2009-02-26-MachineLICMBug.ll
index 7807d49269e64..c421541001c5d 100644
--- a/llvm/test/CodeGen/X86/2009-02-26-MachineLICMBug.ll
+++ b/llvm/test/CodeGen/X86/2009-02-26-MachineLICMBug.ll
@@ -20,7 +20,7 @@ define ptr @t(ptr %desc, i64 %p) nounwind ssp {
 ; CHECK-NEXT:    movq %rdi, %r14
 ; CHECK-NEXT:    orq $2097152, %rbx ## imm = 0x200000
 ; CHECK-NEXT:    andl $15728640, %ebx ## imm = 0xF00000
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_1: ## %bb4
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    xorl %eax, %eax
diff --git a/llvm/test/CodeGen/X86/2009-04-25-CoalescerBug.ll b/llvm/test/CodeGen/X86/2009-04-25-CoalescerBug.ll
index ce28893090c43..1dd30e8263099 100644
--- a/llvm/test/CodeGen/X86/2009-04-25-CoalescerBug.ll
+++ b/llvm/test/CodeGen/X86/2009-04-25-CoalescerBug.ll
@@ -8,7 +8,7 @@ define i64 @test(ptr %tmp13) nounwind {
 ; CHECK-NEXT:    movl (%rdi), %ecx
 ; CHECK-NEXT:    movl %ecx, %eax
 ; CHECK-NEXT:    shrl %eax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %while.cond
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    testb $1, %cl
diff --git a/llvm/test/CodeGen/X86/2009-08-12-badswitch.ll b/llvm/test/CodeGen/X86/2009-08-12-badswitch.ll
index 4b8085a995f08..7050889d71029 100644
--- a/llvm/test/CodeGen/X86/2009-08-12-badswitch.ll
+++ b/llvm/test/CodeGen/X86/2009-08-12-badswitch.ll
@@ -123,7 +123,7 @@ define internal fastcc i32 @foo(i64 %bar) nounwind ssp {
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    popq %rcx
 ; CHECK-NEXT:    retq
-; CHECK-NEXT:    .p2align 2, 0x90
+; CHECK-NEXT:    .p2align 2
 ; CHECK-NEXT:    .data_region jt32
 ; CHECK-NEXT:  .set L0_0_set_3, LBB0_3-LJTI0_0
 ; CHECK-NEXT:  .set L0_0_set_4, LBB0_4-LJTI0_0
diff --git a/llvm/test/CodeGen/X86/2020_12_02_decrementing_loop.ll b/llvm/test/CodeGen/X86/2020_12_02_decrementing_loop.ll
index 43b52898c79a2..22bf4581c6b42 100644
--- a/llvm/test/CodeGen/X86/2020_12_02_decrementing_loop.ll
+++ b/llvm/test/CodeGen/X86/2020_12_02_decrementing_loop.ll
@@ -4,7 +4,7 @@
 define i32 @test_01(ptr %p, i64 %len, i32 %x) {
 ; CHECK-LABEL: test_01:
 ; CHECK:       ## %bb.0: ## %entry
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_1: ## %loop
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    subq $1, %rsi
@@ -44,7 +44,7 @@ failure:                                          ; preds = %backedge
 define i32 @test_01a(ptr %p, i64 %len, i32 %x) {
 ; CHECK-LABEL: test_01a:
 ; CHECK:       ## %bb.0: ## %entry
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB1_1: ## %loop
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    subq $1, %rsi
@@ -84,7 +84,7 @@ failure:                                          ; preds = %backedge
 define i32 @test_02(ptr %p, i64 %len, i32 %x) {
 ; CHECK-LABEL: test_02:
 ; CHECK:       ## %bb.0: ## %entry
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB2_1: ## %loop
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    subq $1, %rsi
@@ -126,7 +126,7 @@ failure:                                          ; preds = %backedge
 define i32 @test_03(ptr %p, i64 %len, i32 %x) {
 ; CHECK-LABEL: test_03:
 ; CHECK:       ## %bb.0: ## %entry
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB3_1: ## %loop
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    subq $1, %rsi
diff --git a/llvm/test/CodeGen/X86/AMX/amx-across-func.ll b/llvm/test/CodeGen/X86/AMX/amx-across-func.ll
index ae0be9b5a5bcd..320c96535abba 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-across-func.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-across-func.ll
@@ -235,7 +235,7 @@ define dso_local i32 @test_loop(i32 %0) nounwind {
 ; CHECK-NEXT:    movl $32, %r15d
 ; CHECK-NEXT:    movw $8, %r12w
 ; CHECK-NEXT:    movl $buf+2048, %r13d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB2_2: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    tileloadd (%r14,%r15), %tmm0
 ; CHECK-NEXT:    movabsq $64, %rax
@@ -300,7 +300,7 @@ define dso_local i32 @test_loop(i32 %0) nounwind {
 ; IPRA-NEXT:    movl $32, %esi
 ; IPRA-NEXT:    movw $8, %di
 ; IPRA-NEXT:    movl $buf+2048, %r8d
-; IPRA-NEXT:    .p2align 4, 0x90
+; IPRA-NEXT:    .p2align 4
 ; IPRA-NEXT:  .LBB2_2: # =>This Inner Loop Header: Depth=1
 ; IPRA-NEXT:    tileloadd (%rdx,%rsi), %tmm0
 ; IPRA-NEXT:    callq foo
@@ -494,7 +494,7 @@ define dso_local void @test_loop2(i32 %0) nounwind {
 ; CHECK-NEXT:    movl $32, %r15d
 ; CHECK-NEXT:    movw $8, %bp
 ; CHECK-NEXT:    movl $buf+2048, %r12d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    callq foo
@@ -535,7 +535,7 @@ define dso_local void @test_loop2(i32 %0) nounwind {
 ; IPRA-NEXT:    movl $32, %ecx
 ; IPRA-NEXT:    movw $8, %dx
 ; IPRA-NEXT:    movl $buf+2048, %esi
-; IPRA-NEXT:    .p2align 4, 0x90
+; IPRA-NEXT:    .p2align 4
 ; IPRA-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
 ; IPRA-NEXT:    callq foo
 ; IPRA-NEXT:    testl %edi, %edi
diff --git a/llvm/test/CodeGen/X86/AMX/amx-ldtilecfg-insert.ll b/llvm/test/CodeGen/X86/AMX/amx-ldtilecfg-insert.ll
index 714590d3c156b..8595024103dbd 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-ldtilecfg-insert.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-ldtilecfg-insert.ll
@@ -241,7 +241,7 @@ define dso_local void @test5(i16 signext %0, i16 signext %1) nounwind {
 ; CHECK-NEXT:    movl $32, %edx
 ; CHECK-NEXT:    leal -1(%rsi), %r8d
 ; CHECK-NEXT:    jmp .LBB4_1
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB4_3: # %if.false
 ; CHECK-NEXT:    # in Loop: Header=BB4_1 Depth=1
 ; CHECK-NEXT:    movl %r8d, %esi
@@ -301,7 +301,7 @@ define dso_local void @test6(i16 signext %0) nounwind {
 ; CHECK-NEXT:    movl $32, %edx
 ; CHECK-NEXT:    xorl %esi, %esi
 ; CHECK-NEXT:    jmp .LBB5_1
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB5_3: # %if.false
 ; CHECK-NEXT:    # in Loop: Header=BB5_1 Depth=1
 ; CHECK-NEXT:    decl %esi
diff --git a/llvm/test/CodeGen/X86/AMX/amx-lower-tile-copy.ll b/llvm/test/CodeGen/X86/AMX/amx-lower-tile-copy.ll
index a0085afbaf025..e9bb68e711f88 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-lower-tile-copy.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-lower-tile-copy.ll
@@ -32,7 +32,7 @@ define dso_local void @test1(ptr%buf) nounwind {
 ; CHECK-NEXT:    movq %rdi, %rbx
 ; CHECK-NEXT:    xorl %r14d, %r14d
 ; CHECK-NEXT:    movl $32, %r15d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # %loop.header
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movabsq $64, %rax
@@ -92,7 +92,7 @@ define dso_local void @test1(ptr%buf) nounwind {
 ; EGPR-NEXT:    movq %rdi, %rbx # encoding: [0x48,0x89,0xfb]
 ; EGPR-NEXT:    xorl %r14d, %r14d # encoding: [0x45,0x31,0xf6]
 ; EGPR-NEXT:    movl $32, %r15d # encoding: [0x41,0xbf,0x20,0x00,0x00,0x00]
-; EGPR-NEXT:    .p2align 4, 0x90
+; EGPR-NEXT:    .p2align 4
 ; EGPR-NEXT:  .LBB0_2: # %loop.header
 ; EGPR-NEXT:    # =>This Inner Loop Header: Depth=1
 ; EGPR-NEXT:    movabsq $64, %rax # encoding: [0x48,0xb8,0x40,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
@@ -179,7 +179,7 @@ define dso_local void @test2(ptr%buf) nounwind {
 ; CHECK-NEXT:    movq %rdi, %rbx
 ; CHECK-NEXT:    xorl %r14d, %r14d
 ; CHECK-NEXT:    movl $32, %r15d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_2: # %loop.header
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    tilezero %tmm0
@@ -231,7 +231,7 @@ define dso_local void @test2(ptr%buf) nounwind {
 ; EGPR-NEXT:    movq %rdi, %rbx # encoding: [0x48,0x89,0xfb]
 ; EGPR-NEXT:    xorl %r14d, %r14d # encoding: [0x45,0x31,0xf6]
 ; EGPR-NEXT:    movl $32, %r15d # encoding: [0x41,0xbf,0x20,0x00,0x00,0x00]
-; EGPR-NEXT:    .p2align 4, 0x90
+; EGPR-NEXT:    .p2align 4
 ; EGPR-NEXT:  .LBB1_2: # %loop.header
 ; EGPR-NEXT:    # =>This Inner Loop Header: Depth=1
 ; EGPR-NEXT:    tilezero %tmm0 # encoding: [0xc4,0xe2,0x7b,0x49,0xc0]
diff --git a/llvm/test/CodeGen/X86/AMX/amx-spill-merge.ll b/llvm/test/CodeGen/X86/AMX/amx-spill-merge.ll
index c7c919c7cbb30..4fb0a4445862f 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-spill-merge.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-spill-merge.ll
@@ -244,7 +244,7 @@ define dso_local void @test3(ptr%buf) nounwind {
 ; CHECK-NEXT:    movq %rdi, %rbx
 ; CHECK-NEXT:    movl $32, %r14d
 ; CHECK-NEXT:    xorl %r15d, %r15d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_2: # %loop.header
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    tilestored %tmm0, (%rbx,%r14)
@@ -299,7 +299,7 @@ define dso_local void @test3(ptr%buf) nounwind {
 ; EGPR-NEXT:    movq %rdi, %rbx # encoding: [0x48,0x89,0xfb]
 ; EGPR-NEXT:    movl $32, %r14d # encoding: [0x41,0xbe,0x20,0x00,0x00,0x00]
 ; EGPR-NEXT:    xorl %r15d, %r15d # encoding: [0x45,0x31,0xff]
-; EGPR-NEXT:    .p2align 4, 0x90
+; EGPR-NEXT:    .p2align 4
 ; EGPR-NEXT:  .LBB1_2: # %loop.header
 ; EGPR-NEXT:    # =>This Inner Loop Header: Depth=1
 ; EGPR-NEXT:    tilestored %tmm0, (%rbx,%r14) # EVEX TO VEX Compression encoding: [0xc4,0xa2,0x7a,0x4b,0x04,0x33]
diff --git a/llvm/test/CodeGen/X86/AMX/amx-tile-basic.ll b/llvm/test/CodeGen/X86/AMX/amx-tile-basic.ll
index 15e7136f4a503..ec087f23248c4 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-tile-basic.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-tile-basic.ll
@@ -85,7 +85,7 @@ define void @PR90954(ptr %0, ptr %1, i32 %2) nounwind {
 ; CHECK-NEXT:    xorl %ebx, %ebx
 ; CHECK-NEXT:    xorl %r14d, %r14d
 ; CHECK-NEXT:    jmp .LBB1_1
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_5: # in Loop: Header=BB1_1 Depth=1
 ; CHECK-NEXT:    incq %r14
 ; CHECK-NEXT:    addl %edx, %ebx
@@ -96,7 +96,7 @@ define void @PR90954(ptr %0, ptr %1, i32 %2) nounwind {
 ; CHECK-NEXT:    xorl %r12d, %r12d
 ; CHECK-NEXT:    xorl %r13d, %r13d
 ; CHECK-NEXT:    jmp .LBB1_2
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_4: # in Loop: Header=BB1_2 Depth=2
 ; CHECK-NEXT:    tilestored %tmm1, (%r15,%rax)
 ; CHECK-NEXT:    incq %r13
diff --git a/llvm/test/CodeGen/X86/MachineSink-Issue98477.ll b/llvm/test/CodeGen/X86/MachineSink-Issue98477.ll
index 4df32854cdae4..4abb1f8e1ca42 100644
--- a/llvm/test/CodeGen/X86/MachineSink-Issue98477.ll
+++ b/llvm/test/CodeGen/X86/MachineSink-Issue98477.ll
@@ -13,7 +13,7 @@ define i32 @main(i1 %tobool.not, i32 %0) {
 ; CHECK-NEXT:  .LBB0_1: # %j.preheader
 ; CHECK-NEXT:    xorl %r9d, %r9d
 ; CHECK-NEXT:    jmp .LBB0_2
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_5: # %if.then4
 ; CHECK-NEXT:    # in Loop: Header=BB0_2 Depth=1
 ; CHECK-NEXT:    movl $1, %eax
diff --git a/llvm/test/CodeGen/X86/MergeConsecutiveStores.ll b/llvm/test/CodeGen/X86/MergeConsecutiveStores.ll
index fa9ed49da1b7f..0103d2bf3cc2c 100644
--- a/llvm/test/CodeGen/X86/MergeConsecutiveStores.ll
+++ b/llvm/test/CodeGen/X86/MergeConsecutiveStores.ll
@@ -17,7 +17,7 @@ define void @merge_const_store(i32 %count, ptr nocapture %p) nounwind uwtable no
 ; X86-NEXT:    jle .LBB0_3
 ; X86-NEXT:  # %bb.1: # %.lr.ph.preheader
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB0_2: # %.lr.ph
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl $67305985, (%ecx) # imm = 0x4030201
@@ -34,7 +34,7 @@ define void @merge_const_store(i32 %count, ptr nocapture %p) nounwind uwtable no
 ; X64-NEXT:    jle .LBB0_3
 ; X64-NEXT:  # %bb.1: # %.lr.ph.preheader
 ; X64-NEXT:    movabsq $578437695752307201, %rax # imm = 0x807060504030201
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB0_2: # %.lr.ph
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movq %rax, (%rsi)
@@ -81,7 +81,7 @@ define void @merge_const_store_no_vec(i32 %count, ptr nocapture %p) noimplicitfl
 ; X86-NEXT:    jle .LBB1_3
 ; X86-NEXT:  # %bb.1: # %.lr.ph.preheader
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB1_2: # %.lr.ph
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl $0, (%ecx)
@@ -102,7 +102,7 @@ define void @merge_const_store_no_vec(i32 %count, ptr nocapture %p) noimplicitfl
 ; X64:       # %bb.0:
 ; X64-NEXT:    testl %edi, %edi
 ; X64-NEXT:    jle .LBB1_2
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB1_1: # %.lr.ph
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movq $0, (%rsi)
@@ -153,7 +153,7 @@ define void @merge_const_store_vec(i32 %count, ptr nocapture %p) nounwind uwtabl
 ; X86-NEXT:  # %bb.1: # %.lr.ph.preheader
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB2_2: # %.lr.ph
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    vmovups %ymm0, (%ecx)
@@ -170,7 +170,7 @@ define void @merge_const_store_vec(i32 %count, ptr nocapture %p) nounwind uwtabl
 ; X64-NEXT:    jle .LBB2_3
 ; X64-NEXT:  # %bb.1: # %.lr.ph.preheader
 ; X64-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB2_2: # %.lr.ph
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    vmovups %ymm0, (%rsi)
@@ -219,7 +219,7 @@ define void @merge_nonconst_store(i32 %count, i8 %zz, ptr nocapture %p) nounwind
 ; X86-BWON-NEXT:  # %bb.1: # %.lr.ph.preheader
 ; X86-BWON-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-BWON-NEXT:    movzbl {{[0-9]+}}(%esp), %edx
-; X86-BWON-NEXT:    .p2align 4, 0x90
+; X86-BWON-NEXT:    .p2align 4
 ; X86-BWON-NEXT:  .LBB3_2: # %.lr.ph
 ; X86-BWON-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-BWON-NEXT:    movl $67305985, (%ecx) # imm = 0x4030201
@@ -240,7 +240,7 @@ define void @merge_nonconst_store(i32 %count, i8 %zz, ptr nocapture %p) nounwind
 ; X86-BWOFF-NEXT:  # %bb.1: # %.lr.ph.preheader
 ; X86-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-BWOFF-NEXT:    movb {{[0-9]+}}(%esp), %dl
-; X86-BWOFF-NEXT:    .p2align 4, 0x90
+; X86-BWOFF-NEXT:    .p2align 4
 ; X86-BWOFF-NEXT:  .LBB3_2: # %.lr.ph
 ; X86-BWOFF-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-BWOFF-NEXT:    movl $67305985, (%ecx) # imm = 0x4030201
@@ -257,7 +257,7 @@ define void @merge_nonconst_store(i32 %count, i8 %zz, ptr nocapture %p) nounwind
 ; X64:       # %bb.0:
 ; X64-NEXT:    testl %edi, %edi
 ; X64-NEXT:    jle .LBB3_2
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB3_1: # %.lr.ph
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl $67305985, (%rdx) # imm = 0x4030201
@@ -310,7 +310,7 @@ define void @merge_loads_i16(i32 %count, ptr noalias nocapture %q, ptr noalias n
 ; X86-BWON-NEXT:  # %bb.1: # %.lr.ph
 ; X86-BWON-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-BWON-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-BWON-NEXT:    .p2align 4, 0x90
+; X86-BWON-NEXT:    .p2align 4
 ; X86-BWON-NEXT:  .LBB4_2: # =>This Inner Loop Header: Depth=1
 ; X86-BWON-NEXT:    movzwl (%edx), %esi
 ; X86-BWON-NEXT:    movw %si, (%ecx)
@@ -333,7 +333,7 @@ define void @merge_loads_i16(i32 %count, ptr noalias nocapture %q, ptr noalias n
 ; X86-BWOFF-NEXT:  # %bb.1: # %.lr.ph
 ; X86-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-BWOFF-NEXT:    .p2align 4, 0x90
+; X86-BWOFF-NEXT:    .p2align 4
 ; X86-BWOFF-NEXT:  .LBB4_2: # =>This Inner Loop Header: Depth=1
 ; X86-BWOFF-NEXT:    movw (%edx), %si
 ; X86-BWOFF-NEXT:    movw %si, (%ecx)
@@ -349,7 +349,7 @@ define void @merge_loads_i16(i32 %count, ptr noalias nocapture %q, ptr noalias n
 ; X64-BWON:       # %bb.0:
 ; X64-BWON-NEXT:    testl %edi, %edi
 ; X64-BWON-NEXT:    jle .LBB4_2
-; X64-BWON-NEXT:    .p2align 4, 0x90
+; X64-BWON-NEXT:    .p2align 4
 ; X64-BWON-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
 ; X64-BWON-NEXT:    movzwl (%rsi), %eax
 ; X64-BWON-NEXT:    movw %ax, (%rdx)
@@ -363,7 +363,7 @@ define void @merge_loads_i16(i32 %count, ptr noalias nocapture %q, ptr noalias n
 ; X64-BWOFF:       # %bb.0:
 ; X64-BWOFF-NEXT:    testl %edi, %edi
 ; X64-BWOFF-NEXT:    jle .LBB4_2
-; X64-BWOFF-NEXT:    .p2align 4, 0x90
+; X64-BWOFF-NEXT:    .p2align 4
 ; X64-BWOFF-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
 ; X64-BWOFF-NEXT:    movw (%rsi), %ax
 ; X64-BWOFF-NEXT:    movw %ax, (%rdx)
@@ -411,7 +411,7 @@ define void @no_merge_loads(i32 %count, ptr noalias nocapture %q, ptr noalias no
 ; X86-BWON-NEXT:  # %bb.1: # %.lr.ph
 ; X86-BWON-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-BWON-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-BWON-NEXT:    .p2align 4, 0x90
+; X86-BWON-NEXT:    .p2align 4
 ; X86-BWON-NEXT:  .LBB5_2: # %a4
 ; X86-BWON-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-BWON-NEXT:    movzbl (%edx), %ebx
@@ -437,7 +437,7 @@ define void @no_merge_loads(i32 %count, ptr noalias nocapture %q, ptr noalias no
 ; X86-BWOFF-NEXT:  # %bb.1: # %.lr.ph
 ; X86-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-BWOFF-NEXT:    .p2align 4, 0x90
+; X86-BWOFF-NEXT:    .p2align 4
 ; X86-BWOFF-NEXT:  .LBB5_2: # %a4
 ; X86-BWOFF-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-BWOFF-NEXT:    movb (%edx), %bl
@@ -456,7 +456,7 @@ define void @no_merge_loads(i32 %count, ptr noalias nocapture %q, ptr noalias no
 ; X64-BWON:       # %bb.0:
 ; X64-BWON-NEXT:    testl %edi, %edi
 ; X64-BWON-NEXT:    jle .LBB5_2
-; X64-BWON-NEXT:    .p2align 4, 0x90
+; X64-BWON-NEXT:    .p2align 4
 ; X64-BWON-NEXT:  .LBB5_1: # %a4
 ; X64-BWON-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-BWON-NEXT:    movzbl (%rsi), %eax
@@ -473,7 +473,7 @@ define void @no_merge_loads(i32 %count, ptr noalias nocapture %q, ptr noalias no
 ; X64-BWOFF:       # %bb.0:
 ; X64-BWOFF-NEXT:    testl %edi, %edi
 ; X64-BWOFF-NEXT:    jle .LBB5_2
-; X64-BWOFF-NEXT:    .p2align 4, 0x90
+; X64-BWOFF-NEXT:    .p2align 4
 ; X64-BWOFF-NEXT:  .LBB5_1: # %a4
 ; X64-BWOFF-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-BWOFF-NEXT:    movb (%rsi), %al
@@ -526,7 +526,7 @@ define void @merge_loads_integer(i32 %count, ptr noalias nocapture %q, ptr noali
 ; X86-NEXT:  # %bb.1: # %.lr.ph
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB6_2: # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl (%edx), %esi
 ; X86-NEXT:    movl 4(%edx), %edi
@@ -546,7 +546,7 @@ define void @merge_loads_integer(i32 %count, ptr noalias nocapture %q, ptr noali
 ; X64:       # %bb.0:
 ; X64-NEXT:    testl %edi, %edi
 ; X64-NEXT:    jle .LBB6_2
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movq (%rsi), %rax
 ; X64-NEXT:    movq %rax, (%rdx)
@@ -590,7 +590,7 @@ define void @merge_loads_vector(i32 %count, ptr noalias nocapture %q, ptr noalia
 ; X86-NEXT:  # %bb.1: # %.lr.ph
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB7_2: # %block4
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    vmovups (%edx), %xmm0
@@ -605,7 +605,7 @@ define void @merge_loads_vector(i32 %count, ptr noalias nocapture %q, ptr noalia
 ; X64:       # %bb.0:
 ; X64-NEXT:    testl %edi, %edi
 ; X64-NEXT:    jle .LBB7_2
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB7_1: # %block4
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    vmovups (%rsi), %xmm0
@@ -659,7 +659,7 @@ define void @merge_loads_no_align(i32 %count, ptr noalias nocapture %q, ptr noal
 ; X86-NEXT:  # %bb.1: # %.lr.ph
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB8_2: # %block4
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    vmovups (%edx), %xmm0
@@ -674,7 +674,7 @@ define void @merge_loads_no_align(i32 %count, ptr noalias nocapture %q, ptr noal
 ; X64:       # %bb.0:
 ; X64-NEXT:    testl %edi, %edi
 ; X64-NEXT:    jle .LBB8_2
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB8_1: # %block4
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    vmovups (%rsi), %xmm0
@@ -737,7 +737,7 @@ define void @MergeLoadStoreBaseIndexOffset(ptr %a, ptr %b, ptr %c, i32 %n) {
 ; X86-BWON-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-BWON-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-BWON-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-BWON-NEXT:    .p2align 4, 0x90
+; X86-BWON-NEXT:    .p2align 4
 ; X86-BWON-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
 ; X86-BWON-NEXT:    movl (%edi,%eax,8), %ebx
 ; X86-BWON-NEXT:    movzwl (%edx,%ebx), %ebx
@@ -770,7 +770,7 @@ define void @MergeLoadStoreBaseIndexOffset(ptr %a, ptr %b, ptr %c, i32 %n) {
 ; X86-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-BWOFF-NEXT:    .p2align 4, 0x90
+; X86-BWOFF-NEXT:    .p2align 4
 ; X86-BWOFF-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
 ; X86-BWOFF-NEXT:    movl (%edi,%eax,8), %ebx
 ; X86-BWOFF-NEXT:    movw (%edx,%ebx), %bx
@@ -791,7 +791,7 @@ define void @MergeLoadStoreBaseIndexOffset(ptr %a, ptr %b, ptr %c, i32 %n) {
 ; X64-BWON:       # %bb.0:
 ; X64-BWON-NEXT:    movl %ecx, %eax
 ; X64-BWON-NEXT:    xorl %ecx, %ecx
-; X64-BWON-NEXT:    .p2align 4, 0x90
+; X64-BWON-NEXT:    .p2align 4
 ; X64-BWON-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
 ; X64-BWON-NEXT:    movq (%rdi,%rcx,8), %r8
 ; X64-BWON-NEXT:    movzwl (%rdx,%r8), %r8d
@@ -806,7 +806,7 @@ define void @MergeLoadStoreBaseIndexOffset(ptr %a, ptr %b, ptr %c, i32 %n) {
 ; X64-BWOFF:       # %bb.0:
 ; X64-BWOFF-NEXT:    movl %ecx, %eax
 ; X64-BWOFF-NEXT:    xorl %ecx, %ecx
-; X64-BWOFF-NEXT:    .p2align 4, 0x90
+; X64-BWOFF-NEXT:    .p2align 4
 ; X64-BWOFF-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
 ; X64-BWOFF-NEXT:    movq (%rdi,%rcx,8), %r8
 ; X64-BWOFF-NEXT:    movw (%rdx,%r8), %r8w
@@ -863,7 +863,7 @@ define void @MergeLoadStoreBaseIndexOffsetComplicated(ptr %a, ptr %b, ptr %c, i6
 ; X86-BWON-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-BWON-NEXT:    movl {{[0-9]+}}(%esp), %ebx
 ; X86-BWON-NEXT:    xorl %ebp, %ebp
-; X86-BWON-NEXT:    .p2align 4, 0x90
+; X86-BWON-NEXT:    .p2align 4
 ; X86-BWON-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
 ; X86-BWON-NEXT:    movsbl (%edi), %ecx
 ; X86-BWON-NEXT:    movzbl (%esi,%ecx), %edx
@@ -909,7 +909,7 @@ define void @MergeLoadStoreBaseIndexOffsetComplicated(ptr %a, ptr %b, ptr %c, i6
 ; X86-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %ebx
 ; X86-BWOFF-NEXT:    xorl %ebp, %ebp
-; X86-BWOFF-NEXT:    .p2align 4, 0x90
+; X86-BWOFF-NEXT:    .p2align 4
 ; X86-BWOFF-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
 ; X86-BWOFF-NEXT:    movsbl (%edi), %ecx
 ; X86-BWOFF-NEXT:    movb (%esi,%ecx), %dl
@@ -939,7 +939,7 @@ define void @MergeLoadStoreBaseIndexOffsetComplicated(ptr %a, ptr %b, ptr %c, i6
 ; X64-BWON-LABEL: MergeLoadStoreBaseIndexOffsetComplicated:
 ; X64-BWON:       # %bb.0:
 ; X64-BWON-NEXT:    xorl %eax, %eax
-; X64-BWON-NEXT:    .p2align 4, 0x90
+; X64-BWON-NEXT:    .p2align 4
 ; X64-BWON-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
 ; X64-BWON-NEXT:    movsbq (%rsi), %r8
 ; X64-BWON-NEXT:    movzwl (%rdx,%r8), %r8d
@@ -954,7 +954,7 @@ define void @MergeLoadStoreBaseIndexOffsetComplicated(ptr %a, ptr %b, ptr %c, i6
 ; X64-BWOFF-LABEL: MergeLoadStoreBaseIndexOffsetComplicated:
 ; X64-BWOFF:       # %bb.0:
 ; X64-BWOFF-NEXT:    xorl %eax, %eax
-; X64-BWOFF-NEXT:    .p2align 4, 0x90
+; X64-BWOFF-NEXT:    .p2align 4
 ; X64-BWOFF-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
 ; X64-BWOFF-NEXT:    movsbq (%rsi), %r8
 ; X64-BWOFF-NEXT:    movw (%rdx,%r8), %r8w
@@ -1011,7 +1011,7 @@ define void @MergeLoadStoreBaseIndexOffsetSext(ptr %a, ptr %b, ptr %c, i32 %n) {
 ; X86-BWON-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-BWON-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-BWON-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-BWON-NEXT:    .p2align 4, 0x90
+; X86-BWON-NEXT:    .p2align 4
 ; X86-BWON-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
 ; X86-BWON-NEXT:    movsbl (%edi,%eax), %ebx
 ; X86-BWON-NEXT:    movzwl (%edx,%ebx), %ebx
@@ -1044,7 +1044,7 @@ define void @MergeLoadStoreBaseIndexOffsetSext(ptr %a, ptr %b, ptr %c, i32 %n) {
 ; X86-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-BWOFF-NEXT:    .p2align 4, 0x90
+; X86-BWOFF-NEXT:    .p2align 4
 ; X86-BWOFF-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
 ; X86-BWOFF-NEXT:    movsbl (%edi,%eax), %ebx
 ; X86-BWOFF-NEXT:    movw (%edx,%ebx), %bx
@@ -1065,7 +1065,7 @@ define void @MergeLoadStoreBaseIndexOffsetSext(ptr %a, ptr %b, ptr %c, i32 %n) {
 ; X64-BWON:       # %bb.0:
 ; X64-BWON-NEXT:    movl %ecx, %eax
 ; X64-BWON-NEXT:    xorl %ecx, %ecx
-; X64-BWON-NEXT:    .p2align 4, 0x90
+; X64-BWON-NEXT:    .p2align 4
 ; X64-BWON-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
 ; X64-BWON-NEXT:    movsbq (%rdi,%rcx), %r8
 ; X64-BWON-NEXT:    movzwl (%rdx,%r8), %r8d
@@ -1080,7 +1080,7 @@ define void @MergeLoadStoreBaseIndexOffsetSext(ptr %a, ptr %b, ptr %c, i32 %n) {
 ; X64-BWOFF:       # %bb.0:
 ; X64-BWOFF-NEXT:    movl %ecx, %eax
 ; X64-BWOFF-NEXT:    xorl %ecx, %ecx
-; X64-BWOFF-NEXT:    .p2align 4, 0x90
+; X64-BWOFF-NEXT:    .p2align 4
 ; X64-BWOFF-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
 ; X64-BWOFF-NEXT:    movsbq (%rdi,%rcx), %r8
 ; X64-BWOFF-NEXT:    movw (%rdx,%r8), %r8w
@@ -1138,7 +1138,7 @@ define void @loadStoreBaseIndexOffsetSextNoSex(ptr %a, ptr %b, ptr %c, i32 %n) {
 ; X86-BWON-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-BWON-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-BWON-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-BWON-NEXT:    .p2align 4, 0x90
+; X86-BWON-NEXT:    .p2align 4
 ; X86-BWON-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
 ; X86-BWON-NEXT:    movsbl (%edi,%eax), %ebx
 ; X86-BWON-NEXT:    movzbl (%edx,%ebx), %ecx
@@ -1180,7 +1180,7 @@ define void @loadStoreBaseIndexOffsetSextNoSex(ptr %a, ptr %b, ptr %c, i32 %n) {
 ; X86-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-BWOFF-NEXT:    .p2align 4, 0x90
+; X86-BWOFF-NEXT:    .p2align 4
 ; X86-BWOFF-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
 ; X86-BWOFF-NEXT:    movsbl (%edi,%eax), %ebx
 ; X86-BWOFF-NEXT:    movb (%edx,%ebx), %cl
@@ -1207,7 +1207,7 @@ define void @loadStoreBaseIndexOffsetSextNoSex(ptr %a, ptr %b, ptr %c, i32 %n) {
 ; X64-BWON:       # %bb.0:
 ; X64-BWON-NEXT:    movl %ecx, %eax
 ; X64-BWON-NEXT:    xorl %ecx, %ecx
-; X64-BWON-NEXT:    .p2align 4, 0x90
+; X64-BWON-NEXT:    .p2align 4
 ; X64-BWON-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
 ; X64-BWON-NEXT:    movsbq (%rdi,%rcx), %r8
 ; X64-BWON-NEXT:    movzbl (%rdx,%r8), %r9d
@@ -1226,7 +1226,7 @@ define void @loadStoreBaseIndexOffsetSextNoSex(ptr %a, ptr %b, ptr %c, i32 %n) {
 ; X64-BWOFF:       # %bb.0:
 ; X64-BWOFF-NEXT:    movl %ecx, %eax
 ; X64-BWOFF-NEXT:    xorl %ecx, %ecx
-; X64-BWOFF-NEXT:    .p2align 4, 0x90
+; X64-BWOFF-NEXT:    .p2align 4
 ; X64-BWOFF-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
 ; X64-BWOFF-NEXT:    movsbq (%rdi,%rcx), %r8
 ; X64-BWOFF-NEXT:    movb (%rdx,%r8), %r9b
@@ -1561,7 +1561,7 @@ define void @merge_const_store_heterogeneous(i32 %count, ptr nocapture %p) nounw
 ; X86-NEXT:    jle .LBB23_3
 ; X86-NEXT:  # %bb.1: # %.lr.ph.preheader
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB23_2: # %.lr.ph
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl $67305985, (%ecx) # imm = 0x4030201
@@ -1578,7 +1578,7 @@ define void @merge_const_store_heterogeneous(i32 %count, ptr nocapture %p) nounw
 ; X64-NEXT:    jle .LBB23_3
 ; X64-NEXT:  # %bb.1: # %.lr.ph.preheader
 ; X64-NEXT:    movabsq $578437695752307201, %rax # imm = 0x807060504030201
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB23_2: # %.lr.ph
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movq %rax, (%rsi)
diff --git a/llvm/test/CodeGen/X86/PR71178-register-coalescer-crash.ll b/llvm/test/CodeGen/X86/PR71178-register-coalescer-crash.ll
index 0ce346f7c2e02..12d66f64cb73d 100644
--- a/llvm/test/CodeGen/X86/PR71178-register-coalescer-crash.ll
+++ b/llvm/test/CodeGen/X86/PR71178-register-coalescer-crash.ll
@@ -8,7 +8,7 @@ define i32 @h(i1 %arg, i32 %arg1) {
 ; CHECK-NEXT:    movabsq $9166129423, %rcx # imm = 0x22258090F
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    jmp .LBB0_1
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_9: # %bb18
 ; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    xorl %eax, %eax
diff --git a/llvm/test/CodeGen/X86/SwitchLowering.ll b/llvm/test/CodeGen/X86/SwitchLowering.ll
index ac80fda69fccd..921b4e3d24d11 100644
--- a/llvm/test/CodeGen/X86/SwitchLowering.ll
+++ b/llvm/test/CodeGen/X86/SwitchLowering.ll
@@ -13,7 +13,7 @@ define ptr @FindChar(ptr %CurPtr) {
 ; CHECK-NEXT:    .cfi_offset %edi, -8
 ; CHECK-NEXT:    xorl %edi, %edi
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %bb
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movzbl (%esi,%edi), %eax
diff --git a/llvm/test/CodeGen/X86/addr-mode-matcher-2.ll b/llvm/test/CodeGen/X86/addr-mode-matcher-2.ll
index daba729bf040f..c810fe137024c 100644
--- a/llvm/test/CodeGen/X86/addr-mode-matcher-2.ll
+++ b/llvm/test/CodeGen/X86/addr-mode-matcher-2.ll
@@ -30,7 +30,7 @@ define void @foo_sext_nsw(i1 zeroext, i32) nounwind {
 ; X86-NEXT:    retl
 ; X86-NEXT:  .LBB0_1: # %.preheader
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB0_2: # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    leal (%eax,%eax,4), %eax
 ; X86-NEXT:    leal 20(,%eax,4), %eax
@@ -49,7 +49,7 @@ define void @foo_sext_nsw(i1 zeroext, i32) nounwind {
 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB0_1: # %.preheader
 ; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB0_2: # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    cltq
 ; X64-NEXT:    shlq $2, %rax
@@ -77,7 +77,7 @@ define void @foo_sext_nuw(i1 zeroext, i32) nounwind {
 ; X86-NEXT:    retl
 ; X86-NEXT:  .LBB1_1: # %.preheader
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB1_2: # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    leal (%eax,%eax,4), %eax
 ; X86-NEXT:    leal 20(,%eax,4), %eax
@@ -96,7 +96,7 @@ define void @foo_sext_nuw(i1 zeroext, i32) nounwind {
 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB1_1: # %.preheader
 ; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB1_2: # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    incl %eax
 ; X64-NEXT:    cltq
@@ -125,7 +125,7 @@ define void @foo_zext_nsw(i1 zeroext, i32) nounwind {
 ; X86-NEXT:    retl
 ; X86-NEXT:  .LBB2_1: # %.preheader
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB2_2: # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    leal (%eax,%eax,4), %eax
 ; X86-NEXT:    leal 20(,%eax,4), %eax
@@ -144,7 +144,7 @@ define void @foo_zext_nsw(i1 zeroext, i32) nounwind {
 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB2_1: # %.preheader
 ; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB2_2: # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    incl %eax
 ; X64-NEXT:    shlq $2, %rax
@@ -173,7 +173,7 @@ define void @foo_zext_nuw(i1 zeroext, i32) nounwind {
 ; X86-NEXT:    retl
 ; X86-NEXT:  .LBB3_1: # %.preheader
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB3_2: # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    leal (%eax,%eax,4), %eax
 ; X86-NEXT:    leal 20(,%eax,4), %eax
@@ -192,7 +192,7 @@ define void @foo_zext_nuw(i1 zeroext, i32) nounwind {
 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB3_1: # %.preheader
 ; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB3_2: # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %eax
 ; X64-NEXT:    shlq $2, %rax
@@ -220,7 +220,7 @@ define void @foo_sext(i1 zeroext, i32) nounwind {
 ; X86-NEXT:    retl
 ; X86-NEXT:  .LBB4_1: # %.preheader
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB4_2: # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    leal (%eax,%eax,4), %eax
 ; X86-NEXT:    leal 20(,%eax,4), %eax
@@ -239,7 +239,7 @@ define void @foo_sext(i1 zeroext, i32) nounwind {
 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB4_1: # %.preheader
 ; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB4_2: # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    incl %eax
 ; X64-NEXT:    cltq
@@ -268,7 +268,7 @@ define void @foo_zext(i1 zeroext, i32) nounwind {
 ; X86-NEXT:    retl
 ; X86-NEXT:  .LBB5_1: # %.preheader
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB5_2: # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    leal (%eax,%eax,4), %eax
 ; X86-NEXT:    leal 20(,%eax,4), %eax
@@ -287,7 +287,7 @@ define void @foo_zext(i1 zeroext, i32) nounwind {
 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB5_1: # %.preheader
 ; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB5_2: # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    incl %eax
 ; X64-NEXT:    shlq $2, %rax
diff --git a/llvm/test/CodeGen/X86/align-basic-block-sections.mir b/llvm/test/CodeGen/X86/align-basic-block-sections.mir
index e7ad5dfe6478f..7521341ff3ea0 100644
--- a/llvm/test/CodeGen/X86/align-basic-block-sections.mir
+++ b/llvm/test/CodeGen/X86/align-basic-block-sections.mir
@@ -132,5 +132,5 @@ body:             |
 ...
 
 # CHECK: .section	.text.test,"ax",@progbits,unique,2
-# CHECK-NEXT: .p2align	8, 0x90
+# CHECK-NEXT: .p2align	8
 # CHECK-NEXT: test.__part.2: # %entry
diff --git a/llvm/test/CodeGen/X86/and-sink.ll b/llvm/test/CodeGen/X86/and-sink.ll
index 965e84844758f..d9a34d743ac65 100644
--- a/llvm/test/CodeGen/X86/and-sink.ll
+++ b/llvm/test/CodeGen/X86/and-sink.ll
@@ -54,7 +54,7 @@ define i32 @and_sink2(i32 %a, i1 %c, i1 %c2) {
 ; CHECK-NEXT:  # %bb.1: # %bb0.preheader
 ; CHECK-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_2: # %bb0
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl $0, B
diff --git a/llvm/test/CodeGen/X86/apx/push2-pop2.ll b/llvm/test/CodeGen/X86/apx/push2-pop2.ll
index 6bd9f525090ee..f5be484be2b1a 100644
--- a/llvm/test/CodeGen/X86/apx/push2-pop2.ll
+++ b/llvm/test/CodeGen/X86/apx/push2-pop2.ll
@@ -286,7 +286,7 @@ define void @lea_in_epilog(i1 %arg, ptr %arg1, ptr %arg2, i64 %arg3, i64 %arg4,
 ; CHECK-NEXT:    xorl %ebp, %ebp
 ; CHECK-NEXT:    xorl %r12d, %r12d
 ; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB6_2: # %bb15
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    incq %r12
@@ -338,7 +338,7 @@ define void @lea_in_epilog(i1 %arg, ptr %arg1, ptr %arg2, i64 %arg3, i64 %arg4,
 ; PPX-NEXT:    xorl %ebp, %ebp
 ; PPX-NEXT:    xorl %r12d, %r12d
 ; PPX-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; PPX-NEXT:    .p2align 4, 0x90
+; PPX-NEXT:    .p2align 4
 ; PPX-NEXT:  .LBB6_2: # %bb15
 ; PPX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; PPX-NEXT:    incq %r12
@@ -390,7 +390,7 @@ define void @lea_in_epilog(i1 %arg, ptr %arg1, ptr %arg2, i64 %arg3, i64 %arg4,
 ; FRAME-NEXT:    xorl %r13d, %r13d
 ; FRAME-NEXT:    xorl %r14d, %r14d
 ; FRAME-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FRAME-NEXT:    .p2align 4, 0x90
+; FRAME-NEXT:    .p2align 4
 ; FRAME-NEXT:  .LBB6_2: # %bb15
 ; FRAME-NEXT:    # =>This Inner Loop Header: Depth=1
 ; FRAME-NEXT:    movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
diff --git a/llvm/test/CodeGen/X86/apx/setzucc.ll b/llvm/test/CodeGen/X86/apx/setzucc.ll
index 084e54235330c..1f2b612a17088 100644
--- a/llvm/test/CodeGen/X86/apx/setzucc.ll
+++ b/llvm/test/CodeGen/X86/apx/setzucc.ll
@@ -53,7 +53,7 @@ define i32 @flags_copy_lowering() nounwind {
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB4_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    addl %edx, 0
diff --git a/llvm/test/CodeGen/X86/assertzext-demanded.ll b/llvm/test/CodeGen/X86/assertzext-demanded.ll
index 3d2120578b49e..33a8c543ed9ab 100644
--- a/llvm/test/CodeGen/X86/assertzext-demanded.ll
+++ b/llvm/test/CodeGen/X86/assertzext-demanded.ll
@@ -7,7 +7,7 @@ define void @simplify_assertzext(ptr %0) {
 ; CHECK:       # %bb.0: # %BB
 ; CHECK-NEXT:    movl $275047, %eax # imm = 0x43267
 ; CHECK-NEXT:    movb $1, %cl
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %CF246
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    testb %cl, %cl
@@ -15,7 +15,7 @@ define void @simplify_assertzext(ptr %0) {
 ; CHECK-NEXT:  # %bb.2: # %CF260
 ; CHECK-NEXT:    orl $278403, %eax # imm = 0x43F83
 ; CHECK-NEXT:    movl %eax, (%rdi)
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_3: # %CF242
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp .LBB0_3
diff --git a/llvm/test/CodeGen/X86/atom-pad-short-functions.ll b/llvm/test/CodeGen/X86/atom-pad-short-functions.ll
index 06800bd244de3..301995dfef0a3 100644
--- a/llvm/test/CodeGen/X86/atom-pad-short-functions.ll
+++ b/llvm/test/CodeGen/X86/atom-pad-short-functions.ll
@@ -110,7 +110,7 @@ define void @test_branch_to_same_bb(i32 %x, i32 %y) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    jle .LBB7_2
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB7_1: # %while.cond
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp .LBB7_1
diff --git a/llvm/test/CodeGen/X86/atomic-bit-test.ll b/llvm/test/CodeGen/X86/atomic-bit-test.ll
index 10b6605c3fb05..8f91f4120842b 100644
--- a/llvm/test/CodeGen/X86/atomic-bit-test.ll
+++ b/llvm/test/CodeGen/X86/atomic-bit-test.ll
@@ -106,7 +106,7 @@ define i64 @bts63() nounwind {
 ; X86-NEXT:    movl $-2147483648, %esi # imm = 0x80000000
 ; X86-NEXT:    movl v64+4, %edx
 ; X86-NEXT:    movl v64, %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB4_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %edx, %ecx
@@ -230,7 +230,7 @@ define i64 @btc63() nounwind {
 ; X86-NEXT:    movl $-2147483648, %esi # imm = 0x80000000
 ; X86-NEXT:    movl v64+4, %edx
 ; X86-NEXT:    movl v64, %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB9_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %edx, %ecx
@@ -360,7 +360,7 @@ define i64 @btr63() nounwind {
 ; X86-NEXT:    movl $-1, %edi
 ; X86-NEXT:    movl v64+4, %edx
 ; X86-NEXT:    movl v64, %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB14_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ebx
@@ -396,7 +396,7 @@ define i16 @multi_use1() nounwind {
 ; X86-LABEL: multi_use1:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movzwl v16, %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB15_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -416,7 +416,7 @@ define i16 @multi_use1() nounwind {
 ; X64-LABEL: multi_use1:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movzwl v16(%rip), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB15_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -471,7 +471,7 @@ define i16 @use_in_diff_bb() nounwind {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movzwl v16, %esi
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB17_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %esi, %ecx
@@ -496,7 +496,7 @@ define i16 @use_in_diff_bb() nounwind {
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    pushq %rbx
 ; X64-NEXT:    movzwl v16(%rip), %ebx
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB17_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %ebx, %ecx
diff --git a/llvm/test/CodeGen/X86/atomic-flags.ll b/llvm/test/CodeGen/X86/atomic-flags.ll
index 00a6308a74969..317af67deb163 100644
--- a/llvm/test/CodeGen/X86/atomic-flags.ll
+++ b/llvm/test/CodeGen/X86/atomic-flags.ll
@@ -144,7 +144,7 @@ define zeroext i1 @xadd_cmp0_i64(ptr %x) nounwind {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    movl (%esi), %eax
 ; X86-NEXT:    movl 4(%esi), %edx
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB2_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ebx
diff --git a/llvm/test/CodeGen/X86/atomic-idempotent.ll b/llvm/test/CodeGen/X86/atomic-idempotent.ll
index d5c46485068a6..55b4d1af094f6 100644
--- a/llvm/test/CodeGen/X86/atomic-idempotent.ll
+++ b/llvm/test/CodeGen/X86/atomic-idempotent.ll
@@ -64,7 +64,7 @@ define i16 @or16(ptr %p) {
 ; X86-SLM:       # %bb.0:
 ; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-SLM-NEXT:    movzwl (%ecx), %eax
-; X86-SLM-NEXT:    .p2align 4, 0x90
+; X86-SLM-NEXT:    .p2align 4
 ; X86-SLM-NEXT:  .LBB1_1: # %atomicrmw.start
 ; X86-SLM-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-SLM-NEXT:    lock cmpxchgw %ax, (%ecx)
@@ -76,7 +76,7 @@ define i16 @or16(ptr %p) {
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-ATOM-NEXT:    movzwl (%ecx), %eax
-; X86-ATOM-NEXT:    .p2align 4, 0x90
+; X86-ATOM-NEXT:    .p2align 4
 ; X86-ATOM-NEXT:  .LBB1_1: # %atomicrmw.start
 ; X86-ATOM-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-ATOM-NEXT:    lock cmpxchgw %ax, (%ecx)
@@ -105,7 +105,7 @@ define i32 @xor32(ptr %p) {
 ; X86-SLM:       # %bb.0:
 ; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-SLM-NEXT:    movl (%ecx), %eax
-; X86-SLM-NEXT:    .p2align 4, 0x90
+; X86-SLM-NEXT:    .p2align 4
 ; X86-SLM-NEXT:  .LBB2_1: # %atomicrmw.start
 ; X86-SLM-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-SLM-NEXT:    lock cmpxchgl %eax, (%ecx)
@@ -117,7 +117,7 @@ define i32 @xor32(ptr %p) {
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-ATOM-NEXT:    movl (%ecx), %eax
-; X86-ATOM-NEXT:    .p2align 4, 0x90
+; X86-ATOM-NEXT:    .p2align 4
 ; X86-ATOM-NEXT:  .LBB2_1: # %atomicrmw.start
 ; X86-ATOM-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-ATOM-NEXT:    lock cmpxchgl %eax, (%ecx)
@@ -146,7 +146,7 @@ define i64 @sub64(ptr %p) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    movl (%esi), %eax
 ; X86-NEXT:    movl 4(%esi), %edx
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB3_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %edx, %ecx
@@ -196,7 +196,7 @@ define i128 @or128(ptr %p) {
 ; X86-GENERIC-NEXT:    movl 8(%edi), %edx
 ; X86-GENERIC-NEXT:    movl (%edi), %ebx
 ; X86-GENERIC-NEXT:    movl 4(%edi), %esi
-; X86-GENERIC-NEXT:    .p2align 4, 0x90
+; X86-GENERIC-NEXT:    .p2align 4
 ; X86-GENERIC-NEXT:  .LBB4_1: # %atomicrmw.start
 ; X86-GENERIC-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-GENERIC-NEXT:    movl %ebx, (%esp)
@@ -257,7 +257,7 @@ define i128 @or128(ptr %p) {
 ; X86-ATOM-NEXT:    movl 8(%edi), %edx
 ; X86-ATOM-NEXT:    movl (%edi), %esi
 ; X86-ATOM-NEXT:    movl 4(%edi), %ebx
-; X86-ATOM-NEXT:    .p2align 4, 0x90
+; X86-ATOM-NEXT:    .p2align 4
 ; X86-ATOM-NEXT:  .LBB4_1: # %atomicrmw.start
 ; X86-ATOM-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-ATOM-NEXT:    movl %esi, (%esp)
@@ -320,7 +320,7 @@ define i32 @and32 (ptr %p) {
 ; X86-SLM:       # %bb.0:
 ; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-SLM-NEXT:    movl (%ecx), %eax
-; X86-SLM-NEXT:    .p2align 4, 0x90
+; X86-SLM-NEXT:    .p2align 4
 ; X86-SLM-NEXT:  .LBB5_1: # %atomicrmw.start
 ; X86-SLM-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-SLM-NEXT:    lock cmpxchgl %eax, (%ecx)
@@ -332,7 +332,7 @@ define i32 @and32 (ptr %p) {
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-ATOM-NEXT:    movl (%ecx), %eax
-; X86-ATOM-NEXT:    .p2align 4, 0x90
+; X86-ATOM-NEXT:    .p2align 4
 ; X86-ATOM-NEXT:  .LBB5_1: # %atomicrmw.start
 ; X86-ATOM-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-ATOM-NEXT:    lock cmpxchgl %eax, (%ecx)
@@ -495,7 +495,7 @@ define void @or64_nouse_seq_cst(ptr %p) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    movl (%esi), %eax
 ; X86-NEXT:    movl 4(%esi), %edx
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB11_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %edx, %ecx
@@ -546,7 +546,7 @@ define void @or128_nouse_seq_cst(ptr %p) {
 ; X86-GENERIC-NEXT:    movl 8(%esi), %edi
 ; X86-GENERIC-NEXT:    movl (%esi), %edx
 ; X86-GENERIC-NEXT:    movl 4(%esi), %ebx
-; X86-GENERIC-NEXT:    .p2align 4, 0x90
+; X86-GENERIC-NEXT:    .p2align 4
 ; X86-GENERIC-NEXT:  .LBB12_1: # %atomicrmw.start
 ; X86-GENERIC-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-GENERIC-NEXT:    movl %edx, (%esp)
@@ -603,7 +603,7 @@ define void @or128_nouse_seq_cst(ptr %p) {
 ; X86-ATOM-NEXT:    movl 8(%esi), %edx
 ; X86-ATOM-NEXT:    movl (%esi), %eax
 ; X86-ATOM-NEXT:    movl 4(%esi), %edi
-; X86-ATOM-NEXT:    .p2align 4, 0x90
+; X86-ATOM-NEXT:    .p2align 4
 ; X86-ATOM-NEXT:  .LBB12_1: # %atomicrmw.start
 ; X86-ATOM-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-ATOM-NEXT:    movl %eax, (%esp)
diff --git a/llvm/test/CodeGen/X86/atomic-minmax-i6432.ll b/llvm/test/CodeGen/X86/atomic-minmax-i6432.ll
index 809457f4d8291..f8cbd0a6a9ee0 100644
--- a/llvm/test/CodeGen/X86/atomic-minmax-i6432.ll
+++ b/llvm/test/CodeGen/X86/atomic-minmax-i6432.ll
@@ -12,7 +12,7 @@ define i64 @atomic_max_i64() nounwind {
 ; LINUX-NEXT:    movl sc64+4, %edx
 ; LINUX-NEXT:    movl sc64, %eax
 ; LINUX-NEXT:    movl $5, %esi
-; LINUX-NEXT:    .p2align 4, 0x90
+; LINUX-NEXT:    .p2align 4
 ; LINUX-NEXT:  .LBB0_1: # %atomicrmw.start
 ; LINUX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; LINUX-NEXT:    cmpl %eax, %esi
@@ -41,7 +41,7 @@ define i64 @atomic_max_i64() nounwind {
 ; PIC-NEXT:    movl (%esi), %eax
 ; PIC-NEXT:    movl 4(%esi), %edx
 ; PIC-NEXT:    movl $5, %edi
-; PIC-NEXT:    .p2align 4, 0x90
+; PIC-NEXT:    .p2align 4
 ; PIC-NEXT:  LBB0_1: ## %atomicrmw.start
 ; PIC-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; PIC-NEXT:    cmpl %eax, %edi
@@ -69,7 +69,7 @@ define i64 @atomic_min_i64() nounwind {
 ; LINUX-NEXT:    pushl %ebx
 ; LINUX-NEXT:    movl sc64+4, %edx
 ; LINUX-NEXT:    movl sc64, %eax
-; LINUX-NEXT:    .p2align 4, 0x90
+; LINUX-NEXT:    .p2align 4
 ; LINUX-NEXT:  .LBB1_1: # %atomicrmw.start
 ; LINUX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; LINUX-NEXT:    cmpl $7, %eax
@@ -95,7 +95,7 @@ define i64 @atomic_min_i64() nounwind {
 ; PIC-NEXT:    movl L_sc64$non_lazy_ptr-L1$pb(%eax), %esi
 ; PIC-NEXT:    movl (%esi), %eax
 ; PIC-NEXT:    movl 4(%esi), %edx
-; PIC-NEXT:    .p2align 4, 0x90
+; PIC-NEXT:    .p2align 4
 ; PIC-NEXT:  LBB1_1: ## %atomicrmw.start
 ; PIC-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; PIC-NEXT:    cmpl $7, %eax
@@ -124,7 +124,7 @@ define i64 @atomic_umax_i64() nounwind {
 ; LINUX-NEXT:    movl sc64+4, %edx
 ; LINUX-NEXT:    movl sc64, %eax
 ; LINUX-NEXT:    movl $7, %esi
-; LINUX-NEXT:    .p2align 4, 0x90
+; LINUX-NEXT:    .p2align 4
 ; LINUX-NEXT:  .LBB2_1: # %atomicrmw.start
 ; LINUX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; LINUX-NEXT:    cmpl %eax, %esi
@@ -153,7 +153,7 @@ define i64 @atomic_umax_i64() nounwind {
 ; PIC-NEXT:    movl (%esi), %eax
 ; PIC-NEXT:    movl 4(%esi), %edx
 ; PIC-NEXT:    movl $7, %edi
-; PIC-NEXT:    .p2align 4, 0x90
+; PIC-NEXT:    .p2align 4
 ; PIC-NEXT:  LBB2_1: ## %atomicrmw.start
 ; PIC-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; PIC-NEXT:    cmpl %eax, %edi
@@ -181,7 +181,7 @@ define i64 @atomic_umin_i64() nounwind {
 ; LINUX-NEXT:    pushl %ebx
 ; LINUX-NEXT:    movl sc64+4, %edx
 ; LINUX-NEXT:    movl sc64, %eax
-; LINUX-NEXT:    .p2align 4, 0x90
+; LINUX-NEXT:    .p2align 4
 ; LINUX-NEXT:  .LBB3_1: # %atomicrmw.start
 ; LINUX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; LINUX-NEXT:    cmpl $9, %eax
@@ -207,7 +207,7 @@ define i64 @atomic_umin_i64() nounwind {
 ; PIC-NEXT:    movl L_sc64$non_lazy_ptr-L3$pb(%eax), %esi
 ; PIC-NEXT:    movl (%esi), %eax
 ; PIC-NEXT:    movl 4(%esi), %edx
-; PIC-NEXT:    .p2align 4, 0x90
+; PIC-NEXT:    .p2align 4
 ; PIC-NEXT:  LBB3_1: ## %atomicrmw.start
 ; PIC-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; PIC-NEXT:    cmpl $9, %eax
@@ -238,7 +238,7 @@ define void @tf_bug(ptr %ptr) nounwind {
 ; LINUX-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; LINUX-NEXT:    movl id+4, %edx
 ; LINUX-NEXT:    movl id, %eax
-; LINUX-NEXT:    .p2align 4, 0x90
+; LINUX-NEXT:    .p2align 4
 ; LINUX-NEXT:  .LBB4_1: # %atomicrmw.start
 ; LINUX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; LINUX-NEXT:    movl %eax, %ebx
@@ -267,7 +267,7 @@ define void @tf_bug(ptr %ptr) nounwind {
 ; PIC-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; PIC-NEXT:    movl (_id-L4$pb)+4(%edi), %edx
 ; PIC-NEXT:    movl _id-L4$pb(%edi), %eax
-; PIC-NEXT:    .p2align 4, 0x90
+; PIC-NEXT:    .p2align 4
 ; PIC-NEXT:  LBB4_1: ## %atomicrmw.start
 ; PIC-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; PIC-NEXT:    movl %eax, %ebx
diff --git a/llvm/test/CodeGen/X86/atomic-non-integer-fp128.ll b/llvm/test/CodeGen/X86/atomic-non-integer-fp128.ll
index 5960787fe30de..e8b4d3f6812d2 100644
--- a/llvm/test/CodeGen/X86/atomic-non-integer-fp128.ll
+++ b/llvm/test/CodeGen/X86/atomic-non-integer-fp128.ll
@@ -16,7 +16,7 @@ define void @store_fp128(ptr %fptr, fp128 %v) {
 ; X64-SSE-NEXT:    movq -{{[0-9]+}}(%rsp), %rcx
 ; X64-SSE-NEXT:    movq (%rdi), %rax
 ; X64-SSE-NEXT:    movq 8(%rdi), %rdx
-; X64-SSE-NEXT:    .p2align 4, 0x90
+; X64-SSE-NEXT:    .p2align 4
 ; X64-SSE-NEXT:  .LBB0_1: # %atomicrmw.start
 ; X64-SSE-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-SSE-NEXT:    lock cmpxchg16b (%rdi)
@@ -73,7 +73,7 @@ define fp128 @exchange_fp128(ptr %fptr, fp128 %x) {
 ; X64-SSE-NEXT:    movq -{{[0-9]+}}(%rsp), %rcx
 ; X64-SSE-NEXT:    movq (%rdi), %rax
 ; X64-SSE-NEXT:    movq 8(%rdi), %rdx
-; X64-SSE-NEXT:    .p2align 4, 0x90
+; X64-SSE-NEXT:    .p2align 4
 ; X64-SSE-NEXT:  .LBB2_1: # %atomicrmw.start
 ; X64-SSE-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-SSE-NEXT:    lock cmpxchg16b (%rdi)
@@ -96,7 +96,7 @@ define fp128 @exchange_fp128(ptr %fptr, fp128 %x) {
 ; X64-AVX-NEXT:    movq -{{[0-9]+}}(%rsp), %rcx
 ; X64-AVX-NEXT:    movq (%rdi), %rax
 ; X64-AVX-NEXT:    movq 8(%rdi), %rdx
-; X64-AVX-NEXT:    .p2align 4, 0x90
+; X64-AVX-NEXT:    .p2align 4
 ; X64-AVX-NEXT:  .LBB2_1: # %atomicrmw.start
 ; X64-AVX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-AVX-NEXT:    lock cmpxchg16b (%rdi)
diff --git a/llvm/test/CodeGen/X86/atomic-non-integer.ll b/llvm/test/CodeGen/X86/atomic-non-integer.ll
index bd794712a3109..ed882cbbc5254 100644
--- a/llvm/test/CodeGen/X86/atomic-non-integer.ll
+++ b/llvm/test/CodeGen/X86/atomic-non-integer.ll
@@ -440,7 +440,7 @@ define double @exchange_double(ptr %fptr, double %x) {
 ; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-SSE1-NEXT:    movl (%esi), %eax
 ; X86-SSE1-NEXT:    movl 4(%esi), %edx
-; X86-SSE1-NEXT:    .p2align 4, 0x90
+; X86-SSE1-NEXT:    .p2align 4
 ; X86-SSE1-NEXT:  .LBB8_1: # %atomicrmw.start
 ; X86-SSE1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-SSE1-NEXT:    lock cmpxchg8b (%esi)
@@ -472,7 +472,7 @@ define double @exchange_double(ptr %fptr, double %x) {
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-SSE2-NEXT:    movl (%esi), %eax
 ; X86-SSE2-NEXT:    movl 4(%esi), %edx
-; X86-SSE2-NEXT:    .p2align 4, 0x90
+; X86-SSE2-NEXT:    .p2align 4
 ; X86-SSE2-NEXT:  .LBB8_1: # %atomicrmw.start
 ; X86-SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-SSE2-NEXT:    lock cmpxchg8b (%esi)
@@ -506,7 +506,7 @@ define double @exchange_double(ptr %fptr, double %x) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl (%esi), %eax
 ; X86-AVX-NEXT:    movl 4(%esi), %edx
-; X86-AVX-NEXT:    .p2align 4, 0x90
+; X86-AVX-NEXT:    .p2align 4
 ; X86-AVX-NEXT:  .LBB8_1: # %atomicrmw.start
 ; X86-AVX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-AVX-NEXT:    lock cmpxchg8b (%esi)
@@ -539,7 +539,7 @@ define double @exchange_double(ptr %fptr, double %x) {
 ; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NOSSE-NEXT:    movl (%esi), %eax
 ; X86-NOSSE-NEXT:    movl 4(%esi), %edx
-; X86-NOSSE-NEXT:    .p2align 4, 0x90
+; X86-NOSSE-NEXT:    .p2align 4
 ; X86-NOSSE-NEXT:  .LBB8_1: # %atomicrmw.start
 ; X86-NOSSE-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NOSSE-NEXT:    lock cmpxchg8b (%esi)
diff --git a/llvm/test/CodeGen/X86/atomic-rm-bit-test-64.ll b/llvm/test/CodeGen/X86/atomic-rm-bit-test-64.ll
index 8ff4f4067dabd..3fe5b70edc718 100644
--- a/llvm/test/CodeGen/X86/atomic-rm-bit-test-64.ll
+++ b/llvm/test/CodeGen/X86/atomic-rm-bit-test-64.ll
@@ -27,7 +27,7 @@ define i64 @atomic_shl2_xor_64_gpr_val(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; CHECK-NEXT:    shlq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rcx
@@ -51,7 +51,7 @@ define i64 @atomic_shl1_neq_xor_64_gpr_val(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movl $1, %edx
 ; CHECK-NEXT:    shlq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB2_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -159,7 +159,7 @@ define i64 @atomic_blsi_xor_64_gpr_val(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    negq %rcx
 ; CHECK-NEXT:    andq %rsi, %rcx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB7_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rdx
@@ -184,7 +184,7 @@ define i64 @atomic_shl1_xor_64_gpr_valz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movl $1, %edx
 ; CHECK-NEXT:    shlq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB8_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -214,7 +214,7 @@ define i64 @atomic_shl2_xor_64_gpr_valz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; CHECK-NEXT:    shlq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB9_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rcx
@@ -243,7 +243,7 @@ define i64 @atomic_shl1_neq_xor_64_gpr_valz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movl $1, %edx
 ; CHECK-NEXT:    shlq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB10_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -276,7 +276,7 @@ define i64 @atomic_shl1_small_mask_xor_64_gpr_valz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movl $1, %edx
 ; CHECK-NEXT:    shlq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB11_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -306,7 +306,7 @@ define i64 @atomic_shl1_mask0_xor_64_gpr_valz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movl $1, %edx
 ; CHECK-NEXT:    shlq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB12_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -336,7 +336,7 @@ define i64 @atomic_shl1_mask1_xor_64_gpr_valz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movl $1, %edx
 ; CHECK-NEXT:    shlq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB13_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -367,7 +367,7 @@ define i64 @atomic_shl1_mask01_xor_64_gpr_valz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movl $1, %edx
 ; CHECK-NEXT:    shlq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB14_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -397,7 +397,7 @@ define i64 @atomic_blsi_xor_64_gpr_valz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    negq %rdx
 ; CHECK-NEXT:    andq %rsi, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB15_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rcx
@@ -427,7 +427,7 @@ define i64 @atomic_shl1_xor_64_gpr_valnz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movl $1, %edx
 ; CHECK-NEXT:    shlq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB16_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -456,7 +456,7 @@ define i64 @atomic_shl2_xor_64_gpr_valnz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; CHECK-NEXT:    shlq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB17_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rcx
@@ -485,7 +485,7 @@ define i64 @atomic_shl1_neq_xor_64_gpr_valnz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movl $1, %edx
 ; CHECK-NEXT:    shlq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB18_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -517,7 +517,7 @@ define i64 @atomic_shl1_small_mask_xor_64_gpr_valnz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movl $1, %edx
 ; CHECK-NEXT:    shlq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB19_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -546,7 +546,7 @@ define i64 @atomic_shl1_mask0_xor_64_gpr_valnz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movl $1, %edx
 ; CHECK-NEXT:    shlq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB20_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -575,7 +575,7 @@ define i64 @atomic_shl1_mask1_xor_64_gpr_valnz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movl $1, %edx
 ; CHECK-NEXT:    shlq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB21_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -605,7 +605,7 @@ define i64 @atomic_shl1_mask01_xor_64_gpr_valnz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movl $1, %edx
 ; CHECK-NEXT:    shlq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB22_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -634,7 +634,7 @@ define i64 @atomic_blsi_xor_64_gpr_valnz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    negq %rdx
 ; CHECK-NEXT:    andq %rsi, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB23_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rcx
@@ -686,7 +686,7 @@ define i64 @atomic_shl2_and_64_gpr_val(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movq %rdx, %rcx
 ; CHECK-NEXT:    notq %rcx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB25_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -711,7 +711,7 @@ define i64 @atomic_shl1_neq_and_64_gpr_val(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movq $-2, %rdx
 ; CHECK-NEXT:    rolq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB26_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -826,7 +826,7 @@ define i64 @atomic_blsi_and_64_gpr_val(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movq %rcx, %rdx
 ; CHECK-NEXT:    notq %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB31_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -853,7 +853,7 @@ define i64 @atomic_shl1_and_64_gpr_valnz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movq $-2, %rdx
 ; CHECK-NEXT:    rolq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB32_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -885,7 +885,7 @@ define i64 @atomic_shl2_and_64_gpr_valnz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movq %rdx, %rcx
 ; CHECK-NEXT:    notq %rcx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB33_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -915,7 +915,7 @@ define i64 @atomic_shl1_neq_and_64_gpr_valnz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movq $-2, %rdx
 ; CHECK-NEXT:    rolq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB34_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -948,7 +948,7 @@ define i64 @atomic_shl1_small_mask_and_64_gpr_valnz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movq $-2, %rdx
 ; CHECK-NEXT:    rolq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB35_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -978,7 +978,7 @@ define i64 @atomic_shl1_mask0_and_64_gpr_valnz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movq $-2, %rdx
 ; CHECK-NEXT:    rolq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB36_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -1008,7 +1008,7 @@ define i64 @atomic_shl1_mask1_and_64_gpr_valnz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movq $-2, %rdx
 ; CHECK-NEXT:    rolq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB37_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -1041,7 +1041,7 @@ define i64 @atomic_shl1_mask01_and_64_gpr_valnz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; CHECK-NEXT:    rolq %cl, %rsi
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB38_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rcx
@@ -1073,7 +1073,7 @@ define i64 @atomic_blsi_and_64_gpr_valnz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movq %rdx, %rcx
 ; CHECK-NEXT:    notq %rcx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB39_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -1137,7 +1137,7 @@ define i64 @atomic_shl2_and_64_gpr_brnz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movq %rdx, %rsi
 ; CHECK-NEXT:    notq %rsi
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB41_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %r8
@@ -1178,7 +1178,7 @@ define i64 @atomic_shl1_neq_and_64_gpr_brnz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movq $-2, %rdx
 ; CHECK-NEXT:    rolq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB42_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
@@ -1354,7 +1354,7 @@ define i64 @atomic_blsi_and_64_gpr_brnz(ptr %v, i64 %c) nounwind {
 ; CHECK-NEXT:    movq %rcx, %rdx
 ; CHECK-NEXT:    notq %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB47_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %r8
@@ -1420,7 +1420,7 @@ define i64 @atomic_shl1_neq_xor_64_const_br(ptr %v) nounwind {
 ; CHECK-LABEL: atomic_shl1_neq_xor_64_const_br:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB49_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rcx
@@ -1483,7 +1483,7 @@ define i64 @atomic_shl1_neq_xor_64_const_brz(ptr %v) nounwind {
 ; CHECK-LABEL: atomic_shl1_neq_xor_64_const_brz:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB51_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rcx
@@ -1546,7 +1546,7 @@ define i64 @atomic_shl1_neq_xor_64_const_brnz(ptr %v) nounwind {
 ; CHECK-LABEL: atomic_shl1_neq_xor_64_const_brnz:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB53_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rcx
@@ -1583,7 +1583,7 @@ define i64 @atomic_and_with_not_arg(ptr %v, i64 %c) nounwind {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movq $-1, %rcx
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB54_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rdx
diff --git a/llvm/test/CodeGen/X86/atomic-rm-bit-test.ll b/llvm/test/CodeGen/X86/atomic-rm-bit-test.ll
index 5594d13a234d0..b4d40fee01e41 100644
--- a/llvm/test/CodeGen/X86/atomic-rm-bit-test.ll
+++ b/llvm/test/CodeGen/X86/atomic-rm-bit-test.ll
@@ -11,7 +11,7 @@ define zeroext i8 @atomic_shl1_or_8_gpr_val(ptr %v, i8 zeroext %c) nounwind {
 ; X86-NEXT:    movl $1, %edx
 ; X86-NEXT:    shll %cl, %edx
 ; X86-NEXT:    movzbl (%esi), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB0_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -31,7 +31,7 @@ define zeroext i8 @atomic_shl1_or_8_gpr_val(ptr %v, i8 zeroext %c) nounwind {
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzbl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB0_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -63,7 +63,7 @@ define zeroext i8 @atomic_shl1_mask0_or_8_gpr_val(ptr %v, i8 zeroext %c) nounwin
 ; X86-NEXT:    movb $1, %ah
 ; X86-NEXT:    shlb %cl, %ah
 ; X86-NEXT:    movb (%esi), %al
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB1_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -86,7 +86,7 @@ define zeroext i8 @atomic_shl1_mask0_or_8_gpr_val(ptr %v, i8 zeroext %c) nounwin
 ; X64-NEXT:    movb $1, %dl
 ; X64-NEXT:    shlb %cl, %dl
 ; X64-NEXT:    movzbl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB1_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -120,7 +120,7 @@ define zeroext i8 @atomic_shl1_mask01_or_8_gpr_val(ptr %v, i8 zeroext %c) nounwi
 ; X86-NEXT:    movb $1, %dl
 ; X86-NEXT:    shlb %cl, %dl
 ; X86-NEXT:    movzbl (%esi), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB2_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -141,7 +141,7 @@ define zeroext i8 @atomic_shl1_mask01_or_8_gpr_val(ptr %v, i8 zeroext %c) nounwi
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shlb %cl, %dl
 ; X64-NEXT:    movzbl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB2_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -169,7 +169,7 @@ define zeroext i8 @atomic_shl1_xor_8_gpr_valz(ptr %v, i8 zeroext %c) nounwind {
 ; X86-NEXT:    movl $1, %edx
 ; X86-NEXT:    shll %cl, %edx
 ; X86-NEXT:    movzbl (%esi), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB3_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -190,7 +190,7 @@ define zeroext i8 @atomic_shl1_xor_8_gpr_valz(ptr %v, i8 zeroext %c) nounwind {
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzbl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB3_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -225,7 +225,7 @@ define zeroext i8 @atomic_shl1_mask0_xor_8_gpr_valz(ptr %v, i8 zeroext %c) nounw
 ; X86-NEXT:    movb $1, %ah
 ; X86-NEXT:    shlb %cl, %ah
 ; X86-NEXT:    movb (%esi), %al
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB4_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -247,7 +247,7 @@ define zeroext i8 @atomic_shl1_mask0_xor_8_gpr_valz(ptr %v, i8 zeroext %c) nounw
 ; X64-NEXT:    movb $1, %dl
 ; X64-NEXT:    shlb %cl, %dl
 ; X64-NEXT:    movzbl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB4_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -283,7 +283,7 @@ define zeroext i8 @atomic_shl1_mask01_xor_8_gpr_valz(ptr %v, i8 zeroext %c) noun
 ; X86-NEXT:    movl $1, %ebx
 ; X86-NEXT:    shll %cl, %ebx
 ; X86-NEXT:    movzbl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB5_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -305,7 +305,7 @@ define zeroext i8 @atomic_shl1_mask01_xor_8_gpr_valz(ptr %v, i8 zeroext %c) noun
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzbl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB5_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -341,7 +341,7 @@ define zeroext i8 @atomic_shl1_and_8_gpr_brnz(ptr %v, i8 zeroext %c) nounwind {
 ; X86-NEXT:    movb %bl, %ah
 ; X86-NEXT:    notb %ah
 ; X86-NEXT:    movb (%edx), %al
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB6_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movb %al, %ch
@@ -370,7 +370,7 @@ define zeroext i8 @atomic_shl1_and_8_gpr_brnz(ptr %v, i8 zeroext %c) nounwind {
 ; X64-NEXT:    movl %edx, %esi
 ; X64-NEXT:    notb %sil
 ; X64-NEXT:    movzbl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB6_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %r8d
@@ -419,7 +419,7 @@ define zeroext i8 @atomic_shl1_mask0_and_8_gpr_brnz(ptr %v, i8 zeroext %c) nounw
 ; X86-NEXT:    movb $-2, %ah
 ; X86-NEXT:    rolb %cl, %ah
 ; X86-NEXT:    movb (%edx), %al
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB7_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movb %al, %ch
@@ -446,7 +446,7 @@ define zeroext i8 @atomic_shl1_mask0_and_8_gpr_brnz(ptr %v, i8 zeroext %c) nounw
 ; X64-NEXT:    movb $-2, %dl
 ; X64-NEXT:    rolb %cl, %dl
 ; X64-NEXT:    movzbl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB7_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -500,7 +500,7 @@ define zeroext i8 @atomic_shl1_mask01_and_8_gpr_brnz(ptr %v, i8 zeroext %c) noun
 ; X86-NEXT:    movl %ebx, %ecx
 ; X86-NEXT:    notb %cl
 ; X86-NEXT:    movb (%edx), %al
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB8_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movb %al, %ch
@@ -530,7 +530,7 @@ define zeroext i8 @atomic_shl1_mask01_and_8_gpr_brnz(ptr %v, i8 zeroext %c) noun
 ; X64-NEXT:    movl %edx, %ecx
 ; X64-NEXT:    notb %cl
 ; X64-NEXT:    movzbl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB8_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %r8d
@@ -576,7 +576,7 @@ define zeroext i8 @atomic_shl1_and_8_gpr_val(ptr %v) nounwind {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movzbl (%ecx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB9_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edx
@@ -590,7 +590,7 @@ define zeroext i8 @atomic_shl1_and_8_gpr_val(ptr %v) nounwind {
 ; X64-LABEL: atomic_shl1_and_8_gpr_val:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movzbl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB9_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -611,7 +611,7 @@ define zeroext i8 @atomic_shl1_or_8_gpr_valnz(ptr %v) nounwind {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movzbl (%ecx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB10_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edx
@@ -626,7 +626,7 @@ define zeroext i8 @atomic_shl1_or_8_gpr_valnz(ptr %v) nounwind {
 ; X64-LABEL: atomic_shl1_or_8_gpr_valnz:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movzbl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB10_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -649,7 +649,7 @@ define zeroext i8 @atomic_shl1_xor_8_gpr_brz(ptr %v) nounwind {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movzbl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB11_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -669,7 +669,7 @@ define zeroext i8 @atomic_shl1_xor_8_gpr_brz(ptr %v) nounwind {
 ; X64-LABEL: atomic_shl1_xor_8_gpr_brz:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movzbl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB11_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -710,7 +710,7 @@ define zeroext i16 @atomic_shl1_xor_16_gpr_val(ptr %v, i16 zeroext %c) nounwind
 ; X86-NEXT:    movl $1, %edx
 ; X86-NEXT:    shll %cl, %edx
 ; X86-NEXT:    movzwl (%esi), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB12_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -732,7 +732,7 @@ define zeroext i16 @atomic_shl1_xor_16_gpr_val(ptr %v, i16 zeroext %c) nounwind
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB12_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -799,7 +799,7 @@ define zeroext i16 @atomic_shl1_mask0_xor_16_gpr_val(ptr %v, i16 zeroext %c) nou
 ; X86-NEXT:    movl $1, %edi
 ; X86-NEXT:    shll %cl, %edi
 ; X86-NEXT:    movzwl (%esi), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB14_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -825,7 +825,7 @@ define zeroext i16 @atomic_shl1_mask0_xor_16_gpr_val(ptr %v, i16 zeroext %c) nou
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB14_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -862,7 +862,7 @@ define zeroext i16 @atomic_shl1_mask1_xor_16_gpr_val(ptr %v, i16 zeroext %c) nou
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB15_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -888,7 +888,7 @@ define zeroext i16 @atomic_shl1_mask1_xor_16_gpr_val(ptr %v, i16 zeroext %c) nou
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB15_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -959,7 +959,7 @@ define zeroext i16 @atomic_blsi_xor_16_gpr_val(ptr %v, i16 zeroext %c) nounwind
 ; X86-NEXT:    negl %ecx
 ; X86-NEXT:    andl %eax, %ecx
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB17_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %esi
@@ -980,7 +980,7 @@ define zeroext i16 @atomic_blsi_xor_16_gpr_val(ptr %v, i16 zeroext %c) nounwind
 ; X64-NEXT:    negl %ecx
 ; X64-NEXT:    andl %esi, %ecx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB17_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %edx
@@ -1010,7 +1010,7 @@ define zeroext i16 @atomic_shl1_xor_16_gpr_valz(ptr %v, i16 zeroext %c) nounwind
 ; X86-NEXT:    movl $1, %edx
 ; X86-NEXT:    shll %cl, %edx
 ; X86-NEXT:    movzwl (%esi), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB18_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -1035,7 +1035,7 @@ define zeroext i16 @atomic_shl1_xor_16_gpr_valz(ptr %v, i16 zeroext %c) nounwind
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB18_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -1075,7 +1075,7 @@ define zeroext i16 @atomic_shl1_small_mask_xor_16_gpr_valz(ptr %v, i16 zeroext %
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movzwl (%edx), %eax
 ; X86-NEXT:    movzwl %si, %esi
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB19_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -1102,7 +1102,7 @@ define zeroext i16 @atomic_shl1_small_mask_xor_16_gpr_valz(ptr %v, i16 zeroext %
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
 ; X64-NEXT:    movzwl %dx, %edx
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB19_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -1143,7 +1143,7 @@ define zeroext i16 @atomic_shl1_mask0_xor_16_gpr_valz(ptr %v, i16 zeroext %c) no
 ; X86-NEXT:    movl $1, %edi
 ; X86-NEXT:    shll %cl, %edi
 ; X86-NEXT:    movzwl (%esi), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB20_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -1169,7 +1169,7 @@ define zeroext i16 @atomic_shl1_mask0_xor_16_gpr_valz(ptr %v, i16 zeroext %c) no
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB20_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -1208,7 +1208,7 @@ define zeroext i16 @atomic_shl1_mask1_xor_16_gpr_valz(ptr %v, i16 zeroext %c) no
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB21_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -1234,7 +1234,7 @@ define zeroext i16 @atomic_shl1_mask1_xor_16_gpr_valz(ptr %v, i16 zeroext %c) no
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB21_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -1276,7 +1276,7 @@ define zeroext i16 @atomic_shl1_mask01_xor_16_gpr_valz(ptr %v, i16 zeroext %c) n
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB22_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -1302,7 +1302,7 @@ define zeroext i16 @atomic_shl1_mask01_xor_16_gpr_valz(ptr %v, i16 zeroext %c) n
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB22_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -1341,7 +1341,7 @@ define zeroext i16 @atomic_blsi_xor_16_gpr_valz(ptr %v, i16 zeroext %c) nounwind
 ; X86-NEXT:    negl %ecx
 ; X86-NEXT:    andl %eax, %ecx
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB23_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %esi
@@ -1365,7 +1365,7 @@ define zeroext i16 @atomic_blsi_xor_16_gpr_valz(ptr %v, i16 zeroext %c) nounwind
 ; X64-NEXT:    negl %ecx
 ; X64-NEXT:    andl %esi, %ecx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB23_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %edx
@@ -1403,7 +1403,7 @@ define zeroext i16 @atomic_shl1_xor_16_gpr_valnz(ptr %v, i16 zeroext %c) nounwin
 ; X86-NEXT:    movl $1, %edx
 ; X86-NEXT:    shll %cl, %edx
 ; X86-NEXT:    movzwl (%esi), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB24_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -1428,7 +1428,7 @@ define zeroext i16 @atomic_shl1_xor_16_gpr_valnz(ptr %v, i16 zeroext %c) nounwin
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB24_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -1468,7 +1468,7 @@ define zeroext i16 @atomic_shl1_small_mask_xor_16_gpr_valnz(ptr %v, i16 zeroext
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movzwl (%edx), %eax
 ; X86-NEXT:    movzwl %si, %esi
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB25_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -1495,7 +1495,7 @@ define zeroext i16 @atomic_shl1_small_mask_xor_16_gpr_valnz(ptr %v, i16 zeroext
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
 ; X64-NEXT:    movzwl %dx, %edx
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB25_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -1536,7 +1536,7 @@ define zeroext i16 @atomic_shl1_mask0_xor_16_gpr_valnz(ptr %v, i16 zeroext %c) n
 ; X86-NEXT:    movl $1, %edi
 ; X86-NEXT:    shll %cl, %edi
 ; X86-NEXT:    movzwl (%esi), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB26_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -1562,7 +1562,7 @@ define zeroext i16 @atomic_shl1_mask0_xor_16_gpr_valnz(ptr %v, i16 zeroext %c) n
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB26_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -1601,7 +1601,7 @@ define zeroext i16 @atomic_shl1_mask1_xor_16_gpr_valnz(ptr %v, i16 zeroext %c) n
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB27_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -1627,7 +1627,7 @@ define zeroext i16 @atomic_shl1_mask1_xor_16_gpr_valnz(ptr %v, i16 zeroext %c) n
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB27_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -1669,7 +1669,7 @@ define zeroext i16 @atomic_shl1_mask01_xor_16_gpr_valnz(ptr %v, i16 zeroext %c)
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB28_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -1695,7 +1695,7 @@ define zeroext i16 @atomic_shl1_mask01_xor_16_gpr_valnz(ptr %v, i16 zeroext %c)
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB28_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -1734,7 +1734,7 @@ define zeroext i16 @atomic_blsi_xor_16_gpr_valnz(ptr %v, i16 zeroext %c) nounwin
 ; X86-NEXT:    negl %ecx
 ; X86-NEXT:    andl %eax, %ecx
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB29_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %esi
@@ -1758,7 +1758,7 @@ define zeroext i16 @atomic_blsi_xor_16_gpr_valnz(ptr %v, i16 zeroext %c) nounwin
 ; X64-NEXT:    negl %ecx
 ; X64-NEXT:    andl %esi, %ecx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB29_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %edx
@@ -1797,7 +1797,7 @@ define zeroext i16 @atomic_shl1_xor_16_gpr_brz(ptr %v, i16 zeroext %c) nounwind
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB30_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -1825,7 +1825,7 @@ define zeroext i16 @atomic_shl1_xor_16_gpr_brz(ptr %v, i16 zeroext %c) nounwind
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB30_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -1878,7 +1878,7 @@ define zeroext i16 @atomic_shl1_small_mask_xor_16_gpr_brz(ptr %v, i16 zeroext %c
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movzwl (%edx), %eax
 ; X86-NEXT:    movzwl %si, %esi
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB31_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -1908,7 +1908,7 @@ define zeroext i16 @atomic_shl1_small_mask_xor_16_gpr_brz(ptr %v, i16 zeroext %c
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
 ; X64-NEXT:    movzwl %dx, %edx
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB31_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -1962,7 +1962,7 @@ define zeroext i16 @atomic_shl1_mask0_xor_16_gpr_brz(ptr %v, i16 zeroext %c) nou
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB32_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -1991,7 +1991,7 @@ define zeroext i16 @atomic_shl1_mask0_xor_16_gpr_brz(ptr %v, i16 zeroext %c) nou
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB32_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -2043,7 +2043,7 @@ define zeroext i16 @atomic_shl1_mask1_xor_16_gpr_brz(ptr %v, i16 zeroext %c) nou
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB33_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -2073,7 +2073,7 @@ define zeroext i16 @atomic_shl1_mask1_xor_16_gpr_brz(ptr %v, i16 zeroext %c) nou
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB33_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -2131,7 +2131,7 @@ define zeroext i16 @atomic_shl1_mask01_xor_16_gpr_brz(ptr %v, i16 zeroext %c) no
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB34_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -2160,7 +2160,7 @@ define zeroext i16 @atomic_shl1_mask01_xor_16_gpr_brz(ptr %v, i16 zeroext %c) no
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB34_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -2213,7 +2213,7 @@ define zeroext i16 @atomic_blsi_xor_16_gpr_brz(ptr %v, i16 zeroext %c) nounwind
 ; X86-NEXT:    negl %esi
 ; X86-NEXT:    andl %ecx, %esi
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB35_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -2241,7 +2241,7 @@ define zeroext i16 @atomic_blsi_xor_16_gpr_brz(ptr %v, i16 zeroext %c) nounwind
 ; X64-NEXT:    negl %ecx
 ; X64-NEXT:    andl %esi, %ecx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB35_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %edx
@@ -2295,7 +2295,7 @@ define zeroext i16 @atomic_shl1_and_16_gpr_val(ptr %v, i16 zeroext %c) nounwind
 ; X86-NEXT:    movl $-2, %edi
 ; X86-NEXT:    roll %cl, %edi
 ; X86-NEXT:    movzwl (%esi), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB36_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -2320,7 +2320,7 @@ define zeroext i16 @atomic_shl1_and_16_gpr_val(ptr %v, i16 zeroext %c) nounwind
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    roll %cl, %esi
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB36_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -2387,7 +2387,7 @@ define zeroext i16 @atomic_shl1_mask0_and_16_gpr_val(ptr %v, i16 zeroext %c) nou
 ; X86-NEXT:    movw $-2, %si
 ; X86-NEXT:    rolw %cl, %si
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB38_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -2412,7 +2412,7 @@ define zeroext i16 @atomic_shl1_mask0_and_16_gpr_val(ptr %v, i16 zeroext %c) nou
 ; X64-NEXT:    movw $-2, %dx
 ; X64-NEXT:    rolw %cl, %dx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB38_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -2450,7 +2450,7 @@ define zeroext i16 @atomic_shl1_mask1_and_16_gpr_val(ptr %v, i16 zeroext %c) nou
 ; X86-NEXT:    movl $-2, %esi
 ; X86-NEXT:    roll %cl, %esi
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB39_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -2476,7 +2476,7 @@ define zeroext i16 @atomic_shl1_mask1_and_16_gpr_val(ptr %v, i16 zeroext %c) nou
 ; X64-NEXT:    movl $-2, %edx
 ; X64-NEXT:    roll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB39_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -2552,7 +2552,7 @@ define zeroext i16 @atomic_blsi_and_16_gpr_val(ptr %v, i16 zeroext %c) nounwind
 ; X86-NEXT:    movl %ecx, %esi
 ; X86-NEXT:    notl %esi
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB41_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -2576,7 +2576,7 @@ define zeroext i16 @atomic_blsi_and_16_gpr_val(ptr %v, i16 zeroext %c) nounwind
 ; X64-NEXT:    movl %ecx, %edx
 ; X64-NEXT:    notl %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB41_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -2610,7 +2610,7 @@ define zeroext i16 @atomic_shl1_and_16_gpr_valnz(ptr %v, i16 zeroext %c) nounwin
 ; X86-NEXT:    movl $-2, %edi
 ; X86-NEXT:    roll %cl, %edi
 ; X86-NEXT:    movzwl (%esi), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB42_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -2638,7 +2638,7 @@ define zeroext i16 @atomic_shl1_and_16_gpr_valnz(ptr %v, i16 zeroext %c) nounwin
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    roll %cl, %esi
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB42_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -2678,7 +2678,7 @@ define zeroext i16 @atomic_shl1_small_mask_and_16_gpr_valnz(ptr %v, i16 zeroext
 ; X86-NEXT:    movw $-2, %si
 ; X86-NEXT:    rolw %cl, %si
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB43_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -2704,7 +2704,7 @@ define zeroext i16 @atomic_shl1_small_mask_and_16_gpr_valnz(ptr %v, i16 zeroext
 ; X64-NEXT:    movw $-2, %dx
 ; X64-NEXT:    rolw %cl, %dx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB43_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -2744,7 +2744,7 @@ define zeroext i16 @atomic_shl1_mask0_and_16_gpr_valnz(ptr %v, i16 zeroext %c) n
 ; X86-NEXT:    movw $-2, %si
 ; X86-NEXT:    rolw %cl, %si
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB44_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -2769,7 +2769,7 @@ define zeroext i16 @atomic_shl1_mask0_and_16_gpr_valnz(ptr %v, i16 zeroext %c) n
 ; X64-NEXT:    movw $-2, %dx
 ; X64-NEXT:    rolw %cl, %dx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB44_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -2809,7 +2809,7 @@ define zeroext i16 @atomic_shl1_mask1_and_16_gpr_valnz(ptr %v, i16 zeroext %c) n
 ; X86-NEXT:    movl $-2, %esi
 ; X86-NEXT:    roll %cl, %esi
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB45_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -2835,7 +2835,7 @@ define zeroext i16 @atomic_shl1_mask1_and_16_gpr_valnz(ptr %v, i16 zeroext %c) n
 ; X64-NEXT:    movl $-2, %edx
 ; X64-NEXT:    roll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB45_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -2881,7 +2881,7 @@ define zeroext i16 @atomic_shl1_mask01_and_16_gpr_valnz(ptr %v, i16 zeroext %c)
 ; X86-NEXT:    movl $-2, %edi
 ; X86-NEXT:    roll %cl, %edi
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB46_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -2910,7 +2910,7 @@ define zeroext i16 @atomic_shl1_mask01_and_16_gpr_valnz(ptr %v, i16 zeroext %c)
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    roll %cl, %esi
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB46_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -2953,7 +2953,7 @@ define zeroext i16 @atomic_blsi_and_16_gpr_valnz(ptr %v, i16 zeroext %c) nounwin
 ; X86-NEXT:    movl %ecx, %esi
 ; X86-NEXT:    notl %esi
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB47_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -2980,7 +2980,7 @@ define zeroext i16 @atomic_blsi_and_16_gpr_valnz(ptr %v, i16 zeroext %c) nounwin
 ; X64-NEXT:    movl %ecx, %edx
 ; X64-NEXT:    notl %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB47_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -3023,7 +3023,7 @@ define zeroext i16 @atomic_shl1_and_16_gpr_brnz(ptr %v, i16 zeroext %c) nounwind
 ; X86-NEXT:    movl $-2, %edi
 ; X86-NEXT:    roll %cl, %edi
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB48_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ebx
@@ -3056,7 +3056,7 @@ define zeroext i16 @atomic_shl1_and_16_gpr_brnz(ptr %v, i16 zeroext %c) nounwind
 ; X64-NEXT:    movl $-2, %esi
 ; X64-NEXT:    roll %cl, %esi
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB48_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %r8d
@@ -3109,7 +3109,7 @@ define zeroext i16 @atomic_shl1_small_mask_and_16_gpr_brnz(ptr %v, i16 zeroext %
 ; X86-NEXT:    movw $-2, %si
 ; X86-NEXT:    rolw %cl, %si
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB49_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -3140,7 +3140,7 @@ define zeroext i16 @atomic_shl1_small_mask_and_16_gpr_brnz(ptr %v, i16 zeroext %
 ; X64-NEXT:    movw $-2, %dx
 ; X64-NEXT:    rolw %cl, %dx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB49_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -3193,7 +3193,7 @@ define zeroext i16 @atomic_shl1_mask0_and_16_gpr_brnz(ptr %v, i16 zeroext %c) no
 ; X86-NEXT:    movw $-2, %si
 ; X86-NEXT:    rolw %cl, %si
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB50_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -3223,7 +3223,7 @@ define zeroext i16 @atomic_shl1_mask0_and_16_gpr_brnz(ptr %v, i16 zeroext %c) no
 ; X64-NEXT:    movw $-2, %dx
 ; X64-NEXT:    rolw %cl, %dx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB50_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -3276,7 +3276,7 @@ define zeroext i16 @atomic_shl1_mask1_and_16_gpr_brnz(ptr %v, i16 zeroext %c) no
 ; X86-NEXT:    movl $-2, %esi
 ; X86-NEXT:    roll %cl, %esi
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB51_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -3308,7 +3308,7 @@ define zeroext i16 @atomic_shl1_mask1_and_16_gpr_brnz(ptr %v, i16 zeroext %c) no
 ; X64-NEXT:    movl $-2, %edx
 ; X64-NEXT:    roll %cl, %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB51_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -3370,7 +3370,7 @@ define zeroext i16 @atomic_shl1_mask01_and_16_gpr_brnz(ptr %v, i16 zeroext %c) n
 ; X86-NEXT:    movl $-2, %edi
 ; X86-NEXT:    roll %cl, %edi
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB52_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -3404,7 +3404,7 @@ define zeroext i16 @atomic_shl1_mask01_and_16_gpr_brnz(ptr %v, i16 zeroext %c) n
 ; X64-NEXT:    movl $-2, %r8d
 ; X64-NEXT:    roll %cl, %r8d
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB52_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -3461,7 +3461,7 @@ define zeroext i16 @atomic_blsi_and_16_gpr_brnz(ptr %v, i16 zeroext %c) nounwind
 ; X86-NEXT:    movl %esi, %edi
 ; X86-NEXT:    notl %edi
 ; X86-NEXT:    movzwl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB53_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ebx
@@ -3494,7 +3494,7 @@ define zeroext i16 @atomic_blsi_and_16_gpr_brnz(ptr %v, i16 zeroext %c) nounwind
 ; X64-NEXT:    movl %ecx, %edx
 ; X64-NEXT:    notl %edx
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB53_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %r8d
@@ -3567,7 +3567,7 @@ define zeroext i16 @atomic_shl1_or_16_const_valnz(ptr %v) nounwind {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movzwl (%ecx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB55_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edx
@@ -3585,7 +3585,7 @@ define zeroext i16 @atomic_shl1_or_16_const_valnz(ptr %v) nounwind {
 ; X64-LABEL: atomic_shl1_or_16_const_valnz:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB55_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -3675,7 +3675,7 @@ define zeroext i16 @atomic_shl1_and_16_const_valz(ptr %v) nounwind {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movzwl (%ecx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB58_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edx
@@ -3694,7 +3694,7 @@ define zeroext i16 @atomic_shl1_and_16_const_valz(ptr %v) nounwind {
 ; X64-LABEL: atomic_shl1_and_16_const_valz:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB58_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -3922,7 +3922,7 @@ define i32 @atomic_blsi_or_32_gpr_val(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    negl %ecx
 ; X86-NEXT:    andl %eax, %ecx
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB65_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %esi
@@ -3940,7 +3940,7 @@ define i32 @atomic_blsi_or_32_gpr_val(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    negl %ecx
 ; X64-NEXT:    andl %esi, %ecx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB65_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %edx
@@ -3968,7 +3968,7 @@ define i32 @atomic_shl1_or_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB66_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -3990,7 +3990,7 @@ define i32 @atomic_shl1_or_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB66_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -4023,7 +4023,7 @@ define i32 @atomic_shl1_small_mask_or_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB67_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -4046,7 +4046,7 @@ define i32 @atomic_shl1_small_mask_or_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB67_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -4079,7 +4079,7 @@ define i32 @atomic_shl1_mask0_or_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB68_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -4101,7 +4101,7 @@ define i32 @atomic_shl1_mask0_or_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB68_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -4134,7 +4134,7 @@ define i32 @atomic_shl1_mask1_or_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB69_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -4156,7 +4156,7 @@ define i32 @atomic_shl1_mask1_or_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB69_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -4190,7 +4190,7 @@ define i32 @atomic_shl1_mask01_or_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB70_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -4213,7 +4213,7 @@ define i32 @atomic_shl1_mask01_or_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB70_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -4246,7 +4246,7 @@ define i32 @atomic_blsi_or_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    negl %edx
 ; X86-NEXT:    andl %eax, %edx
 ; X86-NEXT:    movl (%ecx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB71_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %esi
@@ -4267,7 +4267,7 @@ define i32 @atomic_blsi_or_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    negl %edx
 ; X64-NEXT:    andl %esi, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB71_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -4300,7 +4300,7 @@ define i32 @atomic_shl1_or_32_gpr_valnz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB72_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -4322,7 +4322,7 @@ define i32 @atomic_shl1_or_32_gpr_valnz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB72_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -4354,7 +4354,7 @@ define i32 @atomic_shl1_small_mask_or_32_gpr_valnz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB73_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -4377,7 +4377,7 @@ define i32 @atomic_shl1_small_mask_or_32_gpr_valnz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB73_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -4409,7 +4409,7 @@ define i32 @atomic_shl1_mask0_or_32_gpr_valnz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB74_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -4431,7 +4431,7 @@ define i32 @atomic_shl1_mask0_or_32_gpr_valnz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB74_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -4463,7 +4463,7 @@ define i32 @atomic_shl1_mask1_or_32_gpr_valnz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB75_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -4485,7 +4485,7 @@ define i32 @atomic_shl1_mask1_or_32_gpr_valnz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB75_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -4518,7 +4518,7 @@ define i32 @atomic_shl1_mask01_or_32_gpr_valnz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB76_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -4541,7 +4541,7 @@ define i32 @atomic_shl1_mask01_or_32_gpr_valnz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB76_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -4573,7 +4573,7 @@ define i32 @atomic_blsi_or_32_gpr_valnz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    negl %edx
 ; X86-NEXT:    andl %eax, %edx
 ; X86-NEXT:    movl (%ecx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB77_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %esi
@@ -4594,7 +4594,7 @@ define i32 @atomic_blsi_or_32_gpr_valnz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    negl %edx
 ; X64-NEXT:    andl %esi, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB77_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -4867,7 +4867,7 @@ define i32 @atomic_blsi_or_32_gpr_br(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    negl %esi
 ; X86-NEXT:    andl %ecx, %esi
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB83_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -4893,7 +4893,7 @@ define i32 @atomic_blsi_or_32_gpr_br(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    negl %ecx
 ; X64-NEXT:    andl %esi, %ecx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB83_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %edx
@@ -5179,7 +5179,7 @@ define i32 @atomic_blsi_or_32_gpr_brz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    negl %edi
 ; X86-NEXT:    andl %edx, %edi
 ; X86-NEXT:    movl (%esi), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB89_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -5204,7 +5204,7 @@ define i32 @atomic_blsi_or_32_gpr_brz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    negl %edx
 ; X64-NEXT:    andl %esi, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB89_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -5492,7 +5492,7 @@ define i32 @atomic_blsi_or_32_gpr_brnz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    negl %esi
 ; X86-NEXT:    andl %ecx, %esi
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB95_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -5518,7 +5518,7 @@ define i32 @atomic_blsi_or_32_gpr_brnz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    negl %ecx
 ; X64-NEXT:    andl %esi, %ecx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB95_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %edx
@@ -5564,7 +5564,7 @@ define i32 @atomic_shl1_and_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB96_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -5586,7 +5586,7 @@ define i32 @atomic_shl1_and_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB96_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -5619,7 +5619,7 @@ define i32 @atomic_shl1_small_mask_and_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB97_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -5642,7 +5642,7 @@ define i32 @atomic_shl1_small_mask_and_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB97_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -5675,7 +5675,7 @@ define i32 @atomic_shl1_mask0_and_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB98_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -5697,7 +5697,7 @@ define i32 @atomic_shl1_mask0_and_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB98_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -5730,7 +5730,7 @@ define i32 @atomic_shl1_mask1_and_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB99_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -5752,7 +5752,7 @@ define i32 @atomic_shl1_mask1_and_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB99_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -5786,7 +5786,7 @@ define i32 @atomic_shl1_mask01_and_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB100_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -5809,7 +5809,7 @@ define i32 @atomic_shl1_mask01_and_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB100_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -5969,7 +5969,7 @@ define i32 @atomic_shl1_mask0_and_32_gpr_br(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB104_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -5995,7 +5995,7 @@ define i32 @atomic_shl1_mask0_and_32_gpr_br(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB104_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -6042,7 +6042,7 @@ define i32 @atomic_shl1_mask1_and_32_gpr_br(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB105_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -6068,7 +6068,7 @@ define i32 @atomic_shl1_mask1_and_32_gpr_br(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB105_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -6305,7 +6305,7 @@ define i32 @atomic_shl1_mask0_and_32_gpr_brz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl $1, %edx
 ; X86-NEXT:    shll %cl, %edx
 ; X86-NEXT:    movl (%esi), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB110_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -6330,7 +6330,7 @@ define i32 @atomic_shl1_mask0_and_32_gpr_brz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB110_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -6379,7 +6379,7 @@ define i32 @atomic_shl1_mask1_and_32_gpr_brz(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl $1, %edx
 ; X86-NEXT:    shll %cl, %edx
 ; X86-NEXT:    movl (%esi), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB111_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edi
@@ -6404,7 +6404,7 @@ define i32 @atomic_shl1_mask1_and_32_gpr_brz(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB111_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %esi
@@ -6566,7 +6566,7 @@ define i32 @atomic_shl1_xor_32_const_valz(ptr %v) nounwind {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl (%ecx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB115_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edx
@@ -6583,7 +6583,7 @@ define i32 @atomic_shl1_xor_32_const_valz(ptr %v) nounwind {
 ; X64-LABEL: atomic_shl1_xor_32_const_valz:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB115_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -6609,7 +6609,7 @@ define i32 @atomic_shl1_xor_32_const_valnz(ptr %v) nounwind {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl (%ecx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB116_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edx
@@ -6624,7 +6624,7 @@ define i32 @atomic_shl1_xor_32_const_valnz(ptr %v) nounwind {
 ; X64-LABEL: atomic_shl1_xor_32_const_valnz:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB116_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -6670,7 +6670,7 @@ define i32 @atomic_shl1_and_32_const_valz(ptr %v) nounwind {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl (%ecx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB118_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edx
@@ -6687,7 +6687,7 @@ define i32 @atomic_shl1_and_32_const_valz(ptr %v) nounwind {
 ; X64-LABEL: atomic_shl1_and_32_const_valz:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB118_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -6713,7 +6713,7 @@ define i32 @atomic_shl1_and_32_const_valnz(ptr %v) nounwind {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl (%ecx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB119_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edx
@@ -6728,7 +6728,7 @@ define i32 @atomic_shl1_and_32_const_valnz(ptr %v) nounwind {
 ; X64-LABEL: atomic_shl1_and_32_const_valnz:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB119_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -6835,7 +6835,7 @@ define i32 @atomic_xor_dead_and(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl $1, %esi
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movl (%edx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB122_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
@@ -6855,7 +6855,7 @@ define i32 @atomic_xor_dead_and(ptr %v, i32 %c) nounwind {
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB122_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -6882,7 +6882,7 @@ define i32 @atomic_xor_with_not_arg(ptr %v, i32 %c) nounwind {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    notl %edx
 ; X86-NEXT:    movl (%ecx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB123_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %esi
@@ -6897,7 +6897,7 @@ define i32 @atomic_xor_with_not_arg(ptr %v, i32 %c) nounwind {
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    notl %esi
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB123_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -6920,7 +6920,7 @@ define i16 @atomic_or_with_not_arg(ptr %v, i16 %c) nounwind {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    notl %edx
 ; X86-NEXT:    movzwl (%ecx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB124_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %esi
@@ -6938,7 +6938,7 @@ define i16 @atomic_or_with_not_arg(ptr %v, i16 %c) nounwind {
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    notl %esi
 ; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB124_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
@@ -6963,7 +6963,7 @@ define i8 @atomic_and_with_not_arg(ptr %v, i8 %c) nounwind {
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    notb %dl
 ; X86-NEXT:    movzbl (%ecx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB125_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movb %al, %ah
@@ -6977,7 +6977,7 @@ define i8 @atomic_and_with_not_arg(ptr %v, i8 %c) nounwind {
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    notb %sil
 ; X64-NEXT:    movzbl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB125_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
diff --git a/llvm/test/CodeGen/X86/atomic-xor.ll b/llvm/test/CodeGen/X86/atomic-xor.ll
index 930286c8e5fb3..c648ecdfbe674 100644
--- a/llvm/test/CodeGen/X86/atomic-xor.ll
+++ b/llvm/test/CodeGen/X86/atomic-xor.ll
@@ -32,7 +32,7 @@ define i128 @xor128_signbit_used(ptr %p) nounwind {
 ; X86-NEXT:    movl 8(%edi), %edx
 ; X86-NEXT:    movl (%edi), %ebx
 ; X86-NEXT:    movl 4(%edi), %esi
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB1_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %ebx, (%esp)
@@ -94,7 +94,7 @@ define i64 @xor64_signbit_used(ptr %p) nounwind {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    movl (%esi), %eax
 ; X86-NEXT:    movl 4(%esi), %edx
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB2_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    leal -2147483648(%edx), %ecx
@@ -171,7 +171,7 @@ define i32 @xor32_not_signbit_used(ptr %p) nounwind {
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl (%ecx), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB6_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %edx
@@ -184,7 +184,7 @@ define i32 @xor32_not_signbit_used(ptr %p) nounwind {
 ; X64-LABEL: xor32_not_signbit_used:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB6_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
diff --git a/llvm/test/CodeGen/X86/atomic128.ll b/llvm/test/CodeGen/X86/atomic128.ll
index 949ee9e276a43..8d43c612aa07f 100644
--- a/llvm/test/CodeGen/X86/atomic128.ll
+++ b/llvm/test/CodeGen/X86/atomic128.ll
@@ -47,7 +47,6 @@ define i128 @load_global_with_offset() nounwind {
 ; CHECK-AVX-NEXT:    vmovq %xmm0, %rax
 ; CHECK-AVX-NEXT:    vpextrq $1, %xmm0, %rdx
 ; CHECK-AVX-NEXT:    retq
-;
 entry:
   %0 = load atomic i128, ptr getelementptr inbounds ({i128, i128}, ptr @cmpxchg16b_global, i64 0, i32 1) acquire, align 16
   ret i128 %0
@@ -62,7 +61,7 @@ define void @fetch_and_nand(ptr %p, i128 %bits) {
 ; CHECK-NEXT:    movq %rdx, %r8
 ; CHECK-NEXT:    movq (%rdi), %rax
 ; CHECK-NEXT:    movq 8(%rdi), %rdx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB2_1: ## %atomicrmw.start
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rdx, %rcx
@@ -92,7 +91,7 @@ define void @fetch_and_or(ptr %p, i128 %bits) {
 ; CHECK-NEXT:    movq %rdx, %r8
 ; CHECK-NEXT:    movq (%rdi), %rax
 ; CHECK-NEXT:    movq 8(%rdi), %rdx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB3_1: ## %atomicrmw.start
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rbx
@@ -120,7 +119,7 @@ define void @fetch_and_add(ptr %p, i128 %bits) {
 ; CHECK-NEXT:    movq %rdx, %r8
 ; CHECK-NEXT:    movq (%rdi), %rax
 ; CHECK-NEXT:    movq 8(%rdi), %rdx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB4_1: ## %atomicrmw.start
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rbx
@@ -148,7 +147,7 @@ define void @fetch_and_sub(ptr %p, i128 %bits) {
 ; CHECK-NEXT:    movq %rdx, %r8
 ; CHECK-NEXT:    movq (%rdi), %rax
 ; CHECK-NEXT:    movq 8(%rdi), %rdx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB5_1: ## %atomicrmw.start
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rbx
@@ -176,7 +175,7 @@ define void @fetch_and_min(ptr %p, i128 %bits) {
 ; CHECK-NEXT:    movq %rdx, %r8
 ; CHECK-NEXT:    movq (%rdi), %rax
 ; CHECK-NEXT:    movq 8(%rdi), %rdx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB6_1: ## %atomicrmw.start
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpq %rax, %rsi
@@ -207,7 +206,7 @@ define void @fetch_and_max(ptr %p, i128 %bits) {
 ; CHECK-NEXT:    movq %rdx, %r8
 ; CHECK-NEXT:    movq (%rdi), %rax
 ; CHECK-NEXT:    movq 8(%rdi), %rdx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB7_1: ## %atomicrmw.start
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpq %rax, %rsi
@@ -238,7 +237,7 @@ define void @fetch_and_umin(ptr %p, i128 %bits) {
 ; CHECK-NEXT:    movq %rdx, %r8
 ; CHECK-NEXT:    movq (%rdi), %rax
 ; CHECK-NEXT:    movq 8(%rdi), %rdx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB8_1: ## %atomicrmw.start
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpq %rax, %rsi
@@ -269,7 +268,7 @@ define void @fetch_and_umax(ptr %p, i128 %bits) {
 ; CHECK-NEXT:    movq %rdx, %r8
 ; CHECK-NEXT:    movq (%rdi), %rax
 ; CHECK-NEXT:    movq 8(%rdi), %rdx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB9_1: ## %atomicrmw.start
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpq %rax, %rsi
@@ -349,7 +348,7 @@ define void @atomic_store_seq_cst(ptr %p, i128 %in) {
 ; CHECK-NOAVX-NEXT:    movq %rsi, %rbx
 ; CHECK-NOAVX-NEXT:    movq (%rdi), %rax
 ; CHECK-NOAVX-NEXT:    movq 8(%rdi), %rdx
-; CHECK-NOAVX-NEXT:    .p2align 4, 0x90
+; CHECK-NOAVX-NEXT:    .p2align 4
 ; CHECK-NOAVX-NEXT:  LBB12_1: ## %atomicrmw.start
 ; CHECK-NOAVX-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NOAVX-NEXT:    lock cmpxchg16b (%rdi)
@@ -380,7 +379,7 @@ define void @atomic_store_release(ptr %p, i128 %in) {
 ; CHECK-NOAVX-NEXT:    movq %rsi, %rbx
 ; CHECK-NOAVX-NEXT:    movq (%rdi), %rax
 ; CHECK-NOAVX-NEXT:    movq 8(%rdi), %rdx
-; CHECK-NOAVX-NEXT:    .p2align 4, 0x90
+; CHECK-NOAVX-NEXT:    .p2align 4
 ; CHECK-NOAVX-NEXT:  LBB13_1: ## %atomicrmw.start
 ; CHECK-NOAVX-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NOAVX-NEXT:    lock cmpxchg16b (%rdi)
@@ -410,7 +409,7 @@ define void @atomic_store_relaxed(ptr %p, i128 %in) {
 ; CHECK-NOAVX-NEXT:    movq %rsi, %rbx
 ; CHECK-NOAVX-NEXT:    movq (%rdi), %rax
 ; CHECK-NOAVX-NEXT:    movq 8(%rdi), %rdx
-; CHECK-NOAVX-NEXT:    .p2align 4, 0x90
+; CHECK-NOAVX-NEXT:    .p2align 4
 ; CHECK-NOAVX-NEXT:  LBB14_1: ## %atomicrmw.start
 ; CHECK-NOAVX-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NOAVX-NEXT:    lock cmpxchg16b (%rdi)
diff --git a/llvm/test/CodeGen/X86/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/X86/atomicrmw-cond-sub-clamp.ll
index 04bfb4e367b9d..3e1a631e39b06 100644
--- a/llvm/test/CodeGen/X86/atomicrmw-cond-sub-clamp.ll
+++ b/llvm/test/CodeGen/X86/atomicrmw-cond-sub-clamp.ll
@@ -9,7 +9,7 @@ define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
 ; CHECK-32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK-32-NEXT:    movzbl (%edx), %eax
 ; CHECK-32-NEXT:    jmp .LBB0_1
-; CHECK-32-NEXT:    .p2align 4, 0x90
+; CHECK-32-NEXT:    .p2align 4
 ; CHECK-32-NEXT:  .LBB0_3: # %atomicrmw.start
 ; CHECK-32-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-32-NEXT:    lock cmpxchgb %ah, (%edx)
@@ -29,7 +29,7 @@ define i8 @atomicrmw_usub_cond_i8(ptr %ptr, i8 %val) {
 ; CHECK-64-LABEL: atomicrmw_usub_cond_i8:
 ; CHECK-64:       # %bb.0:
 ; CHECK-64-NEXT:    movzbl (%rdi), %eax
-; CHECK-64-NEXT:    .p2align 4, 0x90
+; CHECK-64-NEXT:    .p2align 4
 ; CHECK-64-NEXT:  .LBB0_1: # %atomicrmw.start
 ; CHECK-64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-64-NEXT:    movzbl %al, %ecx
@@ -55,7 +55,7 @@ define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
 ; CHECK-32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK-32-NEXT:    movzwl (%edx), %eax
 ; CHECK-32-NEXT:    jmp .LBB1_1
-; CHECK-32-NEXT:    .p2align 4, 0x90
+; CHECK-32-NEXT:    .p2align 4
 ; CHECK-32-NEXT:  .LBB1_3: # %atomicrmw.start
 ; CHECK-32-NEXT:    # in Loop: Header=BB1_1 Depth=1
 ; CHECK-32-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -80,7 +80,7 @@ define i16 @atomicrmw_usub_cond_i16(ptr %ptr, i16 %val) {
 ; CHECK-64-LABEL: atomicrmw_usub_cond_i16:
 ; CHECK-64:       # %bb.0:
 ; CHECK-64-NEXT:    movzwl (%rdi), %eax
-; CHECK-64-NEXT:    .p2align 4, 0x90
+; CHECK-64-NEXT:    .p2align 4
 ; CHECK-64-NEXT:  .LBB1_1: # %atomicrmw.start
 ; CHECK-64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-64-NEXT:    movl %eax, %ecx
@@ -107,7 +107,7 @@ define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
 ; CHECK-32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK-32-NEXT:    movl (%edx), %eax
 ; CHECK-32-NEXT:    jmp .LBB2_1
-; CHECK-32-NEXT:    .p2align 4, 0x90
+; CHECK-32-NEXT:    .p2align 4
 ; CHECK-32-NEXT:  .LBB2_3: # %atomicrmw.start
 ; CHECK-32-NEXT:    # in Loop: Header=BB2_1 Depth=1
 ; CHECK-32-NEXT:    lock cmpxchgl %esi, (%edx)
@@ -129,7 +129,7 @@ define i32 @atomicrmw_usub_cond_i32(ptr %ptr, i32 %val) {
 ; CHECK-64-LABEL: atomicrmw_usub_cond_i32:
 ; CHECK-64:       # %bb.0:
 ; CHECK-64-NEXT:    movl (%rdi), %eax
-; CHECK-64-NEXT:    .p2align 4, 0x90
+; CHECK-64-NEXT:    .p2align 4
 ; CHECK-64-NEXT:  .LBB2_1: # %atomicrmw.start
 ; CHECK-64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-64-NEXT:    movl %eax, %ecx
@@ -164,7 +164,7 @@ define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
 ; CHECK-32-NEXT:    movl (%ebp), %eax
 ; CHECK-32-NEXT:    movl 4(%ebp), %edx
 ; CHECK-32-NEXT:    jmp .LBB3_1
-; CHECK-32-NEXT:    .p2align 4, 0x90
+; CHECK-32-NEXT:    .p2align 4
 ; CHECK-32-NEXT:  .LBB3_3: # %atomicrmw.start
 ; CHECK-32-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK-32-NEXT:    lock cmpxchg8b (%ebp)
@@ -195,7 +195,7 @@ define i64 @atomicrmw_usub_cond_i64(ptr %ptr, i64 %val) {
 ; CHECK-64-LABEL: atomicrmw_usub_cond_i64:
 ; CHECK-64:       # %bb.0:
 ; CHECK-64-NEXT:    movq (%rdi), %rax
-; CHECK-64-NEXT:    .p2align 4, 0x90
+; CHECK-64-NEXT:    .p2align 4
 ; CHECK-64-NEXT:  .LBB3_1: # %atomicrmw.start
 ; CHECK-64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-64-NEXT:    movq %rax, %rcx
@@ -219,7 +219,7 @@ define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
 ; CHECK-32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK-32-NEXT:    movzbl (%edx), %eax
 ; CHECK-32-NEXT:    jmp .LBB4_1
-; CHECK-32-NEXT:    .p2align 4, 0x90
+; CHECK-32-NEXT:    .p2align 4
 ; CHECK-32-NEXT:  .LBB4_3: # %atomicrmw.start
 ; CHECK-32-NEXT:    # in Loop: Header=BB4_1 Depth=1
 ; CHECK-32-NEXT:    lock cmpxchgb %bl, (%edx)
@@ -241,7 +241,7 @@ define i8 @atomicrmw_usub_sat_i8(ptr %ptr, i8 %val) {
 ; CHECK-64:       # %bb.0:
 ; CHECK-64-NEXT:    movzbl (%rdi), %eax
 ; CHECK-64-NEXT:    xorl %ecx, %ecx
-; CHECK-64-NEXT:    .p2align 4, 0x90
+; CHECK-64-NEXT:    .p2align 4
 ; CHECK-64-NEXT:  .LBB4_1: # %atomicrmw.start
 ; CHECK-64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-64-NEXT:    movl %eax, %edx
@@ -269,7 +269,7 @@ define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
 ; CHECK-32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK-32-NEXT:    movzwl (%edx), %eax
 ; CHECK-32-NEXT:    jmp .LBB5_1
-; CHECK-32-NEXT:    .p2align 4, 0x90
+; CHECK-32-NEXT:    .p2align 4
 ; CHECK-32-NEXT:  .LBB5_3: # %atomicrmw.start
 ; CHECK-32-NEXT:    # in Loop: Header=BB5_1 Depth=1
 ; CHECK-32-NEXT:    lock cmpxchgw %si, (%edx)
@@ -295,7 +295,7 @@ define i16 @atomicrmw_usub_sat_i16(ptr %ptr, i16 %val) {
 ; CHECK-64:       # %bb.0:
 ; CHECK-64-NEXT:    movzwl (%rdi), %eax
 ; CHECK-64-NEXT:    xorl %ecx, %ecx
-; CHECK-64-NEXT:    .p2align 4, 0x90
+; CHECK-64-NEXT:    .p2align 4
 ; CHECK-64-NEXT:  .LBB5_1: # %atomicrmw.start
 ; CHECK-64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-64-NEXT:    movl %eax, %edx
@@ -322,7 +322,7 @@ define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
 ; CHECK-32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK-32-NEXT:    movl (%edx), %eax
 ; CHECK-32-NEXT:    jmp .LBB6_1
-; CHECK-32-NEXT:    .p2align 4, 0x90
+; CHECK-32-NEXT:    .p2align 4
 ; CHECK-32-NEXT:  .LBB6_3: # %atomicrmw.start
 ; CHECK-32-NEXT:    # in Loop: Header=BB6_1 Depth=1
 ; CHECK-32-NEXT:    lock cmpxchgl %esi, (%edx)
@@ -348,7 +348,7 @@ define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
 ; CHECK-64:       # %bb.0:
 ; CHECK-64-NEXT:    movl (%rdi), %eax
 ; CHECK-64-NEXT:    xorl %ecx, %ecx
-; CHECK-64-NEXT:    .p2align 4, 0x90
+; CHECK-64-NEXT:    .p2align 4
 ; CHECK-64-NEXT:  .LBB6_1: # %atomicrmw.start
 ; CHECK-64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-64-NEXT:    movl %eax, %edx
@@ -381,7 +381,7 @@ define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
 ; CHECK-32-NEXT:    movl (%ebp), %esi
 ; CHECK-32-NEXT:    movl 4(%ebp), %edi
 ; CHECK-32-NEXT:    jmp .LBB7_1
-; CHECK-32-NEXT:    .p2align 4, 0x90
+; CHECK-32-NEXT:    .p2align 4
 ; CHECK-32-NEXT:  .LBB7_3: # %atomicrmw.start
 ; CHECK-32-NEXT:    # in Loop: Header=BB7_1 Depth=1
 ; CHECK-32-NEXT:    movl %esi, %eax
@@ -421,7 +421,7 @@ define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
 ; CHECK-64:       # %bb.0:
 ; CHECK-64-NEXT:    movq (%rdi), %rax
 ; CHECK-64-NEXT:    xorl %ecx, %ecx
-; CHECK-64-NEXT:    .p2align 4, 0x90
+; CHECK-64-NEXT:    .p2align 4
 ; CHECK-64-NEXT:  .LBB7_1: # %atomicrmw.start
 ; CHECK-64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-64-NEXT:    movq %rax, %rdx
diff --git a/llvm/test/CodeGen/X86/atomicrmw-fadd-fp-vector.ll b/llvm/test/CodeGen/X86/atomicrmw-fadd-fp-vector.ll
index 4f8cd5a52ed4c..105ee7f82ee79 100644
--- a/llvm/test/CodeGen/X86/atomicrmw-fadd-fp-vector.ll
+++ b/llvm/test/CodeGen/X86/atomicrmw-fadd-fp-vector.ll
@@ -13,7 +13,7 @@ define <2 x half> @test_atomicrmw_fadd_v2f16_align4(ptr addrspace(1) %ptr, <2 x
 ; CHECK-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    pinsrw $0, 2(%rdi), %xmm1
 ; CHECK-NEXT:    pinsrw $0, (%rdi), %xmm0
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -65,7 +65,7 @@ define <2 x float> @test_atomicrmw_fadd_v2f32_align8(ptr addrspace(1) %ptr, <2 x
 ; CHECK-LABEL: test_atomicrmw_fadd_v2f32_align8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %xmm1, %rax
diff --git a/llvm/test/CodeGen/X86/atomicrmw-uinc-udec-wrap.ll b/llvm/test/CodeGen/X86/atomicrmw-uinc-udec-wrap.ll
index 3126917917fde..f6fc65e3db459 100644
--- a/llvm/test/CodeGen/X86/atomicrmw-uinc-udec-wrap.ll
+++ b/llvm/test/CodeGen/X86/atomicrmw-uinc-udec-wrap.ll
@@ -6,7 +6,7 @@ define i8 @atomicrmw_uinc_wrap_i8(ptr %ptr, i8 %val) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movzbl (%rdi), %eax
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    leal 1(%rax), %edx
@@ -29,7 +29,7 @@ define i16 @atomicrmw_uinc_wrap_i16(ptr %ptr, i16 %val) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movzwl (%rdi), %eax
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    leal 1(%rax), %edx
@@ -51,7 +51,7 @@ define i32 @atomicrmw_uinc_wrap_i32(ptr %ptr, i32 %val) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl (%rdi), %eax
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB2_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    leal 1(%rax), %edx
@@ -73,7 +73,7 @@ define i64 @atomicrmw_uinc_wrap_i64(ptr %ptr, i64 %val) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq (%rdi), %rax
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB3_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    leaq 1(%rax), %rdx
@@ -92,7 +92,7 @@ define i8 @atomicrmw_udec_wrap_i8(ptr %ptr, i8 %val) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movzbl (%rdi), %eax
 ; CHECK-NEXT:    movzbl %sil, %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB4_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %eax, %edx
@@ -114,7 +114,7 @@ define i16 @atomicrmw_udec_wrap_i16(ptr %ptr, i16 %val) {
 ; CHECK-LABEL: atomicrmw_udec_wrap_i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movzwl (%rdi), %eax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB5_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %eax, %ecx
@@ -135,7 +135,7 @@ define i32 @atomicrmw_udec_wrap_i32(ptr %ptr, i32 %val) {
 ; CHECK-LABEL: atomicrmw_udec_wrap_i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl (%rdi), %eax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB6_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    leal -1(%rax), %ecx
@@ -158,7 +158,7 @@ define i64 @atomicrmw_udec_wrap_i64(ptr %ptr, i64 %val) {
 ; CHECK-LABEL: atomicrmw_udec_wrap_i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB7_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    leaq -1(%rax), %rcx
diff --git a/llvm/test/CodeGen/X86/avx-cmp.ll b/llvm/test/CodeGen/X86/avx-cmp.ll
index 4ab9c545ed90d..d31107bfeb7bb 100644
--- a/llvm/test/CodeGen/X86/avx-cmp.ll
+++ b/llvm/test/CodeGen/X86/avx-cmp.ll
@@ -34,7 +34,7 @@ define void @render(double %a0) nounwind {
 ; CHECK-NEXT:    jne .LBB2_5
 ; CHECK-NEXT:  # %bb.1: # %for.cond5.preheader
 ; CHECK-NEXT:    movb $1, %bl
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB2_2: # %for.cond5
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    testb %bl, %bl
diff --git a/llvm/test/CodeGen/X86/avx-vbroadcast.ll b/llvm/test/CodeGen/X86/avx-vbroadcast.ll
index c69886df82bdf..0bfd8921e8b42 100644
--- a/llvm/test/CodeGen/X86/avx-vbroadcast.ll
+++ b/llvm/test/CodeGen/X86/avx-vbroadcast.ll
@@ -923,7 +923,7 @@ define void @fmul_by_v2f32_broadcast() nounwind {
 ; X86-NEXT:    vmovddup {{.*#+}} xmm0 = [3.1E+1,0.0E+0,3.1E+1,0.0E+0]
 ; X86-NEXT:    ## xmm0 = mem[0,0]
 ; X86-NEXT:    ## implicit-def: $xmm1
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  LBB42_1: ## =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
 ; X86-NEXT:    vmulps %xmm0, %xmm2, %xmm2
@@ -937,7 +937,7 @@ define void @fmul_by_v2f32_broadcast() nounwind {
 ; X64-NEXT:    vmovddup {{.*#+}} xmm0 = [3.1E+1,0.0E+0,3.1E+1,0.0E+0]
 ; X64-NEXT:    ## xmm0 = mem[0,0]
 ; X64-NEXT:    ## implicit-def: $xmm1
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  LBB42_1: ## =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
 ; X64-NEXT:    vmulps %xmm0, %xmm2, %xmm2
diff --git a/llvm/test/CodeGen/X86/avx-vzeroupper.ll b/llvm/test/CodeGen/X86/avx-vzeroupper.ll
index 2fa71386c8056..b84b0c2c5cc30 100644
--- a/llvm/test/CodeGen/X86/avx-vzeroupper.ll
+++ b/llvm/test/CodeGen/X86/avx-vzeroupper.ll
@@ -131,7 +131,7 @@ define <4 x float> @test03(<4 x float> %a, <4 x float> %b) nounwind {
 ; VZ-NEXT:    subq $16, %rsp
 ; VZ-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; VZ-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
-; VZ-NEXT:    .p2align 4, 0x90
+; VZ-NEXT:    .p2align 4
 ; VZ-NEXT:  .LBB3_1: # %while.cond
 ; VZ-NEXT:    # =>This Inner Loop Header: Depth=1
 ; VZ-NEXT:    callq foo
@@ -140,7 +140,7 @@ define <4 x float> @test03(<4 x float> %a, <4 x float> %b) nounwind {
 ; VZ-NEXT:  # %bb.2: # %for.body.preheader
 ; VZ-NEXT:    movl $4, %ebx
 ; VZ-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
-; VZ-NEXT:    .p2align 4, 0x90
+; VZ-NEXT:    .p2align 4
 ; VZ-NEXT:  .LBB3_3: # %for.body
 ; VZ-NEXT:    # =>This Inner Loop Header: Depth=1
 ; VZ-NEXT:    callq do_sse
@@ -160,7 +160,7 @@ define <4 x float> @test03(<4 x float> %a, <4 x float> %b) nounwind {
 ; DISABLE-VZ-NEXT:    subq $16, %rsp
 ; DISABLE-VZ-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; DISABLE-VZ-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
-; DISABLE-VZ-NEXT:    .p2align 4, 0x90
+; DISABLE-VZ-NEXT:    .p2align 4
 ; DISABLE-VZ-NEXT:  .LBB3_1: # %while.cond
 ; DISABLE-VZ-NEXT:    # =>This Inner Loop Header: Depth=1
 ; DISABLE-VZ-NEXT:    callq foo
@@ -169,7 +169,7 @@ define <4 x float> @test03(<4 x float> %a, <4 x float> %b) nounwind {
 ; DISABLE-VZ-NEXT:  # %bb.2: # %for.body.preheader
 ; DISABLE-VZ-NEXT:    movl $4, %ebx
 ; DISABLE-VZ-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
-; DISABLE-VZ-NEXT:    .p2align 4, 0x90
+; DISABLE-VZ-NEXT:    .p2align 4
 ; DISABLE-VZ-NEXT:  .LBB3_3: # %for.body
 ; DISABLE-VZ-NEXT:    # =>This Inner Loop Header: Depth=1
 ; DISABLE-VZ-NEXT:    callq do_sse
@@ -189,7 +189,7 @@ define <4 x float> @test03(<4 x float> %a, <4 x float> %b) nounwind {
 ; BDVER2-NEXT:    subq $16, %rsp
 ; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; BDVER2-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
-; BDVER2-NEXT:    .p2align 4, 0x90
+; BDVER2-NEXT:    .p2align 4
 ; BDVER2-NEXT:  .LBB3_1: # %while.cond
 ; BDVER2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; BDVER2-NEXT:    callq foo
@@ -198,7 +198,7 @@ define <4 x float> @test03(<4 x float> %a, <4 x float> %b) nounwind {
 ; BDVER2-NEXT:  # %bb.2: # %for.body.preheader
 ; BDVER2-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
 ; BDVER2-NEXT:    movl $4, %ebx
-; BDVER2-NEXT:    .p2align 4, 0x90
+; BDVER2-NEXT:    .p2align 4
 ; BDVER2-NEXT:  .LBB3_3: # %for.body
 ; BDVER2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; BDVER2-NEXT:    callq do_sse
@@ -218,7 +218,7 @@ define <4 x float> @test03(<4 x float> %a, <4 x float> %b) nounwind {
 ; BTVER2-NEXT:    subq $16, %rsp
 ; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; BTVER2-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
-; BTVER2-NEXT:    .p2align 4, 0x90
+; BTVER2-NEXT:    .p2align 4
 ; BTVER2-NEXT:  .LBB3_1: # %while.cond
 ; BTVER2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; BTVER2-NEXT:    callq foo
@@ -227,7 +227,7 @@ define <4 x float> @test03(<4 x float> %a, <4 x float> %b) nounwind {
 ; BTVER2-NEXT:  # %bb.2: # %for.body.preheader
 ; BTVER2-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
 ; BTVER2-NEXT:    movl $4, %ebx
-; BTVER2-NEXT:    .p2align 4, 0x90
+; BTVER2-NEXT:    .p2align 4
 ; BTVER2-NEXT:  .LBB3_3: # %for.body
 ; BTVER2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; BTVER2-NEXT:    callq do_sse
diff --git a/llvm/test/CodeGen/X86/avx2-vbroadcast.ll b/llvm/test/CodeGen/X86/avx2-vbroadcast.ll
index 9ac0503831eb7..c50af6968f5bb 100644
--- a/llvm/test/CodeGen/X86/avx2-vbroadcast.ll
+++ b/llvm/test/CodeGen/X86/avx2-vbroadcast.ll
@@ -683,7 +683,7 @@ define void @crash() nounwind alwaysinline {
 ; X86-NEXT:    je LBB33_1
 ; X86-NEXT:  ## %bb.2: ## %ret
 ; X86-NEXT:    retl
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  LBB33_1: ## %footer329VF
 ; X86-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    jmp LBB33_1
@@ -695,7 +695,7 @@ define void @crash() nounwind alwaysinline {
 ; X64-NEXT:    je LBB33_1
 ; X64-NEXT:  ## %bb.2: ## %ret
 ; X64-NEXT:    retq
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  LBB33_1: ## %footer329VF
 ; X64-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    jmp LBB33_1
diff --git a/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll b/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll
index ac65a1112be43..d7ecbb41e3b14 100644
--- a/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll
+++ b/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll
@@ -9,7 +9,7 @@ define void @bcast_unfold_add_v16i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpaddd 4096(%rdi,%rax), %zmm0, %zmm1
@@ -41,7 +41,7 @@ define void @bcast_unfold_add_v8i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [2,2,2,2,2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpaddd 4096(%rdi,%rax), %ymm0, %ymm1
@@ -73,7 +73,7 @@ define void @bcast_unfold_add_v4i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB2_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpaddd 4096(%rdi,%rax), %xmm0, %xmm1
@@ -104,7 +104,7 @@ define void @bcast_unfold_add_v8i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB3_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpaddq 8192(%rdi,%rax), %zmm0, %zmm1
@@ -136,7 +136,7 @@ define void @bcast_unfold_add_v4i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB4_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpaddq 8192(%rdi,%rax), %ymm0, %ymm1
@@ -168,7 +168,7 @@ define void @bcast_unfold_add_v2i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} xmm0 = [2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB5_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpaddq 8192(%rdi,%rax), %xmm0, %xmm1
@@ -198,7 +198,7 @@ define void @bcast_unfold_mul_v16i32(ptr %arg) {
 ; CHECK-LABEL: bcast_unfold_mul_v16i32:
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB6_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu64 4096(%rdi,%rax), %zmm0
@@ -231,7 +231,7 @@ define void @bcast_unfold_mul_v8i32(ptr %arg) {
 ; CHECK-LABEL: bcast_unfold_mul_v8i32:
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB7_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu 4096(%rdi,%rax), %ymm0
@@ -264,7 +264,7 @@ define void @bcast_unfold_mul_v4i32(ptr %arg) {
 ; CHECK-LABEL: bcast_unfold_mul_v4i32:
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB8_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu 4096(%rdi,%rax), %xmm0
@@ -296,7 +296,7 @@ define void @bcast_unfold_mul_v8i64(ptr %arg) {
 ; CHECK-LABEL: bcast_unfold_mul_v8i64:
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB9_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu64 8192(%rdi,%rax), %zmm0
@@ -329,7 +329,7 @@ define void @bcast_unfold_mul_v4i64(ptr %arg) {
 ; CHECK-LABEL: bcast_unfold_mul_v4i64:
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB10_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu 8192(%rdi,%rax), %ymm0
@@ -362,7 +362,7 @@ define void @bcast_unfold_mul_v2i64(ptr %arg) {
 ; CHECK-LABEL: bcast_unfold_mul_v2i64:
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB11_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu 8192(%rdi,%rax), %xmm0
@@ -395,7 +395,7 @@ define void @bcast_unfold_or_v16i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB12_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpord 4096(%rdi,%rax), %zmm0, %zmm1
@@ -427,7 +427,7 @@ define void @bcast_unfold_or_v8i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [3,3,3,3,3,3,3,3]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB13_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vorps 4096(%rdi,%rax), %ymm0, %ymm1
@@ -459,7 +459,7 @@ define void @bcast_unfold_or_v4i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [3,3,3,3]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB14_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vorps 4096(%rdi,%rax), %xmm0, %xmm1
@@ -490,7 +490,7 @@ define void @bcast_unfold_or_v8i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [3,3,3,3,3,3,3,3]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB15_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vporq 8192(%rdi,%rax), %zmm0, %zmm1
@@ -522,7 +522,7 @@ define void @bcast_unfold_or_v4i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [3,3,3,3]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB16_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vorps 8192(%rdi,%rax), %ymm0, %ymm1
@@ -555,7 +555,7 @@ define void @bcast_unfold_or_v2i64(ptr %arg) {
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = [3,3]
 ; CHECK-NEXT:    # xmm0 = mem[0,0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB17_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vorps 8192(%rdi,%rax), %xmm0, %xmm1
@@ -586,7 +586,7 @@ define void @bcast_unfold_fneg_v16f32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB18_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpxord 4096(%rdi,%rax), %zmm0, %zmm1
@@ -618,7 +618,7 @@ define void @bcast_unfold_fneg_v8f32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB19_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vxorps 4096(%rdi,%rax), %ymm0, %ymm1
@@ -650,7 +650,7 @@ define void @bcast_unfold_fneg_v4f32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB20_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vxorps 4096(%rdi,%rax), %xmm0, %xmm1
@@ -681,7 +681,7 @@ define void @bcast_unfold_fneg_v8f64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB21_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpxorq 8192(%rdi,%rax), %zmm0, %zmm1
@@ -713,7 +713,7 @@ define void @bcast_unfold_fneg_v4f64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB22_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vxorps 8192(%rdi,%rax), %ymm0, %ymm1
@@ -746,7 +746,7 @@ define void @bcast_unfold_fneg_v2f64(ptr %arg) {
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = [-0.0E+0,-0.0E+0]
 ; CHECK-NEXT:    # xmm0 = mem[0,0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB23_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vxorps 8192(%rdi,%rax), %xmm0, %xmm1
@@ -777,7 +777,7 @@ define void @bcast_unfold_fabs_v16f32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB24_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpandd 4096(%rdi,%rax), %zmm0, %zmm1
@@ -812,7 +812,7 @@ define void @bcast_unfold_fabs_v8f32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB25_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vandps 4096(%rdi,%rax), %ymm0, %ymm1
@@ -847,7 +847,7 @@ define void @bcast_unfold_fabs_v4f32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB26_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vandps 4096(%rdi,%rax), %xmm0, %xmm1
@@ -881,7 +881,7 @@ define void @bcast_unfold_fabs_v8f64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB27_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpandq 8192(%rdi,%rax), %zmm0, %zmm1
@@ -916,7 +916,7 @@ define void @bcast_unfold_fabs_v4f64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [NaN,NaN,NaN,NaN]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB28_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vandps 8192(%rdi,%rax), %ymm0, %ymm1
@@ -952,7 +952,7 @@ define void @bcast_unfold_fabs_v2f64(ptr %arg) {
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = [NaN,NaN]
 ; CHECK-NEXT:    # xmm0 = mem[0,0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB29_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vandps 8192(%rdi,%rax), %xmm0, %xmm1
@@ -986,7 +986,7 @@ define void @bcast_unfold_fadd_v16f32(ptr nocapture %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB30_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vaddps 4096(%rdi,%rax), %zmm0, %zmm1
@@ -1018,7 +1018,7 @@ define void @bcast_unfold_fadd_v8f32(ptr nocapture %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB31_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vaddps 4096(%rdi,%rax), %ymm0, %ymm1
@@ -1050,7 +1050,7 @@ define void @bcast_unfold_fadd_v4f32(ptr nocapture %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB32_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vaddps 4096(%rdi,%rax), %xmm0, %xmm1
@@ -1081,7 +1081,7 @@ define void @bcast_unfold_fadd_v8f64(ptr nocapture %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB33_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vaddpd 8192(%rdi,%rax), %zmm0, %zmm1
@@ -1113,7 +1113,7 @@ define void @bcast_unfold_fadd_v4f64(ptr nocapture %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB34_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vaddpd 8192(%rdi,%rax), %ymm0, %ymm1
@@ -1146,7 +1146,7 @@ define void @bcast_unfold_fadd_v2f64(ptr nocapture %arg) {
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = [2.0E+0,2.0E+0]
 ; CHECK-NEXT:    # xmm0 = mem[0,0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB35_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vaddpd 8192(%rdi,%rax), %xmm0, %xmm1
@@ -1177,7 +1177,7 @@ define void @bcast_unfold_fmul_v16f32(ptr nocapture %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB36_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmulps 4096(%rdi,%rax), %zmm0, %zmm1
@@ -1209,7 +1209,7 @@ define void @bcast_unfold_fmul_v8f32(ptr nocapture %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB37_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmulps 4096(%rdi,%rax), %ymm0, %ymm1
@@ -1241,7 +1241,7 @@ define void @bcast_unfold_fmul_v4f32(ptr nocapture %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB38_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmulps 4096(%rdi,%rax), %xmm0, %xmm1
@@ -1272,7 +1272,7 @@ define void @bcast_unfold_fmul_v8f64(ptr nocapture %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB39_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmulpd 8192(%rdi,%rax), %zmm0, %zmm1
@@ -1304,7 +1304,7 @@ define void @bcast_unfold_fmul_v4f64(ptr nocapture %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB40_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmulpd 8192(%rdi,%rax), %ymm0, %ymm1
@@ -1337,7 +1337,7 @@ define void @bcast_unfold_fmul_v2f64(ptr nocapture %arg) {
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = [3.0E+0,3.0E+0]
 ; CHECK-NEXT:    # xmm0 = mem[0,0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB41_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmulpd 8192(%rdi,%rax), %xmm0, %xmm1
@@ -1368,7 +1368,7 @@ define void @bcast_unfold_fdiv_v16f32(ptr nocapture %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB42_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %zmm1
@@ -1401,7 +1401,7 @@ define void @bcast_unfold_fdiv_v8f32(ptr nocapture %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB43_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %ymm1
@@ -1434,7 +1434,7 @@ define void @bcast_unfold_fdiv_v4f32(ptr nocapture %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB44_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %xmm1
@@ -1466,7 +1466,7 @@ define void @bcast_unfold_fdiv_v8f64(ptr nocapture %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB45_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %zmm1
@@ -1499,7 +1499,7 @@ define void @bcast_unfold_fdiv_v4f64(ptr nocapture %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB46_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %ymm1
@@ -1533,7 +1533,7 @@ define void @bcast_unfold_fdiv_v2f64(ptr nocapture %arg) {
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = [3.0E+0,3.0E+0]
 ; CHECK-NEXT:    # xmm0 = mem[0,0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB47_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %xmm1
@@ -1565,7 +1565,7 @@ define void @bcast_unfold_fma213_v4f32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB48_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %xmm1
@@ -1598,7 +1598,7 @@ define void @bcast_unfold_fma231_v4f32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB49_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %xmm1
@@ -1631,7 +1631,7 @@ define void @bcast_unfold_fma213_v8f32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB50_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %ymm1
@@ -1665,7 +1665,7 @@ define void @bcast_unfold_fma231_v8f32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB51_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %ymm1
@@ -1699,7 +1699,7 @@ define void @bcast_unfold_fma213_v16f32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB52_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %zmm1
@@ -1733,7 +1733,7 @@ define void @bcast_unfold_fma231_v16f32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB53_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %zmm1
@@ -1768,7 +1768,7 @@ define void @bcast_unfold_fma213_v2f64(ptr %arg) {
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = [2.0E+0,2.0E+0]
 ; CHECK-NEXT:    # xmm0 = mem[0,0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB54_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %xmm1
@@ -1802,7 +1802,7 @@ define void @bcast_unfold_fma231_v2f64(ptr %arg) {
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = [2.0E+0,2.0E+0]
 ; CHECK-NEXT:    # xmm0 = mem[0,0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB55_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %xmm1
@@ -1835,7 +1835,7 @@ define void @bcast_unfold_fma213_v4f64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB56_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %ymm1
@@ -1869,7 +1869,7 @@ define void @bcast_unfold_fma231_v4f64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB57_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %ymm1
@@ -1903,7 +1903,7 @@ define void @bcast_unfold_fma213_v8f64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB58_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %zmm1
@@ -1937,7 +1937,7 @@ define void @bcast_unfold_fma231_v8f64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB59_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %zmm1
@@ -1971,7 +1971,7 @@ define void @bcast_unfold_fmax_v4f32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB60_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %xmm1
@@ -2004,7 +2004,7 @@ define void @bcast_unfold_fmax_v8f32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB61_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %ymm1
@@ -2038,7 +2038,7 @@ define void @bcast_unfold_fmax_v16f32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB62_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %zmm1
@@ -2073,7 +2073,7 @@ define void @bcast_unfold_fmax_v2f64(ptr %arg) {
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = [2.0E+0,2.0E+0]
 ; CHECK-NEXT:    # xmm0 = mem[0,0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB63_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %xmm1
@@ -2106,7 +2106,7 @@ define void @bcast_unfold_fmax_v4f64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB64_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %ymm1
@@ -2140,7 +2140,7 @@ define void @bcast_unfold_fmax_v8f64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB65_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %zmm1
@@ -2174,7 +2174,7 @@ define void @bcast_unfold_fmin_v4f32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB66_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %xmm1
@@ -2207,7 +2207,7 @@ define void @bcast_unfold_fmin_v8f32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB67_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %ymm1
@@ -2241,7 +2241,7 @@ define void @bcast_unfold_fmin_v16f32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB68_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %zmm1
@@ -2276,7 +2276,7 @@ define void @bcast_unfold_fmin_v2f64(ptr %arg) {
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = [2.0E+0,2.0E+0]
 ; CHECK-NEXT:    # xmm0 = mem[0,0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB69_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %xmm1
@@ -2309,7 +2309,7 @@ define void @bcast_unfold_fmin_v4f64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB70_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %ymm1
@@ -2343,7 +2343,7 @@ define void @bcast_unfold_fmin_v8f64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB71_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %zmm1
@@ -2377,7 +2377,7 @@ define void @bcast_unfold_smin_v4i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB72_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpminsd 4096(%rdi,%rax), %xmm0, %xmm1
@@ -2409,7 +2409,7 @@ define void @bcast_unfold_smin_v8i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [2,2,2,2,2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB73_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpminsd 4096(%rdi,%rax), %ymm0, %ymm1
@@ -2442,7 +2442,7 @@ define void @bcast_unfold_smin_v16i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB74_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpminsd 4096(%rdi,%rax), %zmm0, %zmm1
@@ -2475,7 +2475,7 @@ define void @bcast_unfold_smin_v2i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} xmm0 = [2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB75_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpminsq 8192(%rdi,%rax), %xmm0, %xmm1
@@ -2507,7 +2507,7 @@ define void @bcast_unfold_smin_v4i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB76_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpminsq 8192(%rdi,%rax), %ymm0, %ymm1
@@ -2540,7 +2540,7 @@ define void @bcast_unfold_smin_v8i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB77_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpminsq 8192(%rdi,%rax), %zmm0, %zmm1
@@ -2573,7 +2573,7 @@ define void @bcast_unfold_smax_v4i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB78_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpmaxsd 4096(%rdi,%rax), %xmm0, %xmm1
@@ -2605,7 +2605,7 @@ define void @bcast_unfold_smax_v8i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [2,2,2,2,2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB79_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpmaxsd 4096(%rdi,%rax), %ymm0, %ymm1
@@ -2638,7 +2638,7 @@ define void @bcast_unfold_smax_v16i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB80_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpmaxsd 4096(%rdi,%rax), %zmm0, %zmm1
@@ -2671,7 +2671,7 @@ define void @bcast_unfold_smax_v2i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} xmm0 = [2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB81_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpmaxsq 8192(%rdi,%rax), %xmm0, %xmm1
@@ -2703,7 +2703,7 @@ define void @bcast_unfold_smax_v4i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB82_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpmaxsq 8192(%rdi,%rax), %ymm0, %ymm1
@@ -2736,7 +2736,7 @@ define void @bcast_unfold_smax_v8i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB83_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpmaxsq 8192(%rdi,%rax), %zmm0, %zmm1
@@ -2769,7 +2769,7 @@ define void @bcast_unfold_umin_v4i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB84_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpminud 4096(%rdi,%rax), %xmm0, %xmm1
@@ -2801,7 +2801,7 @@ define void @bcast_unfold_umin_v8i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [2,2,2,2,2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB85_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpminud 4096(%rdi,%rax), %ymm0, %ymm1
@@ -2834,7 +2834,7 @@ define void @bcast_unfold_umin_v16i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB86_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpminud 4096(%rdi,%rax), %zmm0, %zmm1
@@ -2867,7 +2867,7 @@ define void @bcast_unfold_umin_v2i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} xmm0 = [2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB87_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpminuq 8192(%rdi,%rax), %xmm0, %xmm1
@@ -2899,7 +2899,7 @@ define void @bcast_unfold_umin_v4i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB88_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpminuq 8192(%rdi,%rax), %ymm0, %ymm1
@@ -2932,7 +2932,7 @@ define void @bcast_unfold_umin_v8i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB89_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpminuq 8192(%rdi,%rax), %zmm0, %zmm1
@@ -2965,7 +2965,7 @@ define void @bcast_unfold_umax_v4i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB90_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpmaxud 4096(%rdi,%rax), %xmm0, %xmm1
@@ -2997,7 +2997,7 @@ define void @bcast_unfold_umax_v8i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [2,2,2,2,2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB91_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpmaxud 4096(%rdi,%rax), %ymm0, %ymm1
@@ -3030,7 +3030,7 @@ define void @bcast_unfold_umax_v16i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB92_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpmaxud 4096(%rdi,%rax), %zmm0, %zmm1
@@ -3063,7 +3063,7 @@ define void @bcast_unfold_umax_v2i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} xmm0 = [2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB93_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpmaxuq 8192(%rdi,%rax), %xmm0, %xmm1
@@ -3095,7 +3095,7 @@ define void @bcast_unfold_umax_v4i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB94_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpmaxuq 8192(%rdi,%rax), %ymm0, %ymm1
@@ -3128,7 +3128,7 @@ define void @bcast_unfold_umax_v8i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB95_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpmaxuq 8192(%rdi,%rax), %zmm0, %zmm1
@@ -3161,7 +3161,7 @@ define void @bcast_unfold_pcmpgt_v4i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [1,1,1,1]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB96_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu 4096(%rdi,%rax), %xmm1
@@ -3195,7 +3195,7 @@ define void @bcast_unfold_pcmpgt_v8i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [1,1,1,1,1,1,1,1]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB97_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu 4096(%rdi,%rax), %ymm1
@@ -3230,7 +3230,7 @@ define void @bcast_unfold_pcmpgt_v16i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB98_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu64 4096(%rdi,%rax), %zmm1
@@ -3265,7 +3265,7 @@ define void @bcast_unfold_pcmpgt_v2i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} xmm0 = [1,1]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB99_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu 8192(%rdi,%rax), %xmm1
@@ -3299,7 +3299,7 @@ define void @bcast_unfold_pcmpgt_v4i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [1,1,1,1]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB100_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu 8192(%rdi,%rax), %ymm1
@@ -3334,7 +3334,7 @@ define void @bcast_unfold_pcmpgt_v8i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB101_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu64 8192(%rdi,%rax), %zmm1
@@ -3369,7 +3369,7 @@ define void @bcast_unfold_pcmpeq_v4i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [1,1,1,1]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB102_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu 4096(%rdi,%rax), %xmm1
@@ -3403,7 +3403,7 @@ define void @bcast_unfold_pcmpeq_v8i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [1,1,1,1,1,1,1,1]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB103_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu 4096(%rdi,%rax), %ymm1
@@ -3438,7 +3438,7 @@ define void @bcast_unfold_pcmpeq_v16i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB104_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu64 4096(%rdi,%rax), %zmm1
@@ -3473,7 +3473,7 @@ define void @bcast_unfold_pcmpeq_v2i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} xmm0 = [1,1]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB105_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu 8192(%rdi,%rax), %xmm1
@@ -3507,7 +3507,7 @@ define void @bcast_unfold_pcmpeq_v4i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [1,1,1,1]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB106_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu 8192(%rdi,%rax), %ymm1
@@ -3542,7 +3542,7 @@ define void @bcast_unfold_pcmpeq_v8i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB107_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu64 8192(%rdi,%rax), %zmm1
@@ -3577,7 +3577,7 @@ define void @bcast_unfold_pcmp_v4i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [1,1,1,1]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB108_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu (%rdi,%rax,4), %xmm1
@@ -3612,7 +3612,7 @@ define void @bcast_unfold_pcmp_v8i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [1,1,1,1,1,1,1,1]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB109_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu (%rdi,%rax,4), %ymm1
@@ -3648,7 +3648,7 @@ define void @bcast_unfold_pcmp_v16i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB110_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu64 (%rdi,%rax,4), %zmm1
@@ -3684,7 +3684,7 @@ define void @bcast_unfold_pcmp_v2i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} xmm0 = [1,1]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB111_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu (%rdi,%rax,8), %xmm1
@@ -3719,7 +3719,7 @@ define void @bcast_unfold_pcmp_v4i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [1,1,1,1]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB112_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu (%rdi,%rax,8), %ymm1
@@ -3755,7 +3755,7 @@ define void @bcast_unfold_pcmp_v8i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB113_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu64 (%rdi,%rax,8), %zmm1
@@ -3791,7 +3791,7 @@ define void @bcast_unfold_pcmpu_v4i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB114_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu (%rdi,%rax,4), %xmm1
@@ -3826,7 +3826,7 @@ define void @bcast_unfold_pcmpu_v8i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [2,2,2,2,2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB115_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu (%rdi,%rax,4), %ymm1
@@ -3862,7 +3862,7 @@ define void @bcast_unfold_pcmpu_v16i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB116_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu64 (%rdi,%rax,4), %zmm1
@@ -3898,7 +3898,7 @@ define void @bcast_unfold_pcmpu_v2i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} xmm0 = [2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB117_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu (%rdi,%rax,8), %xmm1
@@ -3933,7 +3933,7 @@ define void @bcast_unfold_pcmpu_v4i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB118_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu (%rdi,%rax,8), %ymm1
@@ -3969,7 +3969,7 @@ define void @bcast_unfold_pcmpu_v8i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB119_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu64 (%rdi,%rax,8), %zmm1
@@ -4006,7 +4006,7 @@ define void @bcast_unfold_cmp_v4f32(ptr %arg) {
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB120_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %xmm2
@@ -4041,7 +4041,7 @@ define void @bcast_unfold_cmp_v8f32(ptr %arg) {
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB121_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %ymm2
@@ -4077,7 +4077,7 @@ define void @bcast_unfold_cmp_v16f32(ptr %arg) {
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} zmm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB122_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovups 4096(%rdi,%rax), %zmm2
@@ -4115,7 +4115,7 @@ define void @bcast_unfold_cmp_v2f64(ptr %arg) {
 ; CHECK-NEXT:    # xmm0 = mem[0,0]
 ; CHECK-NEXT:    vmovddup {{.*#+}} xmm1 = [3.0E+0,3.0E+0]
 ; CHECK-NEXT:    # xmm1 = mem[0,0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB123_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %xmm2
@@ -4150,7 +4150,7 @@ define void @bcast_unfold_cmp_v4f64(ptr %arg) {
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB124_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %ymm2
@@ -4186,7 +4186,7 @@ define void @bcast_unfold_cmp_v8f64(ptr %arg) {
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
 ; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB125_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovupd 8192(%rdi,%rax), %zmm2
@@ -4222,7 +4222,7 @@ define void @bcast_unfold_cmp_v8f32_refold(ptr nocapture %0) {
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
 ; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB126_1: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vcmpgtps 4096(%rdi,%rax), %ymm0, %k1
 ; CHECK-NEXT:    vblendmps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm2 {%k1}
@@ -4254,7 +4254,7 @@ define void @bcast_unfold_ptestm_v4i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB127_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu 4096(%rdi,%rax), %xmm1
@@ -4289,7 +4289,7 @@ define void @bcast_unfold_ptestnm_v4i32(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB128_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu 4096(%rdi,%rax), %xmm1
@@ -4324,7 +4324,7 @@ define void @bcast_unfold_ptestm_v4i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB129_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu 8192(%rdi,%rax), %ymm1
@@ -4360,7 +4360,7 @@ define void @bcast_unfold_ptestnm_v4i64(ptr %arg) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
 ; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB130_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu 8192(%rdi,%rax), %ymm1
@@ -4396,7 +4396,7 @@ define void @bcast_unfold_vpternlog_v16i32(ptr %arg, ptr %arg1) {
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; CHECK-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB131_1: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqu64 4096(%rdi,%rax), %zmm1
diff --git a/llvm/test/CodeGen/X86/avx512-gfni-intrinsics.ll b/llvm/test/CodeGen/X86/avx512-gfni-intrinsics.ll
index 432d27ac04eda..3b7803427ae5e 100644
--- a/llvm/test/CodeGen/X86/avx512-gfni-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512-gfni-intrinsics.ll
@@ -37,7 +37,7 @@ define { <16 x i8>, <16 x i8>, <16 x i8> } @test_vgf2p8affineinvqb_128(<16 x i8>
 ; X86NOBW-NEXT:    vpmovdb %zmm1, %xmm5 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xcd]
 ; X86NOBW-NEXT:    vpand %xmm4, %xmm5, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xd1,0xdb,0xcc]
 ; X86NOBW-NEXT:    vpternlogq $184, %xmm0, %xmm5, %xmm2 # encoding: [0x62,0xf3,0xd5,0x08,0x25,0xd0,0xb8]
-; X86NOBW-NEXT:    # xmm2 = (~xmm2 & xmm5 & xmm0) | (xmm2 & ~xmm5 & ~xmm0) | (xmm2 & ~xmm5 & xmm0) | (xmm2 & xmm5 & xmm0)
+; X86NOBW-NEXT:    # xmm2 = xmm2 ^ (xmm5 & (xmm2 ^ xmm0))
 ; X86NOBW-NEXT:    vmovdqa %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc3]
 ; X86NOBW-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86NOBW-NEXT:    retl # encoding: [0xc3]
@@ -53,7 +53,7 @@ define { <16 x i8>, <16 x i8>, <16 x i8> } @test_vgf2p8affineinvqb_128(<16 x i8>
 ; X64NOBW-NEXT:    vpmovdb %zmm1, %xmm5 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xcd]
 ; X64NOBW-NEXT:    vpand %xmm4, %xmm5, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xd1,0xdb,0xcc]
 ; X64NOBW-NEXT:    vpternlogq $184, %xmm0, %xmm5, %xmm2 # encoding: [0x62,0xf3,0xd5,0x08,0x25,0xd0,0xb8]
-; X64NOBW-NEXT:    # xmm2 = (~xmm2 & xmm5 & xmm0) | (xmm2 & ~xmm5 & ~xmm0) | (xmm2 & ~xmm5 & xmm0) | (xmm2 & xmm5 & xmm0)
+; X64NOBW-NEXT:    # xmm2 = xmm2 ^ (xmm5 & (xmm2 ^ xmm0))
 ; X64NOBW-NEXT:    vmovdqa %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc3]
 ; X64NOBW-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64NOBW-NEXT:    retq # encoding: [0xc3]
@@ -107,7 +107,7 @@ define { <32 x i8>, <32 x i8>, <32 x i8> } @test_vgf2p8affineinvqb_256(<32 x i8>
 ; X86NOBW-NEXT:    vinserti128 $1, %xmm5, %ymm1, %ymm5 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x75,0x38,0xed,0x01]
 ; X86NOBW-NEXT:    vpand %ymm4, %ymm5, %ymm1 # EVEX TO VEX Compression encoding: [0xc5,0xd5,0xdb,0xcc]
 ; X86NOBW-NEXT:    vpternlogq $184, %ymm0, %ymm5, %ymm2 # encoding: [0x62,0xf3,0xd5,0x28,0x25,0xd0,0xb8]
-; X86NOBW-NEXT:    # ymm2 = (~ymm2 & ymm5 & ymm0) | (ymm2 & ~ymm5 & ~ymm0) | (ymm2 & ~ymm5 & ymm0) | (ymm2 & ymm5 & ymm0)
+; X86NOBW-NEXT:    # ymm2 = ymm2 ^ (ymm5 & (ymm2 ^ ymm0))
 ; X86NOBW-NEXT:    vmovdqa %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc3]
 ; X86NOBW-NEXT:    retl # encoding: [0xc3]
 ;
@@ -128,7 +128,7 @@ define { <32 x i8>, <32 x i8>, <32 x i8> } @test_vgf2p8affineinvqb_256(<32 x i8>
 ; X64NOBW-NEXT:    vinserti128 $1, %xmm5, %ymm1, %ymm5 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x75,0x38,0xed,0x01]
 ; X64NOBW-NEXT:    vpand %ymm4, %ymm5, %ymm1 # EVEX TO VEX Compression encoding: [0xc5,0xd5,0xdb,0xcc]
 ; X64NOBW-NEXT:    vpternlogq $184, %ymm0, %ymm5, %ymm2 # encoding: [0x62,0xf3,0xd5,0x28,0x25,0xd0,0xb8]
-; X64NOBW-NEXT:    # ymm2 = (~ymm2 & ymm5 & ymm0) | (ymm2 & ~ymm5 & ~ymm0) | (ymm2 & ~ymm5 & ymm0) | (ymm2 & ymm5 & ymm0)
+; X64NOBW-NEXT:    # ymm2 = ymm2 ^ (ymm5 & (ymm2 ^ ymm0))
 ; X64NOBW-NEXT:    vmovdqa %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc3]
 ; X64NOBW-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i32 %mask to <32 x i1>
@@ -192,7 +192,7 @@ define { <64 x i8>, <64 x i8>, <64 x i8> } @test_vgf2p8affineinvqb_512(<64 x i8>
 ; X86NOBW-NEXT:    vinserti64x4 $1, %ymm1, %zmm5, %zmm5 # encoding: [0x62,0xf3,0xd5,0x48,0x3a,0xe9,0x01]
 ; X86NOBW-NEXT:    vpandq %zmm4, %zmm5, %zmm1 # encoding: [0x62,0xf1,0xd5,0x48,0xdb,0xcc]
 ; X86NOBW-NEXT:    vpternlogq $184, %zmm3, %zmm5, %zmm2 # encoding: [0x62,0xf3,0xd5,0x48,0x25,0xd3,0xb8]
-; X86NOBW-NEXT:    # zmm2 = (~zmm2 & zmm5 & zmm3) | (zmm2 & ~zmm5 & ~zmm3) | (zmm2 & ~zmm5 & zmm3) | (zmm2 & zmm5 & zmm3)
+; X86NOBW-NEXT:    # zmm2 = zmm2 ^ (zmm5 & (zmm2 ^ zmm3))
 ; X86NOBW-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64NOBW-LABEL: test_vgf2p8affineinvqb_512:
@@ -226,7 +226,7 @@ define { <64 x i8>, <64 x i8>, <64 x i8> } @test_vgf2p8affineinvqb_512(<64 x i8>
 ; X64NOBW-NEXT:    vinserti64x4 $1, %ymm1, %zmm5, %zmm5 # encoding: [0x62,0xf3,0xd5,0x48,0x3a,0xe9,0x01]
 ; X64NOBW-NEXT:    vpandq %zmm4, %zmm5, %zmm1 # encoding: [0x62,0xf1,0xd5,0x48,0xdb,0xcc]
 ; X64NOBW-NEXT:    vpternlogq $184, %zmm0, %zmm5, %zmm2 # encoding: [0x62,0xf3,0xd5,0x48,0x25,0xd0,0xb8]
-; X64NOBW-NEXT:    # zmm2 = (~zmm2 & zmm5 & zmm0) | (zmm2 & ~zmm5 & ~zmm0) | (zmm2 & ~zmm5 & zmm0) | (zmm2 & zmm5 & zmm0)
+; X64NOBW-NEXT:    # zmm2 = zmm2 ^ (zmm5 & (zmm2 ^ zmm0))
 ; X64NOBW-NEXT:    vmovdqa64 %zmm3, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc3]
 ; X64NOBW-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i64 %mask to <64 x i1>
@@ -274,7 +274,7 @@ define { <16 x i8>, <16 x i8>, <16 x i8> } @test_vgf2p8affineqb_128(<16 x i8> %s
 ; X86NOBW-NEXT:    vpmovdb %zmm1, %xmm5 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xcd]
 ; X86NOBW-NEXT:    vpand %xmm4, %xmm5, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xd1,0xdb,0xcc]
 ; X86NOBW-NEXT:    vpternlogq $184, %xmm0, %xmm5, %xmm2 # encoding: [0x62,0xf3,0xd5,0x08,0x25,0xd0,0xb8]
-; X86NOBW-NEXT:    # xmm2 = (~xmm2 & xmm5 & xmm0) | (xmm2 & ~xmm5 & ~xmm0) | (xmm2 & ~xmm5 & xmm0) | (xmm2 & xmm5 & xmm0)
+; X86NOBW-NEXT:    # xmm2 = xmm2 ^ (xmm5 & (xmm2 ^ xmm0))
 ; X86NOBW-NEXT:    vmovdqa %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc3]
 ; X86NOBW-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86NOBW-NEXT:    retl # encoding: [0xc3]
@@ -290,7 +290,7 @@ define { <16 x i8>, <16 x i8>, <16 x i8> } @test_vgf2p8affineqb_128(<16 x i8> %s
 ; X64NOBW-NEXT:    vpmovdb %zmm1, %xmm5 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xcd]
 ; X64NOBW-NEXT:    vpand %xmm4, %xmm5, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xd1,0xdb,0xcc]
 ; X64NOBW-NEXT:    vpternlogq $184, %xmm0, %xmm5, %xmm2 # encoding: [0x62,0xf3,0xd5,0x08,0x25,0xd0,0xb8]
-; X64NOBW-NEXT:    # xmm2 = (~xmm2 & xmm5 & xmm0) | (xmm2 & ~xmm5 & ~xmm0) | (xmm2 & ~xmm5 & xmm0) | (xmm2 & xmm5 & xmm0)
+; X64NOBW-NEXT:    # xmm2 = xmm2 ^ (xmm5 & (xmm2 ^ xmm0))
 ; X64NOBW-NEXT:    vmovdqa %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc3]
 ; X64NOBW-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64NOBW-NEXT:    retq # encoding: [0xc3]
@@ -344,7 +344,7 @@ define { <32 x i8>, <32 x i8>, <32 x i8> } @test_vgf2p8affineqb_256(<32 x i8> %s
 ; X86NOBW-NEXT:    vinserti128 $1, %xmm5, %ymm1, %ymm5 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x75,0x38,0xed,0x01]
 ; X86NOBW-NEXT:    vpand %ymm4, %ymm5, %ymm1 # EVEX TO VEX Compression encoding: [0xc5,0xd5,0xdb,0xcc]
 ; X86NOBW-NEXT:    vpternlogq $184, %ymm0, %ymm5, %ymm2 # encoding: [0x62,0xf3,0xd5,0x28,0x25,0xd0,0xb8]
-; X86NOBW-NEXT:    # ymm2 = (~ymm2 & ymm5 & ymm0) | (ymm2 & ~ymm5 & ~ymm0) | (ymm2 & ~ymm5 & ymm0) | (ymm2 & ymm5 & ymm0)
+; X86NOBW-NEXT:    # ymm2 = ymm2 ^ (ymm5 & (ymm2 ^ ymm0))
 ; X86NOBW-NEXT:    vmovdqa %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc3]
 ; X86NOBW-NEXT:    retl # encoding: [0xc3]
 ;
@@ -365,7 +365,7 @@ define { <32 x i8>, <32 x i8>, <32 x i8> } @test_vgf2p8affineqb_256(<32 x i8> %s
 ; X64NOBW-NEXT:    vinserti128 $1, %xmm5, %ymm1, %ymm5 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x75,0x38,0xed,0x01]
 ; X64NOBW-NEXT:    vpand %ymm4, %ymm5, %ymm1 # EVEX TO VEX Compression encoding: [0xc5,0xd5,0xdb,0xcc]
 ; X64NOBW-NEXT:    vpternlogq $184, %ymm0, %ymm5, %ymm2 # encoding: [0x62,0xf3,0xd5,0x28,0x25,0xd0,0xb8]
-; X64NOBW-NEXT:    # ymm2 = (~ymm2 & ymm5 & ymm0) | (ymm2 & ~ymm5 & ~ymm0) | (ymm2 & ~ymm5 & ymm0) | (ymm2 & ymm5 & ymm0)
+; X64NOBW-NEXT:    # ymm2 = ymm2 ^ (ymm5 & (ymm2 ^ ymm0))
 ; X64NOBW-NEXT:    vmovdqa %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc3]
 ; X64NOBW-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i32 %mask to <32 x i1>
@@ -429,7 +429,7 @@ define { <64 x i8>, <64 x i8>, <64 x i8> } @test_vgf2p8affineqb_512(<64 x i8> %s
 ; X86NOBW-NEXT:    vinserti64x4 $1, %ymm1, %zmm5, %zmm5 # encoding: [0x62,0xf3,0xd5,0x48,0x3a,0xe9,0x01]
 ; X86NOBW-NEXT:    vpandq %zmm4, %zmm5, %zmm1 # encoding: [0x62,0xf1,0xd5,0x48,0xdb,0xcc]
 ; X86NOBW-NEXT:    vpternlogq $184, %zmm3, %zmm5, %zmm2 # encoding: [0x62,0xf3,0xd5,0x48,0x25,0xd3,0xb8]
-; X86NOBW-NEXT:    # zmm2 = (~zmm2 & zmm5 & zmm3) | (zmm2 & ~zmm5 & ~zmm3) | (zmm2 & ~zmm5 & zmm3) | (zmm2 & zmm5 & zmm3)
+; X86NOBW-NEXT:    # zmm2 = zmm2 ^ (zmm5 & (zmm2 ^ zmm3))
 ; X86NOBW-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64NOBW-LABEL: test_vgf2p8affineqb_512:
@@ -463,7 +463,7 @@ define { <64 x i8>, <64 x i8>, <64 x i8> } @test_vgf2p8affineqb_512(<64 x i8> %s
 ; X64NOBW-NEXT:    vinserti64x4 $1, %ymm1, %zmm5, %zmm5 # encoding: [0x62,0xf3,0xd5,0x48,0x3a,0xe9,0x01]
 ; X64NOBW-NEXT:    vpandq %zmm4, %zmm5, %zmm1 # encoding: [0x62,0xf1,0xd5,0x48,0xdb,0xcc]
 ; X64NOBW-NEXT:    vpternlogq $184, %zmm0, %zmm5, %zmm2 # encoding: [0x62,0xf3,0xd5,0x48,0x25,0xd0,0xb8]
-; X64NOBW-NEXT:    # zmm2 = (~zmm2 & zmm5 & zmm0) | (zmm2 & ~zmm5 & ~zmm0) | (zmm2 & ~zmm5 & zmm0) | (zmm2 & zmm5 & zmm0)
+; X64NOBW-NEXT:    # zmm2 = zmm2 ^ (zmm5 & (zmm2 ^ zmm0))
 ; X64NOBW-NEXT:    vmovdqa64 %zmm3, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc3]
 ; X64NOBW-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i64 %mask to <64 x i1>
@@ -511,7 +511,7 @@ define <16 x i8> @test_vgf2p8mulb_128_mask(<16 x i8> %src1, <16 x i8> %src2, <16
 ; X86NOBW-NEXT:    # zmm0 {%k1} {z} = -1
 ; X86NOBW-NEXT:    vpmovdb %zmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xc0]
 ; X86NOBW-NEXT:    vpternlogq $202, %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf3,0xf5,0x08,0x25,0xc2,0xca]
-; X86NOBW-NEXT:    # xmm0 = (~xmm0 & ~xmm1 & xmm2) | (~xmm0 & xmm1 & xmm2) | (xmm0 & xmm1 & ~xmm2) | (xmm0 & xmm1 & xmm2)
+; X86NOBW-NEXT:    # xmm0 = xmm2 ^ (xmm0 & (xmm1 ^ xmm2))
 ; X86NOBW-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86NOBW-NEXT:    retl # encoding: [0xc3]
 ;
@@ -523,7 +523,7 @@ define <16 x i8> @test_vgf2p8mulb_128_mask(<16 x i8> %src1, <16 x i8> %src2, <16
 ; X64NOBW-NEXT:    # zmm0 {%k1} {z} = -1
 ; X64NOBW-NEXT:    vpmovdb %zmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xc0]
 ; X64NOBW-NEXT:    vpternlogq $202, %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf3,0xf5,0x08,0x25,0xc2,0xca]
-; X64NOBW-NEXT:    # xmm0 = (~xmm0 & ~xmm1 & xmm2) | (~xmm0 & xmm1 & xmm2) | (xmm0 & xmm1 & ~xmm2) | (xmm0 & xmm1 & xmm2)
+; X64NOBW-NEXT:    # xmm0 = xmm2 ^ (xmm0 & (xmm1 ^ xmm2))
 ; X64NOBW-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64NOBW-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i16 %mask to <16 x i1>
@@ -610,7 +610,7 @@ define <32 x i8> @test_vgf2p8mulb_256_mask(<32 x i8> %src1, <32 x i8> %src2, <32
 ; X86NOBW-NEXT:    vpmovdb %zmm3, %xmm3 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xdb]
 ; X86NOBW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x7d,0x38,0xc3,0x01]
 ; X86NOBW-NEXT:    vpternlogq $202, %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf3,0xf5,0x28,0x25,0xc2,0xca]
-; X86NOBW-NEXT:    # ymm0 = (~ymm0 & ~ymm1 & ymm2) | (~ymm0 & ymm1 & ymm2) | (ymm0 & ymm1 & ~ymm2) | (ymm0 & ymm1 & ymm2)
+; X86NOBW-NEXT:    # ymm0 = ymm2 ^ (ymm0 & (ymm1 ^ ymm2))
 ; X86NOBW-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64NOBW-LABEL: test_vgf2p8mulb_256_mask:
@@ -627,7 +627,7 @@ define <32 x i8> @test_vgf2p8mulb_256_mask(<32 x i8> %src1, <32 x i8> %src2, <32
 ; X64NOBW-NEXT:    vpmovdb %zmm3, %xmm3 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xdb]
 ; X64NOBW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x7d,0x38,0xc3,0x01]
 ; X64NOBW-NEXT:    vpternlogq $202, %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf3,0xf5,0x28,0x25,0xc2,0xca]
-; X64NOBW-NEXT:    # ymm0 = (~ymm0 & ~ymm1 & ymm2) | (~ymm0 & ymm1 & ymm2) | (ymm0 & ymm1 & ~ymm2) | (ymm0 & ymm1 & ymm2)
+; X64NOBW-NEXT:    # ymm0 = ymm2 ^ (ymm0 & (ymm1 ^ ymm2))
 ; X64NOBW-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i32 %mask to <32 x i1>
   %2 = call <32 x i8> @llvm.x86.vgf2p8mulb.256(<32 x i8> %src1, <32 x i8> %src2)
@@ -732,7 +732,7 @@ define <64 x i8> @test_vgf2p8mulb_512_mask(<64 x i8> %src1, <64 x i8> %src2, <64
 ; X86NOBW-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x65,0x38,0xdc,0x01]
 ; X86NOBW-NEXT:    vinserti64x4 $1, %ymm0, %zmm3, %zmm0 # encoding: [0x62,0xf3,0xe5,0x48,0x3a,0xc0,0x01]
 ; X86NOBW-NEXT:    vpternlogq $202, %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf3,0xf5,0x48,0x25,0xc2,0xca]
-; X86NOBW-NEXT:    # zmm0 = (~zmm0 & ~zmm1 & zmm2) | (~zmm0 & zmm1 & zmm2) | (zmm0 & zmm1 & ~zmm2) | (zmm0 & zmm1 & zmm2)
+; X86NOBW-NEXT:    # zmm0 = zmm2 ^ (zmm0 & (zmm1 ^ zmm2))
 ; X86NOBW-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64NOBW-LABEL: test_vgf2p8mulb_512_mask:
@@ -763,7 +763,7 @@ define <64 x i8> @test_vgf2p8mulb_512_mask(<64 x i8> %src1, <64 x i8> %src2, <64
 ; X64NOBW-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x65,0x38,0xdc,0x01]
 ; X64NOBW-NEXT:    vinserti64x4 $1, %ymm0, %zmm3, %zmm0 # encoding: [0x62,0xf3,0xe5,0x48,0x3a,0xc0,0x01]
 ; X64NOBW-NEXT:    vpternlogq $202, %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf3,0xf5,0x48,0x25,0xc2,0xca]
-; X64NOBW-NEXT:    # zmm0 = (~zmm0 & ~zmm1 & zmm2) | (~zmm0 & zmm1 & zmm2) | (zmm0 & zmm1 & ~zmm2) | (zmm0 & zmm1 & zmm2)
+; X64NOBW-NEXT:    # zmm0 = zmm2 ^ (zmm0 & (zmm1 ^ zmm2))
 ; X64NOBW-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i64 %mask to <64 x i1>
   %2 = call <64 x i8> @llvm.x86.vgf2p8mulb.512(<64 x i8> %src1, <64 x i8> %src2)
diff --git a/llvm/test/CodeGen/X86/avx512-i1test.ll b/llvm/test/CodeGen/X86/avx512-i1test.ll
index bf31b2da56b62..3cd733181599e 100644
--- a/llvm/test/CodeGen/X86/avx512-i1test.ll
+++ b/llvm/test/CodeGen/X86/avx512-i1test.ll
@@ -15,7 +15,7 @@ define void @func() {
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:  .LBB0_1: # %bb56
 ; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # %bb33
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    testb %al, %al
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
index 9b94a9cf45ddf..c1ef500d9d3de 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
@@ -7682,7 +7682,7 @@ define <16 x i32>@test_int_x86_avx512_pternlog_d_512(<16 x i32> %x0, <16 x i32>
 ; CHECK-LABEL: test_int_x86_avx512_pternlog_d_512:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf3,0x75,0x48,0x25,0xc2,0x21]
-; CHECK-NEXT:    ## zmm0 = (~zmm0 & ~zmm1 & ~zmm2) | (zmm0 & ~zmm1 & zmm2)
+; CHECK-NEXT:    ## zmm0 = ~(zmm1 | (zmm0 ^ zmm2))
 ; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
   %res = call <16 x i32> @llvm.x86.avx512.mask.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33, i16 -1)
   ret <16 x i32> %res
@@ -7693,14 +7693,14 @@ define <16 x i32>@test_int_x86_avx512_mask_pternlog_d_512(<16 x i32> %x0, <16 x
 ; X86:       ## %bb.0:
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf3,0x75,0x49,0x25,0xc2,0x21]
-; X86-NEXT:    ## zmm0 {%k1} = (~zmm0 & ~zmm1 & ~zmm2) | (zmm0 & ~zmm1 & zmm2)
+; X86-NEXT:    ## zmm0 {%k1} = ~(zmm1 | (zmm0 ^ zmm2))
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pternlog_d_512:
 ; X64:       ## %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf3,0x75,0x49,0x25,0xc2,0x21]
-; X64-NEXT:    ## zmm0 {%k1} = (~zmm0 & ~zmm1 & ~zmm2) | (zmm0 & ~zmm1 & zmm2)
+; X64-NEXT:    ## zmm0 {%k1} = ~(zmm1 | (zmm0 ^ zmm2))
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <16 x i32> @llvm.x86.avx512.mask.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33, i16 %x4)
   ret <16 x i32> %res
@@ -7713,14 +7713,14 @@ define <16 x i32>@test_int_x86_avx512_maskz_pternlog_d_512(<16 x i32> %x0, <16 x
 ; X86:       ## %bb.0:
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm0 {%k1} {z} ## encoding: [0x62,0xf3,0x75,0xc9,0x25,0xc2,0x21]
-; X86-NEXT:    ## zmm0 {%k1} {z} = (~zmm0 & ~zmm1 & ~zmm2) | (zmm0 & ~zmm1 & zmm2)
+; X86-NEXT:    ## zmm0 {%k1} {z} = ~(zmm1 | (zmm0 ^ zmm2))
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_d_512:
 ; X64:       ## %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm0 {%k1} {z} ## encoding: [0x62,0xf3,0x75,0xc9,0x25,0xc2,0x21]
-; X64-NEXT:    ## zmm0 {%k1} {z} = (~zmm0 & ~zmm1 & ~zmm2) | (zmm0 & ~zmm1 & zmm2)
+; X64-NEXT:    ## zmm0 {%k1} {z} = ~(zmm1 | (zmm0 ^ zmm2))
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <16 x i32> @llvm.x86.avx512.maskz.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33, i16 %x4)
   ret <16 x i32> %res
@@ -7732,7 +7732,7 @@ define <8 x i64>@test_int_x86_avx512_pternlog_q_512(<8 x i64> %x0, <8 x i64> %x1
 ; CHECK-LABEL: test_int_x86_avx512_pternlog_q_512:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf3,0xf5,0x48,0x25,0xc2,0x21]
-; CHECK-NEXT:    ## zmm0 = (~zmm0 & ~zmm1 & ~zmm2) | (zmm0 & ~zmm1 & zmm2)
+; CHECK-NEXT:    ## zmm0 = ~(zmm1 | (zmm0 ^ zmm2))
 ; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
   %res = call <8 x i64> @llvm.x86.avx512.mask.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33, i8 -1)
   ret <8 x i64> %res
@@ -7744,14 +7744,14 @@ define <8 x i64>@test_int_x86_avx512_mask_pternlog_q_512(<8 x i64> %x0, <8 x i64
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf3,0xf5,0x49,0x25,0xc2,0x21]
-; X86-NEXT:    ## zmm0 {%k1} = (~zmm0 & ~zmm1 & ~zmm2) | (zmm0 & ~zmm1 & zmm2)
+; X86-NEXT:    ## zmm0 {%k1} = ~(zmm1 | (zmm0 ^ zmm2))
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pternlog_q_512:
 ; X64:       ## %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf3,0xf5,0x49,0x25,0xc2,0x21]
-; X64-NEXT:    ## zmm0 {%k1} = (~zmm0 & ~zmm1 & ~zmm2) | (zmm0 & ~zmm1 & zmm2)
+; X64-NEXT:    ## zmm0 {%k1} = ~(zmm1 | (zmm0 ^ zmm2))
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <8 x i64> @llvm.x86.avx512.mask.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33, i8 %x4)
   ret <8 x i64> %res
@@ -7765,14 +7765,14 @@ define <8 x i64>@test_int_x86_avx512_maskz_pternlog_q_512(<8 x i64> %x0, <8 x i6
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm0 {%k1} {z} ## encoding: [0x62,0xf3,0xf5,0xc9,0x25,0xc2,0x21]
-; X86-NEXT:    ## zmm0 {%k1} {z} = (~zmm0 & ~zmm1 & ~zmm2) | (zmm0 & ~zmm1 & zmm2)
+; X86-NEXT:    ## zmm0 {%k1} {z} = ~(zmm1 | (zmm0 ^ zmm2))
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_q_512:
 ; X64:       ## %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm0 {%k1} {z} ## encoding: [0x62,0xf3,0xf5,0xc9,0x25,0xc2,0x21]
-; X64-NEXT:    ## zmm0 {%k1} {z} = (~zmm0 & ~zmm1 & ~zmm2) | (zmm0 & ~zmm1 & zmm2)
+; X64-NEXT:    ## zmm0 {%k1} {z} = ~(zmm1 | (zmm0 ^ zmm2))
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <8 x i64> @llvm.x86.avx512.maskz.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33, i8 %x4)
   ret <8 x i64> %res
diff --git a/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
index 519f19740ab25..e8e22bae23c92 100644
--- a/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
@@ -12346,7 +12346,7 @@ define <4 x i32>@test_int_x86_avx512_pternlog_d_128(<4 x i32> %x0, <4 x i32> %x1
 ; CHECK-LABEL: test_int_x86_avx512_pternlog_d_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf3,0x75,0x08,0x25,0xc2,0x21]
-; CHECK-NEXT:    # xmm0 = (~xmm0 & ~xmm1 & ~xmm2) | (xmm0 & ~xmm1 & xmm2)
+; CHECK-NEXT:    # xmm0 = ~(xmm1 | (xmm0 ^ xmm2))
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx512.mask.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33, i8 -1)
   ret <4 x i32> %res
@@ -12358,14 +12358,14 @@ define <4 x i32>@test_int_x86_avx512_mask_pternlog_d_128(<4 x i32> %x0, <4 x i32
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf3,0x75,0x09,0x25,0xc2,0x21]
-; X86-NEXT:    # xmm0 {%k1} = (~xmm0 & ~xmm1 & ~xmm2) | (xmm0 & ~xmm1 & xmm2)
+; X86-NEXT:    # xmm0 {%k1} = ~(xmm1 | (xmm0 ^ xmm2))
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pternlog_d_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf3,0x75,0x09,0x25,0xc2,0x21]
-; X64-NEXT:    # xmm0 {%k1} = (~xmm0 & ~xmm1 & ~xmm2) | (xmm0 & ~xmm1 & xmm2)
+; X64-NEXT:    # xmm0 {%k1} = ~(xmm1 | (xmm0 ^ xmm2))
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx512.mask.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33, i8 %x4)
   ret <4 x i32> %res
@@ -12379,14 +12379,14 @@ define <4 x i32>@test_int_x86_avx512_maskz_pternlog_d_128(<4 x i32> %x0, <4 x i3
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x75,0x89,0x25,0xc2,0x21]
-; X86-NEXT:    # xmm0 {%k1} {z} = (~xmm0 & ~xmm1 & ~xmm2) | (xmm0 & ~xmm1 & xmm2)
+; X86-NEXT:    # xmm0 {%k1} {z} = ~(xmm1 | (xmm0 ^ xmm2))
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_d_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x75,0x89,0x25,0xc2,0x21]
-; X64-NEXT:    # xmm0 {%k1} {z} = (~xmm0 & ~xmm1 & ~xmm2) | (xmm0 & ~xmm1 & xmm2)
+; X64-NEXT:    # xmm0 {%k1} {z} = ~(xmm1 | (xmm0 ^ xmm2))
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx512.maskz.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33, i8 %x4)
   ret <4 x i32> %res
@@ -12398,7 +12398,7 @@ define <8 x i32>@test_int_x86_avx512_pternlog_d_256(<8 x i32> %x0, <8 x i32> %x1
 ; CHECK-LABEL: test_int_x86_avx512_pternlog_d_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf3,0x75,0x28,0x25,0xc2,0x21]
-; CHECK-NEXT:    # ymm0 = (~ymm0 & ~ymm1 & ~ymm2) | (ymm0 & ~ymm1 & ymm2)
+; CHECK-NEXT:    # ymm0 = ~(ymm1 | (ymm0 ^ ymm2))
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33, i8 -1)
   ret <8 x i32> %res
@@ -12410,14 +12410,14 @@ define <8 x i32>@test_int_x86_avx512_mask_pternlog_d_256(<8 x i32> %x0, <8 x i32
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf3,0x75,0x29,0x25,0xc2,0x21]
-; X86-NEXT:    # ymm0 {%k1} = (~ymm0 & ~ymm1 & ~ymm2) | (ymm0 & ~ymm1 & ymm2)
+; X86-NEXT:    # ymm0 {%k1} = ~(ymm1 | (ymm0 ^ ymm2))
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pternlog_d_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf3,0x75,0x29,0x25,0xc2,0x21]
-; X64-NEXT:    # ymm0 {%k1} = (~ymm0 & ~ymm1 & ~ymm2) | (ymm0 & ~ymm1 & ymm2)
+; X64-NEXT:    # ymm0 {%k1} = ~(ymm1 | (ymm0 ^ ymm2))
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33, i8 %x4)
   ret <8 x i32> %res
@@ -12431,14 +12431,14 @@ define <8 x i32>@test_int_x86_avx512_maskz_pternlog_d_256(<8 x i32> %x0, <8 x i3
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0x75,0xa9,0x25,0xc2,0x21]
-; X86-NEXT:    # ymm0 {%k1} {z} = (~ymm0 & ~ymm1 & ~ymm2) | (ymm0 & ~ymm1 & ymm2)
+; X86-NEXT:    # ymm0 {%k1} {z} = ~(ymm1 | (ymm0 ^ ymm2))
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_d_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0x75,0xa9,0x25,0xc2,0x21]
-; X64-NEXT:    # ymm0 {%k1} {z} = (~ymm0 & ~ymm1 & ~ymm2) | (ymm0 & ~ymm1 & ymm2)
+; X64-NEXT:    # ymm0 {%k1} {z} = ~(ymm1 | (ymm0 ^ ymm2))
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.maskz.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33, i8 %x4)
   ret <8 x i32> %res
@@ -12450,7 +12450,7 @@ define <2 x i64>@test_int_x86_avx512_pternlog_q_128(<2 x i64> %x0, <2 x i64> %x1
 ; CHECK-LABEL: test_int_x86_avx512_pternlog_q_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf3,0xf5,0x08,0x25,0xc2,0x21]
-; CHECK-NEXT:    # xmm0 = (~xmm0 & ~xmm1 & ~xmm2) | (xmm0 & ~xmm1 & xmm2)
+; CHECK-NEXT:    # xmm0 = ~(xmm1 | (xmm0 ^ xmm2))
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.mask.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33, i8 -1)
   ret <2 x i64> %res
@@ -12462,14 +12462,14 @@ define <2 x i64>@test_int_x86_avx512_mask_pternlog_q_128(<2 x i64> %x0, <2 x i64
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf3,0xf5,0x09,0x25,0xc2,0x21]
-; X86-NEXT:    # xmm0 {%k1} = (~xmm0 & ~xmm1 & ~xmm2) | (xmm0 & ~xmm1 & xmm2)
+; X86-NEXT:    # xmm0 {%k1} = ~(xmm1 | (xmm0 ^ xmm2))
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pternlog_q_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf3,0xf5,0x09,0x25,0xc2,0x21]
-; X64-NEXT:    # xmm0 {%k1} = (~xmm0 & ~xmm1 & ~xmm2) | (xmm0 & ~xmm1 & xmm2)
+; X64-NEXT:    # xmm0 {%k1} = ~(xmm1 | (xmm0 ^ xmm2))
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.mask.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33, i8 %x4)
   ret <2 x i64> %res
@@ -12483,14 +12483,14 @@ define <2 x i64>@test_int_x86_avx512_maskz_pternlog_q_128(<2 x i64> %x0, <2 x i6
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0xf5,0x89,0x25,0xc2,0x21]
-; X86-NEXT:    # xmm0 {%k1} {z} = (~xmm0 & ~xmm1 & ~xmm2) | (xmm0 & ~xmm1 & xmm2)
+; X86-NEXT:    # xmm0 {%k1} {z} = ~(xmm1 | (xmm0 ^ xmm2))
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_q_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0xf5,0x89,0x25,0xc2,0x21]
-; X64-NEXT:    # xmm0 {%k1} {z} = (~xmm0 & ~xmm1 & ~xmm2) | (xmm0 & ~xmm1 & xmm2)
+; X64-NEXT:    # xmm0 {%k1} {z} = ~(xmm1 | (xmm0 ^ xmm2))
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.maskz.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33, i8 %x4)
   ret <2 x i64> %res
@@ -12502,7 +12502,7 @@ define <4 x i64>@test_int_x86_avx512_pternlog_q_256(<4 x i64> %x0, <4 x i64> %x1
 ; CHECK-LABEL: test_int_x86_avx512_pternlog_q_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf3,0xf5,0x28,0x25,0xc2,0x21]
-; CHECK-NEXT:    # ymm0 = (~ymm0 & ~ymm1 & ~ymm2) | (ymm0 & ~ymm1 & ymm2)
+; CHECK-NEXT:    # ymm0 = ~(ymm1 | (ymm0 ^ ymm2))
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.mask.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33, i8 -1)
   ret <4 x i64> %res
@@ -12514,14 +12514,14 @@ define <4 x i64>@test_int_x86_avx512_mask_pternlog_q_256(<4 x i64> %x0, <4 x i64
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf3,0xf5,0x29,0x25,0xc2,0x21]
-; X86-NEXT:    # ymm0 {%k1} = (~ymm0 & ~ymm1 & ~ymm2) | (ymm0 & ~ymm1 & ymm2)
+; X86-NEXT:    # ymm0 {%k1} = ~(ymm1 | (ymm0 ^ ymm2))
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pternlog_q_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf3,0xf5,0x29,0x25,0xc2,0x21]
-; X64-NEXT:    # ymm0 {%k1} = (~ymm0 & ~ymm1 & ~ymm2) | (ymm0 & ~ymm1 & ymm2)
+; X64-NEXT:    # ymm0 {%k1} = ~(ymm1 | (ymm0 ^ ymm2))
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.mask.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33, i8 %x4)
   ret <4 x i64> %res
@@ -12535,14 +12535,14 @@ define <4 x i64>@test_int_x86_avx512_maskz_pternlog_q_256(<4 x i64> %x0, <4 x i6
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0xf5,0xa9,0x25,0xc2,0x21]
-; X86-NEXT:    # ymm0 {%k1} {z} = (~ymm0 & ~ymm1 & ~ymm2) | (ymm0 & ~ymm1 & ymm2)
+; X86-NEXT:    # ymm0 {%k1} {z} = ~(ymm1 | (ymm0 ^ ymm2))
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_q_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0xf5,0xa9,0x25,0xc2,0x21]
-; X64-NEXT:    # ymm0 {%k1} {z} = (~ymm0 & ~ymm1 & ~ymm2) | (ymm0 & ~ymm1 & ymm2)
+; X64-NEXT:    # ymm0 {%k1} {z} = ~(ymm1 | (ymm0 ^ ymm2))
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.maskz.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33, i8 %x4)
   ret <4 x i64> %res
diff --git a/llvm/test/CodeGen/X86/avx512vl-intrinsics.ll b/llvm/test/CodeGen/X86/avx512vl-intrinsics.ll
index 6c7a5d2f86341..6de09c745b32a 100644
--- a/llvm/test/CodeGen/X86/avx512vl-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-intrinsics.ll
@@ -4150,7 +4150,7 @@ define <4 x i32>@test_int_x86_avx512_pternlog_d_128(<4 x i32> %x0, <4 x i32> %x1
 ; CHECK-LABEL: test_int_x86_avx512_pternlog_d_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf3,0x75,0x08,0x25,0xc2,0x21]
-; CHECK-NEXT:    # xmm0 = (~xmm0 & ~xmm1 & ~xmm2) | (xmm0 & ~xmm1 & xmm2)
+; CHECK-NEXT:    # xmm0 = ~(xmm1 | (xmm0 ^ xmm2))
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %1 = call <4 x i32> @llvm.x86.avx512.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33)
   ret <4 x i32> %1
@@ -4162,14 +4162,14 @@ define <4 x i32>@test_int_x86_avx512_mask_pternlog_d_128(<4 x i32> %x0, <4 x i32
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf3,0x75,0x09,0x25,0xc2,0x21]
-; X86-NEXT:    # xmm0 {%k1} = (~xmm0 & ~xmm1 & ~xmm2) | (xmm0 & ~xmm1 & xmm2)
+; X86-NEXT:    # xmm0 {%k1} = ~(xmm1 | (xmm0 ^ xmm2))
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pternlog_d_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf3,0x75,0x09,0x25,0xc2,0x21]
-; X64-NEXT:    # xmm0 {%k1} = (~xmm0 & ~xmm1 & ~xmm2) | (xmm0 & ~xmm1 & xmm2)
+; X64-NEXT:    # xmm0 {%k1} = ~(xmm1 | (xmm0 ^ xmm2))
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x i32> @llvm.x86.avx512.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33)
   %2 = bitcast i8 %x4 to <8 x i1>
@@ -4186,14 +4186,14 @@ define <4 x i32>@test_int_x86_avx512_maskz_pternlog_d_128(<4 x i32> %x0, <4 x i3
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x75,0x89,0x25,0xc2,0x21]
-; X86-NEXT:    # xmm0 {%k1} {z} = (~xmm0 & ~xmm1 & ~xmm2) | (xmm0 & ~xmm1 & xmm2)
+; X86-NEXT:    # xmm0 {%k1} {z} = ~(xmm1 | (xmm0 ^ xmm2))
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_d_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x75,0x89,0x25,0xc2,0x21]
-; X64-NEXT:    # xmm0 {%k1} {z} = (~xmm0 & ~xmm1 & ~xmm2) | (xmm0 & ~xmm1 & xmm2)
+; X64-NEXT:    # xmm0 {%k1} {z} = ~(xmm1 | (xmm0 ^ xmm2))
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x i32> @llvm.x86.avx512.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33)
   %2 = bitcast i8 %x4 to <8 x i1>
@@ -4208,7 +4208,7 @@ define <8 x i32>@test_int_x86_avx512_pternlog_d_256(<8 x i32> %x0, <8 x i32> %x1
 ; CHECK-LABEL: test_int_x86_avx512_pternlog_d_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf3,0x75,0x28,0x25,0xc2,0x21]
-; CHECK-NEXT:    # ymm0 = (~ymm0 & ~ymm1 & ~ymm2) | (ymm0 & ~ymm1 & ymm2)
+; CHECK-NEXT:    # ymm0 = ~(ymm1 | (ymm0 ^ ymm2))
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %1 = call <8 x i32> @llvm.x86.avx512.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33)
   ret <8 x i32> %1
@@ -4220,14 +4220,14 @@ define <8 x i32>@test_int_x86_avx512_mask_pternlog_d_256(<8 x i32> %x0, <8 x i32
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf3,0x75,0x29,0x25,0xc2,0x21]
-; X86-NEXT:    # ymm0 {%k1} = (~ymm0 & ~ymm1 & ~ymm2) | (ymm0 & ~ymm1 & ymm2)
+; X86-NEXT:    # ymm0 {%k1} = ~(ymm1 | (ymm0 ^ ymm2))
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pternlog_d_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf3,0x75,0x29,0x25,0xc2,0x21]
-; X64-NEXT:    # ymm0 {%k1} = (~ymm0 & ~ymm1 & ~ymm2) | (ymm0 & ~ymm1 & ymm2)
+; X64-NEXT:    # ymm0 {%k1} = ~(ymm1 | (ymm0 ^ ymm2))
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i32> @llvm.x86.avx512.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33)
   %2 = bitcast i8 %x4 to <8 x i1>
@@ -4243,14 +4243,14 @@ define <8 x i32>@test_int_x86_avx512_maskz_pternlog_d_256(<8 x i32> %x0, <8 x i3
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0x75,0xa9,0x25,0xc2,0x21]
-; X86-NEXT:    # ymm0 {%k1} {z} = (~ymm0 & ~ymm1 & ~ymm2) | (ymm0 & ~ymm1 & ymm2)
+; X86-NEXT:    # ymm0 {%k1} {z} = ~(ymm1 | (ymm0 ^ ymm2))
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_d_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0x75,0xa9,0x25,0xc2,0x21]
-; X64-NEXT:    # ymm0 {%k1} {z} = (~ymm0 & ~ymm1 & ~ymm2) | (ymm0 & ~ymm1 & ymm2)
+; X64-NEXT:    # ymm0 {%k1} {z} = ~(ymm1 | (ymm0 ^ ymm2))
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i32> @llvm.x86.avx512.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33)
   %2 = bitcast i8 %x4 to <8 x i1>
@@ -4264,7 +4264,7 @@ define <2 x i64>@test_int_x86_avx512_pternlog_q_128(<2 x i64> %x0, <2 x i64> %x1
 ; CHECK-LABEL: test_int_x86_avx512_pternlog_q_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf3,0xf5,0x08,0x25,0xc2,0x21]
-; CHECK-NEXT:    # xmm0 = (~xmm0 & ~xmm1 & ~xmm2) | (xmm0 & ~xmm1 & xmm2)
+; CHECK-NEXT:    # xmm0 = ~(xmm1 | (xmm0 ^ xmm2))
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %1 = call <2 x i64> @llvm.x86.avx512.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33)
   ret <2 x i64> %1
@@ -4276,14 +4276,14 @@ define <2 x i64>@test_int_x86_avx512_mask_pternlog_q_128(<2 x i64> %x0, <2 x i64
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf3,0xf5,0x09,0x25,0xc2,0x21]
-; X86-NEXT:    # xmm0 {%k1} = (~xmm0 & ~xmm1 & ~xmm2) | (xmm0 & ~xmm1 & xmm2)
+; X86-NEXT:    # xmm0 {%k1} = ~(xmm1 | (xmm0 ^ xmm2))
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pternlog_q_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf3,0xf5,0x09,0x25,0xc2,0x21]
-; X64-NEXT:    # xmm0 {%k1} = (~xmm0 & ~xmm1 & ~xmm2) | (xmm0 & ~xmm1 & xmm2)
+; X64-NEXT:    # xmm0 {%k1} = ~(xmm1 | (xmm0 ^ xmm2))
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <2 x i64> @llvm.x86.avx512.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33)
   %2 = bitcast i8 %x4 to <8 x i1>
@@ -4298,14 +4298,14 @@ define <2 x i64>@test_int_x86_avx512_maskz_pternlog_q_128(<2 x i64> %x0, <2 x i6
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0xf5,0x89,0x25,0xc2,0x21]
-; X86-NEXT:    # xmm0 {%k1} {z} = (~xmm0 & ~xmm1 & ~xmm2) | (xmm0 & ~xmm1 & xmm2)
+; X86-NEXT:    # xmm0 {%k1} {z} = ~(xmm1 | (xmm0 ^ xmm2))
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_q_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0xf5,0x89,0x25,0xc2,0x21]
-; X64-NEXT:    # xmm0 {%k1} {z} = (~xmm0 & ~xmm1 & ~xmm2) | (xmm0 & ~xmm1 & xmm2)
+; X64-NEXT:    # xmm0 {%k1} {z} = ~(xmm1 | (xmm0 ^ xmm2))
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <2 x i64> @llvm.x86.avx512.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33)
   %2 = bitcast i8 %x4 to <8 x i1>
@@ -4320,7 +4320,7 @@ define <4 x i64>@test_int_x86_avx512_pternlog_q_256(<4 x i64> %x0, <4 x i64> %x1
 ; CHECK-LABEL: test_int_x86_avx512_pternlog_q_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf3,0xf5,0x28,0x25,0xc2,0x21]
-; CHECK-NEXT:    # ymm0 = (~ymm0 & ~ymm1 & ~ymm2) | (ymm0 & ~ymm1 & ymm2)
+; CHECK-NEXT:    # ymm0 = ~(ymm1 | (ymm0 ^ ymm2))
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %1 = call <4 x i64> @llvm.x86.avx512.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33)
   ret <4 x i64> %1
@@ -4332,14 +4332,14 @@ define <4 x i64>@test_int_x86_avx512_mask_pternlog_q_256(<4 x i64> %x0, <4 x i64
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf3,0xf5,0x29,0x25,0xc2,0x21]
-; X86-NEXT:    # ymm0 {%k1} = (~ymm0 & ~ymm1 & ~ymm2) | (ymm0 & ~ymm1 & ymm2)
+; X86-NEXT:    # ymm0 {%k1} = ~(ymm1 | (ymm0 ^ ymm2))
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pternlog_q_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf3,0xf5,0x29,0x25,0xc2,0x21]
-; X64-NEXT:    # ymm0 {%k1} = (~ymm0 & ~ymm1 & ~ymm2) | (ymm0 & ~ymm1 & ymm2)
+; X64-NEXT:    # ymm0 {%k1} = ~(ymm1 | (ymm0 ^ ymm2))
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x i64> @llvm.x86.avx512.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33)
   %2 = bitcast i8 %x4 to <8 x i1>
@@ -4354,14 +4354,14 @@ define <4 x i64>@test_int_x86_avx512_maskz_pternlog_q_256(<4 x i64> %x0, <4 x i6
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0xf5,0xa9,0x25,0xc2,0x21]
-; X86-NEXT:    # ymm0 {%k1} {z} = (~ymm0 & ~ymm1 & ~ymm2) | (ymm0 & ~ymm1 & ymm2)
+; X86-NEXT:    # ymm0 {%k1} {z} = ~(ymm1 | (ymm0 ^ ymm2))
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_q_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0xf5,0xa9,0x25,0xc2,0x21]
-; X64-NEXT:    # ymm0 {%k1} {z} = (~ymm0 & ~ymm1 & ~ymm2) | (ymm0 & ~ymm1 & ymm2)
+; X64-NEXT:    # ymm0 {%k1} {z} = ~(ymm1 | (ymm0 ^ ymm2))
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x i64> @llvm.x86.avx512.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33)
   %2 = bitcast i8 %x4 to <8 x i1>
diff --git a/llvm/test/CodeGen/X86/avx512vnni-combine.ll b/llvm/test/CodeGen/X86/avx512vnni-combine.ll
index c491a952682d5..b7d950e994241 100644
--- a/llvm/test/CodeGen/X86/avx512vnni-combine.ll
+++ b/llvm/test/CodeGen/X86/avx512vnni-combine.ll
@@ -54,7 +54,7 @@ define <8 x i64> @foo_512(i32 %0, <8 x i64> %1, <8 x i64> %2, ptr %3) {
 ; CHECK-NEXT:    andl $-4, %edx
 ; CHECK-NEXT:    leaq 192(%rsi), %rdi
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_8: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpdpwssd -192(%rdi), %zmm1, %zmm0
 ; CHECK-NEXT:    vpmaddwd -128(%rdi), %zmm1, %zmm2
@@ -75,7 +75,7 @@ define <8 x i64> @foo_512(i32 %0, <8 x i64> %1, <8 x i64> %2, ptr %3) {
 ; CHECK-NEXT:    addq %rcx, %rsi
 ; CHECK-NEXT:    shll $6, %eax
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_5: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpdpwssd (%rsi,%rcx), %zmm1, %zmm0
 ; CHECK-NEXT:    addq $64, %rcx
@@ -179,7 +179,7 @@ define void @bar_512(i32 %0, ptr %1, <8 x i64> %2, ptr %3) {
 ; CHECK-NEXT:    andl $-2, %edi
 ; CHECK-NEXT:    movl $64, %r8d
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB2_7: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovdqa64 (%rsi,%r8), %zmm1
 ; CHECK-NEXT:    vpmaddwd -64(%rdx,%r8), %zmm0, %zmm2
diff --git a/llvm/test/CodeGen/X86/avxvnni-combine.ll b/llvm/test/CodeGen/X86/avxvnni-combine.ll
index 158a1fdf44fc6..45f9a6475244e 100644
--- a/llvm/test/CodeGen/X86/avxvnni-combine.ll
+++ b/llvm/test/CodeGen/X86/avxvnni-combine.ll
@@ -59,7 +59,7 @@ define <2 x i64> @foo_128(i32 %0, <2 x i64> %1, <2 x i64> %2, ptr %3) {
 ; AVX-NEXT:    andl $-4, %edx
 ; AVX-NEXT:    leaq 48(%rsi), %rdi
 ; AVX-NEXT:    xorl %ecx, %ecx
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  .LBB1_8: # =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    {vex} vpdpwssd -48(%rdi), %xmm1, %xmm0
 ; AVX-NEXT:    vpmaddwd -32(%rdi), %xmm1, %xmm2
@@ -80,7 +80,7 @@ define <2 x i64> @foo_128(i32 %0, <2 x i64> %1, <2 x i64> %2, ptr %3) {
 ; AVX-NEXT:    addq %rcx, %rsi
 ; AVX-NEXT:    shll $4, %eax
 ; AVX-NEXT:    xorl %ecx, %ecx
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  .LBB1_5: # =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    {vex} vpdpwssd (%rsi,%rcx), %xmm1, %xmm0
 ; AVX-NEXT:    addq $16, %rcx
@@ -106,7 +106,7 @@ define <2 x i64> @foo_128(i32 %0, <2 x i64> %1, <2 x i64> %2, ptr %3) {
 ; AVX512-NEXT:    andl $-4, %edx
 ; AVX512-NEXT:    leaq 48(%rsi), %rdi
 ; AVX512-NEXT:    xorl %ecx, %ecx
-; AVX512-NEXT:    .p2align 4, 0x90
+; AVX512-NEXT:    .p2align 4
 ; AVX512-NEXT:  .LBB1_8: # =>This Inner Loop Header: Depth=1
 ; AVX512-NEXT:    vpdpwssd -48(%rdi), %xmm1, %xmm0
 ; AVX512-NEXT:    vpmaddwd -32(%rdi), %xmm1, %xmm2
@@ -127,7 +127,7 @@ define <2 x i64> @foo_128(i32 %0, <2 x i64> %1, <2 x i64> %2, ptr %3) {
 ; AVX512-NEXT:    addq %rcx, %rsi
 ; AVX512-NEXT:    shll $4, %eax
 ; AVX512-NEXT:    xorl %ecx, %ecx
-; AVX512-NEXT:    .p2align 4, 0x90
+; AVX512-NEXT:    .p2align 4
 ; AVX512-NEXT:  .LBB1_5: # =>This Inner Loop Header: Depth=1
 ; AVX512-NEXT:    vpdpwssd (%rsi,%rcx), %xmm1, %xmm0
 ; AVX512-NEXT:    addq $16, %rcx
@@ -225,7 +225,7 @@ define void @bar_128(i32 %0, ptr %1, <2 x i64> %2, ptr %3) {
 ; AVX-NEXT:    andl $-2, %edi
 ; AVX-NEXT:    movl $16, %r8d
 ; AVX-NEXT:    xorl %ecx, %ecx
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  .LBB2_7: # =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    vmovdqa (%rsi,%r8), %xmm1
 ; AVX-NEXT:    vpmaddwd -16(%rdx,%r8), %xmm0, %xmm2
@@ -265,7 +265,7 @@ define void @bar_128(i32 %0, ptr %1, <2 x i64> %2, ptr %3) {
 ; AVX512-NEXT:    andl $-2, %edi
 ; AVX512-NEXT:    movl $16, %r8d
 ; AVX512-NEXT:    xorl %ecx, %ecx
-; AVX512-NEXT:    .p2align 4, 0x90
+; AVX512-NEXT:    .p2align 4
 ; AVX512-NEXT:  .LBB2_7: # =>This Inner Loop Header: Depth=1
 ; AVX512-NEXT:    vmovdqa (%rsi,%r8), %xmm1
 ; AVX512-NEXT:    vpmaddwd -16(%rdx,%r8), %xmm0, %xmm2
@@ -406,7 +406,7 @@ define <4 x i64> @foo_256(i32 %0, <4 x i64> %1, <4 x i64> %2, ptr %3) {
 ; AVX-NEXT:    andl $-4, %edx
 ; AVX-NEXT:    leaq 96(%rsi), %rdi
 ; AVX-NEXT:    xorl %ecx, %ecx
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  .LBB4_8: # =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    {vex} vpdpwssd -96(%rdi), %ymm1, %ymm0
 ; AVX-NEXT:    vpmaddwd -64(%rdi), %ymm1, %ymm2
@@ -427,7 +427,7 @@ define <4 x i64> @foo_256(i32 %0, <4 x i64> %1, <4 x i64> %2, ptr %3) {
 ; AVX-NEXT:    addq %rcx, %rsi
 ; AVX-NEXT:    shll $5, %eax
 ; AVX-NEXT:    xorl %ecx, %ecx
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  .LBB4_5: # =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    {vex} vpdpwssd (%rsi,%rcx), %ymm1, %ymm0
 ; AVX-NEXT:    addq $32, %rcx
@@ -453,7 +453,7 @@ define <4 x i64> @foo_256(i32 %0, <4 x i64> %1, <4 x i64> %2, ptr %3) {
 ; AVX512-NEXT:    andl $-4, %edx
 ; AVX512-NEXT:    leaq 96(%rsi), %rdi
 ; AVX512-NEXT:    xorl %ecx, %ecx
-; AVX512-NEXT:    .p2align 4, 0x90
+; AVX512-NEXT:    .p2align 4
 ; AVX512-NEXT:  .LBB4_8: # =>This Inner Loop Header: Depth=1
 ; AVX512-NEXT:    vpdpwssd -96(%rdi), %ymm1, %ymm0
 ; AVX512-NEXT:    vpmaddwd -64(%rdi), %ymm1, %ymm2
@@ -474,7 +474,7 @@ define <4 x i64> @foo_256(i32 %0, <4 x i64> %1, <4 x i64> %2, ptr %3) {
 ; AVX512-NEXT:    addq %rcx, %rsi
 ; AVX512-NEXT:    shll $5, %eax
 ; AVX512-NEXT:    xorl %ecx, %ecx
-; AVX512-NEXT:    .p2align 4, 0x90
+; AVX512-NEXT:    .p2align 4
 ; AVX512-NEXT:  .LBB4_5: # =>This Inner Loop Header: Depth=1
 ; AVX512-NEXT:    vpdpwssd (%rsi,%rcx), %ymm1, %ymm0
 ; AVX512-NEXT:    addq $32, %rcx
@@ -579,7 +579,7 @@ define void @bar_256(i32 %0, ptr %1, <4 x i64> %2, ptr %3) {
 ; AVX-NEXT:    andl $-2, %edi
 ; AVX-NEXT:    movl $32, %r8d
 ; AVX-NEXT:    xorl %ecx, %ecx
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  .LBB5_7: # =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    vmovdqa (%rsi,%r8), %ymm1
 ; AVX-NEXT:    vpmaddwd -32(%rdx,%r8), %ymm0, %ymm2
@@ -620,7 +620,7 @@ define void @bar_256(i32 %0, ptr %1, <4 x i64> %2, ptr %3) {
 ; AVX512-NEXT:    andl $-2, %edi
 ; AVX512-NEXT:    movl $32, %r8d
 ; AVX512-NEXT:    xorl %ecx, %ecx
-; AVX512-NEXT:    .p2align 4, 0x90
+; AVX512-NEXT:    .p2align 4
 ; AVX512-NEXT:  .LBB5_7: # =>This Inner Loop Header: Depth=1
 ; AVX512-NEXT:    vmovdqa (%rsi,%r8), %ymm1
 ; AVX512-NEXT:    vpmaddwd -32(%rdx,%r8), %ymm0, %ymm2
diff --git a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll
index eef2b3db5d694..e9e5bec4a4b15 100644
--- a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll
+++ b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll
@@ -450,7 +450,7 @@ define <8 x i64> @ext_i8_8i64(i8 %a0) {
 ; AVX512-LABEL: ext_i8_8i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    kmovd %edi, %k1
-; AVX512-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512-NEXT:    retq
   %1 = bitcast i8 %a0 to <8 x i1>
   %2 = sext <8 x i1> %1 to <8 x i64>
@@ -507,7 +507,7 @@ define <16 x i32> @ext_i16_16i32(i16 %a0) {
 ; AVX512-LABEL: ext_i16_16i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    kmovd %edi, %k1
-; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512-NEXT:    retq
   %1 = bitcast i16 %a0 to <16 x i1>
   %2 = sext <16 x i1> %1 to <16 x i32>
diff --git a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll
index 5c810797bd2b7..9f6de9a5c5d9b 100644
--- a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll
+++ b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll
@@ -49,7 +49,7 @@ define <2 x i64> @ext_i2_2i64(i2 %a0) {
 ; AVX512F-LABEL: ext_i2_2i64:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    kmovw %edi, %k1
-; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-NEXT:    vpsrlq $63, %xmm0, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -100,7 +100,7 @@ define <4 x i32> @ext_i4_4i32(i4 %a0) {
 ; AVX512F-LABEL: ext_i4_4i32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    kmovw %edi, %k1
-; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -153,7 +153,7 @@ define <8 x i16> @ext_i8_8i16(i8 %a0) {
 ; AVX512F-LABEL: ext_i8_8i16:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    kmovw %edi, %k1
-; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $15, %xmm0, %xmm0
 ; AVX512F-NEXT:    vzeroupper
@@ -283,7 +283,7 @@ define <4 x i64> @ext_i4_4i64(i4 %a0) {
 ; AVX512F-LABEL: ext_i4_4i64:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    kmovw %edi, %k1
-; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-NEXT:    vpsrlq $63, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
 ;
@@ -339,7 +339,7 @@ define <8 x i32> @ext_i8_8i32(i8 %a0) {
 ; AVX512F-LABEL: ext_i8_8i32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    kmovw %edi, %k1
-; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-NEXT:    vpsrld $31, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
 ;
@@ -400,7 +400,7 @@ define <16 x i16> @ext_i16_16i16(i16 %a0) {
 ; AVX512F-LABEL: ext_i16_16i16:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    kmovw %edi, %k1
-; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $15, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
@@ -564,14 +564,14 @@ define <8 x i64> @ext_i8_8i64(i8 %a0) {
 ; AVX512F-LABEL: ext_i8_8i64:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    kmovw %edi, %k1
-; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-NEXT:    vpsrlq $63, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: ext_i8_8i64:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    kmovd %edi, %k1
-; AVX512VLBW-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512VLBW-NEXT:    vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512VLBW-NEXT:    vpsrlq $63, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    retq
   %1 = bitcast i8 %a0 to <8 x i1>
@@ -638,14 +638,14 @@ define <16 x i32> @ext_i16_16i32(i16 %a0) {
 ; AVX512F-LABEL: ext_i16_16i32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    kmovw %edi, %k1
-; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-NEXT:    vpsrld $31, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: ext_i16_16i32:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    kmovd %edi, %k1
-; AVX512VLBW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512VLBW-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512VLBW-NEXT:    vpsrld $31, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    retq
   %1 = bitcast i16 %a0 to <16 x i1>
@@ -726,10 +726,10 @@ define <32 x i16> @ext_i32_32i16(i32 %a0) {
 ; AVX512F-NEXT:    kmovw %edi, %k1
 ; AVX512F-NEXT:    shrl $16, %edi
 ; AVX512F-NEXT:    kmovw %edi, %k2
-; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $15, %ymm0, %ymm0
-; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm1 {%k2} {z} = -1
 ; AVX512F-NEXT:    vpmovdw %zmm1, %ymm1
 ; AVX512F-NEXT:    vpsrlw $15, %ymm1, %ymm1
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
diff --git a/llvm/test/CodeGen/X86/block-placement.ll b/llvm/test/CodeGen/X86/block-placement.ll
index e479ad20d6e4c..675293410dfe7 100644
--- a/llvm/test/CodeGen/X86/block-placement.ll
+++ b/llvm/test/CodeGen/X86/block-placement.ll
@@ -251,7 +251,7 @@ define i32 @test_loop_align(i32 %i, ptr %a) {
 ; pass.
 ; CHECK-LABEL: test_loop_align:
 ; CHECK: %entry
-; CHECK: .p2align [[ALIGN:[0-9]+]],
+; CHECK: .p2align [[ALIGN:[0-9]+]]
 ; CHECK-NEXT: %body
 ; CHECK: %exit
 
@@ -276,9 +276,9 @@ define i32 @test_nested_loop_align(i32 %i, ptr %a, ptr %b) {
 ; Check that we provide nested loop body alignment.
 ; CHECK-LABEL: test_nested_loop_align:
 ; CHECK: %entry
-; CHECK: .p2align [[ALIGN]],
+; CHECK: .p2align [[ALIGN]]
 ; CHECK-NEXT: %loop.body.1
-; CHECK: .p2align [[ALIGN]],
+; CHECK: .p2align [[ALIGN]]
 ; CHECK-NEXT: %inner.loop.body
 ; CHECK-NOT: .p2align
 ; CHECK: %exit
diff --git a/llvm/test/CodeGen/X86/break-false-dep.ll b/llvm/test/CodeGen/X86/break-false-dep.ll
index 8ff7fb2d351ad..0a367e1916ff6 100644
--- a/llvm/test/CodeGen/X86/break-false-dep.ll
+++ b/llvm/test/CodeGen/X86/break-false-dep.ll
@@ -138,7 +138,7 @@ define dso_local float @loopdep1(i32 %m) nounwind uwtable readnone ssp {
 ; SSE-LINUX-NEXT:    movl $1, %eax
 ; SSE-LINUX-NEXT:    xorps %xmm0, %xmm0
 ; SSE-LINUX-NEXT:    xorps %xmm1, %xmm1
-; SSE-LINUX-NEXT:    .p2align 4, 0x90
+; SSE-LINUX-NEXT:    .p2align 4
 ; SSE-LINUX-NEXT:  .LBB6_3: # %for.body
 ; SSE-LINUX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE-LINUX-NEXT:    xorps %xmm2, %xmm2
@@ -167,7 +167,7 @@ define dso_local float @loopdep1(i32 %m) nounwind uwtable readnone ssp {
 ; SSE-WIN-NEXT:    movl $1, %eax
 ; SSE-WIN-NEXT:    xorps %xmm0, %xmm0
 ; SSE-WIN-NEXT:    xorps %xmm1, %xmm1
-; SSE-WIN-NEXT:    .p2align 4, 0x90
+; SSE-WIN-NEXT:    .p2align 4
 ; SSE-WIN-NEXT:  .LBB6_3: # %for.body
 ; SSE-WIN-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE-WIN-NEXT:    xorps %xmm2, %xmm2
@@ -196,7 +196,7 @@ define dso_local float @loopdep1(i32 %m) nounwind uwtable readnone ssp {
 ; AVX1-NEXT:    movl $1, %eax
 ; AVX1-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB6_3: # %for.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vcvtsi2ss %eax, %xmm4, %xmm2
@@ -223,7 +223,7 @@ define dso_local float @loopdep1(i32 %m) nounwind uwtable readnone ssp {
 ; AVX512VL-NEXT:    movl $1, %eax
 ; AVX512VL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT:    .p2align 4, 0x90
+; AVX512VL-NEXT:    .p2align 4
 ; AVX512VL-NEXT:  .LBB6_3: # %for.body
 ; AVX512VL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512VL-NEXT:    vcvtsi2ss %eax, %xmm3, %xmm2
@@ -276,7 +276,7 @@ define i64 @loopdep2(ptr nocapture %x, ptr nocapture %y) nounwind {
 ; SSE-LINUX:       # %bb.0: # %entry
 ; SSE-LINUX-NEXT:    movq (%rdi), %rax
 ; SSE-LINUX-NEXT:    movl $1, %ecx
-; SSE-LINUX-NEXT:    .p2align 4, 0x90
+; SSE-LINUX-NEXT:    .p2align 4
 ; SSE-LINUX-NEXT:  .LBB7_1: # %loop
 ; SSE-LINUX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE-LINUX-NEXT:    xorps %xmm0, %xmm0
@@ -310,7 +310,7 @@ define i64 @loopdep2(ptr nocapture %x, ptr nocapture %y) nounwind {
 ; SSE-WIN-NEXT:    movaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; SSE-WIN-NEXT:    movq (%rcx), %rax
 ; SSE-WIN-NEXT:    movl $1, %ecx
-; SSE-WIN-NEXT:    .p2align 4, 0x90
+; SSE-WIN-NEXT:    .p2align 4
 ; SSE-WIN-NEXT:  .LBB7_1: # %loop
 ; SSE-WIN-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE-WIN-NEXT:    xorps %xmm0, %xmm0
@@ -355,7 +355,7 @@ define i64 @loopdep2(ptr nocapture %x, ptr nocapture %y) nounwind {
 ; AVX-NEXT:    vmovaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX-NEXT:    movq (%rcx), %rax
 ; AVX-NEXT:    movl $1, %ecx
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  .LBB7_1: # %loop
 ; AVX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
@@ -418,12 +418,12 @@ define dso_local void @loopdep3() {
 ; SSE-LINUX-LABEL: loopdep3:
 ; SSE-LINUX:       # %bb.0: # %entry
 ; SSE-LINUX-NEXT:    xorl %eax, %eax
-; SSE-LINUX-NEXT:    .p2align 4, 0x90
+; SSE-LINUX-NEXT:    .p2align 4
 ; SSE-LINUX-NEXT:  .LBB8_1: # %for.cond1.preheader
 ; SSE-LINUX-NEXT:    # =>This Loop Header: Depth=1
 ; SSE-LINUX-NEXT:    # Child Loop BB8_2 Depth 2
 ; SSE-LINUX-NEXT:    movq $-4096, %rcx # imm = 0xF000
-; SSE-LINUX-NEXT:    .p2align 4, 0x90
+; SSE-LINUX-NEXT:    .p2align 4
 ; SSE-LINUX-NEXT:  .LBB8_2: # %for.body3
 ; SSE-LINUX-NEXT:    # Parent Loop BB8_1 Depth=1
 ; SSE-LINUX-NEXT:    # => This Inner Loop Header: Depth=2
@@ -478,13 +478,13 @@ define dso_local void @loopdep3() {
 ; SSE-WIN-NEXT:    leaq y(%rip), %r8
 ; SSE-WIN-NEXT:    leaq z(%rip), %r9
 ; SSE-WIN-NEXT:    leaq w(%rip), %r10
-; SSE-WIN-NEXT:    .p2align 4, 0x90
+; SSE-WIN-NEXT:    .p2align 4
 ; SSE-WIN-NEXT:  .LBB8_1: # %for.cond1.preheader
 ; SSE-WIN-NEXT:    # =>This Loop Header: Depth=1
 ; SSE-WIN-NEXT:    # Child Loop BB8_2 Depth 2
 ; SSE-WIN-NEXT:    movq %rcx, %r11
 ; SSE-WIN-NEXT:    xorl %esi, %esi
-; SSE-WIN-NEXT:    .p2align 4, 0x90
+; SSE-WIN-NEXT:    .p2align 4
 ; SSE-WIN-NEXT:  .LBB8_2: # %for.body3
 ; SSE-WIN-NEXT:    # Parent Loop BB8_1 Depth=1
 ; SSE-WIN-NEXT:    # => This Inner Loop Header: Depth=2
@@ -554,13 +554,13 @@ define dso_local void @loopdep3() {
 ; AVX-NEXT:    leaq y(%rip), %r8
 ; AVX-NEXT:    leaq z(%rip), %r9
 ; AVX-NEXT:    leaq w(%rip), %r10
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  .LBB8_1: # %for.cond1.preheader
 ; AVX-NEXT:    # =>This Loop Header: Depth=1
 ; AVX-NEXT:    # Child Loop BB8_2 Depth 2
 ; AVX-NEXT:    movq %rcx, %r11
 ; AVX-NEXT:    xorl %esi, %esi
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  .LBB8_2: # %for.body3
 ; AVX-NEXT:    # Parent Loop BB8_1 Depth=1
 ; AVX-NEXT:    # => This Inner Loop Header: Depth=2
@@ -1114,7 +1114,7 @@ define i64 @loopclearence(ptr nocapture %x, ptr nocapture %y) nounwind {
 ; SSE-LINUX:       # %bb.0: # %entry
 ; SSE-LINUX-NEXT:    movq (%rdi), %rax
 ; SSE-LINUX-NEXT:    movl $1, %ecx
-; SSE-LINUX-NEXT:    .p2align 4, 0x90
+; SSE-LINUX-NEXT:    .p2align 4
 ; SSE-LINUX-NEXT:  .LBB12_1: # %loop
 ; SSE-LINUX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE-LINUX-NEXT:    xorps %xmm4, %xmm4
@@ -1155,7 +1155,7 @@ define i64 @loopclearence(ptr nocapture %x, ptr nocapture %y) nounwind {
 ; SSE-WIN-NEXT:    movaps %xmm8, (%rsp) # 16-byte Spill
 ; SSE-WIN-NEXT:    movq (%rcx), %rax
 ; SSE-WIN-NEXT:    movl $1, %ecx
-; SSE-WIN-NEXT:    .p2align 4, 0x90
+; SSE-WIN-NEXT:    .p2align 4
 ; SSE-WIN-NEXT:  .LBB12_1: # %loop
 ; SSE-WIN-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE-WIN-NEXT:    xorps %xmm4, %xmm4
@@ -1205,7 +1205,7 @@ define i64 @loopclearence(ptr nocapture %x, ptr nocapture %y) nounwind {
 ; AVX-NEXT:    vmovaps %xmm8, (%rsp) # 16-byte Spill
 ; AVX-NEXT:    movq (%rcx), %rax
 ; AVX-NEXT:    movl $1, %ecx
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  .LBB12_1: # %loop
 ; AVX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    vcvtsi2sd %rcx, %xmm5, %xmm4
@@ -1288,7 +1288,7 @@ define dso_local void @loopclearance2(ptr nocapture %y, ptr %x, double %c1, doub
 ; SSE-LINUX-NEXT:    #NO_APP
 ; SSE-LINUX-NEXT:    movl $1, %eax
 ; SSE-LINUX-NEXT:    xorl %ecx, %ecx
-; SSE-LINUX-NEXT:    .p2align 4, 0x90
+; SSE-LINUX-NEXT:    .p2align 4
 ; SSE-LINUX-NEXT:  .LBB13_1: # %inner_loop
 ; SSE-LINUX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE-LINUX-NEXT:    movq %rcx, %r8
@@ -1360,7 +1360,7 @@ define dso_local void @loopclearance2(ptr nocapture %y, ptr %x, double %c1, doub
 ; SSE-WIN-NEXT:    #NO_APP
 ; SSE-WIN-NEXT:    movl $1, %r8d
 ; SSE-WIN-NEXT:    xorl %r9d, %r9d
-; SSE-WIN-NEXT:    .p2align 4, 0x90
+; SSE-WIN-NEXT:    .p2align 4
 ; SSE-WIN-NEXT:  .LBB13_1: # %inner_loop
 ; SSE-WIN-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE-WIN-NEXT:    movq %r9, %r10
@@ -1443,7 +1443,7 @@ define dso_local void @loopclearance2(ptr nocapture %y, ptr %x, double %c1, doub
 ; AVX1-NEXT:    #NO_APP
 ; AVX1-NEXT:    movl $1, %r8d
 ; AVX1-NEXT:    xorl %r9d, %r9d
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB13_1: # %inner_loop
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    movq %r9, %r10
@@ -1523,7 +1523,7 @@ define dso_local void @loopclearance2(ptr nocapture %y, ptr %x, double %c1, doub
 ; AVX512VL-NEXT:    #NO_APP
 ; AVX512VL-NEXT:    movl $1, %r8d
 ; AVX512VL-NEXT:    xorl %r9d, %r9d
-; AVX512VL-NEXT:    .p2align 4, 0x90
+; AVX512VL-NEXT:    .p2align 4
 ; AVX512VL-NEXT:  .LBB13_1: # %inner_loop
 ; AVX512VL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512VL-NEXT:    movq %r9, %r10
diff --git a/llvm/test/CodeGen/X86/callbr-asm-blockplacement.ll b/llvm/test/CodeGen/X86/callbr-asm-blockplacement.ll
index 941f331833c19..7f91b846447b3 100644
--- a/llvm/test/CodeGen/X86/callbr-asm-blockplacement.ll
+++ b/llvm/test/CodeGen/X86/callbr-asm-blockplacement.ll
@@ -29,7 +29,7 @@ define i32 @foo(i32 %arg, ptr %arg3) nounwind {
 ; CHECK-NEXT:    leaq global(%rax,%rax,2), %r14
 ; CHECK-NEXT:    leaq global+4(%rax,%rax,2), %r15
 ; CHECK-NEXT:    xorl %r13d, %r13d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # %bb8
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    callq bar@PLT
diff --git a/llvm/test/CodeGen/X86/cast-vsel.ll b/llvm/test/CodeGen/X86/cast-vsel.ll
index 3067ea674c00d..d5ccc152abc23 100644
--- a/llvm/test/CodeGen/X86/cast-vsel.ll
+++ b/llvm/test/CodeGen/X86/cast-vsel.ll
@@ -278,7 +278,7 @@ define dso_local void @example25() nounwind {
 ; SSE2-LABEL: example25:
 ; SSE2:       # %bb.0: # %vector.ph
 ; SSE2-NEXT:    movq $-4096, %rax # imm = 0xF000
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB5_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movaps da+4112(%rax), %xmm0
@@ -303,7 +303,7 @@ define dso_local void @example25() nounwind {
 ; SSE41-LABEL: example25:
 ; SSE41:       # %bb.0: # %vector.ph
 ; SSE41-NEXT:    movq $-4096, %rax # imm = 0xF000
-; SSE41-NEXT:    .p2align 4, 0x90
+; SSE41-NEXT:    .p2align 4
 ; SSE41-NEXT:  .LBB5_1: # %vector.body
 ; SSE41-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE41-NEXT:    movaps da+4112(%rax), %xmm0
@@ -329,7 +329,7 @@ define dso_local void @example25() nounwind {
 ; AVX1:       # %bb.0: # %vector.ph
 ; AVX1-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; AVX1-NEXT:    vbroadcastss {{.*#+}} ymm0 = [1,1,1,1,1,1,1,1]
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB5_1: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vmovups da+4096(%rax), %ymm1
@@ -348,7 +348,7 @@ define dso_local void @example25() nounwind {
 ; AVX2-LABEL: example25:
 ; AVX2:       # %bb.0: # %vector.ph
 ; AVX2-NEXT:    movq $-4096, %rax # imm = 0xF000
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB5_1: # %vector.body
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    vmovups da+4096(%rax), %ymm0
@@ -402,7 +402,7 @@ define dso_local void @example24(i16 signext %x, i16 signext %y) nounwind {
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; SSE2-NEXT:    movq $-4096, %rax # imm = 0xF000
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB6_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movaps da+4096(%rax), %xmm2
@@ -434,7 +434,7 @@ define dso_local void @example24(i16 signext %x, i16 signext %y) nounwind {
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,0,0]
 ; SSE41-NEXT:    movq $-4096, %rax # imm = 0xF000
-; SSE41-NEXT:    .p2align 4, 0x90
+; SSE41-NEXT:    .p2align 4
 ; SSE41-NEXT:  .LBB6_1: # %vector.body
 ; SSE41-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE41-NEXT:    movaps da+4096(%rax), %xmm0
@@ -469,7 +469,7 @@ define dso_local void @example24(i16 signext %x, i16 signext %y) nounwind {
 ; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm2
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxwd %xmm3, %xmm3
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB6_1: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vmovups da+4096(%rax), %ymm4
@@ -494,7 +494,7 @@ define dso_local void @example24(i16 signext %x, i16 signext %y) nounwind {
 ; AVX2-NEXT:    movq $-4096, %rax # imm = 0xF000
 ; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
 ; AVX2-NEXT:    vpmovsxwd %xmm1, %ymm1
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB6_1: # %vector.body
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    vmovups da+4096(%rax), %ymm2
diff --git a/llvm/test/CodeGen/X86/cmpxchg-clobber-flags.ll b/llvm/test/CodeGen/X86/cmpxchg-clobber-flags.ll
index 29751dcfca5d6..4877d15e44a4f 100644
--- a/llvm/test/CodeGen/X86/cmpxchg-clobber-flags.ll
+++ b/llvm/test/CodeGen/X86/cmpxchg-clobber-flags.ll
@@ -127,12 +127,12 @@ define i32 @test_control_flow(ptr %p, i32 %i, i32 %j) nounwind {
 ; X86-ALL-NEXT:    jle .LBB1_6
 ; X86-ALL-NEXT:  # %bb.1: # %loop_start
 ; X86-ALL-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-ALL-NEXT:    .p2align 4, 0x90
+; X86-ALL-NEXT:    .p2align 4
 ; X86-ALL-NEXT:  .LBB1_2: # %while.condthread-pre-split.i
 ; X86-ALL-NEXT:    # =>This Loop Header: Depth=1
 ; X86-ALL-NEXT:    # Child Loop BB1_3 Depth 2
 ; X86-ALL-NEXT:    movl (%ecx), %edx
-; X86-ALL-NEXT:    .p2align 4, 0x90
+; X86-ALL-NEXT:    .p2align 4
 ; X86-ALL-NEXT:  .LBB1_3: # %while.cond.i
 ; X86-ALL-NEXT:    # Parent Loop BB1_2 Depth=1
 ; X86-ALL-NEXT:    # => This Inner Loop Header: Depth=2
@@ -154,12 +154,12 @@ define i32 @test_control_flow(ptr %p, i32 %i, i32 %j) nounwind {
 ; X64-ALL-NEXT:    movl %esi, %eax
 ; X64-ALL-NEXT:    cmpl %edx, %esi
 ; X64-ALL-NEXT:    jle .LBB1_5
-; X64-ALL-NEXT:    .p2align 4, 0x90
+; X64-ALL-NEXT:    .p2align 4
 ; X64-ALL-NEXT:  .LBB1_1: # %while.condthread-pre-split.i
 ; X64-ALL-NEXT:    # =>This Loop Header: Depth=1
 ; X64-ALL-NEXT:    # Child Loop BB1_2 Depth 2
 ; X64-ALL-NEXT:    movl (%rdi), %ecx
-; X64-ALL-NEXT:    .p2align 4, 0x90
+; X64-ALL-NEXT:    .p2align 4
 ; X64-ALL-NEXT:  .LBB1_2: # %while.cond.i
 ; X64-ALL-NEXT:    # Parent Loop BB1_1 Depth=1
 ; X64-ALL-NEXT:    # => This Inner Loop Header: Depth=2
diff --git a/llvm/test/CodeGen/X86/cmpxchg-i128-i1.ll b/llvm/test/CodeGen/X86/cmpxchg-i128-i1.ll
index 3fb561d00f97d..bb07640dd285f 100644
--- a/llvm/test/CodeGen/X86/cmpxchg-i128-i1.ll
+++ b/llvm/test/CodeGen/X86/cmpxchg-i128-i1.ll
@@ -113,7 +113,7 @@ define i128 @cmpxchg_use_eflags_and_val(ptr %addr, i128 %offset) {
 ; CHECK-NEXT:    vmovdqa (%rdi), %xmm0
 ; CHECK-NEXT:    vpextrq $1, %xmm0, %rdx
 ; CHECK-NEXT:    vmovq %xmm0, %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB4_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rdx, %r9
diff --git a/llvm/test/CodeGen/X86/coalesce-esp.ll b/llvm/test/CodeGen/X86/coalesce-esp.ll
index 34964c0afcc69..0c495c9cf2b00 100644
--- a/llvm/test/CodeGen/X86/coalesce-esp.ll
+++ b/llvm/test/CodeGen/X86/coalesce-esp.ll
@@ -27,7 +27,7 @@ define void @_ZSt17__gslice_to_indexjRKSt8valarrayIjES2_RS0_(i32 %__o, ptr nocap
 ; CHECK-NEXT:  .LBB0_1: # %bb4.preheader
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    jmp .LBB0_2
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_4: # %bb7.backedge
 ; CHECK-NEXT:    # in Loop: Header=BB0_2 Depth=1
 ; CHECK-NEXT:    addl $-4, %edx
diff --git a/llvm/test/CodeGen/X86/coalescer-breaks-subreg-to-reg-liveness-reduced.ll b/llvm/test/CodeGen/X86/coalescer-breaks-subreg-to-reg-liveness-reduced.ll
index a5295d44b07c8..e9f529eea7d3f 100644
--- a/llvm/test/CodeGen/X86/coalescer-breaks-subreg-to-reg-liveness-reduced.ll
+++ b/llvm/test/CodeGen/X86/coalescer-breaks-subreg-to-reg-liveness-reduced.ll
@@ -46,7 +46,7 @@ define void @foo(ptr %arg3, i1 %icmp16) #0 {
 ; CHECK-NEXT:    xorl %r12d, %r12d
 ; CHECK-NEXT:    # implicit-def: $r13
 ; CHECK-NEXT:    jmp .LBB0_2
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %bb5
 ; CHECK-NEXT:    # in Loop: Header=BB0_2 Depth=1
 ; CHECK-NEXT:    orl $1, %r12d
diff --git a/llvm/test/CodeGen/X86/coalescer-commute1.ll b/llvm/test/CodeGen/X86/coalescer-commute1.ll
index 93c6756131fd0..28502782cf642 100644
--- a/llvm/test/CodeGen/X86/coalescer-commute1.ll
+++ b/llvm/test/CodeGen/X86/coalescer-commute1.ll
@@ -13,7 +13,7 @@ define void @runcont(ptr %source) nounwind  {
 ; CHECK-NEXT:    movl (%ecx), %ecx
 ; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_1: ## %bb
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vcvtsi2ssl (%eax,%edx,4), %xmm2, %xmm1
diff --git a/llvm/test/CodeGen/X86/coalescer-commute4.ll b/llvm/test/CodeGen/X86/coalescer-commute4.ll
index 72ca9f17841b6..0ec99dcba88ae 100644
--- a/llvm/test/CodeGen/X86/coalescer-commute4.ll
+++ b/llvm/test/CodeGen/X86/coalescer-commute4.ll
@@ -15,7 +15,7 @@ define float @foo(ptr %x, ptr %y, i32 %c) nounwind  {
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    xorl %esi, %esi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_3: ## %bb
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    xorps %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/coalescer-dead-flag-verifier-error.ll b/llvm/test/CodeGen/X86/coalescer-dead-flag-verifier-error.ll
index 4a396599a4444..4d41c8406f6e0 100644
--- a/llvm/test/CodeGen/X86/coalescer-dead-flag-verifier-error.ll
+++ b/llvm/test/CodeGen/X86/coalescer-dead-flag-verifier-error.ll
@@ -11,7 +11,7 @@ define void @_ZNK4llvm5APInt21multiplicativeInverseERKS0_(ptr %r) {
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    xorl %ecx, %ecx
 ; CHECK-NEXT:    jmp .LBB0_1
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_4: # %_ZNK4llvm5APInt13getActiveBitsEv.exit.i.i
 ; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    movl %edx, %edx
@@ -26,7 +26,7 @@ define void @_ZNK4llvm5APInt21multiplicativeInverseERKS0_(ptr %r) {
 ; CHECK-NEXT:    movq %rcx, %r8
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    jne .LBB0_4
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_3: # %for.body.i.i.i.i.i.3
 ; CHECK-NEXT:    # Parent Loop BB0_1 Depth=1
 ; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
@@ -72,7 +72,7 @@ define void @_ZNK4llvm5APInt21multiplicativeInverseERKS0__assert(ptr %r) {
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    xorl %ecx, %ecx
 ; CHECK-NEXT:    jmp .LBB1_1
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_4: # %_ZNK4llvm5APInt13getActiveBitsEv.exit.i.i
 ; CHECK-NEXT:    # in Loop: Header=BB1_1 Depth=1
 ; CHECK-NEXT:    movl %edx, %edx
@@ -89,7 +89,7 @@ define void @_ZNK4llvm5APInt21multiplicativeInverseERKS0__assert(ptr %r) {
 ; CHECK-NEXT:    xorl %r8d, %r8d
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    jne .LBB1_4
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_3: # %for.body.i.i.i.i.i.3
 ; CHECK-NEXT:    # Parent Loop BB1_1 Depth=1
 ; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
diff --git a/llvm/test/CodeGen/X86/code-align-loops.ll b/llvm/test/CodeGen/X86/code-align-loops.ll
index 164df14df4173..68ae49792ed18 100644
--- a/llvm/test/CodeGen/X86/code-align-loops.ll
+++ b/llvm/test/CodeGen/X86/code-align-loops.ll
@@ -20,19 +20,19 @@
 ; The difference between test1 and test2 is test2 only set one loop metadata node for the second loop.
 
 ; CHECK-LABEL: test1:
-; ALIGN: .p2align 6, 0x90
+; ALIGN: .p2align 6
 ; ALIGN-NEXT: .LBB0_2: # %for.body
-; ALIGN: .p2align 9, 0x90
+; ALIGN: .p2align 9
 ; ALIGN-NEXT: .LBB0_3: # %for.body
 
-; ALIGN32: .p2align 6, 0x90
+; ALIGN32: .p2align 6
 ; ALIGN32-NEXT: .LBB0_2: # %for.body
-; ALIGN32: .p2align 9, 0x90
+; ALIGN32: .p2align 9
 ; ALIGN32-NEXT: .LBB0_3: # %for.body
 
-; ALIGN256: .p2align 8, 0x90
+; ALIGN256: .p2align 8
 ; ALIGN256-NEXT: .LBB0_2: # %for.body
-; ALIGN256: .p2align 9, 0x90
+; ALIGN256: .p2align 9
 ; ALIGN256-NEXT: .LBB0_3: # %for.body
 
 define void @test1(i32 %a) nounwind {
@@ -59,19 +59,19 @@ for.body5:                                        ; preds = %for.body, %for.body
 }
 
 ; CHECK-LABEL: test2:
-; ALIGN: .p2align 4, 0x90
+; ALIGN: .p2align 4
 ; ALIGN-NEXT: .LBB1_2: # %for.body
-; ALIGN: .p2align 9, 0x90
+; ALIGN: .p2align 9
 ; ALIGN-NEXT: .LBB1_3: # %for.body
 
-; ALIGN32: .p2align 5, 0x90
+; ALIGN32: .p2align 5
 ; ALIGN32-NEXT: .LBB1_2: # %for.body
-; ALIGN32: .p2align 9, 0x90
+; ALIGN32: .p2align 9
 ; ALIGN32-NEXT: .LBB1_3: # %for.body
 
-; ALIGN256: .p2align 8, 0x90
+; ALIGN256: .p2align 8
 ; ALIGN256-NEXT: .LBB1_2: # %for.body
-; ALIGN256: .p2align 9, 0x90
+; ALIGN256: .p2align 9
 ; ALIGN256-NEXT: .LBB1_3: # %for.body
 define void @test2(i32 %a) nounwind {
 entry:
@@ -111,7 +111,7 @@ for.body5:                                        ; preds = %for.body, %for.body
 ;     }
 ; }
 ; CHECK-LABEL: test3_multilatch:
-; ALIGN: .p2align 6, 0x90
+; ALIGN: .p2align 6
 ; ALIGN-NEXT: .LBB2_1: # %while.cond
 define dso_local i32 @test3_multilatch() #0 {
 entry:
@@ -147,7 +147,7 @@ while.end:                                        ; preds = %while.cond
 }
 
 ; CHECK-LABEL: test4_multilatch:
-; ALIGN: .p2align 6, 0x90
+; ALIGN: .p2align 6
 ; ALIGN-NEXT: .LBB3_4: # %bb4
 define void @test4_multilatch(i32 %a, i32 %b, i32 %c, i32 %d) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/X86/code_placement_align_all.ll b/llvm/test/CodeGen/X86/code_placement_align_all.ll
index 11dc59a3bab9d..0e57b0e8ebd32 100644
--- a/llvm/test/CodeGen/X86/code_placement_align_all.ll
+++ b/llvm/test/CodeGen/X86/code_placement_align_all.ll
@@ -1,9 +1,9 @@
 ; RUN: llc  -mcpu=corei7 -mtriple=x86_64-linux -align-all-blocks=16 < %s | FileCheck %s
 
 ;CHECK-LABEL: foo:
-;CHECK: .p2align  16, 0x90
-;CHECK: .p2align  16, 0x90
-;CHECK: .p2align  16, 0x90
+;CHECK: .p2align  16
+;CHECK: .p2align  16
+;CHECK: .p2align  16
 ;CHECK: ret
 define i32 @foo(i32 %t, i32 %l) nounwind readnone ssp uwtable {
   %1 = icmp eq i32 %t, 0
diff --git a/llvm/test/CodeGen/X86/code_placement_ext_tsp_size.ll b/llvm/test/CodeGen/X86/code_placement_ext_tsp_size.ll
new file mode 100644
index 0000000000000..59eaf2586f173
--- /dev/null
+++ b/llvm/test/CodeGen/X86/code_placement_ext_tsp_size.ll
@@ -0,0 +1,131 @@
+; RUN: llc -mcpu=corei7 -mtriple=x86_64-linux -apply-ext-tsp-for-size=true  < %s | FileCheck %s -check-prefix=CHECK-PERF
+; RUN: llc -mcpu=corei7 -mtriple=x86_64-linux -apply-ext-tsp-for-size=false < %s | FileCheck %s -check-prefix=CHECK-SIZE
+
+define void @func1() minsize {
+;
+; +-----+
+; | b0  | -+
+; +-----+  |
+;   |      |
+;   | 10   |
+;   v      |
+; +-----+  |
+; | b1  |  | 10000
+; +-----+  |
+;   |      |
+;   | 10   |
+;   v      |
+; +-----+  |
+; | b2  | <+
+; +-----+
+;
+; CHECK-PERF-LABEL: func1:
+; CHECK-PERF: %b0
+; CHECK-PERF: %b1
+; CHECK-PERF: %b2
+;
+; CHECK-SIZE-LABEL: func1:
+; CHECK-SIZE: %b0
+; CHECK-SIZE: %b2
+; CHECK-SIZE: %b1
+
+b0:
+  %call = call zeroext i1 @a()
+  br i1 %call, label %b1, label %b2, !prof !1
+
+b1:
+  call void @d()
+  call void @d()
+  call void @d()
+  br label %b2
+
+b2:
+  call void @e()
+  ret void
+}
+
+define void @func_loop() minsize !prof !9 {
+; Test that the algorithm can rotate loops in the presence of profile data.
+;
+;                  +--------+
+;                  | entry  |
+;                  +--------+
+;                    |
+;                    | 1
+;                    v
+; +--------+  16   +--------+
+; | if.then| <---- | header | <+
+; +--------+       +--------+  |
+;   |                |         |
+;   |                | 160     |
+;   |                v         |
+;   |              +--------+  |
+;   |              | if.else|  | 175
+;   |              +--------+  |
+;   |                |         |
+;   |                | 160     |
+;   |                v         |
+;   |        16    +--------+  |
+;   +------------> | if.end | -+
+;                  +--------+
+;                    |
+;                    | 1
+;                    v
+;                  +--------+
+;                  |  end   |
+;                  +--------+
+;
+; CHECK-PERF-LABEL: func_loop:
+; CHECK-PERF: %entry
+; CHECK-PERF: %header
+; CHECK-PERF: %if.then
+; CHECK-PERF: %if.else
+; CHECK-PERF: %if.end
+; CHECK-PERF: %end
+;
+; CHECK-SIZE-LABEL: func_loop:
+; CHECK-SIZE: %entry
+; CHECK-SIZE: %header
+; CHECK-SIZE: %if.else
+; CHECK-SIZE: %if.end
+; CHECK-SIZE: %if.then
+; CHECK-SIZE: %end
+
+entry:
+  br label %header
+
+header:
+  call void @e()
+  %call = call zeroext i1 @a()
+  br i1 %call, label %if.then, label %if.else, !prof !10
+
+if.then:
+  call void @f()
+  br label %if.end
+
+if.else:
+  call void @g()
+  br label %if.end
+
+if.end:
+  call void @h()
+  %call2 = call zeroext i1 @a()
+  br i1 %call2, label %header, label %end
+
+end:
+  ret void
+}
+
+
+declare zeroext i1 @a()
+declare void @b()
+declare void @c()
+declare void @d()
+declare void @e()
+declare void @g()
+declare void @f()
+declare void @h()
+
+!1 = !{!"branch_weights", i32 10, i32 10000}
+!9 = !{!"function_entry_count", i64 1}
+!10 = !{!"branch_weights", i32 16, i32 160}
diff --git a/llvm/test/CodeGen/X86/combine-pmuldq.ll b/llvm/test/CodeGen/X86/combine-pmuldq.ll
index aa3bea2791416..2a52b9eabc7b4 100644
--- a/llvm/test/CodeGen/X86/combine-pmuldq.ll
+++ b/llvm/test/CodeGen/X86/combine-pmuldq.ll
@@ -157,7 +157,7 @@ define <8 x i64> @combine_zext_pmuludq_256(<8 x i32> %a) {
 define void @PR39398(i32 %a0) {
 ; SSE-LABEL: PR39398:
 ; SSE:       # %bb.0: # %bb
-; SSE-NEXT:    .p2align 4, 0x90
+; SSE-NEXT:    .p2align 4
 ; SSE-NEXT:  .LBB5_1: # %bb10
 ; SSE-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE-NEXT:    cmpl $232, %edi
@@ -167,7 +167,7 @@ define void @PR39398(i32 %a0) {
 ;
 ; AVX-LABEL: PR39398:
 ; AVX:       # %bb.0: # %bb
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  .LBB5_1: # %bb10
 ; AVX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    cmpl $232, %edi
@@ -338,7 +338,7 @@ define <8 x i32> @PR49658_zext(ptr %ptr, i32 %mul) {
 ; SSE-NEXT:    pxor %xmm0, %xmm0
 ; SSE-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
 ; SSE-NEXT:    pxor %xmm1, %xmm1
-; SSE-NEXT:    .p2align 4, 0x90
+; SSE-NEXT:    .p2align 4
 ; SSE-NEXT:  .LBB7_1: # %loop
 ; SSE-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE-NEXT:    pmovzxdq {{.*#+}} xmm3 = mem[0],zero,mem[1],zero
@@ -367,7 +367,7 @@ define <8 x i32> @PR49658_zext(ptr %ptr, i32 %mul) {
 ; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB7_1: # %loop
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = mem[0],zero,mem[1],zero
@@ -396,7 +396,7 @@ define <8 x i32> @PR49658_zext(ptr %ptr, i32 %mul) {
 ; AVX2-NEXT:    vpbroadcastq %xmm0, %ymm1
 ; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB7_1: # %loop
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
@@ -418,7 +418,7 @@ define <8 x i32> @PR49658_zext(ptr %ptr, i32 %mul) {
 ; AVX512VL-NEXT:    vpbroadcastq %rax, %zmm1
 ; AVX512VL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
-; AVX512VL-NEXT:    .p2align 4, 0x90
+; AVX512VL-NEXT:    .p2align 4
 ; AVX512VL-NEXT:  .LBB7_1: # %loop
 ; AVX512VL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} zmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
@@ -437,7 +437,7 @@ define <8 x i32> @PR49658_zext(ptr %ptr, i32 %mul) {
 ; AVX512DQVL-NEXT:    vpbroadcastq %rax, %zmm1
 ; AVX512DQVL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
-; AVX512DQVL-NEXT:    .p2align 4, 0x90
+; AVX512DQVL-NEXT:    .p2align 4
 ; AVX512DQVL-NEXT:  .LBB7_1: # %loop
 ; AVX512DQVL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512DQVL-NEXT:    vpmovzxdq {{.*#+}} zmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
@@ -482,7 +482,7 @@ define <8 x i32> @PR49658_sext(ptr %ptr, i32 %mul) {
 ; SSE-NEXT:    movdqa %xmm2, %xmm3
 ; SSE-NEXT:    psrlq $32, %xmm3
 ; SSE-NEXT:    pxor %xmm1, %xmm1
-; SSE-NEXT:    .p2align 4, 0x90
+; SSE-NEXT:    .p2align 4
 ; SSE-NEXT:  .LBB8_1: # %loop
 ; SSE-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE-NEXT:    pmovsxdq 2097176(%rdi,%rax), %xmm5
@@ -543,7 +543,7 @@ define <8 x i32> @PR49658_sext(ptr %ptr, i32 %mul) {
 ; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB8_1: # %loop
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vpmovsxdq 2097152(%rdi,%rax), %xmm3
@@ -572,7 +572,7 @@ define <8 x i32> @PR49658_sext(ptr %ptr, i32 %mul) {
 ; AVX2-NEXT:    vpbroadcastq %xmm0, %ymm1
 ; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB8_1: # %loop
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    vpmovsxdq 2097168(%rdi,%rax), %ymm2
@@ -594,7 +594,7 @@ define <8 x i32> @PR49658_sext(ptr %ptr, i32 %mul) {
 ; AVX512VL-NEXT:    vpbroadcastq %rax, %zmm1
 ; AVX512VL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
-; AVX512VL-NEXT:    .p2align 4, 0x90
+; AVX512VL-NEXT:    .p2align 4
 ; AVX512VL-NEXT:  .LBB8_1: # %loop
 ; AVX512VL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} zmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
@@ -613,7 +613,7 @@ define <8 x i32> @PR49658_sext(ptr %ptr, i32 %mul) {
 ; AVX512DQVL-NEXT:    vpbroadcastq %rax, %zmm1
 ; AVX512DQVL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
-; AVX512DQVL-NEXT:    .p2align 4, 0x90
+; AVX512DQVL-NEXT:    .p2align 4
 ; AVX512DQVL-NEXT:  .LBB8_1: # %loop
 ; AVX512DQVL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512DQVL-NEXT:    vpmovzxdq {{.*#+}} zmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
diff --git a/llvm/test/CodeGen/X86/constant-pool-sharing.ll b/llvm/test/CodeGen/X86/constant-pool-sharing.ll
index 062d87ed035fd..9e0114c017419 100644
--- a/llvm/test/CodeGen/X86/constant-pool-sharing.ll
+++ b/llvm/test/CodeGen/X86/constant-pool-sharing.ll
@@ -12,7 +12,7 @@ define void @share_v4i32_v4f32(ptr %p, ptr %q, i1 %t) nounwind {
 ; SSE-LINUX-LABEL: share_v4i32_v4f32:
 ; SSE-LINUX:       # %bb.0: # %entry
 ; SSE-LINUX-NEXT:    movaps {{.*#+}} xmm0 = [1073741824,1073741824,1073741824,1073741824]
-; SSE-LINUX-NEXT:    .p2align 4, 0x90
+; SSE-LINUX-NEXT:    .p2align 4
 ; SSE-LINUX-NEXT:  .LBB0_1: # %loop
 ; SSE-LINUX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE-LINUX-NEXT:    movaps %xmm0, (%rdi)
@@ -25,7 +25,7 @@ define void @share_v4i32_v4f32(ptr %p, ptr %q, i1 %t) nounwind {
 ; SSE-MSVC-LABEL: share_v4i32_v4f32:
 ; SSE-MSVC:       # %bb.0: # %entry
 ; SSE-MSVC-NEXT:    movaps {{.*#+}} xmm0 = [1073741824,1073741824,1073741824,1073741824]
-; SSE-MSVC-NEXT:    .p2align 4, 0x90
+; SSE-MSVC-NEXT:    .p2align 4
 ; SSE-MSVC-NEXT:  .LBB0_1: # %loop
 ; SSE-MSVC-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE-MSVC-NEXT:    movaps %xmm0, (%rcx)
@@ -39,7 +39,7 @@ define void @share_v4i32_v4f32(ptr %p, ptr %q, i1 %t) nounwind {
 ; AVX-LINUX:       # %bb.0: # %entry
 ; AVX-LINUX-NEXT:    vbroadcastss {{.*#+}} xmm0 = [1073741824,1073741824,1073741824,1073741824]
 ; AVX-LINUX-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1073741824,1073741824,1073741824,1073741824]
-; AVX-LINUX-NEXT:    .p2align 4, 0x90
+; AVX-LINUX-NEXT:    .p2align 4
 ; AVX-LINUX-NEXT:  .LBB0_1: # %loop
 ; AVX-LINUX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX-LINUX-NEXT:    vmovaps %xmm0, (%rdi)
@@ -53,7 +53,7 @@ define void @share_v4i32_v4f32(ptr %p, ptr %q, i1 %t) nounwind {
 ; AVX-MSVC:       # %bb.0: # %entry
 ; AVX-MSVC-NEXT:    vbroadcastss {{.*#+}} xmm0 = [1073741824,1073741824,1073741824,1073741824]
 ; AVX-MSVC-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1073741824,1073741824,1073741824,1073741824]
-; AVX-MSVC-NEXT:    .p2align 4, 0x90
+; AVX-MSVC-NEXT:    .p2align 4
 ; AVX-MSVC-NEXT:  .LBB0_1: # %loop
 ; AVX-MSVC-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX-MSVC-NEXT:    vmovaps %xmm0, (%rcx)
diff --git a/llvm/test/CodeGen/X86/copy-eflags.ll b/llvm/test/CodeGen/X86/copy-eflags.ll
index 6af80860401af..e1711ccdbe13f 100644
--- a/llvm/test/CodeGen/X86/copy-eflags.ll
+++ b/llvm/test/CodeGen/X86/copy-eflags.ll
@@ -211,7 +211,7 @@ define dso_local void @PR37100(i8 %arg1, i16 %arg2, i64 %arg3, i8 %arg4, ptr %pt
 ; X32-NEXT:    movb {{[0-9]+}}(%esp), %ch
 ; X32-NEXT:    movb {{[0-9]+}}(%esp), %cl
 ; X32-NEXT:    jmp .LBB3_1
-; X32-NEXT:    .p2align 4, 0x90
+; X32-NEXT:    .p2align 4
 ; X32-NEXT:  .LBB3_5: # %bb1
 ; X32-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; X32-NEXT:    movl %esi, %eax
@@ -250,7 +250,7 @@ define dso_local void @PR37100(i8 %arg1, i16 %arg2, i64 %arg3, i8 %arg4, ptr %pt
 ; X64-NEXT:    movq %rdx, %r10
 ; X64-NEXT:    movl {{[0-9]+}}(%rsp), %esi
 ; X64-NEXT:    movzbl %cl, %ecx
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB3_1: # %bb1
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movsbq %dil, %rax
diff --git a/llvm/test/CodeGen/X86/dag-update-nodetomatch.ll b/llvm/test/CodeGen/X86/dag-update-nodetomatch.ll
index adb7319fe80b1..bf6b09674e187 100644
--- a/llvm/test/CodeGen/X86/dag-update-nodetomatch.ll
+++ b/llvm/test/CodeGen/X86/dag-update-nodetomatch.ll
@@ -144,7 +144,7 @@ define void @_Z2x6v() local_unnamed_addr {
 ; CHECK-NEXT:    movq x0@GOTPCREL(%rip), %r15
 ; CHECK-NEXT:    movq %rsi, %r12
 ; CHECK-NEXT:    jmp .LBB1_2
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_15: # %for.cond1.for.inc3_crit_edge
 ; CHECK-NEXT:    # in Loop: Header=BB1_2 Depth=1
 ; CHECK-NEXT:    movl %edx, (%r8)
@@ -220,7 +220,7 @@ define void @_Z2x6v() local_unnamed_addr {
 ; CHECK-NEXT:    subq %rdx, %rax
 ; CHECK-NEXT:    addq %r13, %r10
 ; CHECK-NEXT:    leaq (%r11,%r10,8), %r10
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_12: # %vector.body
 ; CHECK-NEXT:    # Parent Loop BB1_2 Depth=1
 ; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
@@ -237,7 +237,7 @@ define void @_Z2x6v() local_unnamed_addr {
 ; CHECK-NEXT:    cmpq %rdx, %rbp
 ; CHECK-NEXT:    movq %r13, %rdx
 ; CHECK-NEXT:    je .LBB1_15
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_14: # %for.body2
 ; CHECK-NEXT:    # Parent Loop BB1_2 Depth=1
 ; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
diff --git a/llvm/test/CodeGen/X86/debug-spilled-snippet.ll b/llvm/test/CodeGen/X86/debug-spilled-snippet.ll
new file mode 100644
index 0000000000000..96d5d9812325f
--- /dev/null
+++ b/llvm/test/CodeGen/X86/debug-spilled-snippet.ll
@@ -0,0 +1,55 @@
+; RUN: llc -mtriple i386 %s -stop-after=livedebugvalues -o - | FileCheck %s
+
+; There should be multiple debug values for this variable after regalloc. The
+; value has been spilled, but we shouldn't lose track of the location because
+; of this.
+
+; CHECK-COUNT-4: DBG_VALUE $ebp, 0, !6, !DIExpression(DW_OP_constu, 16, DW_OP_minus), debug-location !10
+
+define void @main(i32 %call, i32 %xor.i, i1 %tobool4.not, i32 %.pre) #0 !dbg !4 {
+entry:
+  %tobool1.not = icmp ne i32 %call, 0
+  %spec.select = zext i1 %tobool1.not to i32
+  br label %for.body5
+
+for.cond.loopexit.loopexit:                       ; preds = %for.body5
+    #dbg_value(i32 %spec.select, !6, !DIExpression(), !10)
+  %tobool.not.i53 = icmp eq i32 %spec.select, 0
+  br i1 %tobool.not.i53, label %transparent_crc.exit57, label %if.then.i54
+
+for.body5:                                        ; preds = %for.body5, %entry
+  %0 = phi i32 [ 0, %entry ], [ %xor1.i40.i, %for.body5 ]
+  %xor6.i = xor i32 %.pre, %0
+  %shr7.i = ashr i32 %xor6.i, 1
+  %xor17.i = xor i32 %shr7.i, %call
+  %shr18.i = ashr i32 %xor17.i, 1
+  %xor.i.i = xor i32 %shr18.i, %xor.i
+  %arrayidx.i.i = getelementptr [0 x i32], ptr null, i32 0, i32 %xor.i.i
+  %xor1.i40.i = xor i32 %xor.i.i, %call
+  br i1 %tobool4.not, label %for.cond.loopexit.loopexit, label %for.body5
+
+if.then.i54:                                      ; preds = %for.cond.loopexit.loopexit
+  store i64 0, ptr null, align 4
+  br label %transparent_crc.exit57
+
+transparent_crc.exit57:                           ; preds = %if.then.i54, %for.cond.loopexit.loopexit
+  ret void
+}
+
+attributes #0 = { "frame-pointer"="all" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 20.0.0git.prerel", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, globals: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "xx.c", directory: "/path", checksumkind: CSK_MD5, checksum: "c4b2fc62bca9171ad484c91fb78b8842")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 20, type: !5, scopeLine: 20, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!5 = !DISubroutineType(types: !2)
+!6 = !DILocalVariable(name: "flag", arg: 2, scope: !7, file: !1, line: 8, type: !9)
+!7 = distinct !DISubprogram(name: "transparent_crc", scope: !1, file: !1, line: 8, type: !8, scopeLine: 8, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!8 = distinct !DISubroutineType(types: !2)
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DILocation(line: 0, scope: !7, inlinedAt: !11)
+!11 = distinct !DILocation(line: 28, column: 3, scope: !4)
diff --git a/llvm/test/CodeGen/X86/debug-spilled-snippet.mir b/llvm/test/CodeGen/X86/debug-spilled-snippet.mir
new file mode 100644
index 0000000000000..015701f71fefc
--- /dev/null
+++ b/llvm/test/CodeGen/X86/debug-spilled-snippet.mir
@@ -0,0 +1,100 @@
+# RUN: llc -mtriple i386 -start-before=greedy -stop-after=livedebugvars %s -o - | FileCheck %s
+
+# There should be multiple debug values for this variable after regalloc. The
+# value has been spilled, but we shouldn't lose track of the location because
+# of this.
+
+# CHECK-COUNT-4: DBG_VALUE $ebp, 0, !6, !DIExpression(DW_OP_constu, 16, DW_OP_minus), debug-location !10
+
+--- |
+  
+  define void @main() #0 !dbg !4 {
+  entry:
+  #dbg_value(i32 undef, !6, !DIExpression(), !10)
+    ret void
+  }
+  
+  attributes #0 = { "frame-pointer"="all" }
+  
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!3}
+  
+  !0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 20.0.0git.prerel", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, globals: !2, splitDebugInlining: false, nameTableKind: None)
+  !1 = !DIFile(filename: "xx.c", directory: "/path", checksumkind: CSK_MD5, checksum: "c4b2fc62bca9171ad484c91fb78b8842")
+  !2 = !{}
+  !3 = !{i32 2, !"Debug Info Version", i32 3}
+  !4 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 20, type: !5, scopeLine: 20, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+  !5 = !DISubroutineType(types: !2)
+  !6 = !DILocalVariable(name: "flag", arg: 2, scope: !7, file: !1, line: 8, type: !9)
+  !7 = distinct !DISubprogram(name: "transparent_crc", scope: !1, file: !1, line: 8, type: !8, scopeLine: 8, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+  !8 = distinct !DISubroutineType(types: !2)
+  !9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+  !10 = !DILocation(line: 0, scope: !7, inlinedAt: !11)
+  !11 = distinct !DILocation(line: 28, column: 3, scope: !4)
+
+...
+---
+name:            main
+alignment:       16
+tracksRegLiveness: true
+hasWinCFI:       false
+noPhis:          true
+fixedStack:
+  - { id: 0, type: default, offset: 0, size: 4, alignment: 4, stack-id: default, 
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, type: default, offset: 4, size: 4, alignment: 4, stack-id: default, 
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 2, type: default, offset: 8, size: 1, alignment: 4, stack-id: default, 
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 3, type: default, offset: 12, size: 4, alignment: 4, stack-id: default, 
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.0:
+    successors: %bb.2(0x80000000)
+  
+    %0:gr32 = MOV32rm %fixed-stack.3, 1, $noreg, 0, $noreg :: (load (s32) from %fixed-stack.3)
+    %1:gr8 = MOV8rm %fixed-stack.2, 1, $noreg, 0, $noreg :: (load (s8) from %fixed-stack.2, align 4)
+    %2:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s32) from %fixed-stack.0)
+    %3:gr32_abcd = MOV32r0 implicit-def dead $eflags
+    TEST32rr %2, %2, implicit-def $eflags
+    %3.sub_8bit:gr32_abcd = SETCCr 5, implicit $eflags
+    %4:gr32 = COPY %2
+    %4:gr32 = SAR32ri %4, 1, implicit-def dead $eflags
+    %5:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (load (s32) from %fixed-stack.1)
+    %6:gr32 = MOV32r0 implicit-def dead $eflags
+    JMP_1 %bb.2
+  
+  bb.1:
+    successors: %bb.4(0x30000000), %bb.3(0x50000000)
+  
+    DBG_VALUE %3, $noreg, !6, !DIExpression(), debug-location !10
+    TEST32rr %3, %3, implicit-def $eflags
+    JCC_1 %bb.4, 4, implicit $eflags
+    JMP_1 %bb.3
+  
+  bb.2:
+    successors: %bb.1(0x04000000), %bb.2(0x7c000000)
+  
+    %6:gr32 = XOR32rr %6, %0, implicit-def dead $eflags
+    %6:gr32 = SAR32ri %6, 2, implicit-def dead $eflags
+    %6:gr32 = XOR32rr %6, %4, implicit-def dead $eflags
+    %6:gr32 = XOR32rr %6, %5, implicit-def dead $eflags
+    %6:gr32 = XOR32rr %6, %2, implicit-def dead $eflags
+    TEST8ri %1, 1, implicit-def $eflags
+    JCC_1 %bb.1, 5, implicit $eflags
+    JMP_1 %bb.2
+  
+  bb.3:
+    successors: %bb.4(0x80000000)
+  
+    MOV32mi $noreg, 1, $noreg, 4, $noreg, 0 :: (store (s32) into `ptr null` + 4)
+    MOV32mi $noreg, 1, $noreg, 0, $noreg, 0 :: (store (s32) into `ptr null`)
+  
+  bb.4:
+    RET 0
+
+...
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
index c350ed64280dd..f66d81c781fe0 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
@@ -417,7 +417,7 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB4_3: # %udiv-do-while
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %edx, %esi
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
index 16dc1d6b446cf..6fdde0b14a984 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
@@ -377,7 +377,7 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB4_3: # %udiv-do-while
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
diff --git a/llvm/test/CodeGen/X86/fdiv-combine.ll b/llvm/test/CodeGen/X86/fdiv-combine.ll
index 322ef6b393922..1abcdc3cca9b7 100644
--- a/llvm/test/CodeGen/X86/fdiv-combine.ll
+++ b/llvm/test/CodeGen/X86/fdiv-combine.ll
@@ -128,7 +128,7 @@ define void @PR24141() {
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    # implicit-def: $xmm0
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB8_1: # %while.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    callq g@PLT
diff --git a/llvm/test/CodeGen/X86/fixup-lea.ll b/llvm/test/CodeGen/X86/fixup-lea.ll
index 35c3976408f64..548e74de39295 100644
--- a/llvm/test/CodeGen/X86/fixup-lea.ll
+++ b/llvm/test/CodeGen/X86/fixup-lea.ll
@@ -165,7 +165,7 @@ define void @foo_nosize(i32 inreg %dns) {
 ; SLOW-LABEL: foo_nosize:
 ; SLOW:       # %bb.0: # %entry
 ; SLOW-NEXT:    movw $-1, %cx
-; SLOW-NEXT:    .p2align 4, 0x90
+; SLOW-NEXT:    .p2align 4
 ; SLOW-NEXT:  .LBB6_1: # %for.body
 ; SLOW-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SLOW-NEXT:    movzwl %cx, %edx
@@ -178,7 +178,7 @@ define void @foo_nosize(i32 inreg %dns) {
 ; FAST-LABEL: foo_nosize:
 ; FAST:       # %bb.0: # %entry
 ; FAST-NEXT:    movw $-1, %cx
-; FAST-NEXT:    .p2align 4, 0x90
+; FAST-NEXT:    .p2align 4
 ; FAST-NEXT:  .LBB6_1: # %for.body
 ; FAST-NEXT:    # =>This Inner Loop Header: Depth=1
 ; FAST-NEXT:    movzwl %cx, %edx
@@ -205,7 +205,7 @@ define void @bar_nosize(i32 inreg %dns) {
 ; SLOW-LABEL: bar_nosize:
 ; SLOW:       # %bb.0: # %entry
 ; SLOW-NEXT:    movw $1, %cx
-; SLOW-NEXT:    .p2align 4, 0x90
+; SLOW-NEXT:    .p2align 4
 ; SLOW-NEXT:  .LBB7_1: # %for.body
 ; SLOW-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SLOW-NEXT:    movzwl %cx, %edx
@@ -218,7 +218,7 @@ define void @bar_nosize(i32 inreg %dns) {
 ; FAST-LABEL: bar_nosize:
 ; FAST:       # %bb.0: # %entry
 ; FAST-NEXT:    movw $1, %cx
-; FAST-NEXT:    .p2align 4, 0x90
+; FAST-NEXT:    .p2align 4
 ; FAST-NEXT:  .LBB7_1: # %for.body
 ; FAST-NEXT:    # =>This Inner Loop Header: Depth=1
 ; FAST-NEXT:    movzwl %cx, %edx
diff --git a/llvm/test/CodeGen/X86/fma-commute-loop.ll b/llvm/test/CodeGen/X86/fma-commute-loop.ll
index d26aaaa71e28d..ae0efa320f1e4 100644
--- a/llvm/test/CodeGen/X86/fma-commute-loop.ll
+++ b/llvm/test/CodeGen/X86/fma-commute-loop.ll
@@ -30,7 +30,7 @@ define void @eggs(ptr %arg, ptr %arg1, ptr %arg2, ptr %arg3, ptr %arg4, ptr %arg
 ; CHECK-NEXT:    vxorpd %xmm5, %xmm5, %xmm5
 ; CHECK-NEXT:    vxorpd %xmm6, %xmm6, %xmm6
 ; CHECK-NEXT:    vxorpd %xmm7, %xmm7, %xmm7
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_1: ## %bb15
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vbroadcastsd (%r11,%rbx,8), %zmm9
diff --git a/llvm/test/CodeGen/X86/fma-intrinsics-phi-213-to-231.ll b/llvm/test/CodeGen/X86/fma-intrinsics-phi-213-to-231.ll
index a509e93ef3f83..5f1c5a5690ef2 100644
--- a/llvm/test/CodeGen/X86/fma-intrinsics-phi-213-to-231.ll
+++ b/llvm/test/CodeGen/X86/fma-intrinsics-phi-213-to-231.ll
@@ -7,7 +7,7 @@ define <2 x double> @fmaddsubpd_loop_128(i32 %iter, <2 x double> %a, <2 x double
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB0_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfmaddsub231pd {{.*#+}} xmm2 = (xmm0 * xmm1) +/- xmm2
@@ -44,7 +44,7 @@ define <2 x double> @fmsubaddpd_loop_128(i32 %iter, <2 x double> %a, <2 x double
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB1_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfmsubadd231pd {{.*#+}} xmm2 = (xmm0 * xmm1) -/+ xmm2
@@ -81,7 +81,7 @@ define <2 x double> @fmaddpd_loop_128(i32 %iter, <2 x double> %a, <2 x double> %
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB2_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB2_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfmadd231pd {{.*#+}} xmm2 = (xmm0 * xmm1) + xmm2
@@ -118,7 +118,7 @@ define <2 x double> @fmsubpd_loop_128(i32 %iter, <2 x double> %a, <2 x double> %
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB3_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB3_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfmsub231pd {{.*#+}} xmm2 = (xmm0 * xmm1) - xmm2
@@ -155,7 +155,7 @@ define <2 x double> @fnmaddpd_loop_128(i32 %iter, <2 x double> %a, <2 x double>
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB4_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB4_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfnmadd231pd {{.*#+}} xmm2 = -(xmm0 * xmm1) + xmm2
@@ -192,7 +192,7 @@ define <2 x double> @fnmsubpd_loop_128(i32 %iter, <2 x double> %a, <2 x double>
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB5_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB5_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfnmsub231pd {{.*#+}} xmm2 = -(xmm0 * xmm1) - xmm2
@@ -236,7 +236,7 @@ define <4 x float> @fmaddsubps_loop_128(i32 %iter, <4 x float> %a, <4 x float> %
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB6_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB6_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfmaddsub231ps {{.*#+}} xmm2 = (xmm0 * xmm1) +/- xmm2
@@ -273,7 +273,7 @@ define <4 x float> @fmsubaddps_loop_128(i32 %iter, <4 x float> %a, <4 x float> %
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB7_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB7_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfmsubadd231ps {{.*#+}} xmm2 = (xmm0 * xmm1) -/+ xmm2
@@ -310,7 +310,7 @@ define <4 x float> @fmaddps_loop_128(i32 %iter, <4 x float> %a, <4 x float> %b,
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB8_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB8_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfmadd231ps {{.*#+}} xmm2 = (xmm0 * xmm1) + xmm2
@@ -347,7 +347,7 @@ define <4 x float> @fmsubps_loop_128(i32 %iter, <4 x float> %a, <4 x float> %b,
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB9_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB9_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfmsub231ps {{.*#+}} xmm2 = (xmm0 * xmm1) - xmm2
@@ -384,7 +384,7 @@ define <4 x float> @fnmaddps_loop_128(i32 %iter, <4 x float> %a, <4 x float> %b,
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB10_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB10_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfnmadd231ps {{.*#+}} xmm2 = -(xmm0 * xmm1) + xmm2
@@ -421,7 +421,7 @@ define <4 x float> @fnmsubps_loop_128(i32 %iter, <4 x float> %a, <4 x float> %b,
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB11_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB11_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfnmsub231ps {{.*#+}} xmm2 = -(xmm0 * xmm1) - xmm2
@@ -465,7 +465,7 @@ define <4 x double> @fmaddsubpd_loop_256(i32 %iter, <4 x double> %a, <4 x double
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB12_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB12_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfmaddsub231pd {{.*#+}} ymm2 = (ymm0 * ymm1) +/- ymm2
@@ -502,7 +502,7 @@ define <4 x double> @fmsubaddpd_loop_256(i32 %iter, <4 x double> %a, <4 x double
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB13_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB13_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfmsubadd231pd {{.*#+}} ymm2 = (ymm0 * ymm1) -/+ ymm2
@@ -539,7 +539,7 @@ define <4 x double> @fmaddpd_loop_256(i32 %iter, <4 x double> %a, <4 x double> %
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB14_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB14_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfmadd231pd {{.*#+}} ymm2 = (ymm0 * ymm1) + ymm2
@@ -576,7 +576,7 @@ define <4 x double> @fmsubpd_loop_256(i32 %iter, <4 x double> %a, <4 x double> %
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB15_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB15_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfmsub231pd {{.*#+}} ymm2 = (ymm0 * ymm1) - ymm2
@@ -613,7 +613,7 @@ define <4 x double> @fnmaddpd_loop_256(i32 %iter, <4 x double> %a, <4 x double>
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB16_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB16_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfnmadd231pd {{.*#+}} ymm2 = -(ymm0 * ymm1) + ymm2
@@ -650,7 +650,7 @@ define <4 x double> @fnmsubpd_loop_256(i32 %iter, <4 x double> %a, <4 x double>
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB17_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB17_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfnmsub231pd {{.*#+}} ymm2 = -(ymm0 * ymm1) - ymm2
@@ -694,7 +694,7 @@ define <8 x float> @fmaddsubps_loop_256(i32 %iter, <8 x float> %a, <8 x float> %
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB18_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB18_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfmaddsub231ps {{.*#+}} ymm2 = (ymm0 * ymm1) +/- ymm2
@@ -731,7 +731,7 @@ define <8 x float> @fmsubaddps_loop_256(i32 %iter, <8 x float> %a, <8 x float> %
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB19_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB19_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfmsubadd231ps {{.*#+}} ymm2 = (ymm0 * ymm1) -/+ ymm2
@@ -768,7 +768,7 @@ define <8 x float> @fmaddps_loop_256(i32 %iter, <8 x float> %a, <8 x float> %b,
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB20_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB20_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfmadd231ps {{.*#+}} ymm2 = (ymm0 * ymm1) + ymm2
@@ -805,7 +805,7 @@ define <8 x float> @fmsubps_loop_256(i32 %iter, <8 x float> %a, <8 x float> %b,
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB21_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB21_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfmsub231ps {{.*#+}} ymm2 = (ymm0 * ymm1) - ymm2
@@ -842,7 +842,7 @@ define <8 x float> @fnmaddps_loop_256(i32 %iter, <8 x float> %a, <8 x float> %b,
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB22_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB22_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfnmadd231ps {{.*#+}} ymm2 = -(ymm0 * ymm1) + ymm2
@@ -879,7 +879,7 @@ define <8 x float> @fnmsubps_loop_256(i32 %iter, <8 x float> %a, <8 x float> %b,
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl %edi, %eax
 ; CHECK-NEXT:    jge .LBB23_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB23_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vfnmsub231ps {{.*#+}} ymm2 = -(ymm0 * ymm1) - ymm2
diff --git a/llvm/test/CodeGen/X86/fold-call-3.ll b/llvm/test/CodeGen/X86/fold-call-3.ll
index 691f46b9eeb0e..1db5cba8892d1 100644
--- a/llvm/test/CodeGen/X86/fold-call-3.ll
+++ b/llvm/test/CodeGen/X86/fold-call-3.ll
@@ -22,7 +22,7 @@ define void @_Z25RawPointerPerformanceTestPvRN5clang6ActionE(ptr %Val, ptr %Acti
 ; CHECK-NEXT:    movq %rsi, %rbx
 ; CHECK-NEXT:    movq %rdi, %rax
 ; CHECK-NEXT:    xorl %ebp, %ebp
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_2: ## %bb
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq (%rbx), %rcx
@@ -52,7 +52,7 @@ define void @_Z25RawPointerPerformanceTestPvRN5clang6ActionE(ptr %Val, ptr %Acti
 ; pre-RA-NEXT:    movq %rsi, %rbx
 ; pre-RA-NEXT:    movq %rdi, %rax
 ; pre-RA-NEXT:    xorl %ebp, %ebp
-; pre-RA-NEXT:    .p2align 4, 0x90
+; pre-RA-NEXT:    .p2align 4
 ; pre-RA-NEXT:  LBB0_2: ## %bb
 ; pre-RA-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; pre-RA-NEXT:    movq (%rbx), %rcx
diff --git a/llvm/test/CodeGen/X86/fold-loop-of-urem.ll b/llvm/test/CodeGen/X86/fold-loop-of-urem.ll
index c4b130a8b4e71..c1beb7c803b2b 100644
--- a/llvm/test/CodeGen/X86/fold-loop-of-urem.ll
+++ b/llvm/test/CodeGen/X86/fold-loop-of-urem.ll
@@ -24,7 +24,7 @@ define void @simple_urem_to_sel(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    xorl %r15d, %r15d
 ; CHECK-NEXT:    xorl %r14d, %r14d
 ; CHECK-NEXT:    xorl %r12d, %r12d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %r14d, %edi
@@ -71,7 +71,7 @@ define void @simple_urem_to_sel_fail_not_in_loop(i32 %N, i32 %rem_amt) nounwind
 ; CHECK-NEXT:  # %bb.3: # %for.body.preheader
 ; CHECK-NEXT:    movl %edi, %r14d
 ; CHECK-NEXT:    xorl %ebp, %ebp
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_4: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %ebp, %edi
@@ -129,7 +129,7 @@ define void @simple_urem_to_sel_inner_loop(i32 %N, i32 %M) nounwind {
 ; CHECK-NEXT:    xorl %r15d, %r15d
 ; CHECK-NEXT:    xorl %r13d, %r13d
 ; CHECK-NEXT:    jmp .LBB2_2
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB2_5: # %for.inner.cond.cleanup
 ; CHECK-NEXT:    # in Loop: Header=BB2_2 Depth=1
 ; CHECK-NEXT:    incl %r15d
@@ -147,7 +147,7 @@ define void @simple_urem_to_sel_inner_loop(i32 %N, i32 %M) nounwind {
 ; CHECK-NEXT:  # %bb.3: # %for.inner.body.preheader
 ; CHECK-NEXT:    # in Loop: Header=BB2_2 Depth=1
 ; CHECK-NEXT:    movl %r12d, %ebx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB2_4: # %for.inner.body
 ; CHECK-NEXT:    # Parent Loop BB2_2 Depth=1
 ; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
@@ -208,7 +208,7 @@ define void @simple_urem_to_sel_inner_loop_fail_not_invariant(i32 %N, i32 %M) no
 ; CHECK-NEXT:    movl %edi, %ebp
 ; CHECK-NEXT:    xorl %r14d, %r14d
 ; CHECK-NEXT:    jmp .LBB3_2
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB3_5: # %for.inner.cond.cleanup
 ; CHECK-NEXT:    # in Loop: Header=BB3_2 Depth=1
 ; CHECK-NEXT:    incl %r14d
@@ -224,7 +224,7 @@ define void @simple_urem_to_sel_inner_loop_fail_not_invariant(i32 %N, i32 %M) no
 ; CHECK-NEXT:    # in Loop: Header=BB3_2 Depth=1
 ; CHECK-NEXT:    movl %eax, %r15d
 ; CHECK-NEXT:    movl %ebx, %r12d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB3_4: # %for.inner.body
 ; CHECK-NEXT:    # Parent Loop BB3_2 Depth=1
 ; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
@@ -288,7 +288,7 @@ define void @simple_urem_to_sel_nested2(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    xorl %r14d, %r14d
 ; CHECK-NEXT:    xorl %r12d, %r12d
 ; CHECK-NEXT:    jmp .LBB4_2
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB4_5: # %for.body1
 ; CHECK-NEXT:    # in Loop: Header=BB4_2 Depth=1
 ; CHECK-NEXT:    movl %r14d, %edi
@@ -363,7 +363,7 @@ define void @simple_urem_fail_bad_incr3(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    movl %esi, %ebx
 ; CHECK-NEXT:    jmp .LBB5_2
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB5_6: # %for.body1
 ; CHECK-NEXT:    # in Loop: Header=BB5_2 Depth=1
 ; CHECK-NEXT:    movl %ebp, %eax
@@ -392,7 +392,7 @@ define void @simple_urem_fail_bad_incr3(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    movl %eax, %ebp
 ; CHECK-NEXT:    incl %ebp
 ; CHECK-NEXT:    jmp .LBB5_6
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB5_5: # %for.body2
 ; CHECK-NEXT:    # in Loop: Header=BB5_2 Depth=1
 ; CHECK-NEXT:    xorl %ebp, %ebp
@@ -442,7 +442,7 @@ define void @simple_urem_to_sel_vec(<2 x i64> %rem_amt) nounwind {
 ; CHECK-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-NEXT:    pxor %xmm0, %xmm0
 ; CHECK-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB6_1: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movdqa %xmm1, (%rsp) # 16-byte Spill
@@ -496,7 +496,7 @@ define void @simple_urem_fail_bad_incr(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    movl %edi, %ebp
 ; CHECK-NEXT:    xorl %r14d, %r14d
 ; CHECK-NEXT:    jmp .LBB7_2
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB7_4: # %for.body.tail
 ; CHECK-NEXT:    # in Loop: Header=BB7_2 Depth=1
 ; CHECK-NEXT:    movl %r14d, %eax
@@ -566,7 +566,7 @@ define void @simple_urem_to_sel_second_acc(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    xorl %r12d, %r12d
 ; CHECK-NEXT:    xorl %r14d, %r14d
 ; CHECK-NEXT:    xorl %r13d, %r13d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB8_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %r14d, %edi
@@ -618,7 +618,7 @@ define void @simple_urem_fail_srem(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    movl %esi, %ebx
 ; CHECK-NEXT:    movl %edi, %ebp
 ; CHECK-NEXT:    xorl %r14d, %r14d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB9_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %r14d, %eax
@@ -663,7 +663,7 @@ define void @simple_urem_fail_missing_nuw(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    movl %esi, %ebx
 ; CHECK-NEXT:    movl %edi, %ebp
 ; CHECK-NEXT:    xorl %r14d, %r14d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB10_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %r14d, %eax
@@ -708,7 +708,7 @@ define void @simple_urem_fail_bad_incr2(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    movl %esi, %ebx
 ; CHECK-NEXT:    movl %edi, %ebp
 ; CHECK-NEXT:    xorl %r14d, %r14d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB11_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %r14d, %eax
@@ -753,7 +753,7 @@ define void @simple_urem_non_zero_entry4(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    movl %esi, %ebx
 ; CHECK-NEXT:    movl %edi, %ebp
 ; CHECK-NEXT:    movl $4, %r14d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB12_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %r14d, %eax
@@ -799,7 +799,7 @@ define void @simple_urem_skip_const_rem_amt(i32 %N) nounwind {
 ; CHECK-NEXT:    addl $-4, %ebx
 ; CHECK-NEXT:    movl $4, %ebp
 ; CHECK-NEXT:    movl $2938661835, %r14d # imm = 0xAF286BCB
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB13_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %ebp, %eax
@@ -855,7 +855,7 @@ define void @simple_urem_fail_no_preheader_non_canonical(i32 %N, i32 %rem_amt) n
 ; CHECK-NEXT:    jmp .LBB14_3
 ; CHECK-NEXT:  .LBB14_1:
 ; CHECK-NEXT:    xorl %r14d, %r14d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB14_3: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %r14d, %eax
@@ -913,7 +913,7 @@ define void @simple_urem_multi_latch_non_canonical(i32 %N, i32 %rem_amt) nounwin
 ; CHECK-NEXT:    xorl %r14d, %r14d
 ; CHECK-NEXT:    xorl %r13d, %r13d
 ; CHECK-NEXT:    jmp .LBB15_2
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB15_3: # %for.body.backedge
 ; CHECK-NEXT:    # in Loop: Header=BB15_2 Depth=1
 ; CHECK-NEXT:    incl %r14d
@@ -1038,7 +1038,7 @@ define void @simple_urem_fail_intermediate_inc(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    movl %edi, %r14d
 ; CHECK-NEXT:    negl %r14d
 ; CHECK-NEXT:    movl $1, %r15d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB17_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %r15d, %eax
@@ -1078,7 +1078,7 @@ for.body:
 define void @weird_loop(i64 %sub.ptr.div.i56) personality ptr null {
 ; CHECK-LABEL: weird_loop:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB18_1: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp .LBB18_1
@@ -1107,7 +1107,7 @@ define void @simple_urem_to_sel_non_zero_start_fail(i32 %N, i32 %rem_amt) nounwi
 ; CHECK-NEXT:    movl %esi, %ebx
 ; CHECK-NEXT:    movl %edi, %ebp
 ; CHECK-NEXT:    movl $2, %r14d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB19_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %r14d, %eax
@@ -1157,7 +1157,7 @@ define void @simple_urem_to_sel_non_zero_start_okay(i32 %N, i32 %rem_amt_in) nou
 ; CHECK-NEXT:    movl $2, %r14d
 ; CHECK-NEXT:    xorl %r15d, %r15d
 ; CHECK-NEXT:    movl $2, %r12d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB20_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %r14d, %edi
@@ -1207,7 +1207,7 @@ define void @simple_urem_to_sel_non_zero_start_through_add(i32 %N, i32 %rem_amt_
 ; CHECK-NEXT:    orl $16, %ebx
 ; CHECK-NEXT:    negl %r14d
 ; CHECK-NEXT:    movl $7, %r15d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB21_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %r15d, %eax
@@ -1259,7 +1259,7 @@ define void @simple_urem_to_sel_non_zero_start_through_add_fail_missing_nuw(i32
 ; CHECK-NEXT:    orl $16, %ebx
 ; CHECK-NEXT:    negl %r14d
 ; CHECK-NEXT:    movl $7, %r15d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB22_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %r15d, %eax
@@ -1310,7 +1310,7 @@ define void @simple_urem_to_sel_non_zero_start_through_add_fail_no_simplify_rem(
 ; CHECK-NEXT:    movl %edi, %r14d
 ; CHECK-NEXT:    negl %r14d
 ; CHECK-NEXT:    movl $7, %r15d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB23_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %r15d, %eax
@@ -1363,7 +1363,7 @@ define void @simple_urem_to_sel_non_zero_start_through_sub(i32 %N, i32 %rem_amt,
 ; CHECK-NEXT:    xorl %r15d, %r15d
 ; CHECK-NEXT:    xorl %r14d, %r14d
 ; CHECK-NEXT:    xorl %r12d, %r12d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB24_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %r14d, %edi
@@ -1412,7 +1412,7 @@ define void @simple_urem_to_sel_non_zero_start_through_sub_no_simplfy(i32 %N, i3
 ; CHECK-NEXT:    movl %edi, %r14d
 ; CHECK-NEXT:    negl %r14d
 ; CHECK-NEXT:    addl $-2, %r15d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB25_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %r15d, %eax
diff --git a/llvm/test/CodeGen/X86/fp-une-cmp.ll b/llvm/test/CodeGen/X86/fp-une-cmp.ll
index f1b1732f9092c..dd32f23d3b9cb 100644
--- a/llvm/test/CodeGen/X86/fp-une-cmp.ll
+++ b/llvm/test/CodeGen/X86/fp-une-cmp.ll
@@ -109,7 +109,7 @@ define void @pr27750(ptr %b, float %x, i1 %y) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorps %xmm1, %xmm1
 ; CHECK-NEXT:    jmp .LBB3_1
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB3_3: # %for.end
 ; CHECK-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; CHECK-NEXT:    ucomiss %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/hoist-invariant-load.ll b/llvm/test/CodeGen/X86/hoist-invariant-load.ll
index ee23c922174d6..68e10c0c98871 100644
--- a/llvm/test/CodeGen/X86/hoist-invariant-load.ll
+++ b/llvm/test/CodeGen/X86/hoist-invariant-load.ll
@@ -43,7 +43,7 @@ define void @test(ptr %x) uwtable ssp {
 ; CHECK-NEXT:    movl $10000, %ebp ## imm = 0x2710
 ; CHECK-NEXT:    movq L_OBJC_SELECTOR_REFERENCES_(%rip), %r14
 ; CHECK-NEXT:    movq _objc_msgSend@GOTPCREL(%rip), %r15
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_1: ## %for.body
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rbx, %rdi
@@ -94,7 +94,7 @@ define void @test_unordered(ptr %x) uwtable ssp {
 ; CHECK-NEXT:    movl $10000, %ebp ## imm = 0x2710
 ; CHECK-NEXT:    movq L_OBJC_SELECTOR_REFERENCES_(%rip), %r14
 ; CHECK-NEXT:    movq _objc_msgSend@GOTPCREL(%rip), %r15
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB1_1: ## %for.body
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rbx, %rdi
@@ -139,7 +139,7 @@ define void @test_volatile(ptr %x) uwtable ssp {
 ; CHECK-NEXT:    movq %rdi, %rbx
 ; CHECK-NEXT:    movl $10000, %ebp ## imm = 0x2710
 ; CHECK-NEXT:    movq _objc_msgSend@GOTPCREL(%rip), %r14
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB2_1: ## %for.body
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq L_OBJC_SELECTOR_REFERENCES_(%rip), %rsi
@@ -182,7 +182,7 @@ define void @test_seq_cst(ptr %x) uwtable ssp {
 ; CHECK-NEXT:    movq %rdi, %rbx
 ; CHECK-NEXT:    movl $10000, %ebp ## imm = 0x2710
 ; CHECK-NEXT:    movq _objc_msgSend@GOTPCREL(%rip), %r14
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB3_1: ## %for.body
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq L_OBJC_SELECTOR_REFERENCES_(%rip), %rsi
@@ -219,7 +219,7 @@ define void @test_multi_def(ptr dereferenceable(8) align(8) %x1,
 ; CHECK-NEXT:    xorl %r8d, %r8d
 ; CHECK-NEXT:    movq (%rdi), %rdx
 ; CHECK-NEXT:    movq (%rsi), %rsi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB4_2: ## %for.body
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    mulxq %rsi, %r9, %rdi
@@ -267,7 +267,7 @@ define void @test_div_def(ptr dereferenceable(8) align(8) %x1,
 ; CHECK-NEXT:    xorl %r9d, %r9d
 ; CHECK-NEXT:    movl (%rdi), %edi
 ; CHECK-NEXT:    movl (%rsi), %esi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB5_2: ## %for.body
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %edi, %eax
diff --git a/llvm/test/CodeGen/X86/i128-mul.ll b/llvm/test/CodeGen/X86/i128-mul.ll
index 9f58ed0843348..cffd88c55bb0a 100644
--- a/llvm/test/CodeGen/X86/i128-mul.ll
+++ b/llvm/test/CodeGen/X86/i128-mul.ll
@@ -115,7 +115,7 @@ define i64 @mul1(i64 %n, ptr nocapture %z, ptr nocapture %x, i64 %y) nounwind {
 ; X86-NOBMI-NEXT:    xorl %edx, %edx
 ; X86-NOBMI-NEXT:    xorl %ecx, %ecx
 ; X86-NOBMI-NEXT:    movl $0, (%esp) # 4-byte Folded Spill
-; X86-NOBMI-NEXT:    .p2align 4, 0x90
+; X86-NOBMI-NEXT:    .p2align 4
 ; X86-NOBMI-NEXT:  .LBB1_2: # %for.body
 ; X86-NOBMI-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NOBMI-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -194,7 +194,7 @@ define i64 @mul1(i64 %n, ptr nocapture %z, ptr nocapture %x, i64 %y) nounwind {
 ; X86-BMI-NEXT:    xorl %eax, %eax
 ; X86-BMI-NEXT:    xorl %ebx, %ebx
 ; X86-BMI-NEXT:    xorl %ebp, %ebp
-; X86-BMI-NEXT:    .p2align 4, 0x90
+; X86-BMI-NEXT:    .p2align 4
 ; X86-BMI-NEXT:  .LBB1_2: # %for.body
 ; X86-BMI-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-BMI-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -263,7 +263,7 @@ define i64 @mul1(i64 %n, ptr nocapture %z, ptr nocapture %x, i64 %y) nounwind {
 ; X64-NOBMI-NEXT:    movq %rdx, %r8
 ; X64-NOBMI-NEXT:    xorl %r10d, %r10d
 ; X64-NOBMI-NEXT:    xorl %r9d, %r9d
-; X64-NOBMI-NEXT:    .p2align 4, 0x90
+; X64-NOBMI-NEXT:    .p2align 4
 ; X64-NOBMI-NEXT:  .LBB1_2: # %for.body
 ; X64-NOBMI-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NOBMI-NEXT:    movq %rcx, %rax
@@ -287,7 +287,7 @@ define i64 @mul1(i64 %n, ptr nocapture %z, ptr nocapture %x, i64 %y) nounwind {
 ; X64-BMI-NEXT:    movq %rdx, %rax
 ; X64-BMI-NEXT:    xorl %r9d, %r9d
 ; X64-BMI-NEXT:    xorl %r8d, %r8d
-; X64-BMI-NEXT:    .p2align 4, 0x90
+; X64-BMI-NEXT:    .p2align 4
 ; X64-BMI-NEXT:  .LBB1_2: # %for.body
 ; X64-BMI-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-BMI-NEXT:    movq %rcx, %rdx
diff --git a/llvm/test/CodeGen/X86/i386-shrink-wrapping.ll b/llvm/test/CodeGen/X86/i386-shrink-wrapping.ll
index d282a8f42622b..7bb3bf42d3361 100644
--- a/llvm/test/CodeGen/X86/i386-shrink-wrapping.ll
+++ b/llvm/test/CodeGen/X86/i386-shrink-wrapping.ll
@@ -30,7 +30,7 @@ define i32 @eflagsLiveInPrologue() #0 {
 ; ENABLE-NEXT:    movl (%eax), %eax
 ; ENABLE-NEXT:    testl %eax, %eax
 ; ENABLE-NEXT:    je LBB0_4
-; ENABLE-NEXT:    .p2align 4, 0x90
+; ENABLE-NEXT:    .p2align 4
 ; ENABLE-NEXT:  LBB0_3: ## %for.body
 ; ENABLE-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; ENABLE-NEXT:    jmp LBB0_3
@@ -70,7 +70,7 @@ define i32 @eflagsLiveInPrologue() #0 {
 ; DISABLE-NEXT:    movl (%eax), %eax
 ; DISABLE-NEXT:    testl %eax, %eax
 ; DISABLE-NEXT:    je LBB0_4
-; DISABLE-NEXT:    .p2align 4, 0x90
+; DISABLE-NEXT:    .p2align 4
 ; DISABLE-NEXT:  LBB0_3: ## %for.body
 ; DISABLE-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; DISABLE-NEXT:    jmp LBB0_3
diff --git a/llvm/test/CodeGen/X86/icmp-shift-opt.ll b/llvm/test/CodeGen/X86/icmp-shift-opt.ll
index 1673649f5c07b..c52b3ed6c926d 100644
--- a/llvm/test/CodeGen/X86/icmp-shift-opt.ll
+++ b/llvm/test/CodeGen/X86/icmp-shift-opt.ll
@@ -18,7 +18,7 @@ define i128 @opt_setcc_lt_power_of_2(i128 %a) nounwind {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB0_1: # %loop
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    addl $1, %edi
@@ -47,7 +47,7 @@ define i128 @opt_setcc_lt_power_of_2(i128 %a) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rsi, %rdx
 ; X64-NEXT:    movq %rdi, %rax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB0_1: # %loop
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    addq $1, %rax
diff --git a/llvm/test/CodeGen/X86/ifunc-asm.ll b/llvm/test/CodeGen/X86/ifunc-asm.ll
index 6cfd37604ff5e..a4c47da7f4c65 100644
--- a/llvm/test/CodeGen/X86/ifunc-asm.ll
+++ b/llvm/test/CodeGen/X86/ifunc-asm.ll
@@ -8,7 +8,7 @@ entry:
 ; ELF:             .type foo_resolver,@function
 ; ELF-NEXT:    foo_resolver:
 
-; MACHO:           .p2align 4, 0x90
+; MACHO:           .p2align 4
 ; MACHO-NEXT:  _foo_resolver
 
 
@@ -23,10 +23,10 @@ entry:
 ; MACHO-NEXT:      .quad _foo_ifunc.stub_helper
 ; MACHO-NEXT:      .section __TEXT,__text,regular,pure_instructions
 ; MACHO-NEXT:      .globl _foo_ifunc
-; MACHO-NEXT:      .p2align 0, 0x90
+; MACHO-NEXT:      .p2align 0
 ; MACHO-NEXT:  _foo_ifunc:
 ; MACHO-NEXT:      jmpl   *_foo_ifunc.lazy_pointer(%rip)
-; MACHO-NEXT:      .p2align 0, 0x90
+; MACHO-NEXT:      .p2align 0
 ; MACHO-NEXT:  _foo_ifunc.stub_helper:
 ; MACHO-NEXT:      pushq   %rax
 ; MACHO-NEXT:      pushq   %rdi
diff --git a/llvm/test/CodeGen/X86/innermost-loop-alignment.ll b/llvm/test/CodeGen/X86/innermost-loop-alignment.ll
index 113d5bffe983b..482c8141b97ba 100644
--- a/llvm/test/CodeGen/X86/innermost-loop-alignment.ll
+++ b/llvm/test/CodeGen/X86/innermost-loop-alignment.ll
@@ -25,12 +25,12 @@ define void @test(i32 %n, i32 %m) {
 ; DEFAULT-NEXT:    xorl %esi, %esi
 ; DEFAULT-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; DEFAULT-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; DEFAULT-NEXT:    .p2align 4, 0x90
+; DEFAULT-NEXT:    .p2align 4
 ; DEFAULT-NEXT:  .LBB0_1: # %outer
 ; DEFAULT-NEXT:    # =>This Loop Header: Depth=1
 ; DEFAULT-NEXT:    # Child Loop BB0_2 Depth 2
 ; DEFAULT-NEXT:    movl %edi, %ebp
-; DEFAULT-NEXT:    .p2align 4, 0x90
+; DEFAULT-NEXT:    .p2align 4
 ; DEFAULT-NEXT:  .LBB0_2: # %inner
 ; DEFAULT-NEXT:    # Parent Loop BB0_1 Depth=1
 ; DEFAULT-NEXT:    # => This Inner Loop Header: Depth=2
@@ -74,12 +74,12 @@ define void @test(i32 %n, i32 %m) {
 ; ALIGN32-NEXT:    xorl %esi, %esi
 ; ALIGN32-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; ALIGN32-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; ALIGN32-NEXT:    .p2align 4, 0x90
+; ALIGN32-NEXT:    .p2align 4
 ; ALIGN32-NEXT:  .LBB0_1: # %outer
 ; ALIGN32-NEXT:    # =>This Loop Header: Depth=1
 ; ALIGN32-NEXT:    # Child Loop BB0_2 Depth 2
 ; ALIGN32-NEXT:    movl %edi, %ebp
-; ALIGN32-NEXT:    .p2align 5, 0x90
+; ALIGN32-NEXT:    .p2align 5
 ; ALIGN32-NEXT:  .LBB0_2: # %inner
 ; ALIGN32-NEXT:    # Parent Loop BB0_1 Depth=1
 ; ALIGN32-NEXT:    # => This Inner Loop Header: Depth=2
@@ -123,12 +123,12 @@ define void @test(i32 %n, i32 %m) {
 ; ALIGN64-NEXT:    xorl %esi, %esi
 ; ALIGN64-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; ALIGN64-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; ALIGN64-NEXT:    .p2align 5, 0x90
+; ALIGN64-NEXT:    .p2align 5
 ; ALIGN64-NEXT:  .LBB0_1: # %outer
 ; ALIGN64-NEXT:    # =>This Loop Header: Depth=1
 ; ALIGN64-NEXT:    # Child Loop BB0_2 Depth 2
 ; ALIGN64-NEXT:    movl %edi, %ebp
-; ALIGN64-NEXT:    .p2align 6, 0x90
+; ALIGN64-NEXT:    .p2align 6
 ; ALIGN64-NEXT:  .LBB0_2: # %inner
 ; ALIGN64-NEXT:    # Parent Loop BB0_1 Depth=1
 ; ALIGN64-NEXT:    # => This Inner Loop Header: Depth=2
diff --git a/llvm/test/CodeGen/X86/ins_subreg_coalesce-3.ll b/llvm/test/CodeGen/X86/ins_subreg_coalesce-3.ll
index 3660741fef364..296694ca3c72d 100644
--- a/llvm/test/CodeGen/X86/ins_subreg_coalesce-3.ll
+++ b/llvm/test/CodeGen/X86/ins_subreg_coalesce-3.ll
@@ -23,7 +23,7 @@ define void @FontChange(i1 %foo) nounwind {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    testb $1, %dil
 ; CHECK-NEXT:    je .LBB0_9
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %bb366
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    testb $1, %dil
@@ -31,7 +31,7 @@ define void @FontChange(i1 %foo) nounwind {
 ; CHECK-NEXT:  # %bb.2: # %bb428
 ; CHECK-NEXT:    testb $1, %dil
 ; CHECK-NEXT:    je .LBB0_9
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_3: # %bb650
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpb $0, 0
diff --git a/llvm/test/CodeGen/X86/issue76416.ll b/llvm/test/CodeGen/X86/issue76416.ll
index d0f7fe684a840..7193e54a6ad55 100644
--- a/llvm/test/CodeGen/X86/issue76416.ll
+++ b/llvm/test/CodeGen/X86/issue76416.ll
@@ -12,7 +12,7 @@ define dso_local void @vga_load_state() #0 {
 ; CHECK-NEXT:    movl $0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    cmpl $3, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    jg .LBB0_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    xorl %eax, %eax
@@ -23,7 +23,7 @@ define dso_local void @vga_load_state() #0 {
 ; CHECK-NEXT:    jle .LBB0_2
 ; CHECK-NEXT:  .LBB0_3: # %for.end
 ; CHECK-NEXT:    movl $0, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_4: # %for.cond1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    #APP
diff --git a/llvm/test/CodeGen/X86/kcfi-patchable-function-prefix.ll b/llvm/test/CodeGen/X86/kcfi-patchable-function-prefix.ll
index e5d541f8ed9d9..1b7bd7835e890 100644
--- a/llvm/test/CodeGen/X86/kcfi-patchable-function-prefix.ll
+++ b/llvm/test/CodeGen/X86/kcfi-patchable-function-prefix.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -mtriple=x86_64-unknown-linux-gnu -verify-machineinstrs < %s | FileCheck %s
 
-; CHECK:          .p2align 4, 0x90
+; CHECK:          .p2align 4
 ; CHECK-LABEL:    __cfi_f1:
 ; CHECK-COUNT-11:   nop
 ; CHECK-NEXT:       movl $12345678, %eax
@@ -13,7 +13,7 @@ define void @f1(ptr noundef %x) !kcfi_type !1 {
   ret void
 }
 
-; CHECK:          .p2align 4, 0x90
+; CHECK:          .p2align 4
 ; CHECK-NOT:      __cfi_f2:
 ; CHECK-NOT:        nop
 ; CHECK-LABEL:    f2:
@@ -23,7 +23,7 @@ define void @f2(ptr noundef %x) {
   ret void
 }
 
-; CHECK:          .p2align 4, 0x90
+; CHECK:          .p2align 4
 ; CHECK-LABEL:    __cfi_f3:
 ; CHECK-NOT:        nop
 ; CHECK-NEXT:       movl $12345678, %eax
@@ -35,7 +35,7 @@ define void @f3(ptr noundef %x) #0 !kcfi_type !1 {
   ret void
 }
 
-; CHECK:          .p2align 4, 0x90
+; CHECK:          .p2align 4
 ; CHECK-NOT:      __cfi_f4:
 ; CHECK-COUNT-16:   nop
 ; CHECK-LABEL:    f4:
diff --git a/llvm/test/CodeGen/X86/kcfi.ll b/llvm/test/CodeGen/X86/kcfi.ll
index 566a88b76c4fc..059efcc71b0eb 100644
--- a/llvm/test/CodeGen/X86/kcfi.ll
+++ b/llvm/test/CodeGen/X86/kcfi.ll
@@ -2,7 +2,7 @@
 ; RUN: llc -mtriple=x86_64-unknown-linux-gnu -verify-machineinstrs -stop-after=finalize-isel < %s | FileCheck %s --check-prefixes=MIR,ISEL
 ; RUN: llc -mtriple=x86_64-unknown-linux-gnu -verify-machineinstrs -stop-after=kcfi < %s | FileCheck %s --check-prefixes=MIR,KCFI
 
-; ASM:       .p2align 4, 0x90
+; ASM:       .p2align 4
 ; ASM:       .type __cfi_f1,@function
 ; ASM-LABEL: __cfi_f1:
 ; ASM-NEXT:    nop
diff --git a/llvm/test/CodeGen/X86/known-bits.ll b/llvm/test/CodeGen/X86/known-bits.ll
index f7b2538e1de2f..58a0595e4322a 100644
--- a/llvm/test/CodeGen/X86/known-bits.ll
+++ b/llvm/test/CodeGen/X86/known-bits.ll
@@ -14,13 +14,13 @@ define void @knownbits_zext_in_reg(ptr) nounwind {
 ; X86-NEXT:    shrl $14, %edx
 ; X86-NEXT:    movzbl %al, %ecx
 ; X86-NEXT:    xorl %ebx, %ebx
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB0_1: # %CF
 ; X86-NEXT:    # =>This Loop Header: Depth=1
 ; X86-NEXT:    # Child Loop BB0_2 Depth 2
 ; X86-NEXT:    movl %ecx, %eax
 ; X86-NEXT:    divb %dl
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB0_2: # %CF237
 ; X86-NEXT:    # Parent Loop BB0_1 Depth=1
 ; X86-NEXT:    # => This Inner Loop Header: Depth=2
@@ -37,13 +37,13 @@ define void @knownbits_zext_in_reg(ptr) nounwind {
 ; X64-NEXT:    shrl $14, %edx
 ; X64-NEXT:    movzbl %cl, %ecx
 ; X64-NEXT:    xorl %esi, %esi
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB0_1: # %CF
 ; X64-NEXT:    # =>This Loop Header: Depth=1
 ; X64-NEXT:    # Child Loop BB0_2 Depth 2
 ; X64-NEXT:    movl %ecx, %eax
 ; X64-NEXT:    divb %dl
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB0_2: # %CF237
 ; X64-NEXT:    # Parent Loop BB0_1 Depth=1
 ; X64-NEXT:    # => This Inner Loop Header: Depth=2
diff --git a/llvm/test/CodeGen/X86/lea-opt-cse2.ll b/llvm/test/CodeGen/X86/lea-opt-cse2.ll
index 258a12a8b54e9..e39d01f1447f8 100644
--- a/llvm/test/CodeGen/X86/lea-opt-cse2.ll
+++ b/llvm/test/CodeGen/X86/lea-opt-cse2.ll
@@ -7,7 +7,7 @@
 define void @foo(ptr nocapture %ctx, i32 %n) local_unnamed_addr #0 {
 ; X64-LABEL: foo:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB0_1: # %loop
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl (%rdi), %eax
@@ -32,7 +32,7 @@ define void @foo(ptr nocapture %ctx, i32 %n) local_unnamed_addr #0 {
 ; X86-NEXT:    .cfi_offset %edi, -8
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB0_1: # %loop
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl (%eax), %edx
diff --git a/llvm/test/CodeGen/X86/lea-opt-cse4.ll b/llvm/test/CodeGen/X86/lea-opt-cse4.ll
index ef8e030d6caf2..4fa9acd99bb2f 100644
--- a/llvm/test/CodeGen/X86/lea-opt-cse4.ll
+++ b/llvm/test/CodeGen/X86/lea-opt-cse4.ll
@@ -59,7 +59,7 @@ define void @foo(ptr nocapture %ctx, i32 %n) local_unnamed_addr #0 {
 define void @foo_loop(ptr nocapture %ctx, i32 %n) local_unnamed_addr #0 {
 ; X64-LABEL: foo_loop:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB1_1: # %loop
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl (%rdi), %ecx
@@ -88,7 +88,7 @@ define void @foo_loop(ptr nocapture %ctx, i32 %n) local_unnamed_addr #0 {
 ; X86-NEXT:    .cfi_offset %edi, -8
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB1_1: # %loop
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl (%eax), %esi
diff --git a/llvm/test/CodeGen/X86/licm-symbol.ll b/llvm/test/CodeGen/X86/licm-symbol.ll
index 2e62ed9c8f6d9..e6b3f34ee9bb6 100644
--- a/llvm/test/CodeGen/X86/licm-symbol.ll
+++ b/llvm/test/CodeGen/X86/licm-symbol.ll
@@ -6,7 +6,7 @@
 ; CHECK: pushl
 ; CHECK: movl  $176, %esi
 ; CHECK: addl  L___sF$non_lazy_ptr, %esi
-; CHECK: .p2align  4, 0x90
+; CHECK: .p2align  4
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
 target triple = "i386-apple-darwin8"
diff --git a/llvm/test/CodeGen/X86/loop-search.ll b/llvm/test/CodeGen/X86/loop-search.ll
index 4646452ffdc65..0d5f97d21fb3a 100644
--- a/llvm/test/CodeGen/X86/loop-search.ll
+++ b/llvm/test/CodeGen/X86/loop-search.ll
@@ -12,7 +12,7 @@ define zeroext i1 @search(i32 %needle, ptr nocapture readonly %haystack, i32 %co
 ; CHECK-NEXT:  ## %bb.1: ## %for.body.preheader
 ; CHECK-NEXT:    movslq %edx, %rax
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_2: ## %for.body
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpl %edi, (%rsi,%rcx,4)
diff --git a/llvm/test/CodeGen/X86/loop-strength-reduce5.ll b/llvm/test/CodeGen/X86/loop-strength-reduce5.ll
index 8b512fc94f78b..ebae51fa2aa46 100644
--- a/llvm/test/CodeGen/X86/loop-strength-reduce5.ll
+++ b/llvm/test/CodeGen/X86/loop-strength-reduce5.ll
@@ -13,7 +13,7 @@ define void @foo(i32 %N) nounwind {
 ; CHECK-NEXT:  # %bb.1: # %bb.preheader
 ; CHECK-NEXT:    xorl %ecx, %ecx
 ; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # %bb
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movw %dx, X
diff --git a/llvm/test/CodeGen/X86/loop-strength-reduce7.ll b/llvm/test/CodeGen/X86/loop-strength-reduce7.ll
index 3687194972f27..11473bacc56cf 100644
--- a/llvm/test/CodeGen/X86/loop-strength-reduce7.ll
+++ b/llvm/test/CodeGen/X86/loop-strength-reduce7.ll
@@ -13,13 +13,13 @@ define fastcc void @outer_loop(ptr nocapture %gfp, ptr nocapture %xr, i32 %targ_
 ; CHECK-NEXT:    pushl %esi
 ; CHECK-NEXT:    movl $88, %eax
 ; CHECK-NEXT:    movl $168, %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_2: ## %bb28.i37
 ; CHECK-NEXT:    ## =>This Loop Header: Depth=1
 ; CHECK-NEXT:    ## Child Loop BB0_3 Depth 2
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    movl %eax, %esi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_3: ## %bb29.i38
 ; CHECK-NEXT:    ## Parent Loop BB0_2 Depth=1
 ; CHECK-NEXT:    ## => This Inner Loop Header: Depth=2
diff --git a/llvm/test/CodeGen/X86/lsr-addrecloops.ll b/llvm/test/CodeGen/X86/lsr-addrecloops.ll
index 963405c8b0b3d..98c8f587784c2 100644
--- a/llvm/test/CodeGen/X86/lsr-addrecloops.ll
+++ b/llvm/test/CodeGen/X86/lsr-addrecloops.ll
@@ -16,7 +16,7 @@ define void @in4dob_(ptr nocapture writeonly %0, ptr nocapture readonly %1, ptr
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    jmp .LBB0_1
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_20: # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    incq %r10
 ; CHECK-NEXT:    addq %r9, %rax
@@ -46,7 +46,7 @@ define void @in4dob_(ptr nocapture writeonly %0, ptr nocapture readonly %1, ptr
 ; CHECK-NEXT:    andq $-8, %r10
 ; CHECK-NEXT:    xorl %r9d, %r9d
 ; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_6: # %vector.body807
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    leaq (%rdi,%r9), %r11
@@ -68,7 +68,7 @@ define void @in4dob_(ptr nocapture writeonly %0, ptr nocapture readonly %1, ptr
 ; CHECK-NEXT:    addq %rdi, %r9
 ; CHECK-NEXT:    xorl %r10d, %r10d
 ; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_9: # %vector.body807.epil
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    leaq (%r9,%r10), %r11
@@ -95,7 +95,7 @@ define void @in4dob_(ptr nocapture writeonly %0, ptr nocapture readonly %1, ptr
 ; CHECK-NEXT:    andq $-8, %rdx
 ; CHECK-NEXT:    xorl %ecx, %ecx
 ; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_14: # %vector.body847
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    leaq (%rdi,%rcx), %r8
@@ -117,7 +117,7 @@ define void @in4dob_(ptr nocapture writeonly %0, ptr nocapture readonly %1, ptr
 ; CHECK-NEXT:    leaq 96(%rcx,%rdi), %rcx
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_17: # %vector.body847.epil
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    leaq (%rcx,%rdx), %rdi
diff --git a/llvm/test/CodeGen/X86/lsr-interesting-step.ll b/llvm/test/CodeGen/X86/lsr-interesting-step.ll
index b15c8c4295998..049d2640d25db 100644
--- a/llvm/test/CodeGen/X86/lsr-interesting-step.ll
+++ b/llvm/test/CodeGen/X86/lsr-interesting-step.ll
@@ -10,12 +10,12 @@ define void @foo() nounwind {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movl $2, %eax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %bb7
 ; CHECK-NEXT:    # =>This Loop Header: Depth=1
 ; CHECK-NEXT:    # Child Loop BB0_2 Depth 2
 ; CHECK-NEXT:    movl $2, %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # %bb11
 ; CHECK-NEXT:    # Parent Loop BB0_1 Depth=1
 ; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
diff --git a/llvm/test/CodeGen/X86/lsr-loop-exit-cond.ll b/llvm/test/CodeGen/X86/lsr-loop-exit-cond.ll
index 14dfc046c029a..4cd206adc31de 100644
--- a/llvm/test/CodeGen/X86/lsr-loop-exit-cond.ll
+++ b/llvm/test/CodeGen/X86/lsr-loop-exit-cond.ll
@@ -22,7 +22,7 @@ define void @t(ptr nocapture %in, ptr nocapture %out, ptr nocapture %rk, i32 %r)
 ; GENERIC-NEXT:    movq _Te1@GOTPCREL(%rip), %rax
 ; GENERIC-NEXT:    movq _Te3@GOTPCREL(%rip), %r10
 ; GENERIC-NEXT:    movq %rcx, %r11
-; GENERIC-NEXT:    .p2align 4, 0x90
+; GENERIC-NEXT:    .p2align 4
 ; GENERIC-NEXT:  LBB0_1: ## %bb
 ; GENERIC-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; GENERIC-NEXT:    movzbl %r8b, %r14d
@@ -105,7 +105,7 @@ define void @t(ptr nocapture %in, ptr nocapture %out, ptr nocapture %rk, i32 %r)
 ; ATOM-NEXT:    movq _Te3@GOTPCREL(%rip), %r10
 ; ATOM-NEXT:    decl %ecx
 ; ATOM-NEXT:    movq %rcx, %r11
-; ATOM-NEXT:    .p2align 4, 0x90
+; ATOM-NEXT:    .p2align 4
 ; ATOM-NEXT:  LBB0_1: ## %bb
 ; ATOM-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; ATOM-NEXT:    movl %r15d, %ebx
@@ -315,7 +315,7 @@ define i32 @f(i32 %i, ptr nocapture %a) nounwind uwtable readonly ssp {
 ; GENERIC-NEXT:    leaq (%rsi,%rax,4), %rcx
 ; GENERIC-NEXT:    xorl %eax, %eax
 ; GENERIC-NEXT:    xorl %edx, %edx
-; GENERIC-NEXT:    .p2align 4, 0x90
+; GENERIC-NEXT:    .p2align 4
 ; GENERIC-NEXT:  LBB1_2: ## %for.body
 ; GENERIC-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; GENERIC-NEXT:    movl (%rcx), %esi
@@ -339,7 +339,7 @@ define i32 @f(i32 %i, ptr nocapture %a) nounwind uwtable readonly ssp {
 ; ATOM-NEXT:    xorl %edx, %edx
 ; ATOM-NEXT:    leaq (%rsi,%rax,4), %rcx
 ; ATOM-NEXT:    xorl %eax, %eax
-; ATOM-NEXT:    .p2align 4, 0x90
+; ATOM-NEXT:    .p2align 4
 ; ATOM-NEXT:  LBB1_2: ## %for.body
 ; ATOM-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; ATOM-NEXT:    movl (%rcx), %esi
diff --git a/llvm/test/CodeGen/X86/lsr-negative-stride.ll b/llvm/test/CodeGen/X86/lsr-negative-stride.ll
index 26c6128ab48db..1d5e208f3a326 100644
--- a/llvm/test/CodeGen/X86/lsr-negative-stride.ll
+++ b/llvm/test/CodeGen/X86/lsr-negative-stride.ll
@@ -23,12 +23,12 @@ define i32 @t(i32 %a, i32 %b) nounwind {
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    movl %edx, %eax
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # %bb.outer
 ; CHECK-NEXT:    # =>This Loop Header: Depth=1
 ; CHECK-NEXT:    # Child Loop BB0_3 Depth 2
 ; CHECK-NEXT:    movl %edx, %eax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_3: # %bb
 ; CHECK-NEXT:    # Parent Loop BB0_2 Depth=1
 ; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
@@ -40,7 +40,7 @@ define i32 @t(i32 %a, i32 %b) nounwind {
 ; CHECK-NEXT:    movl %eax, %edx
 ; CHECK-NEXT:    jne .LBB0_3
 ; CHECK-NEXT:    jmp .LBB0_6
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_5: # %cond_false
 ; CHECK-NEXT:    # in Loop: Header=BB0_2 Depth=1
 ; CHECK-NEXT:    subl %edx, %ecx
diff --git a/llvm/test/CodeGen/X86/lsr-sort.ll b/llvm/test/CodeGen/X86/lsr-sort.ll
index 9c7f5efaf74eb..37cb6f996d34f 100644
--- a/llvm/test/CodeGen/X86/lsr-sort.ll
+++ b/llvm/test/CodeGen/X86/lsr-sort.ll
@@ -9,7 +9,7 @@ define dso_local i32 @foo(i32 %N) nounwind {
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testl %edi, %edi
 ; CHECK-NEXT:    jle .LBB0_2
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %bb
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movw %ax, X(%rip)
diff --git a/llvm/test/CodeGen/X86/lsr-static-addr.ll b/llvm/test/CodeGen/X86/lsr-static-addr.ll
index a06ead491bf7d..c67ac7d909fa7 100644
--- a/llvm/test/CodeGen/X86/lsr-static-addr.ll
+++ b/llvm/test/CodeGen/X86/lsr-static-addr.ll
@@ -12,7 +12,7 @@ define void @foo(i64 %n) nounwind {
 ; CHECK-NEXT:  # %bb.1: # %for.body.preheader
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = [2.2999999999999998E+0,0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
@@ -31,7 +31,7 @@ define void @foo(i64 %n) nounwind {
 ; ATOM-NEXT:  # %bb.1: # %for.body.preheader
 ; ATOM-NEXT:    xorl %eax, %eax
 ; ATOM-NEXT:    movsd {{.*#+}} xmm0 = [2.2999999999999998E+0,0.0E+0]
-; ATOM-NEXT:    .p2align 4, 0x90
+; ATOM-NEXT:    .p2align 4
 ; ATOM-NEXT:  .LBB0_2: # %for.body
 ; ATOM-NEXT:    # =>This Inner Loop Header: Depth=1
 ; ATOM-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
diff --git a/llvm/test/CodeGen/X86/machine-cp.ll b/llvm/test/CodeGen/X86/machine-cp.ll
index f84960485840d..c84a1159ad56a 100644
--- a/llvm/test/CodeGen/X86/machine-cp.ll
+++ b/llvm/test/CodeGen/X86/machine-cp.ll
@@ -11,7 +11,7 @@ define i32 @t1(i32 %a, i32 %b) nounwind  {
 ; CHECK-NEXT:    je LBB0_4
 ; CHECK-NEXT:  ## %bb.1: ## %while.body.preheader
 ; CHECK-NEXT:    movl %esi, %edx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_2: ## %while.body
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %edx, %ecx
@@ -62,7 +62,7 @@ define i32 @t3(i64 %a, i64 %b) nounwind  {
 ; CHECK-NEXT:    je LBB2_4
 ; CHECK-NEXT:  ## %bb.1: ## %while.body.preheader
 ; CHECK-NEXT:    movq %rsi, %rdx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB2_2: ## %while.body
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rdx, %rcx
diff --git a/llvm/test/CodeGen/X86/machine-cse.ll b/llvm/test/CodeGen/X86/machine-cse.ll
index 431031136e4a4..e953f67651d55 100644
--- a/llvm/test/CodeGen/X86/machine-cse.ll
+++ b/llvm/test/CodeGen/X86/machine-cse.ll
@@ -68,7 +68,7 @@ define void @commute(i32 %test_case, i32 %scale) nounwind ssp {
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    callq printf@PLT
 ; CHECK-NEXT:    addq $8, %rsp
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_3: # %for.body53.us
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp .LBB1_3
@@ -141,7 +141,7 @@ define ptr @bsd_memchr(ptr %s, i32 %a, i32 %c, i64 %n) nounwind ssp {
 ; CHECK-NEXT:  # %bb.1: # %preheader
 ; CHECK-NEXT:    movq %rdi, %rax
 ; CHECK-NEXT:    movzbl %dl, %edx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB3_2: # %do.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpl %edx, %esi
diff --git a/llvm/test/CodeGen/X86/madd.ll b/llvm/test/CodeGen/X86/madd.ll
index f3b117c626c8c..bdb7c307a5759 100644
--- a/llvm/test/CodeGen/X86/madd.ll
+++ b/llvm/test/CodeGen/X86/madd.ll
@@ -12,7 +12,7 @@ define i32 @_Z10test_shortPsS_i_128(ptr nocapture readonly, ptr nocapture readon
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    xorl %ecx, %ecx
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB0_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
@@ -37,7 +37,7 @@ define i32 @_Z10test_shortPsS_i_128(ptr nocapture readonly, ptr nocapture readon
 ; AVX-NEXT:    movl %edx, %eax
 ; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    xorl %ecx, %ecx
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  .LBB0_1: # %vector.body
 ; AVX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
@@ -91,7 +91,7 @@ define i32 @_Z10test_shortPsS_i_256(ptr nocapture readonly, ptr nocapture readon
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    xorl %ecx, %ecx
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB1_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movdqu (%rdi,%rcx,2), %xmm2
@@ -115,7 +115,7 @@ define i32 @_Z10test_shortPsS_i_256(ptr nocapture readonly, ptr nocapture readon
 ; AVX1-NEXT:    movl %edx, %eax
 ; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    xorl %ecx, %ecx
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB1_1: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vmovdqu (%rsi,%rcx,2), %xmm1
@@ -141,7 +141,7 @@ define i32 @_Z10test_shortPsS_i_256(ptr nocapture readonly, ptr nocapture readon
 ; AVX256-NEXT:    movl %edx, %eax
 ; AVX256-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX256-NEXT:    xorl %ecx, %ecx
-; AVX256-NEXT:    .p2align 4, 0x90
+; AVX256-NEXT:    .p2align 4
 ; AVX256-NEXT:  .LBB1_1: # %vector.body
 ; AVX256-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX256-NEXT:    vmovdqu (%rsi,%rcx,2), %xmm1
@@ -200,7 +200,7 @@ define i32 @_Z10test_shortPsS_i_512(ptr nocapture readonly, ptr nocapture readon
 ; SSE2-NEXT:    xorl %ecx, %ecx
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB2_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movdqu (%rdi,%rcx,2), %xmm3
@@ -231,7 +231,7 @@ define i32 @_Z10test_shortPsS_i_512(ptr nocapture readonly, ptr nocapture readon
 ; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    xorl %ecx, %ecx
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB2_1: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vmovdqu (%rsi,%rcx,2), %xmm2
@@ -265,7 +265,7 @@ define i32 @_Z10test_shortPsS_i_512(ptr nocapture readonly, ptr nocapture readon
 ; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    xorl %ecx, %ecx
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB2_1: # %vector.body
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    vmovdqu (%rsi,%rcx,2), %ymm2
@@ -291,7 +291,7 @@ define i32 @_Z10test_shortPsS_i_512(ptr nocapture readonly, ptr nocapture readon
 ; AVX512-NEXT:    movl %edx, %eax
 ; AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    xorl %ecx, %ecx
-; AVX512-NEXT:    .p2align 4, 0x90
+; AVX512-NEXT:    .p2align 4
 ; AVX512-NEXT:  .LBB2_1: # %vector.body
 ; AVX512-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512-NEXT:    vmovdqu (%rsi,%rcx,2), %ymm1
@@ -356,7 +356,7 @@ define i32 @_Z10test_shortPsS_i_1024(ptr nocapture readonly, ptr nocapture reado
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB3_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movdqu (%rdi,%rcx,2), %xmm5
@@ -400,7 +400,7 @@ define i32 @_Z10test_shortPsS_i_1024(ptr nocapture readonly, ptr nocapture reado
 ; AVX1-NEXT:    xorl %ecx, %ecx
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB3_1: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vmovdqu (%rsi,%rcx,2), %xmm3
@@ -448,7 +448,7 @@ define i32 @_Z10test_shortPsS_i_1024(ptr nocapture readonly, ptr nocapture reado
 ; AVX2-NEXT:    xorl %ecx, %ecx
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB3_1: # %vector.body
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    vmovdqu (%rsi,%rcx,2), %ymm3
@@ -480,7 +480,7 @@ define i32 @_Z10test_shortPsS_i_1024(ptr nocapture readonly, ptr nocapture reado
 ; AVX512F-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX512F-NEXT:    xorl %ecx, %ecx
 ; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    .p2align 4, 0x90
+; AVX512F-NEXT:    .p2align 4
 ; AVX512F-NEXT:  .LBB3_1: # %vector.body
 ; AVX512F-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512F-NEXT:    vmovdqu (%rsi,%rcx,2), %ymm2
@@ -512,7 +512,7 @@ define i32 @_Z10test_shortPsS_i_1024(ptr nocapture readonly, ptr nocapture reado
 ; AVX512BW-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX512BW-NEXT:    xorl %ecx, %ecx
 ; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    .p2align 4, 0x90
+; AVX512BW-NEXT:    .p2align 4
 ; AVX512BW-NEXT:  .LBB3_1: # %vector.body
 ; AVX512BW-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512BW-NEXT:    vmovdqu64 (%rsi,%rcx,2), %zmm2
@@ -576,7 +576,7 @@ define i32 @_Z9test_charPcS_i_128(ptr nocapture readonly, ptr nocapture readonly
 ; SSE2-NEXT:    movl %edx, %eax
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    xorl %ecx, %ecx
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB4_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
@@ -605,7 +605,7 @@ define i32 @_Z9test_charPcS_i_128(ptr nocapture readonly, ptr nocapture readonly
 ; AVX-NEXT:    movl %edx, %eax
 ; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    xorl %ecx, %ecx
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  .LBB4_1: # %vector.body
 ; AVX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    vpmovsxbd (%rdi,%rcx), %xmm1
@@ -659,7 +659,7 @@ define i32 @_Z9test_charPcS_i_256(ptr nocapture readonly, ptr nocapture readonly
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    xorl %ecx, %ecx
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB5_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
@@ -687,7 +687,7 @@ define i32 @_Z9test_charPcS_i_256(ptr nocapture readonly, ptr nocapture readonly
 ; AVX1-NEXT:    movl %edx, %eax
 ; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    xorl %ecx, %ecx
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB5_1: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vpmovsxbw (%rdi,%rcx), %xmm1
@@ -714,7 +714,7 @@ define i32 @_Z9test_charPcS_i_256(ptr nocapture readonly, ptr nocapture readonly
 ; AVX256-NEXT:    movl %edx, %eax
 ; AVX256-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX256-NEXT:    xorl %ecx, %ecx
-; AVX256-NEXT:    .p2align 4, 0x90
+; AVX256-NEXT:    .p2align 4
 ; AVX256-NEXT:  .LBB5_1: # %vector.body
 ; AVX256-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX256-NEXT:    vpmovsxbw (%rdi,%rcx), %xmm1
@@ -774,7 +774,7 @@ define i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocapture readonly
 ; SSE2-NEXT:    xorl %ecx, %ecx
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB6_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movdqu (%rdi,%rcx), %xmm3
@@ -811,7 +811,7 @@ define i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocapture readonly
 ; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    xorl %ecx, %ecx
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB6_1: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vpmovsxbw 8(%rdi,%rcx), %xmm2
@@ -847,7 +847,7 @@ define i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocapture readonly
 ; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    xorl %ecx, %ecx
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB6_1: # %vector.body
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    vpmovsxbw (%rdi,%rcx), %ymm2
@@ -874,7 +874,7 @@ define i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocapture readonly
 ; AVX512-NEXT:    movl %edx, %eax
 ; AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    xorl %ecx, %ecx
-; AVX512-NEXT:    .p2align 4, 0x90
+; AVX512-NEXT:    .p2align 4
 ; AVX512-NEXT:  .LBB6_1: # %vector.body
 ; AVX512-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512-NEXT:    vpmovsxbw (%rdi,%rcx), %ymm1
@@ -940,7 +940,7 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB7_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movdqu (%rdi,%rcx), %xmm7
@@ -996,7 +996,7 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
 ; AVX1-NEXT:    xorl %ecx, %ecx
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB7_1: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vpmovsxbw 24(%rdi,%rcx), %xmm3
@@ -1048,7 +1048,7 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
 ; AVX2-NEXT:    xorl %ecx, %ecx
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB7_1: # %vector.body
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    vpmovsxbw 16(%rdi,%rcx), %ymm3
@@ -1082,7 +1082,7 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
 ; AVX512F-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX512F-NEXT:    xorl %ecx, %ecx
 ; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    .p2align 4, 0x90
+; AVX512F-NEXT:    .p2align 4
 ; AVX512F-NEXT:  .LBB7_1: # %vector.body
 ; AVX512F-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512F-NEXT:    vpmovsxbw (%rdi,%rcx), %ymm2
@@ -1116,7 +1116,7 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
 ; AVX512BW-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX512BW-NEXT:    xorl %ecx, %ecx
 ; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    .p2align 4, 0x90
+; AVX512BW-NEXT:    .p2align 4
 ; AVX512BW-NEXT:  .LBB7_1: # %vector.body
 ; AVX512BW-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512BW-NEXT:    vpmovsxbw (%rdi,%rcx), %zmm2
@@ -1181,7 +1181,7 @@ define i32 @test_unsigned_short_128(ptr nocapture readonly, ptr nocapture readon
 ; SSE2-NEXT:    movl %edx, %eax
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    xorl %ecx, %ecx
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB8_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
@@ -1207,7 +1207,7 @@ define i32 @test_unsigned_short_128(ptr nocapture readonly, ptr nocapture readon
 ; AVX-NEXT:    movl %edx, %eax
 ; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    xorl %ecx, %ecx
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  .LBB8_1: # %vector.body
 ; AVX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
@@ -1261,7 +1261,7 @@ define i32 @test_unsigned_short_256(ptr nocapture readonly, ptr nocapture readon
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    xorl %ecx, %ecx
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB9_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movdqu (%rdi,%rcx,2), %xmm2
@@ -1291,7 +1291,7 @@ define i32 @test_unsigned_short_256(ptr nocapture readonly, ptr nocapture readon
 ; AVX1-NEXT:    movl %edx, %eax
 ; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    xorl %ecx, %ecx
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB9_1: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
@@ -1323,7 +1323,7 @@ define i32 @test_unsigned_short_256(ptr nocapture readonly, ptr nocapture readon
 ; AVX256-NEXT:    movl %edx, %eax
 ; AVX256-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX256-NEXT:    xorl %ecx, %ecx
-; AVX256-NEXT:    .p2align 4, 0x90
+; AVX256-NEXT:    .p2align 4
 ; AVX256-NEXT:  .LBB9_1: # %vector.body
 ; AVX256-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX256-NEXT:    vpmovzxwd {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
@@ -1384,7 +1384,7 @@ define i32 @test_unsigned_short_512(ptr nocapture readonly, ptr nocapture readon
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB10_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movdqu (%rdi,%rcx,2), %xmm4
@@ -1427,7 +1427,7 @@ define i32 @test_unsigned_short_512(ptr nocapture readonly, ptr nocapture readon
 ; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    xorl %ecx, %ecx
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB10_1: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
@@ -1473,7 +1473,7 @@ define i32 @test_unsigned_short_512(ptr nocapture readonly, ptr nocapture readon
 ; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    xorl %ecx, %ecx
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB10_1: # %vector.body
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
@@ -1504,7 +1504,7 @@ define i32 @test_unsigned_short_512(ptr nocapture readonly, ptr nocapture readon
 ; AVX512-NEXT:    movl %edx, %eax
 ; AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    xorl %ecx, %ecx
-; AVX512-NEXT:    .p2align 4, 0x90
+; AVX512-NEXT:    .p2align 4
 ; AVX512-NEXT:  .LBB10_1: # %vector.body
 ; AVX512-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512-NEXT:    vpmovzxwd {{.*#+}} zmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
@@ -1573,7 +1573,7 @@ define i32 @test_unsigned_short_1024(ptr nocapture readonly, ptr nocapture reado
 ; SSE2-NEXT:    pxor %xmm6, %xmm6
 ; SSE2-NEXT:    pxor %xmm5, %xmm5
 ; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB11_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movdqu 48(%rdi,%rcx,2), %xmm8
@@ -1642,7 +1642,7 @@ define i32 @test_unsigned_short_1024(ptr nocapture readonly, ptr nocapture reado
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB11_1: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
@@ -1716,7 +1716,7 @@ define i32 @test_unsigned_short_1024(ptr nocapture readonly, ptr nocapture reado
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB11_1: # %vector.body
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
@@ -1758,7 +1758,7 @@ define i32 @test_unsigned_short_1024(ptr nocapture readonly, ptr nocapture reado
 ; AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    xorl %ecx, %ecx
 ; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    .p2align 4, 0x90
+; AVX512-NEXT:    .p2align 4
 ; AVX512-NEXT:  .LBB11_1: # %vector.body
 ; AVX512-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512-NEXT:    vpmovzxwd {{.*#+}} zmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
@@ -2742,7 +2742,7 @@ define i64 @sum_and_sum_of_squares(ptr %a, i32 %n) {
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB33_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movq {{.*#+}} xmm4 = mem[0],zero
@@ -2780,7 +2780,7 @@ define i64 @sum_and_sum_of_squares(ptr %a, i32 %n) {
 ; AVX1-NEXT:    movl %esi, %eax
 ; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB33_1: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
@@ -2823,7 +2823,7 @@ define i64 @sum_and_sum_of_squares(ptr %a, i32 %n) {
 ; AVX256-NEXT:    movl %esi, %eax
 ; AVX256-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX256-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX256-NEXT:    .p2align 4, 0x90
+; AVX256-NEXT:    .p2align 4
 ; AVX256-NEXT:  .LBB33_1: # %vector.body
 ; AVX256-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX256-NEXT:    vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
@@ -2900,7 +2900,7 @@ define i32 @sum_of_square_differences(ptr %a, ptr %b, i32 %n) {
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    xorl %ecx, %ecx
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB34_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
@@ -2927,7 +2927,7 @@ define i32 @sum_of_square_differences(ptr %a, ptr %b, i32 %n) {
 ; AVX1-NEXT:    movl %edx, %eax
 ; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    xorl %ecx, %ecx
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB34_1: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
@@ -2955,7 +2955,7 @@ define i32 @sum_of_square_differences(ptr %a, ptr %b, i32 %n) {
 ; AVX256-NEXT:    movl %edx, %eax
 ; AVX256-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX256-NEXT:    xorl %ecx, %ecx
-; AVX256-NEXT:    .p2align 4, 0x90
+; AVX256-NEXT:    .p2align 4
 ; AVX256-NEXT:  .LBB34_1: # %vector.body
 ; AVX256-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX256-NEXT:    vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
@@ -3105,7 +3105,7 @@ define i32 @add_used_by_loop_phi(ptr %a, ptr %b, i64 %offset_a, i64 %offset_b, i
 ; SSE2-NEXT:    xorl %eax, %eax
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB38_1: # %loop
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movdqu (%rdi,%rax), %xmm3
@@ -3143,7 +3143,7 @@ define i32 @add_used_by_loop_phi(ptr %a, ptr %b, i64 %offset_a, i64 %offset_b, i
 ; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    xorl %eax, %eax
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB38_1: # %loop
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vpmovsxbw 8(%rdi,%rax), %xmm2
@@ -3180,7 +3180,7 @@ define i32 @add_used_by_loop_phi(ptr %a, ptr %b, i64 %offset_a, i64 %offset_b, i
 ; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    xorl %eax, %eax
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB38_1: # %loop
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    vpmovsxbw (%rdi,%rax), %ymm2
@@ -3208,7 +3208,7 @@ define i32 @add_used_by_loop_phi(ptr %a, ptr %b, i64 %offset_a, i64 %offset_b, i
 ; AVX512-NEXT:    addq %rcx, %rsi
 ; AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    xorl %eax, %eax
-; AVX512-NEXT:    .p2align 4, 0x90
+; AVX512-NEXT:    .p2align 4
 ; AVX512-NEXT:  .LBB38_1: # %loop
 ; AVX512-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512-NEXT:    vpmovsxbw (%rdi,%rax), %ymm1
diff --git a/llvm/test/CodeGen/X86/masked-iv-safe.ll b/llvm/test/CodeGen/X86/masked-iv-safe.ll
index 4e4ad3a0161e0..a4f5e52a27d8a 100644
--- a/llvm/test/CodeGen/X86/masked-iv-safe.ll
+++ b/llvm/test/CodeGen/X86/masked-iv-safe.ll
@@ -11,7 +11,7 @@ define void @count_up(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = [1.0000000000000001E-1,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = [2.2999999999999998E+0,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
@@ -57,7 +57,7 @@ define void @count_down(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = [1.0000000000000001E-1,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = [2.2999999999999998E+0,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
@@ -103,7 +103,7 @@ define void @count_up_signed(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = [1.0000000000000001E-1,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = [2.2999999999999998E+0,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB2_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
@@ -151,7 +151,7 @@ define void @count_down_signed(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = [1.0000000000000001E-1,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = [2.2999999999999998E+0,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB3_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
@@ -199,7 +199,7 @@ define void @another_count_up(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = [1.0000000000000001E-1,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = [2.2999999999999998E+0,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB4_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
@@ -251,7 +251,7 @@ define void @another_count_down(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
 ; CHECK-NEXT:    movq %rdi, %rcx
 ; CHECK-NEXT:    movq %rdi, %rdx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB5_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
@@ -303,7 +303,7 @@ define void @another_count_up_signed(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = [1.0000000000000001E-1,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = [2.2999999999999998E+0,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB6_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
@@ -351,7 +351,7 @@ define void @another_count_down_signed(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = [1.0000000000000001E-1,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = [2.2999999999999998E+0,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB7_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
diff --git a/llvm/test/CodeGen/X86/masked-iv-unsafe.ll b/llvm/test/CodeGen/X86/masked-iv-unsafe.ll
index f10db3424c2ea..42bd6e9b75447 100644
--- a/llvm/test/CodeGen/X86/masked-iv-unsafe.ll
+++ b/llvm/test/CodeGen/X86/masked-iv-unsafe.ll
@@ -11,7 +11,7 @@ define void @count_up(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = [1.0000000000000001E-1,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = [2.2999999999999998E+0,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movzbl %al, %ecx
@@ -64,7 +64,7 @@ define void @count_down(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = [1.0000000000000001E-1,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = [2.2999999999999998E+0,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movzbl %al, %ecx
@@ -120,7 +120,7 @@ define void @count_up_signed(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = [1.0000000000000001E-1,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = [2.2999999999999998E+0,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB2_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rdx, %rsi
@@ -180,7 +180,7 @@ define void @count_down_signed(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = [1.0000000000000001E-1,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = [2.2999999999999998E+0,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB3_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rdx, %rsi
@@ -238,7 +238,7 @@ define void @another_count_up(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = [1.0000000000000001E-1,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = [2.2999999999999998E+0,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB4_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movzbl %al, %ecx
@@ -291,7 +291,7 @@ define void @another_count_down(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = [1.0000000000000001E-1,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = [2.2999999999999998E+0,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB5_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movzbl %sil, %eax
@@ -347,7 +347,7 @@ define void @another_count_up_signed(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
 ; CHECK-NEXT:    xorl %ecx, %ecx
 ; CHECK-NEXT:    movq %rdi, %rdx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB6_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %r8
@@ -409,7 +409,7 @@ define void @another_count_down_signed(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = [1.0000000000000001E-1,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = [2.2999999999999998E+0,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB7_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rsi, %rdx
@@ -469,7 +469,7 @@ define void @yet_another_count_down(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
 ; CHECK-NEXT:    movq %rdi, %rcx
 ; CHECK-NEXT:    movq %rdi, %rdx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB8_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
@@ -521,7 +521,7 @@ define void @yet_another_count_up(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = [1.0000000000000001E-1,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = [2.2999999999999998E+0,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB9_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movzbl %al, %ecx
@@ -575,7 +575,7 @@ define void @still_another_count_down(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = [1.0000000000000001E-1,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = [2.2999999999999998E+0,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB10_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movzbl %al, %ecx
@@ -630,7 +630,7 @@ define void @yet_another_count_up_signed(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = [2.2999999999999998E+0,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
 ; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB11_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rcx, %rsi
@@ -690,7 +690,7 @@ define void @yet_another_count_down_signed(ptr %d, i64 %n) nounwind {
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = [1.0000000000000001E-1,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = [2.2999999999999998E+0,0.0E+0]
 ; CHECK-NEXT:    movsd {{.*#+}} xmm2 = [4.5E+0,0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB12_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rdx, %rsi
diff --git a/llvm/test/CodeGen/X86/merge_store.ll b/llvm/test/CodeGen/X86/merge_store.ll
index 1cf3b9a83bac1..afe0ef969a40e 100644
--- a/llvm/test/CodeGen/X86/merge_store.ll
+++ b/llvm/test/CodeGen/X86/merge_store.ll
@@ -6,7 +6,7 @@ define void @merge_store(ptr nocapture %a) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    movabsq $4294967297, %rcx # imm = 0x100000001
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rcx, (%rdi,%rax,4)
diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
index 1413a19b34935..e3d2ac659d438 100644
--- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
@@ -177,7 +177,7 @@ define dso_local i32 @_Z9test_charPcS_i_256(ptr nocapture readonly, ptr nocaptur
 ; CHECK-SKX-NEXT:    xorl %ecx, %ecx
 ; CHECK-SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; CHECK-SKX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; CHECK-SKX-NEXT:    .p2align 4, 0x90
+; CHECK-SKX-NEXT:    .p2align 4
 ; CHECK-SKX-NEXT:  .LBB8_1: # %vector.body
 ; CHECK-SKX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-SKX-NEXT:    vpmovsxbw 16(%rdi,%rcx), %ymm3
@@ -212,7 +212,7 @@ define dso_local i32 @_Z9test_charPcS_i_256(ptr nocapture readonly, ptr nocaptur
 ; CHECK-AVX512-NEXT:    xorl %ecx, %ecx
 ; CHECK-AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; CHECK-AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; CHECK-AVX512-NEXT:    .p2align 4, 0x90
+; CHECK-AVX512-NEXT:    .p2align 4
 ; CHECK-AVX512-NEXT:  .LBB8_1: # %vector.body
 ; CHECK-AVX512-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-AVX512-NEXT:    vpmovsxbw 16(%rdi,%rcx), %ymm3
@@ -247,7 +247,7 @@ define dso_local i32 @_Z9test_charPcS_i_256(ptr nocapture readonly, ptr nocaptur
 ; CHECK-VBMI-NEXT:    xorl %ecx, %ecx
 ; CHECK-VBMI-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; CHECK-VBMI-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; CHECK-VBMI-NEXT:    .p2align 4, 0x90
+; CHECK-VBMI-NEXT:    .p2align 4
 ; CHECK-VBMI-NEXT:  .LBB8_1: # %vector.body
 ; CHECK-VBMI-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-VBMI-NEXT:    vpmovsxbw 16(%rdi,%rcx), %ymm3
@@ -317,7 +317,7 @@ define dso_local i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocaptur
 ; CHECK-SKX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; CHECK-SKX-NEXT:    xorl %ecx, %ecx
 ; CHECK-SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-SKX-NEXT:    .p2align 4, 0x90
+; CHECK-SKX-NEXT:    .p2align 4
 ; CHECK-SKX-NEXT:  .LBB9_1: # %vector.body
 ; CHECK-SKX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-SKX-NEXT:    vpmovsxbw (%rdi,%rcx), %zmm2
@@ -347,7 +347,7 @@ define dso_local i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocaptur
 ; CHECK-AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; CHECK-AVX512-NEXT:    xorl %ecx, %ecx
 ; CHECK-AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX512-NEXT:    .p2align 4, 0x90
+; CHECK-AVX512-NEXT:    .p2align 4
 ; CHECK-AVX512-NEXT:  .LBB9_1: # %vector.body
 ; CHECK-AVX512-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-AVX512-NEXT:    vpmovsxbw (%rdi,%rcx), %zmm2
@@ -377,7 +377,7 @@ define dso_local i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocaptur
 ; CHECK-VBMI-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; CHECK-VBMI-NEXT:    xorl %ecx, %ecx
 ; CHECK-VBMI-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-VBMI-NEXT:    .p2align 4, 0x90
+; CHECK-VBMI-NEXT:    .p2align 4
 ; CHECK-VBMI-NEXT:  .LBB9_1: # %vector.body
 ; CHECK-VBMI-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-VBMI-NEXT:    vpmovsxbw (%rdi,%rcx), %zmm2
@@ -445,7 +445,7 @@ define dso_local i32 @sad_16i8_256() "min-legal-vector-width"="256" {
 ; CHECK-SKX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; CHECK-SKX-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; CHECK-SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-SKX-NEXT:    .p2align 4, 0x90
+; CHECK-SKX-NEXT:    .p2align 4
 ; CHECK-SKX-NEXT:  .LBB10_1: # %vector.body
 ; CHECK-SKX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-SKX-NEXT:    vmovdqu a+1024(%rax), %xmm2
@@ -470,7 +470,7 @@ define dso_local i32 @sad_16i8_256() "min-legal-vector-width"="256" {
 ; CHECK-AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; CHECK-AVX512-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; CHECK-AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX512-NEXT:    .p2align 4, 0x90
+; CHECK-AVX512-NEXT:    .p2align 4
 ; CHECK-AVX512-NEXT:  .LBB10_1: # %vector.body
 ; CHECK-AVX512-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-AVX512-NEXT:    vmovdqu a+1024(%rax), %xmm2
@@ -495,7 +495,7 @@ define dso_local i32 @sad_16i8_256() "min-legal-vector-width"="256" {
 ; CHECK-VBMI-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; CHECK-VBMI-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; CHECK-VBMI-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-VBMI-NEXT:    .p2align 4, 0x90
+; CHECK-VBMI-NEXT:    .p2align 4
 ; CHECK-VBMI-NEXT:  .LBB10_1: # %vector.body
 ; CHECK-VBMI-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-VBMI-NEXT:    vmovdqu a+1024(%rax), %xmm2
@@ -555,7 +555,7 @@ define dso_local i32 @sad_16i8_512() "min-legal-vector-width"="512" {
 ; CHECK-SKX:       # %bb.0: # %entry
 ; CHECK-SKX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; CHECK-SKX-NEXT:    movq $-1024, %rax # imm = 0xFC00
-; CHECK-SKX-NEXT:    .p2align 4, 0x90
+; CHECK-SKX-NEXT:    .p2align 4
 ; CHECK-SKX-NEXT:  .LBB11_1: # %vector.body
 ; CHECK-SKX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-SKX-NEXT:    vmovdqu a+1024(%rax), %xmm1
@@ -580,7 +580,7 @@ define dso_local i32 @sad_16i8_512() "min-legal-vector-width"="512" {
 ; CHECK-AVX512:       # %bb.0: # %entry
 ; CHECK-AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; CHECK-AVX512-NEXT:    movq $-1024, %rax # imm = 0xFC00
-; CHECK-AVX512-NEXT:    .p2align 4, 0x90
+; CHECK-AVX512-NEXT:    .p2align 4
 ; CHECK-AVX512-NEXT:  .LBB11_1: # %vector.body
 ; CHECK-AVX512-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-AVX512-NEXT:    vmovdqu a+1024(%rax), %xmm1
@@ -605,7 +605,7 @@ define dso_local i32 @sad_16i8_512() "min-legal-vector-width"="512" {
 ; CHECK-VBMI:       # %bb.0: # %entry
 ; CHECK-VBMI-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; CHECK-VBMI-NEXT:    movq $-1024, %rax # imm = 0xFC00
-; CHECK-VBMI-NEXT:    .p2align 4, 0x90
+; CHECK-VBMI-NEXT:    .p2align 4
 ; CHECK-VBMI-NEXT:  .LBB11_1: # %vector.body
 ; CHECK-VBMI-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-VBMI-NEXT:    vmovdqu a+1024(%rax), %xmm1
diff --git a/llvm/test/CodeGen/X86/mmx-arith.ll b/llvm/test/CodeGen/X86/mmx-arith.ll
index 5bb3b17fdf421..73d459ba77026 100644
--- a/llvm/test/CodeGen/X86/mmx-arith.ll
+++ b/llvm/test/CodeGen/X86/mmx-arith.ll
@@ -409,7 +409,7 @@ define <1 x i64> @test3(ptr %a, ptr %b, i32 %count) nounwind {
 ; X86-NEXT:    xorl %ebx, %ebx
 ; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB3_3: # %bb26
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
@@ -442,7 +442,7 @@ define <1 x i64> @test3(ptr %a, ptr %b, i32 %count) nounwind {
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    testl %edx, %edx
 ; X64-NEXT:    je .LBB3_2
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB3_1: # %bb26
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movslq %ecx, %rcx
diff --git a/llvm/test/CodeGen/X86/negative-stride-fptosi-user.ll b/llvm/test/CodeGen/X86/negative-stride-fptosi-user.ll
index d0d46b5f11836..96080d2d87cdf 100644
--- a/llvm/test/CodeGen/X86/negative-stride-fptosi-user.ll
+++ b/llvm/test/CodeGen/X86/negative-stride-fptosi-user.ll
@@ -19,7 +19,7 @@ define void @foo(i32 %N) nounwind {
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    movl %edi, %ebx
 ; CHECK-NEXT:    xorl %ebp, %ebp
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # %bb
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/optimize-max-0.ll b/llvm/test/CodeGen/X86/optimize-max-0.ll
index 81dafdffe3116..283c00e17f21a 100644
--- a/llvm/test/CodeGen/X86/optimize-max-0.ll
+++ b/llvm/test/CodeGen/X86/optimize-max-0.ll
@@ -41,7 +41,7 @@ define void @foo(ptr %r, i32 %s, i32 %w, i32 %x, ptr %j, i32 %d) nounwind {
 ; CHECK-NEXT:    xorl %ecx, %ecx
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK-NEXT:    xorl %esi, %esi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_4: ## %bb6
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movzbl (%eax,%esi,2), %ebx
@@ -91,7 +91,7 @@ define void @foo(ptr %r, i32 %s, i32 %w, i32 %x, ptr %j, i32 %d) nounwind {
 ; CHECK-NEXT:    addl %esi, %ecx
 ; CHECK-NEXT:    xorl %esi, %esi
 ; CHECK-NEXT:    xorl %edi, %edi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_9: ## %bb13
 ; CHECK-NEXT:    ## =>This Loop Header: Depth=1
 ; CHECK-NEXT:    ## Child Loop BB0_10 Depth 2
@@ -102,7 +102,7 @@ define void @foo(ptr %r, i32 %s, i32 %w, i32 %x, ptr %j, i32 %d) nounwind {
 ; CHECK-NEXT:    imull {{[0-9]+}}(%esp), %edi
 ; CHECK-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi ## 4-byte Folded Reload
 ; CHECK-NEXT:    xorl %esi, %esi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_10: ## %bb14
 ; CHECK-NEXT:    ## Parent Loop BB0_9 Depth=1
 ; CHECK-NEXT:    ## => This Inner Loop Header: Depth=2
@@ -150,7 +150,7 @@ define void @foo(ptr %r, i32 %s, i32 %w, i32 %x, ptr %j, i32 %d) nounwind {
 ; CHECK-NEXT:    leal 15(%ebp), %eax
 ; CHECK-NEXT:    andl $-16, %eax
 ; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_17: ## %bb23
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    subl $4, %esp
@@ -183,7 +183,7 @@ define void @foo(ptr %r, i32 %s, i32 %w, i32 %x, ptr %j, i32 %d) nounwind {
 ; CHECK-NEXT:    andl $-16, %eax
 ; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_21: ## %bb30
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    subl $4, %esp
@@ -479,7 +479,7 @@ define void @bar(ptr %r, i32 %s, i32 %w, i32 %x, ptr %j, i32 %d) nounwind {
 ; CHECK-NEXT:    xorl %ecx, %ecx
 ; CHECK-NEXT:    movl %esi, %edx
 ; CHECK-NEXT:    xorl %esi, %esi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB1_4: ## %bb6
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movzbl (%eax,%esi,2), %ebx
@@ -524,7 +524,7 @@ define void @bar(ptr %r, i32 %s, i32 %w, i32 %x, ptr %j, i32 %d) nounwind {
 ; CHECK-NEXT:    addl %edx, %eax
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    xorl %ebx, %ebx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB1_9: ## %bb13
 ; CHECK-NEXT:    ## =>This Loop Header: Depth=1
 ; CHECK-NEXT:    ## Child Loop BB1_10 Depth 2
@@ -535,7 +535,7 @@ define void @bar(ptr %r, i32 %s, i32 %w, i32 %x, ptr %j, i32 %d) nounwind {
 ; CHECK-NEXT:    imull {{[0-9]+}}(%esp), %ebx
 ; CHECK-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx ## 4-byte Folded Reload
 ; CHECK-NEXT:    xorl %esi, %esi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB1_10: ## %bb14
 ; CHECK-NEXT:    ## Parent Loop BB1_9 Depth=1
 ; CHECK-NEXT:    ## => This Inner Loop Header: Depth=2
@@ -584,7 +584,7 @@ define void @bar(ptr %r, i32 %s, i32 %w, i32 %x, ptr %j, i32 %d) nounwind {
 ; CHECK-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax ## 4-byte Reload
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK-NEXT:    leal (%edx,%eax), %ebp
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB1_17: ## %bb23
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    subl $4, %esp
@@ -622,7 +622,7 @@ define void @bar(ptr %r, i32 %s, i32 %w, i32 %x, ptr %j, i32 %d) nounwind {
 ; CHECK-NEXT:    andl $-16, %eax
 ; CHECK-NEXT:    movl %eax, (%esp) ## 4-byte Spill
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB1_21: ## %bb30
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    subl $4, %esp
diff --git a/llvm/test/CodeGen/X86/optimize-max-1.ll b/llvm/test/CodeGen/X86/optimize-max-1.ll
index 4a49dc70d4995..1ac48586bf900 100644
--- a/llvm/test/CodeGen/X86/optimize-max-1.ll
+++ b/llvm/test/CodeGen/X86/optimize-max-1.ll
@@ -10,7 +10,7 @@ define void @fs(ptr nocapture %p, i64 %n) nounwind {
 ; CHECK-LABEL: fs:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %bb
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq $0, (%rdi,%rax,8)
@@ -40,7 +40,7 @@ define void @bs(ptr nocapture %p, i64 %n) nounwind {
 ; CHECK-LABEL: bs:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_1: # %bb
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq $0, (%rdi,%rax,8)
@@ -70,7 +70,7 @@ define void @fu(ptr nocapture %p, i64 %n) nounwind {
 ; CHECK-LABEL: fu:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB2_1: # %bb
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq $0, (%rdi,%rax,8)
@@ -100,7 +100,7 @@ define void @bu(ptr nocapture %p, i64 %n) nounwind {
 ; CHECK-LABEL: bu:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB3_1: # %bb
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq $0, (%rdi,%rax,8)
diff --git a/llvm/test/CodeGen/X86/optimize-max-2.ll b/llvm/test/CodeGen/X86/optimize-max-2.ll
index c83fe63883aae..3533bfbc61e6d 100644
--- a/llvm/test/CodeGen/X86/optimize-max-2.ll
+++ b/llvm/test/CodeGen/X86/optimize-max-2.ll
@@ -14,7 +14,7 @@ define void @foo(ptr nocapture %p, i64 %x, i64 %y) nounwind {
 ; CHECK-NEXT:    cmovneq %rdx, %rax
 ; CHECK-NEXT:    cmpq %rsi, %rax
 ; CHECK-NEXT:    cmovbeq %rsi, %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %bb4
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
diff --git a/llvm/test/CodeGen/X86/or-address.ll b/llvm/test/CodeGen/X86/or-address.ll
index ef1ea0ef5de95..6e59d3f70c332 100644
--- a/llvm/test/CodeGen/X86/or-address.ll
+++ b/llvm/test/CodeGen/X86/or-address.ll
@@ -12,7 +12,7 @@ define void @test(ptr nocapture %array, i32 %r0) nounwind ssp noredzone {
 ; CHECK-NEXT:    xorl %ecx, %ecx
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    xorl %r8d, %r8d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_1: ## %bb
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpb $4, %r8b
@@ -75,7 +75,7 @@ define void @test1(ptr nocapture %array, i32 %r0, i8 signext %k, i8 signext %i0)
 ; CHECK-NEXT:    ## kill: def $ecx killed $ecx def $rcx
 ; CHECK-NEXT:    movb $32, %al
 ; CHECK-NEXT:    xorl %r8d, %r8d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB1_1: ## %for.body
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpb $4, %cl
diff --git a/llvm/test/CodeGen/X86/overflowing-iv-codegen.ll b/llvm/test/CodeGen/X86/overflowing-iv-codegen.ll
index 3e4539d333474..e5dd5b23bea96 100644
--- a/llvm/test/CodeGen/X86/overflowing-iv-codegen.ll
+++ b/llvm/test/CodeGen/X86/overflowing-iv-codegen.ll
@@ -6,7 +6,7 @@ define i32 @test_01(ptr %p, i64 %len, i32 %x) {
 ; CHECK-LABEL: test_01:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addq $-4, %rdi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    subq $1, %rsi
@@ -48,7 +48,7 @@ define i32 @test_02(ptr %p, i64 %len, i32 %x) {
 ; CHECK-LABEL: test_02:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addq $-4, %rdi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    subq $1, %rsi
@@ -90,7 +90,7 @@ define i32 @test_02_nopoison(ptr %p, i64 %len, i32 %x) {
 ; CHECK-LABEL: test_02_nopoison:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addq $-4, %rdi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB2_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    subq $1, %rsi
@@ -134,7 +134,7 @@ define i32 @test_03(ptr %p, i64 %len, i32 %x) {
 ; CHECK-LABEL: test_03:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addq $-4, %rdi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB3_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    subq $1, %rsi
@@ -176,7 +176,7 @@ define i32 @test_03_nopoison(ptr %p, i64 %len, i32 %x) {
 ; CHECK-LABEL: test_03_nopoison:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addq $-4, %rdi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB4_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    subq $1, %rsi
diff --git a/llvm/test/CodeGen/X86/patchable-prologue.ll b/llvm/test/CodeGen/X86/patchable-prologue.ll
index aec76a359d267..a6a6c0d20a1af 100644
--- a/llvm/test/CodeGen/X86/patchable-prologue.ll
+++ b/llvm/test/CodeGen/X86/patchable-prologue.ll
@@ -12,7 +12,7 @@ define void @f0() "patchable-function"="prologue-short-redirect" {
 ; CHECK-LABEL: _f0{{>?}}:
 ; CHECK-NEXT:  66 90 	nop
 
-; CHECK-ALIGN: 	.p2align	4, 0x90
+; CHECK-ALIGN: 	.p2align	4
 ; CHECK-ALIGN: _f0:
 
 ; X86: f0:
@@ -35,7 +35,7 @@ define void @f1() "patchable-function"="prologue-short-redirect" "frame-pointer"
 ; CHECK-NEXT: 66 90     nop
 ; CHECK-NEXT: 55		pushq	%rbp
 
-; CHECK-ALIGN: 	.p2align	4, 0x90
+; CHECK-ALIGN: 	.p2align	4
 ; CHECK-ALIGN: _f1:
 
 ; X86: f1:
@@ -58,7 +58,7 @@ define void @f2() "patchable-function"="prologue-short-redirect" {
 ; CHECK-LABEL: _f2
 ; CHECK-NEXT: 48 81 ec a8 00 00 00 	subq	$168, %rsp
 
-; CHECK-ALIGN: 	.p2align	4, 0x90
+; CHECK-ALIGN: 	.p2align	4
 ; CHECK-ALIGN: _f2:
 
 ; X86: f2:
@@ -82,7 +82,7 @@ define void @f3() "patchable-function"="prologue-short-redirect" optsize {
 ; CHECK-LABEL: _f3
 ; CHECK-NEXT: 66 90 	nop
 
-; CHECK-ALIGN: 	.p2align	4, 0x90
+; CHECK-ALIGN: 	.p2align	4
 ; CHECK-ALIGN: _f3:
 
 ; X86: f3:
@@ -140,7 +140,7 @@ bb21:
 ; CHECK-LABEL: _emptyfunc
 ; CHECK-NEXT: 0f 0b 	ud2
 
-; CHECK-ALIGN: 	.p2align	4, 0x90
+; CHECK-ALIGN: 	.p2align	4
 ; CHECK-ALIGN: _emptyfunc:
 
 ; X86: emptyfunc:
@@ -166,7 +166,7 @@ define i32 @emptyfunc() "patchable-function"="prologue-short-redirect" {
 ;   } while ((++(*b++)));
 ; }
 
-; CHECK-ALIGN: 	.p2align	4, 0x90
+; CHECK-ALIGN: 	.p2align	4
 ; CHECK-ALIGN: _jmp_to_start:
 
 ; X86: jmp_to_start:
diff --git a/llvm/test/CodeGen/X86/pcsections-atomics.ll b/llvm/test/CodeGen/X86/pcsections-atomics.ll
index cfc9d50763af4..672ebc1ec7275 100644
--- a/llvm/test/CodeGen/X86/pcsections-atomics.ll
+++ b/llvm/test/CodeGen/X86/pcsections-atomics.ll
@@ -706,7 +706,7 @@ define void @atomic8_nand_monotonic(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:  .Lpcsection16:
 ; O1-NEXT:    movzbl (%rdi), %eax
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB16_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
@@ -727,7 +727,7 @@ define void @atomic8_nand_monotonic(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:  .Lpcsection16:
 ; O2-NEXT:    movzbl (%rdi), %eax
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB16_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
@@ -748,7 +748,7 @@ define void @atomic8_nand_monotonic(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:  .Lpcsection16:
 ; O3-NEXT:    movzbl (%rdi), %eax
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB16_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
@@ -1044,7 +1044,7 @@ define void @atomic8_nand_acquire(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:  .Lpcsection27:
 ; O1-NEXT:    movzbl (%rdi), %eax
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB23_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
@@ -1065,7 +1065,7 @@ define void @atomic8_nand_acquire(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:  .Lpcsection27:
 ; O2-NEXT:    movzbl (%rdi), %eax
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB23_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
@@ -1086,7 +1086,7 @@ define void @atomic8_nand_acquire(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:  .Lpcsection27:
 ; O3-NEXT:    movzbl (%rdi), %eax
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB23_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
@@ -1382,7 +1382,7 @@ define void @atomic8_nand_release(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:  .Lpcsection38:
 ; O1-NEXT:    movzbl (%rdi), %eax
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB30_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
@@ -1403,7 +1403,7 @@ define void @atomic8_nand_release(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:  .Lpcsection38:
 ; O2-NEXT:    movzbl (%rdi), %eax
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB30_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
@@ -1424,7 +1424,7 @@ define void @atomic8_nand_release(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:  .Lpcsection38:
 ; O3-NEXT:    movzbl (%rdi), %eax
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB30_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
@@ -1720,7 +1720,7 @@ define void @atomic8_nand_acq_rel(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:  .Lpcsection49:
 ; O1-NEXT:    movzbl (%rdi), %eax
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB37_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
@@ -1741,7 +1741,7 @@ define void @atomic8_nand_acq_rel(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:  .Lpcsection49:
 ; O2-NEXT:    movzbl (%rdi), %eax
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB37_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
@@ -1762,7 +1762,7 @@ define void @atomic8_nand_acq_rel(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:  .Lpcsection49:
 ; O3-NEXT:    movzbl (%rdi), %eax
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB37_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
@@ -2058,7 +2058,7 @@ define void @atomic8_nand_seq_cst(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:  .Lpcsection60:
 ; O1-NEXT:    movzbl (%rdi), %eax
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB44_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
@@ -2079,7 +2079,7 @@ define void @atomic8_nand_seq_cst(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:  .Lpcsection60:
 ; O2-NEXT:    movzbl (%rdi), %eax
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB44_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
@@ -2100,7 +2100,7 @@ define void @atomic8_nand_seq_cst(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:  .Lpcsection60:
 ; O3-NEXT:    movzbl (%rdi), %eax
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB44_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
@@ -3151,7 +3151,7 @@ define void @atomic16_nand_monotonic(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:  .Lpcsection109:
 ; O1-NEXT:    movzwl (%rdi), %eax
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB64_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
@@ -3176,7 +3176,7 @@ define void @atomic16_nand_monotonic(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:  .Lpcsection109:
 ; O2-NEXT:    movzwl (%rdi), %eax
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB64_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
@@ -3201,7 +3201,7 @@ define void @atomic16_nand_monotonic(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:  .Lpcsection109:
 ; O3-NEXT:    movzwl (%rdi), %eax
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB64_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
@@ -3505,7 +3505,7 @@ define void @atomic16_nand_acquire(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:  .Lpcsection122:
 ; O1-NEXT:    movzwl (%rdi), %eax
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB71_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
@@ -3530,7 +3530,7 @@ define void @atomic16_nand_acquire(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:  .Lpcsection122:
 ; O2-NEXT:    movzwl (%rdi), %eax
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB71_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
@@ -3555,7 +3555,7 @@ define void @atomic16_nand_acquire(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:  .Lpcsection122:
 ; O3-NEXT:    movzwl (%rdi), %eax
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB71_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
@@ -3859,7 +3859,7 @@ define void @atomic16_nand_release(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:  .Lpcsection135:
 ; O1-NEXT:    movzwl (%rdi), %eax
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB78_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
@@ -3884,7 +3884,7 @@ define void @atomic16_nand_release(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:  .Lpcsection135:
 ; O2-NEXT:    movzwl (%rdi), %eax
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB78_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
@@ -3909,7 +3909,7 @@ define void @atomic16_nand_release(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:  .Lpcsection135:
 ; O3-NEXT:    movzwl (%rdi), %eax
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB78_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
@@ -4213,7 +4213,7 @@ define void @atomic16_nand_acq_rel(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:  .Lpcsection148:
 ; O1-NEXT:    movzwl (%rdi), %eax
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB85_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
@@ -4238,7 +4238,7 @@ define void @atomic16_nand_acq_rel(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:  .Lpcsection148:
 ; O2-NEXT:    movzwl (%rdi), %eax
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB85_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
@@ -4263,7 +4263,7 @@ define void @atomic16_nand_acq_rel(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:  .Lpcsection148:
 ; O3-NEXT:    movzwl (%rdi), %eax
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB85_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
@@ -4567,7 +4567,7 @@ define void @atomic16_nand_seq_cst(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:  .Lpcsection161:
 ; O1-NEXT:    movzwl (%rdi), %eax
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB92_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
@@ -4592,7 +4592,7 @@ define void @atomic16_nand_seq_cst(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:  .Lpcsection161:
 ; O2-NEXT:    movzwl (%rdi), %eax
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB92_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
@@ -4617,7 +4617,7 @@ define void @atomic16_nand_seq_cst(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:  .Lpcsection161:
 ; O3-NEXT:    movzwl (%rdi), %eax
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB92_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
@@ -5623,7 +5623,7 @@ define void @atomic32_nand_monotonic(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:  .Lpcsection197:
 ; O1-NEXT:    movl (%rdi), %eax
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB112_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
@@ -5644,7 +5644,7 @@ define void @atomic32_nand_monotonic(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:  .Lpcsection197:
 ; O2-NEXT:    movl (%rdi), %eax
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB112_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
@@ -5665,7 +5665,7 @@ define void @atomic32_nand_monotonic(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:  .Lpcsection197:
 ; O3-NEXT:    movl (%rdi), %eax
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB112_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
@@ -5961,7 +5961,7 @@ define void @atomic32_nand_acquire(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:  .Lpcsection208:
 ; O1-NEXT:    movl (%rdi), %eax
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB119_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
@@ -5982,7 +5982,7 @@ define void @atomic32_nand_acquire(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:  .Lpcsection208:
 ; O2-NEXT:    movl (%rdi), %eax
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB119_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
@@ -6003,7 +6003,7 @@ define void @atomic32_nand_acquire(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:  .Lpcsection208:
 ; O3-NEXT:    movl (%rdi), %eax
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB119_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
@@ -6299,7 +6299,7 @@ define void @atomic32_nand_release(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:  .Lpcsection219:
 ; O1-NEXT:    movl (%rdi), %eax
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB126_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
@@ -6320,7 +6320,7 @@ define void @atomic32_nand_release(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:  .Lpcsection219:
 ; O2-NEXT:    movl (%rdi), %eax
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB126_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
@@ -6341,7 +6341,7 @@ define void @atomic32_nand_release(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:  .Lpcsection219:
 ; O3-NEXT:    movl (%rdi), %eax
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB126_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
@@ -6637,7 +6637,7 @@ define void @atomic32_nand_acq_rel(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:  .Lpcsection230:
 ; O1-NEXT:    movl (%rdi), %eax
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB133_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
@@ -6658,7 +6658,7 @@ define void @atomic32_nand_acq_rel(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:  .Lpcsection230:
 ; O2-NEXT:    movl (%rdi), %eax
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB133_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
@@ -6679,7 +6679,7 @@ define void @atomic32_nand_acq_rel(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:  .Lpcsection230:
 ; O3-NEXT:    movl (%rdi), %eax
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB133_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
@@ -6975,7 +6975,7 @@ define void @atomic32_nand_seq_cst(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:  .Lpcsection241:
 ; O1-NEXT:    movl (%rdi), %eax
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB140_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
@@ -6996,7 +6996,7 @@ define void @atomic32_nand_seq_cst(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:  .Lpcsection241:
 ; O2-NEXT:    movl (%rdi), %eax
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB140_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
@@ -7017,7 +7017,7 @@ define void @atomic32_nand_seq_cst(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:  .Lpcsection241:
 ; O3-NEXT:    movl (%rdi), %eax
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB140_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
@@ -8145,7 +8145,7 @@ define void @atomic64_nand_monotonic(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:  .Lpcsection292:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB162_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
@@ -8166,7 +8166,7 @@ define void @atomic64_nand_monotonic(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:  .Lpcsection292:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB162_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
@@ -8187,7 +8187,7 @@ define void @atomic64_nand_monotonic(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:  .Lpcsection292:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB162_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
@@ -8486,7 +8486,7 @@ define void @atomic64_nand_acquire(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:  .Lpcsection303:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB169_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
@@ -8507,7 +8507,7 @@ define void @atomic64_nand_acquire(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:  .Lpcsection303:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB169_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
@@ -8528,7 +8528,7 @@ define void @atomic64_nand_acquire(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:  .Lpcsection303:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB169_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
@@ -8827,7 +8827,7 @@ define void @atomic64_nand_release(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:  .Lpcsection314:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB176_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
@@ -8848,7 +8848,7 @@ define void @atomic64_nand_release(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:  .Lpcsection314:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB176_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
@@ -8869,7 +8869,7 @@ define void @atomic64_nand_release(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:  .Lpcsection314:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB176_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
@@ -9168,7 +9168,7 @@ define void @atomic64_nand_acq_rel(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:  .Lpcsection325:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB183_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
@@ -9189,7 +9189,7 @@ define void @atomic64_nand_acq_rel(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:  .Lpcsection325:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB183_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
@@ -9210,7 +9210,7 @@ define void @atomic64_nand_acq_rel(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:  .Lpcsection325:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB183_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
@@ -9509,7 +9509,7 @@ define void @atomic64_nand_seq_cst(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:  .Lpcsection336:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB190_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
@@ -9530,7 +9530,7 @@ define void @atomic64_nand_seq_cst(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:  .Lpcsection336:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB190_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
@@ -9551,7 +9551,7 @@ define void @atomic64_nand_seq_cst(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:  .Lpcsection336:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB190_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
@@ -10561,7 +10561,7 @@ define void @atomic128_store_unordered(ptr %a) {
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:  .Lpcsection396:
 ; O1-NEXT:    movl $42, %ebx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB203_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:  .Lpcsection397:
@@ -10588,7 +10588,7 @@ define void @atomic128_store_unordered(ptr %a) {
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:  .Lpcsection396:
 ; O2-NEXT:    movl $42, %ebx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB203_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:  .Lpcsection397:
@@ -10615,7 +10615,7 @@ define void @atomic128_store_unordered(ptr %a) {
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:  .Lpcsection396:
 ; O3-NEXT:    movl $42, %ebx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB203_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:  .Lpcsection397:
@@ -10688,7 +10688,7 @@ define void @atomic128_store_monotonic(ptr %a) {
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:  .Lpcsection402:
 ; O1-NEXT:    movl $42, %ebx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB204_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:  .Lpcsection403:
@@ -10715,7 +10715,7 @@ define void @atomic128_store_monotonic(ptr %a) {
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:  .Lpcsection402:
 ; O2-NEXT:    movl $42, %ebx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB204_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:  .Lpcsection403:
@@ -10742,7 +10742,7 @@ define void @atomic128_store_monotonic(ptr %a) {
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:  .Lpcsection402:
 ; O3-NEXT:    movl $42, %ebx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB204_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:  .Lpcsection403:
@@ -10815,7 +10815,7 @@ define void @atomic128_store_release(ptr %a) {
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:  .Lpcsection408:
 ; O1-NEXT:    movl $42, %ebx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB205_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:  .Lpcsection409:
@@ -10842,7 +10842,7 @@ define void @atomic128_store_release(ptr %a) {
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:  .Lpcsection408:
 ; O2-NEXT:    movl $42, %ebx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB205_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:  .Lpcsection409:
@@ -10869,7 +10869,7 @@ define void @atomic128_store_release(ptr %a) {
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:  .Lpcsection408:
 ; O3-NEXT:    movl $42, %ebx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB205_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:  .Lpcsection409:
@@ -10942,7 +10942,7 @@ define void @atomic128_store_seq_cst(ptr %a) {
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:  .Lpcsection414:
 ; O1-NEXT:    movl $42, %ebx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB206_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:  .Lpcsection415:
@@ -10969,7 +10969,7 @@ define void @atomic128_store_seq_cst(ptr %a) {
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:  .Lpcsection414:
 ; O2-NEXT:    movl $42, %ebx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB206_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:  .Lpcsection415:
@@ -10996,7 +10996,7 @@ define void @atomic128_store_seq_cst(ptr %a) {
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:  .Lpcsection414:
 ; O3-NEXT:    movl $42, %ebx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB206_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:  .Lpcsection415:
@@ -11108,7 +11108,7 @@ define void @atomic128_xchg_monotonic(ptr %a) {
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:  .Lpcsection421:
 ; O1-NEXT:    movl $42, %ebx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB208_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:  .Lpcsection422:
@@ -11135,7 +11135,7 @@ define void @atomic128_xchg_monotonic(ptr %a) {
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:  .Lpcsection421:
 ; O2-NEXT:    movl $42, %ebx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB208_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:  .Lpcsection422:
@@ -11162,7 +11162,7 @@ define void @atomic128_xchg_monotonic(ptr %a) {
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:  .Lpcsection421:
 ; O3-NEXT:    movl $42, %ebx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB208_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:  .Lpcsection422:
@@ -11233,7 +11233,7 @@ define void @atomic128_add_monotonic(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection426:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB209_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
@@ -11262,7 +11262,7 @@ define void @atomic128_add_monotonic(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection426:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB209_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
@@ -11291,7 +11291,7 @@ define void @atomic128_add_monotonic(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection426:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB209_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
@@ -11366,7 +11366,7 @@ define void @atomic128_sub_monotonic(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection432:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB210_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
@@ -11395,7 +11395,7 @@ define void @atomic128_sub_monotonic(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection432:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB210_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
@@ -11424,7 +11424,7 @@ define void @atomic128_sub_monotonic(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection432:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB210_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
@@ -11501,7 +11501,7 @@ define void @atomic128_and_monotonic(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection438:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB211_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ebx
@@ -11529,7 +11529,7 @@ define void @atomic128_and_monotonic(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection438:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB211_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ebx
@@ -11557,7 +11557,7 @@ define void @atomic128_and_monotonic(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection438:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB211_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ebx
@@ -11629,7 +11629,7 @@ define void @atomic128_or_monotonic(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection444:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB212_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
@@ -11656,7 +11656,7 @@ define void @atomic128_or_monotonic(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection444:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB212_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
@@ -11683,7 +11683,7 @@ define void @atomic128_or_monotonic(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection444:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB212_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
@@ -11754,7 +11754,7 @@ define void @atomic128_xor_monotonic(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection449:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB213_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
@@ -11781,7 +11781,7 @@ define void @atomic128_xor_monotonic(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection449:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB213_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
@@ -11808,7 +11808,7 @@ define void @atomic128_xor_monotonic(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection449:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB213_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
@@ -11887,7 +11887,7 @@ define void @atomic128_nand_monotonic(ptr %a) {
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:  .Lpcsection455:
 ; O1-NEXT:    movq $-1, %rcx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB214_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ebx
@@ -11917,7 +11917,7 @@ define void @atomic128_nand_monotonic(ptr %a) {
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:  .Lpcsection455:
 ; O2-NEXT:    movq $-1, %rcx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB214_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ebx
@@ -11947,7 +11947,7 @@ define void @atomic128_nand_monotonic(ptr %a) {
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:  .Lpcsection455:
 ; O3-NEXT:    movq $-1, %rcx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB214_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ebx
@@ -12023,7 +12023,7 @@ define void @atomic128_xchg_acquire(ptr %a) {
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:  .Lpcsection462:
 ; O1-NEXT:    movl $42, %ebx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB215_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:  .Lpcsection463:
@@ -12050,7 +12050,7 @@ define void @atomic128_xchg_acquire(ptr %a) {
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:  .Lpcsection462:
 ; O2-NEXT:    movl $42, %ebx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB215_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:  .Lpcsection463:
@@ -12077,7 +12077,7 @@ define void @atomic128_xchg_acquire(ptr %a) {
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:  .Lpcsection462:
 ; O3-NEXT:    movl $42, %ebx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB215_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:  .Lpcsection463:
@@ -12148,7 +12148,7 @@ define void @atomic128_add_acquire(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection467:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB216_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
@@ -12177,7 +12177,7 @@ define void @atomic128_add_acquire(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection467:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB216_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
@@ -12206,7 +12206,7 @@ define void @atomic128_add_acquire(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection467:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB216_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
@@ -12281,7 +12281,7 @@ define void @atomic128_sub_acquire(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection473:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB217_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
@@ -12310,7 +12310,7 @@ define void @atomic128_sub_acquire(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection473:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB217_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
@@ -12339,7 +12339,7 @@ define void @atomic128_sub_acquire(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection473:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB217_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
@@ -12416,7 +12416,7 @@ define void @atomic128_and_acquire(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection479:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB218_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ebx
@@ -12444,7 +12444,7 @@ define void @atomic128_and_acquire(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection479:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB218_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ebx
@@ -12472,7 +12472,7 @@ define void @atomic128_and_acquire(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection479:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB218_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ebx
@@ -12544,7 +12544,7 @@ define void @atomic128_or_acquire(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection485:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB219_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
@@ -12571,7 +12571,7 @@ define void @atomic128_or_acquire(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection485:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB219_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
@@ -12598,7 +12598,7 @@ define void @atomic128_or_acquire(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection485:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB219_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
@@ -12669,7 +12669,7 @@ define void @atomic128_xor_acquire(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection490:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB220_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
@@ -12696,7 +12696,7 @@ define void @atomic128_xor_acquire(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection490:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB220_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
@@ -12723,7 +12723,7 @@ define void @atomic128_xor_acquire(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection490:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB220_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
@@ -12802,7 +12802,7 @@ define void @atomic128_nand_acquire(ptr %a) {
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:  .Lpcsection496:
 ; O1-NEXT:    movq $-1, %rcx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB221_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ebx
@@ -12832,7 +12832,7 @@ define void @atomic128_nand_acquire(ptr %a) {
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:  .Lpcsection496:
 ; O2-NEXT:    movq $-1, %rcx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB221_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ebx
@@ -12862,7 +12862,7 @@ define void @atomic128_nand_acquire(ptr %a) {
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:  .Lpcsection496:
 ; O3-NEXT:    movq $-1, %rcx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB221_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ebx
@@ -12938,7 +12938,7 @@ define void @atomic128_xchg_release(ptr %a) {
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:  .Lpcsection503:
 ; O1-NEXT:    movl $42, %ebx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB222_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:  .Lpcsection504:
@@ -12965,7 +12965,7 @@ define void @atomic128_xchg_release(ptr %a) {
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:  .Lpcsection503:
 ; O2-NEXT:    movl $42, %ebx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB222_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:  .Lpcsection504:
@@ -12992,7 +12992,7 @@ define void @atomic128_xchg_release(ptr %a) {
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:  .Lpcsection503:
 ; O3-NEXT:    movl $42, %ebx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB222_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:  .Lpcsection504:
@@ -13062,7 +13062,7 @@ define void @atomic128_add_release(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection508:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB223_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
@@ -13091,7 +13091,7 @@ define void @atomic128_add_release(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection508:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB223_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
@@ -13120,7 +13120,7 @@ define void @atomic128_add_release(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection508:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB223_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
@@ -13195,7 +13195,7 @@ define void @atomic128_sub_release(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection514:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB224_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
@@ -13224,7 +13224,7 @@ define void @atomic128_sub_release(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection514:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB224_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
@@ -13253,7 +13253,7 @@ define void @atomic128_sub_release(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection514:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB224_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
@@ -13330,7 +13330,7 @@ define void @atomic128_and_release(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection520:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB225_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ebx
@@ -13358,7 +13358,7 @@ define void @atomic128_and_release(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection520:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB225_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ebx
@@ -13386,7 +13386,7 @@ define void @atomic128_and_release(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection520:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB225_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ebx
@@ -13458,7 +13458,7 @@ define void @atomic128_or_release(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection526:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB226_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
@@ -13485,7 +13485,7 @@ define void @atomic128_or_release(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection526:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB226_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
@@ -13512,7 +13512,7 @@ define void @atomic128_or_release(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection526:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB226_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
@@ -13583,7 +13583,7 @@ define void @atomic128_xor_release(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection531:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB227_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
@@ -13610,7 +13610,7 @@ define void @atomic128_xor_release(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection531:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB227_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
@@ -13637,7 +13637,7 @@ define void @atomic128_xor_release(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection531:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB227_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
@@ -13716,7 +13716,7 @@ define void @atomic128_nand_release(ptr %a) {
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:  .Lpcsection537:
 ; O1-NEXT:    movq $-1, %rcx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB228_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ebx
@@ -13746,7 +13746,7 @@ define void @atomic128_nand_release(ptr %a) {
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:  .Lpcsection537:
 ; O2-NEXT:    movq $-1, %rcx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB228_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ebx
@@ -13776,7 +13776,7 @@ define void @atomic128_nand_release(ptr %a) {
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:  .Lpcsection537:
 ; O3-NEXT:    movq $-1, %rcx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB228_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ebx
@@ -13852,7 +13852,7 @@ define void @atomic128_xchg_acq_rel(ptr %a) {
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:  .Lpcsection544:
 ; O1-NEXT:    movl $42, %ebx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB229_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:  .Lpcsection545:
@@ -13879,7 +13879,7 @@ define void @atomic128_xchg_acq_rel(ptr %a) {
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:  .Lpcsection544:
 ; O2-NEXT:    movl $42, %ebx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB229_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:  .Lpcsection545:
@@ -13906,7 +13906,7 @@ define void @atomic128_xchg_acq_rel(ptr %a) {
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:  .Lpcsection544:
 ; O3-NEXT:    movl $42, %ebx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB229_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:  .Lpcsection545:
@@ -13977,7 +13977,7 @@ define void @atomic128_add_acq_rel(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection549:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB230_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
@@ -14006,7 +14006,7 @@ define void @atomic128_add_acq_rel(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection549:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB230_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
@@ -14035,7 +14035,7 @@ define void @atomic128_add_acq_rel(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection549:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB230_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
@@ -14110,7 +14110,7 @@ define void @atomic128_sub_acq_rel(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection555:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB231_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
@@ -14139,7 +14139,7 @@ define void @atomic128_sub_acq_rel(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection555:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB231_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
@@ -14168,7 +14168,7 @@ define void @atomic128_sub_acq_rel(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection555:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB231_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
@@ -14245,7 +14245,7 @@ define void @atomic128_and_acq_rel(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection561:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB232_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ebx
@@ -14273,7 +14273,7 @@ define void @atomic128_and_acq_rel(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection561:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB232_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ebx
@@ -14301,7 +14301,7 @@ define void @atomic128_and_acq_rel(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection561:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB232_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ebx
@@ -14373,7 +14373,7 @@ define void @atomic128_or_acq_rel(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection567:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB233_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
@@ -14400,7 +14400,7 @@ define void @atomic128_or_acq_rel(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection567:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB233_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
@@ -14427,7 +14427,7 @@ define void @atomic128_or_acq_rel(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection567:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB233_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
@@ -14498,7 +14498,7 @@ define void @atomic128_xor_acq_rel(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection572:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB234_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
@@ -14525,7 +14525,7 @@ define void @atomic128_xor_acq_rel(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection572:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB234_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
@@ -14552,7 +14552,7 @@ define void @atomic128_xor_acq_rel(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection572:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB234_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
@@ -14631,7 +14631,7 @@ define void @atomic128_nand_acq_rel(ptr %a) {
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:  .Lpcsection578:
 ; O1-NEXT:    movq $-1, %rcx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB235_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ebx
@@ -14661,7 +14661,7 @@ define void @atomic128_nand_acq_rel(ptr %a) {
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:  .Lpcsection578:
 ; O2-NEXT:    movq $-1, %rcx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB235_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ebx
@@ -14691,7 +14691,7 @@ define void @atomic128_nand_acq_rel(ptr %a) {
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:  .Lpcsection578:
 ; O3-NEXT:    movq $-1, %rcx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB235_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ebx
@@ -14767,7 +14767,7 @@ define void @atomic128_xchg_seq_cst(ptr %a) {
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:  .Lpcsection585:
 ; O1-NEXT:    movl $42, %ebx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB236_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:  .Lpcsection586:
@@ -14794,7 +14794,7 @@ define void @atomic128_xchg_seq_cst(ptr %a) {
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:  .Lpcsection585:
 ; O2-NEXT:    movl $42, %ebx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB236_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:  .Lpcsection586:
@@ -14821,7 +14821,7 @@ define void @atomic128_xchg_seq_cst(ptr %a) {
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:  .Lpcsection585:
 ; O3-NEXT:    movl $42, %ebx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB236_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:  .Lpcsection586:
@@ -14892,7 +14892,7 @@ define void @atomic128_add_seq_cst(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection590:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB237_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
@@ -14921,7 +14921,7 @@ define void @atomic128_add_seq_cst(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection590:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB237_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
@@ -14950,7 +14950,7 @@ define void @atomic128_add_seq_cst(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection590:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB237_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
@@ -15025,7 +15025,7 @@ define void @atomic128_sub_seq_cst(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection596:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB238_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
@@ -15054,7 +15054,7 @@ define void @atomic128_sub_seq_cst(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection596:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB238_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
@@ -15083,7 +15083,7 @@ define void @atomic128_sub_seq_cst(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection596:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB238_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
@@ -15160,7 +15160,7 @@ define void @atomic128_and_seq_cst(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection602:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB239_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ebx
@@ -15188,7 +15188,7 @@ define void @atomic128_and_seq_cst(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection602:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB239_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ebx
@@ -15216,7 +15216,7 @@ define void @atomic128_and_seq_cst(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection602:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB239_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ebx
@@ -15288,7 +15288,7 @@ define void @atomic128_or_seq_cst(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection608:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB240_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
@@ -15315,7 +15315,7 @@ define void @atomic128_or_seq_cst(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection608:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB240_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
@@ -15342,7 +15342,7 @@ define void @atomic128_or_seq_cst(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection608:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB240_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
@@ -15413,7 +15413,7 @@ define void @atomic128_xor_seq_cst(ptr %a) {
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:  .Lpcsection613:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB241_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
@@ -15440,7 +15440,7 @@ define void @atomic128_xor_seq_cst(ptr %a) {
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:  .Lpcsection613:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB241_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
@@ -15467,7 +15467,7 @@ define void @atomic128_xor_seq_cst(ptr %a) {
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:  .Lpcsection613:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB241_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
@@ -15546,7 +15546,7 @@ define void @atomic128_nand_seq_cst(ptr %a) {
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:  .Lpcsection619:
 ; O1-NEXT:    movq $-1, %rcx
-; O1-NEXT:    .p2align 4, 0x90
+; O1-NEXT:    .p2align 4
 ; O1-NEXT:  .LBB242_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ebx
@@ -15576,7 +15576,7 @@ define void @atomic128_nand_seq_cst(ptr %a) {
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:  .Lpcsection619:
 ; O2-NEXT:    movq $-1, %rcx
-; O2-NEXT:    .p2align 4, 0x90
+; O2-NEXT:    .p2align 4
 ; O2-NEXT:  .LBB242_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ebx
@@ -15606,7 +15606,7 @@ define void @atomic128_nand_seq_cst(ptr %a) {
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:  .Lpcsection619:
 ; O3-NEXT:    movq $-1, %rcx
-; O3-NEXT:    .p2align 4, 0x90
+; O3-NEXT:    .p2align 4
 ; O3-NEXT:  .LBB242_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ebx
diff --git a/llvm/test/CodeGen/X86/peep-test-0.ll b/llvm/test/CodeGen/X86/peep-test-0.ll
index 71dadf7edea3a..3a0ef86b64ee3 100644
--- a/llvm/test/CodeGen/X86/peep-test-0.ll
+++ b/llvm/test/CodeGen/X86/peep-test-0.ll
@@ -8,7 +8,7 @@ define void @loop(i64 %n, ptr nocapture %d) nounwind {
 ; CHECK-NEXT:    shlq $4, %rax
 ; CHECK-NEXT:    addq %rsi, %rax
 ; CHECK-NEXT:    movsd {{.*#+}} xmm0 = [3.0E+0,0.0E+0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %bb
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
diff --git a/llvm/test/CodeGen/X86/peep-test-1.ll b/llvm/test/CodeGen/X86/peep-test-1.ll
index 0d86bd4ec8c45..730a7fdccbd7c 100644
--- a/llvm/test/CodeGen/X86/peep-test-1.ll
+++ b/llvm/test/CodeGen/X86/peep-test-1.ll
@@ -6,7 +6,7 @@ define void @foo(i32 %n, ptr nocapture %p) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %bb
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    fldl (%eax,%ecx,8)
diff --git a/llvm/test/CodeGen/X86/peephole-copy.ll b/llvm/test/CodeGen/X86/peephole-copy.ll
index 69112d0a1f471..e7ca7c5cc6393 100644
--- a/llvm/test/CodeGen/X86/peephole-copy.ll
+++ b/llvm/test/CodeGen/X86/peephole-copy.ll
@@ -17,7 +17,7 @@ define void @foo(ptr %p1, ptr %p2, ptr %p3, ptr %p4) {
 ; CHECK-NEXT:    movl (%rax), %eax
 ; CHECK-NEXT:    shll %cl, %eax
 ; CHECK-NEXT:    movl %r8d, (%rdx)
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %loop2.header
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl $5, %edi
diff --git a/llvm/test/CodeGen/X86/pic-load-remat.ll b/llvm/test/CodeGen/X86/pic-load-remat.ll
index 7737093d2ba07..2587b0379a0f7 100644
--- a/llvm/test/CodeGen/X86/pic-load-remat.ll
+++ b/llvm/test/CodeGen/X86/pic-load-remat.ll
@@ -12,7 +12,7 @@ define void @f() nounwind  {
 ; CHECK-NEXT:    movdqa {{.*#+}} xmm0 = [21183,21183,21183,21183,21183,21183,21183,21183]
 ; CHECK-NEXT:    paddsw %xmm0, %xmm0
 ; CHECK-NEXT:    paddw %xmm1, %xmm0
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_1: ## %bb
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movdqa %xmm0, 0
diff --git a/llvm/test/CodeGen/X86/postalloc-coalescing.ll b/llvm/test/CodeGen/X86/postalloc-coalescing.ll
index 68266fc4e158a..7430d7a9baf26 100644
--- a/llvm/test/CodeGen/X86/postalloc-coalescing.ll
+++ b/llvm/test/CodeGen/X86/postalloc-coalescing.ll
@@ -7,7 +7,7 @@ define fastcc i32 @_Z18yy_get_next_bufferv() nounwind {
 ; CHECK-NEXT:    movl $42, %eax
 ; CHECK-NEXT:    cmpl $-1, %eax
 ; CHECK-NEXT:    je .LBB0_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %bb116
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movb %al, 0
diff --git a/llvm/test/CodeGen/X86/pr111170.ll b/llvm/test/CodeGen/X86/pr111170.ll
new file mode 100644
index 0000000000000..145bf7119edcb
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr111170.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=i686-pc-windows-msvc -mcpu=corei7-avx | FileCheck %s
+
+define void @PR111170(<16 x i32> %x_load, ptr %offsetsPtr.i) {
+; CHECK-LABEL: PR111170:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm2 = [2.80259693E-44,2.80259693E-44,2.80259693E-44,2.80259693E-44]
+; CHECK-NEXT:    vpmulld %xmm2, %xmm1, %xmm3
+; CHECK-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; CHECK-NEXT:    vpmulld %xmm2, %xmm1, %xmm1
+; CHECK-NEXT:    vpmulld %xmm2, %xmm0, %xmm4
+; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; CHECK-NEXT:    vpmulld %xmm2, %xmm0, %xmm0
+; CHECK-NEXT:    vmovdqu %xmm0, 16(%eax)
+; CHECK-NEXT:    vmovdqu %xmm4, (%eax)
+; CHECK-NEXT:    vmovdqu %xmm1, 48(%eax)
+; CHECK-NEXT:    vmovdqu %xmm3, 32(%eax)
+; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vmovdqu %xmm0, 16
+; CHECK-NEXT:    vmovdqu %xmm0, 0
+; CHECK-NEXT:    vmovdqu %xmm0, 48
+; CHECK-NEXT:    vmovdqu %xmm0, 32
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retl
+  %mul__x_load = mul <16 x i32> <i32 20, i32 20, i32 20, i32 20, i32 20, i32 20, i32 20, i32 20, i32 20, i32 20, i32 20, i32 20, i32 20, i32 20, i32 20, i32 20>, %x_load
+  store <16 x i32> %mul__x_load, ptr %offsetsPtr.i, align 4
+  %blend1.i12.i = call <8 x float> @llvm.x86.avx.blendv.ps.256(<8 x float> zeroinitializer, <8 x float> <float 0x36E4000000000000, float 0x36E4000000000000, float 0x36E4000000000000, float 0x36E4000000000000, float 0x36E4000000000000, float 0x36E4000000000000, float 0x36E4000000000000, float 0x36E4000000000000>, <8 x float> zeroinitializer)
+  %blend.i13.i = shufflevector <8 x float> zeroinitializer, <8 x float> %blend1.i12.i, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %blendAsInt.i14.i = bitcast <16 x float> %blend.i13.i to <16 x i32>
+  store <16 x i32> %blendAsInt.i14.i, ptr null, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/pr14314.ll b/llvm/test/CodeGen/X86/pr14314.ll
index c7dabf1f661cf..bb1608e3cd9b2 100644
--- a/llvm/test/CodeGen/X86/pr14314.ll
+++ b/llvm/test/CodeGen/X86/pr14314.ll
@@ -13,7 +13,7 @@ define i64 @atomicSub(ptr %a, i64 %b) nounwind {
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ebp
 ; CHECK-NEXT:    movl (%ebp), %eax
 ; CHECK-NEXT:    movl 4(%ebp), %edx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %eax, %ebx
diff --git a/llvm/test/CodeGen/X86/pr22338.ll b/llvm/test/CodeGen/X86/pr22338.ll
index 9ab5248a94602..bf3967ca8fede 100644
--- a/llvm/test/CodeGen/X86/pr22338.ll
+++ b/llvm/test/CodeGen/X86/pr22338.ll
@@ -21,7 +21,7 @@ define i32 @fn(i32 %a0, i32 %a1) {
 ; X86-NEXT:    addb %dl, %dl
 ; X86-NEXT:    movl %edx, %ecx
 ; X86-NEXT:    shll %cl, %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB0_1: # %bb1
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    testl %ebx, %ebx
@@ -46,7 +46,7 @@ define i32 @fn(i32 %a0, i32 %a1) {
 ; X64-NEXT:    addb %dl, %dl
 ; X64-NEXT:    movl %edx, %ecx
 ; X64-NEXT:    shll %cl, %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB0_1: # %bb1
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    testl %esi, %esi
diff --git a/llvm/test/CodeGen/X86/pr30562.ll b/llvm/test/CodeGen/X86/pr30562.ll
index 2745514cff8d4..e05a8672b1f81 100644
--- a/llvm/test/CodeGen/X86/pr30562.ll
+++ b/llvm/test/CodeGen/X86/pr30562.ll
@@ -10,7 +10,7 @@ define i32 @foo(ptr nocapture %perm, i32 %n) {
 ; CHECK-NEXT:    movl %esi, %ecx
 ; CHECK-NEXT:    andl $1, %ecx
 ; CHECK-NEXT:    movaps {{.*#+}} xmm1 = [2,3]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
diff --git a/llvm/test/CodeGen/X86/pr32108.ll b/llvm/test/CodeGen/X86/pr32108.ll
index dc14746440ae5..32f8a7657a3f1 100644
--- a/llvm/test/CodeGen/X86/pr32108.ll
+++ b/llvm/test/CodeGen/X86/pr32108.ll
@@ -4,7 +4,7 @@
 define void @pr32108() {
 ; CHECK-LABEL: pr32108:
 ; CHECK:       # %bb.0: # %BB
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %CF244
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp .LBB0_1
diff --git a/llvm/test/CodeGen/X86/pr33290.ll b/llvm/test/CodeGen/X86/pr33290.ll
index 0e6f2e92f1400..3991080d75bc0 100644
--- a/llvm/test/CodeGen/X86/pr33290.ll
+++ b/llvm/test/CodeGen/X86/pr33290.ll
@@ -10,7 +10,7 @@ define dso_local void @e() {
 ; X86-LABEL: e:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl b, %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB0_1: # %for.cond
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movzbl c, %ecx
@@ -22,7 +22,7 @@ define dso_local void @e() {
 ; X64-LABEL: e:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movq b(%rip), %rax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB0_1: # %for.cond
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movzbl c(%rip), %ecx
diff --git a/llvm/test/CodeGen/X86/pr33747.ll b/llvm/test/CodeGen/X86/pr33747.ll
index c8ba2b2e3a790..f6df42450e187 100644
--- a/llvm/test/CodeGen/X86/pr33747.ll
+++ b/llvm/test/CodeGen/X86/pr33747.ll
@@ -12,10 +12,10 @@ define void @PR33747(ptr nocapture) {
 ; CHECK-NEXT:    setne %al
 ; CHECK-NEXT:    testb %cl, %al
 ; CHECK-NEXT:    je .LBB0_2
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp .LBB0_1
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp .LBB0_2
   %2 = getelementptr inbounds i32, ptr %0, i64 6
diff --git a/llvm/test/CodeGen/X86/pr37916.ll b/llvm/test/CodeGen/X86/pr37916.ll
index 2f8f0fdae1848..e6639a11ca5ea 100644
--- a/llvm/test/CodeGen/X86/pr37916.ll
+++ b/llvm/test/CodeGen/X86/pr37916.ll
@@ -7,7 +7,7 @@
 define void @fn1() local_unnamed_addr {
 ; CHECK-LABEL: fn1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %if.end
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl a+4, %eax
diff --git a/llvm/test/CodeGen/X86/pr38185.ll b/llvm/test/CodeGen/X86/pr38185.ll
index e062302d4f0b5..d5591c50738fa 100644
--- a/llvm/test/CodeGen/X86/pr38185.ll
+++ b/llvm/test/CodeGen/X86/pr38185.ll
@@ -5,7 +5,7 @@ define void @foo(ptr %a, ptr %b, ptr noalias %c, i64 %s) {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
diff --git a/llvm/test/CodeGen/X86/pr38217.ll b/llvm/test/CodeGen/X86/pr38217.ll
index f1538f3598aec..02d2d9e02721a 100644
--- a/llvm/test/CodeGen/X86/pr38217.ll
+++ b/llvm/test/CodeGen/X86/pr38217.ll
@@ -11,7 +11,7 @@ define dso_local void @_Z12d2s_bufferedmPc(i64 %arg, ptr nocapture %arg1) {
 ; CHECK-NEXT:  # %bb.1: # %bb2.preheader
 ; CHECK-NEXT:    xorl %ecx, %ecx
 ; CHECK-NEXT:    movabsq $3777893186295716171, %r8 # imm = 0x346DC5D63886594B
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # %bb2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rdi, %rax
diff --git a/llvm/test/CodeGen/X86/pr38539.ll b/llvm/test/CodeGen/X86/pr38539.ll
index fb169a3777fb8..b633c28a214b7 100644
--- a/llvm/test/CodeGen/X86/pr38539.ll
+++ b/llvm/test/CodeGen/X86/pr38539.ll
@@ -168,7 +168,7 @@ define void @f() nounwind {
 ; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X86-NEXT:    xorl %ecx, %ecx
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB0_12: # %udiv-do-while
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %ebx, %esi
diff --git a/llvm/test/CodeGen/X86/pr38795.ll b/llvm/test/CodeGen/X86/pr38795.ll
index f64c70e8fc79a..c3c96e8228797 100644
--- a/llvm/test/CodeGen/X86/pr38795.ll
+++ b/llvm/test/CodeGen/X86/pr38795.ll
@@ -30,7 +30,7 @@ define dso_local void @fn() {
 ; CHECK-NEXT:    # kill: killed $al
 ; CHECK-NEXT:    # implicit-def: $ebp
 ; CHECK-NEXT:    jmp .LBB0_1
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_15: # %for.inc
 ; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    movb %dl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
@@ -49,7 +49,7 @@ define dso_local void @fn() {
 ; CHECK-NEXT:    testl %edi, %edi
 ; CHECK-NEXT:    jne .LBB0_10
 ; CHECK-NEXT:    jmp .LBB0_6
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_3: # %if.end
 ; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    movl %ecx, %eax
@@ -73,7 +73,7 @@ define dso_local void @fn() {
 ; CHECK-NEXT:    # implicit-def: $eax
 ; CHECK-NEXT:    movb %dh, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
 ; CHECK-NEXT:    jne .LBB0_15
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  # %bb.5: # %for.cond35
 ; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    testl %edi, %edi
@@ -98,16 +98,16 @@ define dso_local void @fn() {
 ; CHECK-NEXT:    # implicit-def: $dl
 ; CHECK-NEXT:    # implicit-def: $ebp
 ; CHECK-NEXT:    jmp .LBB0_19
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_7: # %if.end21
 ; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    # implicit-def: $ebp
 ; CHECK-NEXT:    jmp .LBB0_8
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_6: # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    xorl %edi, %edi
 ; CHECK-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %dh # 1-byte Reload
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_19: # %for.cond47
 ; CHECK-NEXT:    # Parent Loop BB0_1 Depth=1
 ; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
@@ -131,7 +131,7 @@ define dso_local void @fn() {
 ; CHECK-NEXT:    xorl %ecx, %ecx
 ; CHECK-NEXT:    xorl %ebp, %ebp
 ; CHECK-NEXT:    jmp .LBB0_15
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_9: # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    # implicit-def: $eax
 ; CHECK-NEXT:    testb %bl, %bl
@@ -264,7 +264,7 @@ define void @verifier_error_reduced_issue38788(i1 %cmp11) {
 ; CHECK-NEXT:    xorl %ecx, %ecx
 ; CHECK-NEXT:    xorl %ebx, %ebx
 ; CHECK-NEXT:    jmp .LBB1_1
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_7: # %if.end26
 ; CHECK-NEXT:    # in Loop: Header=BB1_1 Depth=1
 ; CHECK-NEXT:    movl %ecx, %edx
@@ -280,7 +280,7 @@ define void @verifier_error_reduced_issue38788(i1 %cmp11) {
 ; CHECK-NEXT:  # %bb.2: # in Loop: Header=BB1_1 Depth=1
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    jmp .LBB1_5
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_3: # %if.end
 ; CHECK-NEXT:    # in Loop: Header=BB1_1 Depth=1
 ; CHECK-NEXT:    testb $1, %al
@@ -293,7 +293,7 @@ define void @verifier_error_reduced_issue38788(i1 %cmp11) {
 ; CHECK-NEXT:    movl $0, %ebx
 ; CHECK-NEXT:    jne .LBB1_8
 ; CHECK-NEXT:    jmp .LBB1_5
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_4: # in Loop: Header=BB1_1 Depth=1
 ; CHECK-NEXT:    movl %ebx, %eax
 ; CHECK-NEXT:    xorl %ebx, %ebx
diff --git a/llvm/test/CodeGen/X86/pr42565.ll b/llvm/test/CodeGen/X86/pr42565.ll
index 33acf02135eab..10071064057b8 100644
--- a/llvm/test/CodeGen/X86/pr42565.ll
+++ b/llvm/test/CodeGen/X86/pr42565.ll
@@ -7,7 +7,7 @@ define void @HUF_writeCTable_wksp()  {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movl $2, %eax
 ; CHECK-NEXT:    movb $-2, %cl
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    leal 1(%rcx), %edx
diff --git a/llvm/test/CodeGen/X86/pr42909.ll b/llvm/test/CodeGen/X86/pr42909.ll
index 47b85e144919d..8ff7caf463854 100644
--- a/llvm/test/CodeGen/X86/pr42909.ll
+++ b/llvm/test/CodeGen/X86/pr42909.ll
@@ -4,7 +4,7 @@
 define void @autogen_SD31033(ptr %a0) {
 ; CHECK-LABEL: autogen_SD31033:
 ; CHECK:       # %bb.0: # %BB
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %CF
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp .LBB0_1
diff --git a/llvm/test/CodeGen/X86/pr43529.ll b/llvm/test/CodeGen/X86/pr43529.ll
index 3e9976c90449e..ac5f7c84f82c3 100644
--- a/llvm/test/CodeGen/X86/pr43529.ll
+++ b/llvm/test/CodeGen/X86/pr43529.ll
@@ -12,7 +12,7 @@ define i32 @a() nounwind {
 ; CHECK-NEXT:    calll d@PLT
 ; CHECK-NEXT:    cmpl $a, %esi
 ; CHECK-NEXT:    jbe .LBB0_2
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %for.cond
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp .LBB0_1
diff --git a/llvm/test/CodeGen/X86/pr44140.ll b/llvm/test/CodeGen/X86/pr44140.ll
index 9455895a99b87..02525d73a786d 100644
--- a/llvm/test/CodeGen/X86/pr44140.ll
+++ b/llvm/test/CodeGen/X86/pr44140.ll
@@ -17,7 +17,7 @@ define i32 @main() {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 592
 ; CHECK-NEXT:    vmovaps {{.*#+}} xmm6 = [1010101010101010101,2020202020202020202]
 ; CHECK-NEXT:    xorl %esi, %esi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_1: # %fake-loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm0
diff --git a/llvm/test/CodeGen/X86/pr44412.ll b/llvm/test/CodeGen/X86/pr44412.ll
index 67579a5bb7c52..546dbcc156129 100644
--- a/llvm/test/CodeGen/X86/pr44412.ll
+++ b/llvm/test/CodeGen/X86/pr44412.ll
@@ -10,7 +10,7 @@ define void @bar(i32 %0, i32 %1) nounwind {
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    movl %edi, %ebx
 ; CHECK-NEXT:    decl %ebx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %ebx, %edi
 ; CHECK-NEXT:    callq foo@PLT
@@ -43,7 +43,7 @@ define void @baz(i32 %0, i32 %1) nounwind {
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    movl %edi, %ebx
 ; CHECK-NEXT:    decl %ebx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_2: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %ebx, %edi
 ; CHECK-NEXT:    callq foo@PLT
diff --git a/llvm/test/CodeGen/X86/pr47874.ll b/llvm/test/CodeGen/X86/pr47874.ll
index 2da3585357a1c..ce3aaca59fae8 100644
--- a/llvm/test/CodeGen/X86/pr47874.ll
+++ b/llvm/test/CodeGen/X86/pr47874.ll
@@ -11,7 +11,7 @@ define void @a(ptr %arg, i32 %arg1) {
 ; SSE2-NEXT:  ## %bb.1: ## %bb2
 ; SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
 ; SSE2-NEXT:    movl %esi, %eax
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  LBB0_2: ## %bb6
 ; SSE2-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    ## InlineAsm Start
@@ -32,7 +32,7 @@ define void @a(ptr %arg, i32 %arg1) {
 ; AVX-NEXT:  ## %bb.1: ## %bb2
 ; AVX-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
 ; AVX-NEXT:    movl %esi, %eax
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  LBB0_2: ## %bb6
 ; AVX-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    ## InlineAsm Start
@@ -77,7 +77,7 @@ define void @b(ptr %arg, i64 %arg1) {
 ; SSE2-NEXT:    jle LBB1_3
 ; SSE2-NEXT:  ## %bb.1: ## %bb2
 ; SSE2-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  LBB1_2: ## %bb6
 ; SSE2-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    ## InlineAsm Start
@@ -97,7 +97,7 @@ define void @b(ptr %arg, i64 %arg1) {
 ; AVX-NEXT:    jle LBB1_3
 ; AVX-NEXT:  ## %bb.1: ## %bb2
 ; AVX-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  LBB1_2: ## %bb6
 ; AVX-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    ## InlineAsm Start
@@ -142,7 +142,7 @@ define void @c(ptr %arg, ptr %arg1, i32 %arg2) {
 ; SSE2-NEXT:  ## %bb.1: ## %bb4
 ; SSE2-NEXT:    movl %edx, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
 ; SSE2-NEXT:    movl %edx, %eax
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  LBB2_2: ## %bb8
 ; SSE2-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    ## InlineAsm Start
@@ -163,7 +163,7 @@ define void @c(ptr %arg, ptr %arg1, i32 %arg2) {
 ; AVX-NEXT:  ## %bb.1: ## %bb4
 ; AVX-NEXT:    movl %edx, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
 ; AVX-NEXT:    movl %edx, %eax
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  LBB2_2: ## %bb8
 ; AVX-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    ## InlineAsm Start
@@ -210,7 +210,7 @@ define void @d(ptr %arg, ptr %arg1, i64 %arg2) {
 ; SSE2-NEXT:    jle LBB3_3
 ; SSE2-NEXT:  ## %bb.1: ## %bb3
 ; SSE2-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  LBB3_2: ## %bb6
 ; SSE2-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    ## InlineAsm Start
@@ -230,7 +230,7 @@ define void @d(ptr %arg, ptr %arg1, i64 %arg2) {
 ; AVX-NEXT:    jle LBB3_3
 ; AVX-NEXT:  ## %bb.1: ## %bb3
 ; AVX-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  LBB3_2: ## %bb6
 ; AVX-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    ## InlineAsm Start
diff --git a/llvm/test/CodeGen/X86/pr49393.ll b/llvm/test/CodeGen/X86/pr49393.ll
index 3fb6a82f3fbb2..512177246b5d9 100644
--- a/llvm/test/CodeGen/X86/pr49393.ll
+++ b/llvm/test/CodeGen/X86/pr49393.ll
@@ -9,7 +9,7 @@ define void @f() {
 ; CHECK-NEXT:    movapd %xmm0, %xmm1
 ; CHECK-NEXT:    mulsd %xmm0, %xmm1
 ; CHECK-NEXT:    subsd %xmm0, %xmm1
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %for.cond
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    imull %eax, %eax
diff --git a/llvm/test/CodeGen/X86/pr49451.ll b/llvm/test/CodeGen/X86/pr49451.ll
index cc4607c5c55f3..173c41140ebef 100644
--- a/llvm/test/CodeGen/X86/pr49451.ll
+++ b/llvm/test/CodeGen/X86/pr49451.ll
@@ -15,7 +15,7 @@ define void @func_6(i8 %uc_8, i64 %uli_10) nounwind {
 ; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    xorl %ebx, %ebx
 ; X86-NEXT:    # implicit-def: $si
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB0_1: # %for.body612
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    testb %dl, %dl
@@ -37,7 +37,7 @@ define void @func_6(i8 %uc_8, i64 %uli_10) nounwind {
 ; X64-NEXT:    movl $23090, %eax # imm = 0x5A32
 ; X64-NEXT:    xorl %ecx, %ecx
 ; X64-NEXT:    # implicit-def: $dx
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB0_1: # %for.body612
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    testb %cl, %cl
diff --git a/llvm/test/CodeGen/X86/pr50374.ll b/llvm/test/CodeGen/X86/pr50374.ll
index 0db038f7d8595..edb52c6e27bb6 100644
--- a/llvm/test/CodeGen/X86/pr50374.ll
+++ b/llvm/test/CodeGen/X86/pr50374.ll
@@ -4,7 +4,7 @@
 define void @PR50374() {
 ; CHECK-LABEL: PR50374:
 ; CHECK:       # %bb.0: # %while.84.body.preheader
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %vector.body1999
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp .LBB0_1
diff --git a/llvm/test/CodeGen/X86/pr50782.ll b/llvm/test/CodeGen/X86/pr50782.ll
index c6dab7dc26fd5..591a33446d4e3 100644
--- a/llvm/test/CodeGen/X86/pr50782.ll
+++ b/llvm/test/CodeGen/X86/pr50782.ll
@@ -52,7 +52,7 @@ define void @h(float %i) {
 ; CHECK-NEXT:    fildl 12(%esi)
 ; CHECK-NEXT:    movl _c, %edx
 ; CHECK-NEXT:    jmp LBB0_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_5: # %for.inc
 ; CHECK-NEXT:    # in Loop: Header=BB0_3 Depth=1
 ; CHECK-NEXT:    fxch %st(5)
diff --git a/llvm/test/CodeGen/X86/pr51371.ll b/llvm/test/CodeGen/X86/pr51371.ll
index 9b63269e96207..8952f0967cde0 100644
--- a/llvm/test/CodeGen/X86/pr51371.ll
+++ b/llvm/test/CodeGen/X86/pr51371.ll
@@ -9,7 +9,7 @@ define void @pmuldq(ptr nocapture %0, i32 %1, i64 %2) {
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    movd %esi, %xmm0
 ; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movdqa (%rdi), %xmm1
 ; CHECK-NEXT:    pmuldq %xmm0, %xmm1
@@ -51,7 +51,7 @@ define void @pmuludq(ptr nocapture %0, i32 %1, i64 %2) {
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    movd %esi, %xmm0
 ; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_2: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movdqa (%rdi), %xmm1
 ; CHECK-NEXT:    pmuludq %xmm0, %xmm1
diff --git a/llvm/test/CodeGen/X86/pr5145.ll b/llvm/test/CodeGen/X86/pr5145.ll
index 16398cb57061f..5cb0ac9bc9821 100644
--- a/llvm/test/CodeGen/X86/pr5145.ll
+++ b/llvm/test/CodeGen/X86/pr5145.ll
@@ -6,7 +6,7 @@ define void @atomic_maxmin_i8() {
 ; CHECK-LABEL: atomic_maxmin_i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movzbl sc8(%rip), %eax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %atomicrmw.start14
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpb $6, %al
@@ -18,7 +18,7 @@ define void @atomic_maxmin_i8() {
 ; CHECK-NEXT:    jne .LBB0_1
 ; CHECK-NEXT:  # %bb.2: # %atomicrmw.end13
 ; CHECK-NEXT:    movzbl sc8(%rip), %eax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_3: # %atomicrmw.start8
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpb $7, %al
@@ -30,7 +30,7 @@ define void @atomic_maxmin_i8() {
 ; CHECK-NEXT:    jne .LBB0_3
 ; CHECK-NEXT:  # %bb.4: # %atomicrmw.end7
 ; CHECK-NEXT:    movzbl sc8(%rip), %eax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_5: # %atomicrmw.start2
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpb $8, %al
@@ -42,7 +42,7 @@ define void @atomic_maxmin_i8() {
 ; CHECK-NEXT:    jne .LBB0_5
 ; CHECK-NEXT:  # %bb.6: # %atomicrmw.end1
 ; CHECK-NEXT:    movzbl sc8(%rip), %eax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_7: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpb $9, %al
diff --git a/llvm/test/CodeGen/X86/pr51615.ll b/llvm/test/CodeGen/X86/pr51615.ll
index 8b2cea73fd36c..a062aa138a1e5 100644
--- a/llvm/test/CodeGen/X86/pr51615.ll
+++ b/llvm/test/CodeGen/X86/pr51615.ll
@@ -83,13 +83,13 @@ define void @elts_from_consecutive_loads(ptr %arg, ptr %arg12, ptr %arg13, float
 ; ALL-LABEL: elts_from_consecutive_loads:
 ; ALL:       # %bb.0: # %bb
 ; ALL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; ALL-NEXT:    .p2align 4, 0x90
+; ALL-NEXT:    .p2align 4
 ; ALL-NEXT:  .LBB3_1: # %bb16
 ; ALL-NEXT:    # =>This Loop Header: Depth=1
 ; ALL-NEXT:    # Child Loop BB3_2 Depth 2
 ; ALL-NEXT:    testb $1, %cl
 ; ALL-NEXT:    je .LBB3_1
-; ALL-NEXT:    .p2align 4, 0x90
+; ALL-NEXT:    .p2align 4
 ; ALL-NEXT:  .LBB3_2: # %bb17
 ; ALL-NEXT:    # Parent Loop BB3_1 Depth=1
 ; ALL-NEXT:    # => This Inner Loop Header: Depth=2
diff --git a/llvm/test/CodeGen/X86/pr53842.ll b/llvm/test/CodeGen/X86/pr53842.ll
index 26d1a977c84c5..a962a9b3ddf2f 100644
--- a/llvm/test/CodeGen/X86/pr53842.ll
+++ b/llvm/test/CodeGen/X86/pr53842.ll
@@ -14,7 +14,7 @@ define void @PR53842() {
 ; CHECK-NEXT:    vpcmpeqq %ymm3, %ymm2, %ymm2
 ; CHECK-NEXT:    vpcmpeqq %ymm3, %ymm1, %ymm1
 ; CHECK-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpsubq %zmm1, %zmm0, %zmm0
diff --git a/llvm/test/CodeGen/X86/pr53990-incorrect-machine-sink.ll b/llvm/test/CodeGen/X86/pr53990-incorrect-machine-sink.ll
index ac5ee4635d506..f2f6e6934abeb 100644
--- a/llvm/test/CodeGen/X86/pr53990-incorrect-machine-sink.ll
+++ b/llvm/test/CodeGen/X86/pr53990-incorrect-machine-sink.ll
@@ -16,7 +16,7 @@ define void @test(i1 %c, ptr %p, ptr noalias %p2) nounwind {
 ; CHECK-NEXT:    movq (%rsi), %r14
 ; CHECK-NEXT:    movb $1, %r15b
 ; CHECK-NEXT:    jmp .LBB0_1
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_4: # %sink
 ; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    movq %r14, (%rbx)
diff --git a/llvm/test/CodeGen/X86/pr55648.ll b/llvm/test/CodeGen/X86/pr55648.ll
index 6e99643aecbd3..da62b07915c86 100644
--- a/llvm/test/CodeGen/X86/pr55648.ll
+++ b/llvm/test/CodeGen/X86/pr55648.ll
@@ -4,7 +4,7 @@
 define void @PR55648() #0 {
 ; CHECK-LABEL: PR55648:
 ; CHECK:       # %bb.0: # %bb
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %bb38
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp .LBB0_1
diff --git a/llvm/test/CodeGen/X86/pr61923.ll b/llvm/test/CodeGen/X86/pr61923.ll
index 576b029cd0329..f9290b9743f47 100644
--- a/llvm/test/CodeGen/X86/pr61923.ll
+++ b/llvm/test/CodeGen/X86/pr61923.ll
@@ -14,7 +14,7 @@ define void @test_loop(ptr align 1 %src, ptr align 1 %dest, i32 %len) {
 ; CHECK-NEXT:    movl %edx, %eax
 ; CHECK-NEXT:    andl $-32, %eax
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # %memcmp.loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vmovups (%rsi,%rcx), %ymm0
diff --git a/llvm/test/CodeGen/X86/pr63108.ll b/llvm/test/CodeGen/X86/pr63108.ll
index b1576851ed023..53f6fa697c452 100644
--- a/llvm/test/CodeGen/X86/pr63108.ll
+++ b/llvm/test/CodeGen/X86/pr63108.ll
@@ -17,7 +17,7 @@ define i32 @PR63108() {
 ; SSE-NEXT:    pxor %xmm0, %xmm0
 ; SSE-NEXT:    movd {{.*#+}} xmm1 = [57339,0,0,0]
 ; SSE-NEXT:    xorl %eax, %eax
-; SSE-NEXT:    .p2align 4, 0x90
+; SSE-NEXT:    .p2align 4
 ; SSE-NEXT:  .LBB0_3: # %vector.body
 ; SSE-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE-NEXT:    movdqa %xmm1, %xmm2
@@ -52,7 +52,7 @@ define i32 @PR63108() {
 ; AVX1-NEXT:  .LBB0_2: # %vector.body.preheader
 ; AVX1-NEXT:    vmovss {{.*#+}} xmm0 = [57339,0,0,0]
 ; AVX1-NEXT:    xorl %eax, %eax
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB0_3: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vmovaps %ymm0, %ymm1
@@ -89,7 +89,7 @@ define i32 @PR63108() {
 ; AVX2-NEXT:  .LBB0_2: # %vector.body.preheader
 ; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = [57339,0,0,0]
 ; AVX2-NEXT:    xorl %eax, %eax
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB0_3: # %vector.body
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    vmovdqa %ymm0, %ymm1
@@ -126,7 +126,7 @@ define i32 @PR63108() {
 ; AVX512-NEXT:  .LBB0_2: # %vector.body.preheader
 ; AVX512-NEXT:    vmovd {{.*#+}} xmm0 = [57339,0,0,0]
 ; AVX512-NEXT:    xorl %eax, %eax
-; AVX512-NEXT:    .p2align 4, 0x90
+; AVX512-NEXT:    .p2align 4
 ; AVX512-NEXT:  .LBB0_3: # %vector.body
 ; AVX512-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512-NEXT:    vmovdqa %ymm0, %ymm1
diff --git a/llvm/test/CodeGen/X86/pr63692.ll b/llvm/test/CodeGen/X86/pr63692.ll
index fb3198743a394..8cbd24240e1d8 100644
--- a/llvm/test/CodeGen/X86/pr63692.ll
+++ b/llvm/test/CodeGen/X86/pr63692.ll
@@ -6,7 +6,7 @@ define void @prefault(ptr noundef %range_start, ptr noundef readnone %range_end)
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    cmpq %rsi, %rdi
 ; CHECK-NEXT:    jae .LBB0_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %while.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    lock orb $0, (%rdi)
diff --git a/llvm/test/CodeGen/X86/pr65895.ll b/llvm/test/CodeGen/X86/pr65895.ll
index ced60ee73cc84..0990b10fa936d 100644
--- a/llvm/test/CodeGen/X86/pr65895.ll
+++ b/llvm/test/CodeGen/X86/pr65895.ll
@@ -16,7 +16,7 @@ define i32 @PR65895() {
 ; CHECK-NEXT:    je .LBB0_3
 ; CHECK-NEXT:  # %bb.1: # %for.body.lr.ph
 ; CHECK-NEXT:    movb %al, b(%rip)
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp .LBB0_2
diff --git a/llvm/test/CodeGen/X86/pr68539.ll b/llvm/test/CodeGen/X86/pr68539.ll
index 8c7e7792dc5f4..5e3a7b8dbafb2 100644
--- a/llvm/test/CodeGen/X86/pr68539.ll
+++ b/llvm/test/CodeGen/X86/pr68539.ll
@@ -4,7 +4,7 @@
 define i32 @main(i1 %arg) {
 ; CHECK-LABEL: main:
 ; CHECK:       # %bb.0: # %bb
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp .LBB0_1
diff --git a/llvm/test/CodeGen/X86/pr93000.ll b/llvm/test/CodeGen/X86/pr93000.ll
index 0bd5da48847e8..d55332f81c2ca 100644
--- a/llvm/test/CodeGen/X86/pr93000.ll
+++ b/llvm/test/CodeGen/X86/pr93000.ll
@@ -6,7 +6,7 @@ define void @PR93000(ptr %a0, ptr %a1, ptr %a2, <32 x i16> %a3) {
 ; CHECK:       # %bb.0: # %Entry
 ; CHECK-NEXT:    movl (%rdi), %eax
 ; CHECK-NEXT:    addq $4, %rdi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %Loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    kmovd %eax, %k1
diff --git a/llvm/test/CodeGen/X86/promote-sra-by-itself.ll b/llvm/test/CodeGen/X86/promote-sra-by-itself.ll
index e19f30b386339..d57411f495b69 100644
--- a/llvm/test/CodeGen/X86/promote-sra-by-itself.ll
+++ b/llvm/test/CodeGen/X86/promote-sra-by-itself.ll
@@ -19,7 +19,7 @@ define i16 @basic(ptr %p) {
 define void @crash(i1 %cond, ptr %p) {
 ; CHECK-LABEL: crash:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    testb $1, %dil
diff --git a/llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll b/llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll
index c0bfb71e189cd..beb42f55b709c 100644
--- a/llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll
+++ b/llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll
@@ -84,7 +84,7 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    movq _syCTRO@GOTPCREL(%rip), %rax
 ; CHECK-NEXT:    movl $1, %r13d
 ; CHECK-NEXT:    movb $1, %cl
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_9: ## %do.body
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl $0, (%rax)
@@ -108,7 +108,7 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
 ; CHECK-NEXT:    testb %r12b, %r12b
 ; CHECK-NEXT:    je LBB0_54
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_20: ## %while.cond197.backedge
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
 ; CHECK-NEXT:    decl %r13d
@@ -141,7 +141,7 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
 ; CHECK-NEXT:    ## implicit-def: $rax
 ; CHECK-NEXT:    jmp LBB0_28
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_31: ## %do.body479.backedge
 ; CHECK-NEXT:    ## in Loop: Header=BB0_28 Depth=2
 ; CHECK-NEXT:    leaq 1(%rbx), %rax
@@ -165,7 +165,7 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    testb %r12b, %r12b
 ; CHECK-NEXT:    jne LBB0_31
 ; CHECK-NEXT:    jmp LBB0_33
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_14: ## %while.body200
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
 ; CHECK-NEXT:    leal 1(%rbp), %eax
@@ -223,7 +223,7 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    ## in Loop: Header=BB0_13 Depth=1
 ; CHECK-NEXT:    movq _SyFgets.yank@GOTPCREL(%rip), %rax
 ; CHECK-NEXT:    movb $0, (%rax)
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_37: ## %for.cond534
 ; CHECK-NEXT:    ## Parent Loop BB0_13 Depth=1
 ; CHECK-NEXT:    ## => This Inner Loop Header: Depth=2
@@ -235,15 +235,15 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    movb $0, (%rbx)
 ; CHECK-NEXT:    leaq LJTI0_0(%rip), %rdx
 ; CHECK-NEXT:    jmp LBB0_20
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_41: ## %while.cond864
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp LBB0_41
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_42: ## %while.cond962
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp LBB0_42
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_24: ## %for.cond357
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp LBB0_24
@@ -266,7 +266,7 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    subq %rbx, %r14
 ; CHECK-NEXT:    movq _syHistory@GOTPCREL(%rip), %rax
 ; CHECK-NEXT:    leaq 8189(%r14,%rax), %rax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_48: ## %for.body1723
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    decq %rax
@@ -279,11 +279,11 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    je LBB0_40
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_17: ## %for.body643.us
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp LBB0_17
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_40: ## %while.cond661
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp LBB0_40
@@ -300,7 +300,7 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:  ## %bb.51: ## %while.body1679.preheader
 ; CHECK-NEXT:    incl %ebp
 ; CHECK-NEXT:    xorl %ebx, %ebx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_52: ## %while.body1679
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq (%r14), %rdi
diff --git a/llvm/test/CodeGen/X86/rdrand.ll b/llvm/test/CodeGen/X86/rdrand.ll
index 164145a789943..52f767a4a419c 100644
--- a/llvm/test/CodeGen/X86/rdrand.ll
+++ b/llvm/test/CodeGen/X86/rdrand.ll
@@ -89,7 +89,7 @@ define void @loop(ptr %p, i32 %n) nounwind {
 ; X86-NEXT:  # %bb.1: # %while.body.preheader
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB3_2: # %while.body
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    rdrandl %esi
@@ -108,7 +108,7 @@ define void @loop(ptr %p, i32 %n) nounwind {
 ; X64-NEXT:  # %bb.1: # %while.body.preheader
 ; X64-NEXT:    movl %esi, %eax
 ; X64-NEXT:    xorl %ecx, %ecx
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB3_2: # %while.body
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    rdrandl %edx
diff --git a/llvm/test/CodeGen/X86/retpoline.ll b/llvm/test/CodeGen/X86/retpoline.ll
index d06ddc64044d0..ea9c9bc4fae22 100644
--- a/llvm/test/CodeGen/X86/retpoline.ll
+++ b/llvm/test/CodeGen/X86/retpoline.ll
@@ -426,7 +426,7 @@ latch:
 ; X64-NEXT:          pause
 ; X64-NEXT:          lfence
 ; X64-NEXT:          jmp     [[CAPTURE_SPEC]]
-; X64-NEXT:          .p2align        4, 0x90
+; X64-NEXT:          .p2align        4
 ; X64-NEXT:  {{.*}}                                  # Block address taken
 ; X64-NEXT:                                          # %entry
 ; X64-NEXT:  [[CALL_TARGET]]:
@@ -445,7 +445,7 @@ latch:
 ; X86-NEXT:          pause
 ; X86-NEXT:          lfence
 ; X86-NEXT:          jmp     [[CAPTURE_SPEC]]
-; X86-NEXT:          .p2align        4, 0x90
+; X86-NEXT:          .p2align        4
 ; X86-NEXT:  {{.*}}                                  # Block address taken
 ; X86-NEXT:                                          # %entry
 ; X86-NEXT:  [[CALL_TARGET]]:
@@ -464,7 +464,7 @@ latch:
 ; X86-NEXT:          pause
 ; X86-NEXT:          lfence
 ; X86-NEXT:          jmp     [[CAPTURE_SPEC]]
-; X86-NEXT:          .p2align        4, 0x90
+; X86-NEXT:          .p2align        4
 ; X86-NEXT:  {{.*}}                                  # Block address taken
 ; X86-NEXT:                                          # %entry
 ; X86-NEXT:  [[CALL_TARGET]]:
@@ -483,7 +483,7 @@ latch:
 ; X86-NEXT:          pause
 ; X86-NEXT:          lfence
 ; X86-NEXT:          jmp     [[CAPTURE_SPEC]]
-; X86-NEXT:          .p2align        4, 0x90
+; X86-NEXT:          .p2align        4
 ; X86-NEXT:  {{.*}}                                  # Block address taken
 ; X86-NEXT:                                          # %entry
 ; X86-NEXT:  [[CALL_TARGET]]:
@@ -502,7 +502,7 @@ latch:
 ; X86-NEXT:          pause
 ; X86-NEXT:          lfence
 ; X86-NEXT:          jmp     [[CAPTURE_SPEC]]
-; X86-NEXT:          .p2align        4, 0x90
+; X86-NEXT:          .p2align        4
 ; X86-NEXT:  {{.*}}                                  # Block address taken
 ; X86-NEXT:                                          # %entry
 ; X86-NEXT:  [[CALL_TARGET]]:
diff --git a/llvm/test/CodeGen/X86/reverse_branches.ll b/llvm/test/CodeGen/X86/reverse_branches.ll
index d874a47356e3c..93c82a4524ef9 100644
--- a/llvm/test/CodeGen/X86/reverse_branches.ll
+++ b/llvm/test/CodeGen/X86/reverse_branches.ll
@@ -37,7 +37,7 @@ define i32 @test_branches_order() uwtable ssp {
 ; CHECK-NEXT:    leaq -{{[0-9]+}}(%rsp), %r14
 ; CHECK-NEXT:    movq %rsp, %r15
 ; CHECK-NEXT:    jmp LBB0_1
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_6: ## %for.inc9
 ; CHECK-NEXT:    ## in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    incl %ebx
@@ -51,7 +51,7 @@ define i32 @test_branches_order() uwtable ssp {
 ; CHECK-NEXT:    movl $-1, %ebp
 ; CHECK-NEXT:    movq %r15, %rdi
 ; CHECK-NEXT:    movq %r14, %r12
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_3: ## %for.cond1
 ; CHECK-NEXT:    ## Parent Loop BB0_1 Depth=1
 ; CHECK-NEXT:    ## => This Inner Loop Header: Depth=2
@@ -75,7 +75,7 @@ define i32 @test_branches_order() uwtable ssp {
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    movq %rsp, %rcx
 ; CHECK-NEXT:    jmp LBB0_8
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_15: ## %for.inc38
 ; CHECK-NEXT:    ## in Loop: Header=BB0_8 Depth=1
 ; CHECK-NEXT:    incl %eax
@@ -91,7 +91,7 @@ define i32 @test_branches_order() uwtable ssp {
 ; CHECK-NEXT:    xorl %esi, %esi
 ; CHECK-NEXT:    xorl %edi, %edi
 ; CHECK-NEXT:    jmp LBB0_10
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_14: ## %exit
 ; CHECK-NEXT:    ## in Loop: Header=BB0_10 Depth=2
 ; CHECK-NEXT:    addq %rsi, %r8
@@ -109,7 +109,7 @@ define i32 @test_branches_order() uwtable ssp {
 ; CHECK-NEXT:  ## %bb.11: ## %for.body20
 ; CHECK-NEXT:    ## in Loop: Header=BB0_10 Depth=2
 ; CHECK-NEXT:    movq $-1000, %r8 ## imm = 0xFC18
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_12: ## %do.body.i
 ; CHECK-NEXT:    ## Parent Loop BB0_8 Depth=1
 ; CHECK-NEXT:    ## Parent Loop BB0_10 Depth=2
diff --git a/llvm/test/CodeGen/X86/sad.ll b/llvm/test/CodeGen/X86/sad.ll
index ca319687da54d..245516974d15b 100644
--- a/llvm/test/CodeGen/X86/sad.ll
+++ b/llvm/test/CodeGen/X86/sad.ll
@@ -14,7 +14,7 @@ define dso_local i32 @sad_16i8() nounwind {
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB0_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movdqu a+1024(%rax), %xmm2
@@ -39,7 +39,7 @@ define dso_local i32 @sad_16i8() nounwind {
 ; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB0_1: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vmovdqu a+1024(%rax), %xmm2
@@ -67,7 +67,7 @@ define dso_local i32 @sad_16i8() nounwind {
 ; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB0_1: # %vector.body
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    vmovdqu a+1024(%rax), %xmm2
@@ -91,7 +91,7 @@ define dso_local i32 @sad_16i8() nounwind {
 ; AVX512:       # %bb.0: # %entry
 ; AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    movq $-1024, %rax # imm = 0xFC00
-; AVX512-NEXT:    .p2align 4, 0x90
+; AVX512-NEXT:    .p2align 4
 ; AVX512-NEXT:  .LBB0_1: # %vector.body
 ; AVX512-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512-NEXT:    vmovdqu a+1024(%rax), %xmm1
@@ -154,7 +154,7 @@ define dso_local i32 @sad_32i8() nounwind {
 ; SSE2-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB1_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movdqa a+1024(%rax), %xmm3
@@ -184,7 +184,7 @@ define dso_local i32 @sad_32i8() nounwind {
 ; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB1_1: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vmovdqa a+1024(%rax), %xmm2
@@ -220,7 +220,7 @@ define dso_local i32 @sad_32i8() nounwind {
 ; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB1_1: # %vector.body
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    vmovdqa a+1024(%rax), %ymm2
@@ -247,7 +247,7 @@ define dso_local i32 @sad_32i8() nounwind {
 ; AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    .p2align 4, 0x90
+; AVX512-NEXT:    .p2align 4
 ; AVX512-NEXT:  .LBB1_1: # %vector.body
 ; AVX512-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512-NEXT:    vmovdqa a+1024(%rax), %ymm2
@@ -315,7 +315,7 @@ define dso_local i32 @sad_avx64i8() nounwind {
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB2_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movdqa a+1024(%rax), %xmm5
@@ -359,7 +359,7 @@ define dso_local i32 @sad_avx64i8() nounwind {
 ; AVX1-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB2_1: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vmovdqa a+1024(%rax), %xmm3
@@ -411,7 +411,7 @@ define dso_local i32 @sad_avx64i8() nounwind {
 ; AVX2-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB2_1: # %vector.body
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    vmovdqa a+1024(%rax), %ymm3
@@ -444,7 +444,7 @@ define dso_local i32 @sad_avx64i8() nounwind {
 ; AVX512F-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX512F-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    .p2align 4, 0x90
+; AVX512F-NEXT:    .p2align 4
 ; AVX512F-NEXT:  .LBB2_1: # %vector.body
 ; AVX512F-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512F-NEXT:    vmovdqa a+1024(%rax), %ymm2
@@ -476,7 +476,7 @@ define dso_local i32 @sad_avx64i8() nounwind {
 ; AVX512BW-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX512BW-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    .p2align 4, 0x90
+; AVX512BW-NEXT:    .p2align 4
 ; AVX512BW-NEXT:  .LBB2_1: # %vector.body
 ; AVX512BW-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512BW-NEXT:    vmovdqa64 a+1024(%rax), %zmm2
@@ -545,7 +545,7 @@ define dso_local i32 @sad_2i8() nounwind {
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; SSE2-NEXT:    movd {{.*#+}} xmm1 = [65535,0,0,0]
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB3_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
@@ -567,7 +567,7 @@ define dso_local i32 @sad_2i8() nounwind {
 ; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  .LBB3_1: # %vector.body
 ; AVX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
@@ -618,7 +618,7 @@ define dso_local i32 @sad_4i8() nounwind {
 ; SSE2:       # %bb.0: # %entry
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    movq $-1024, %rax # imm = 0xFC00
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB4_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
@@ -639,7 +639,7 @@ define dso_local i32 @sad_4i8() nounwind {
 ; AVX:       # %bb.0: # %entry
 ; AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    movq $-1024, %rax # imm = 0xFC00
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  .LBB4_1: # %vector.body
 ; AVX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
diff --git a/llvm/test/CodeGen/X86/saddo-redundant-add.ll b/llvm/test/CodeGen/X86/saddo-redundant-add.ll
index e89d81a842318..f132ef8baab92 100644
--- a/llvm/test/CodeGen/X86/saddo-redundant-add.ll
+++ b/llvm/test/CodeGen/X86/saddo-redundant-add.ll
@@ -6,7 +6,7 @@ define void @redundant_add(i64 %n) {
 ; CHECK-LABEL: redundant_add:
 ; CHECK:       ## %bb.0: ## %entry
 ; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_1: ## %exit_check
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpq %rdi, %rax
diff --git a/llvm/test/CodeGen/X86/same-align-bytes-with-llasm-llobj.ll b/llvm/test/CodeGen/X86/same-align-bytes-with-llasm-llobj.ll
new file mode 100644
index 0000000000000..419bf627b39e6
--- /dev/null
+++ b/llvm/test/CodeGen/X86/same-align-bytes-with-llasm-llobj.ll
@@ -0,0 +1,46 @@
+; RUN: llc %s -o - -filetype=obj --mcpu=znver2 | llvm-objdump -dr  - | FileCheck %s
+; RUN: llc %s -o - -filetype=asm --mcpu=znver2 | llvm-mc - -o - --mcpu=znver2 -filetype=obj -triple x86_64-unknown-linux-gnu | llvm-objdump -dr  - | FileCheck %s
+; RUN: llc %s -o - -filetype=asm --mcpu=znver2 | FileCheck %s --check-prefix=ASM
+
+;; Check that we produce a push, then an align-to-16-bytes p2align.
+;
+; ASM:        # %bb.0:
+; ASM-NEXT:   pushq   %rax
+; ASM-NEXT:   .cfi_def_cfa_offset 16
+; ASM-NEXT:   .p2align 4{{$}}
+
+;; When we assemble the file, either using the built-in asssembler or going
+;; via a textual assembly file, we should get the same padding between the
+;; initial push and the next block for alignment. It's a single 15 byte
+;; nop.
+
+; CHECK:        0:   50
+; CHECK-NEXT:   66 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 nopw %cs:(%rax,%rax)
+
+;; Note that we specify a CPU to ensure the same nop patterns are selected
+;; between llvm-mc and llc, just in case defaults changed, which one isn't
+;; important.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: noreturn nounwind uwtable
+define dso_local void @b() local_unnamed_addr {
+entry:
+  br label %for.cond
+
+for.cond:
+  tail call void (...) @a()
+  br label %for.cond
+}
+
+declare void @a(...) local_unnamed_addr
+
+!llvm.module.flags = !{!0, !1, !2, !3}
+!llvm.ident = !{!4}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 8, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{!"clang"}
diff --git a/llvm/test/CodeGen/X86/scalar_widen_div.ll b/llvm/test/CodeGen/X86/scalar_widen_div.ll
index 23e3d0c5c4321..1d98b4f62069d 100644
--- a/llvm/test/CodeGen/X86/scalar_widen_div.ll
+++ b/llvm/test/CodeGen/X86/scalar_widen_div.ll
@@ -394,7 +394,7 @@ define void @test_int_div(ptr %dest, ptr %old, i32 %n) {
 ; CHECK-NEXT:  # %bb.1: # %bb.nph
 ; CHECK-NEXT:    movl %edx, %ecx
 ; CHECK-NEXT:    xorl %r10d, %r10d
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB12_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl (%rdi,%r10), %r8d
diff --git a/llvm/test/CodeGen/X86/setcc-lowering.ll b/llvm/test/CodeGen/X86/setcc-lowering.ll
index 5bbc73eacd968..bf31df20887d0 100644
--- a/llvm/test/CodeGen/X86/setcc-lowering.ll
+++ b/llvm/test/CodeGen/X86/setcc-lowering.ll
@@ -54,7 +54,7 @@ define void @pr26232(i64 %a, <16 x i1> %b) nounwind {
 ; AVX1-LABEL: pr26232:
 ; AVX1:       # %bb.0: # %allocas
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB1_1: # %for_loop599
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    cmpq $65536, %rdi # imm = 0x10000
@@ -71,7 +71,7 @@ define void @pr26232(i64 %a, <16 x i1> %b) nounwind {
 ;
 ; AVX2-LABEL: pr26232:
 ; AVX2:       # %bb.0: # %allocas
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB1_1: # %for_loop599
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    cmpq $65536, %rdi # imm = 0x10000
@@ -93,7 +93,7 @@ define void @pr26232(i64 %a, <16 x i1> %b) nounwind {
 ; KNL-32-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; KNL-32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; KNL-32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; KNL-32-NEXT:    .p2align 4, 0x90
+; KNL-32-NEXT:    .p2align 4
 ; KNL-32-NEXT:  .LBB1_1: # %for_loop599
 ; KNL-32-NEXT:    # =>This Inner Loop Header: Depth=1
 ; KNL-32-NEXT:    cmpl $65536, %ecx # imm = 0x10000
diff --git a/llvm/test/CodeGen/X86/setcc-non-simple-type.ll b/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
index 76cf2423a254f..2ac2be5545dfd 100644
--- a/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
+++ b/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
@@ -48,14 +48,14 @@ define void @failing(ptr %0, ptr %1) nounwind {
 ; CHECK-NEXT:    movdqa {{.*#+}} xmm0 = [0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]
 ; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = [1,1]
 ; CHECK-NEXT:    movdqa {{.*#+}} xmm2 = [2,2]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %vector.ph
 ; CHECK-NEXT:    # =>This Loop Header: Depth=1
 ; CHECK-NEXT:    # Child Loop BB0_2 Depth 2
 ; CHECK-NEXT:    xorpd %xmm3, %xmm3
 ; CHECK-NEXT:    movq $-1024, %rsi # imm = 0xFC00
 ; CHECK-NEXT:    movdqa %xmm0, %xmm4
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # %vector.body
 ; CHECK-NEXT:    # Parent Loop BB0_1 Depth=1
 ; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
@@ -108,14 +108,14 @@ define void @failing(ptr %0, ptr %1) nounwind {
 ; CHECK-AVX2-NEXT:    vpmovsxbq {{.*#+}} xmm0 = [0,1]
 ; CHECK-AVX2-NEXT:    vpmovsxbq {{.*#+}} xmm1 = [1,1]
 ; CHECK-AVX2-NEXT:    vpmovsxbq {{.*#+}} xmm2 = [2,2]
-; CHECK-AVX2-NEXT:    .p2align 4, 0x90
+; CHECK-AVX2-NEXT:    .p2align 4
 ; CHECK-AVX2-NEXT:  .LBB0_1: # %vector.ph
 ; CHECK-AVX2-NEXT:    # =>This Loop Header: Depth=1
 ; CHECK-AVX2-NEXT:    # Child Loop BB0_2 Depth 2
 ; CHECK-AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; CHECK-AVX2-NEXT:    movq $-1024, %rsi # imm = 0xFC00
 ; CHECK-AVX2-NEXT:    vmovdqa %xmm0, %xmm4
-; CHECK-AVX2-NEXT:    .p2align 4, 0x90
+; CHECK-AVX2-NEXT:    .p2align 4
 ; CHECK-AVX2-NEXT:  .LBB0_2: # %vector.body
 ; CHECK-AVX2-NEXT:    # Parent Loop BB0_1 Depth=1
 ; CHECK-AVX2-NEXT:    # => This Inner Loop Header: Depth=2
diff --git a/llvm/test/CodeGen/X86/shift-parts.ll b/llvm/test/CodeGen/X86/shift-parts.ll
index c48f7c5a0c0ba..62d86909ffca2 100644
--- a/llvm/test/CodeGen/X86/shift-parts.ll
+++ b/llvm/test/CodeGen/X86/shift-parts.ll
@@ -13,7 +13,7 @@ define i32 @int87(i32 %uint64p_8, i1 %cond) nounwind {
 ; CHECK-NEXT:    movq g_144+16(%rip), %rcx
 ; CHECK-NEXT:    movzbl %sil, %edx
 ; CHECK-NEXT:    shll $6, %edx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %for.cond
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    testb $64, %dl
diff --git a/llvm/test/CodeGen/X86/sink-out-of-loop.ll b/llvm/test/CodeGen/X86/sink-out-of-loop.ll
index 55d8c7bb03697..4634d34a7536c 100644
--- a/llvm/test/CodeGen/X86/sink-out-of-loop.ll
+++ b/llvm/test/CodeGen/X86/sink-out-of-loop.ll
@@ -9,19 +9,19 @@ define i32 @sink_succ(i32 %argc, ptr nocapture %argv) nounwind uwtable ssp {
 ; CHECK-LABEL: sink_succ:
 ; CHECK:       ## %bb.0: ## %entry
 ; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_1: ## %preheader
 ; CHECK-NEXT:    ## =>This Loop Header: Depth=1
 ; CHECK-NEXT:    ## Child Loop BB0_2 Depth 2
 ; CHECK-NEXT:    ## Child Loop BB0_3 Depth 3
 ; CHECK-NEXT:    movl $1, %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_2: ## %for.body1.lr
 ; CHECK-NEXT:    ## Parent Loop BB0_1 Depth=1
 ; CHECK-NEXT:    ## => This Loop Header: Depth=2
 ; CHECK-NEXT:    ## Child Loop BB0_3 Depth 3
 ; CHECK-NEXT:    movl %ecx, %edx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_3: ## %for.body1
 ; CHECK-NEXT:    ## Parent Loop BB0_1 Depth=1
 ; CHECK-NEXT:    ## Parent Loop BB0_2 Depth=2
@@ -40,7 +40,7 @@ define i32 @sink_succ(i32 %argc, ptr nocapture %argv) nounwind uwtable ssp {
 ; CHECK-NEXT:    jne LBB0_1
 ; CHECK-NEXT:  ## %bb.6: ## %for.body2.preheader
 ; CHECK-NEXT:    movl $2048, %eax ## imm = 0x800
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_7: ## %for.body2
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    decl %eax
@@ -91,7 +91,7 @@ define i32 @sink_out_of_loop(i32 %n, ptr %output) {
 ; CHECK-LABEL: sink_out_of_loop:
 ; CHECK:       ## %bb.0: ## %entry
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB1_1: ## %loop
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %ecx, %eax
diff --git a/llvm/test/CodeGen/X86/speculative-load-hardening.ll b/llvm/test/CodeGen/X86/speculative-load-hardening.ll
index 45da777ea4e2a..5fd1f77e166d4 100644
--- a/llvm/test/CodeGen/X86/speculative-load-hardening.ll
+++ b/llvm/test/CodeGen/X86/speculative-load-hardening.ll
@@ -217,7 +217,7 @@ define void @test_basic_loop(i32 %a, i32 %b, ptr %ptr1, ptr %ptr2) nounwind spec
 ; X64-NEXT:    movl %esi, %ebp
 ; X64-NEXT:    cmovneq %r15, %rax
 ; X64-NEXT:    xorl %r12d, %r12d
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB2_3: # %l.header
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movslq (%r14), %rcx
@@ -267,7 +267,7 @@ define void @test_basic_loop(i32 %a, i32 %b, ptr %ptr1, ptr %ptr2) nounwind spec
 ; X64-LFENCE-NEXT:    movl %esi, %ebp
 ; X64-LFENCE-NEXT:    lfence
 ; X64-LFENCE-NEXT:    xorl %r15d, %r15d
-; X64-LFENCE-NEXT:    .p2align 4, 0x90
+; X64-LFENCE-NEXT:    .p2align 4
 ; X64-LFENCE-NEXT:  .LBB2_2: # %l.header
 ; X64-LFENCE-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-LFENCE-NEXT:    lfence
@@ -331,11 +331,11 @@ define void @test_basic_nested_loop(i32 %a, i32 %b, i32 %c, ptr %ptr1, ptr %ptr2
 ; X64-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
 ; X64-NEXT:    testl %r15d, %r15d
 ; X64-NEXT:    jle .LBB3_4
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB3_5: # %l2.header.preheader
 ; X64-NEXT:    cmovleq %r12, %rax
 ; X64-NEXT:    xorl %r15d, %r15d
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB3_6: # %l2.header
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movslq (%r14), %rcx
@@ -358,12 +358,12 @@ define void @test_basic_nested_loop(i32 %a, i32 %b, i32 %c, ptr %ptr1, ptr %ptr2
 ; X64-NEXT:  # %bb.11: # in Loop: Header=BB3_6 Depth=1
 ; X64-NEXT:    cmovgeq %r12, %rax
 ; X64-NEXT:    jmp .LBB3_6
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB3_7:
 ; X64-NEXT:    cmovlq %r12, %rax
 ; X64-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %r15d # 4-byte Reload
 ; X64-NEXT:    jmp .LBB3_8
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB3_4:
 ; X64-NEXT:    cmovgq %r12, %rax
 ; X64-NEXT:  .LBB3_8: # %l1.latch
@@ -432,7 +432,7 @@ define void @test_basic_nested_loop(i32 %a, i32 %b, i32 %c, ptr %ptr1, ptr %ptr2
 ; X64-LFENCE-NEXT:    lfence
 ; X64-LFENCE-NEXT:    xorl %r12d, %r12d
 ; X64-LFENCE-NEXT:    jmp .LBB3_2
-; X64-LFENCE-NEXT:    .p2align 4, 0x90
+; X64-LFENCE-NEXT:    .p2align 4
 ; X64-LFENCE-NEXT:  .LBB3_5: # %l1.latch
 ; X64-LFENCE-NEXT:    # in Loop: Header=BB3_2 Depth=1
 ; X64-LFENCE-NEXT:    lfence
@@ -452,7 +452,7 @@ define void @test_basic_nested_loop(i32 %a, i32 %b, i32 %c, ptr %ptr1, ptr %ptr2
 ; X64-LFENCE-NEXT:    # in Loop: Header=BB3_2 Depth=1
 ; X64-LFENCE-NEXT:    lfence
 ; X64-LFENCE-NEXT:    xorl %r13d, %r13d
-; X64-LFENCE-NEXT:    .p2align 4, 0x90
+; X64-LFENCE-NEXT:    .p2align 4
 ; X64-LFENCE-NEXT:  .LBB3_4: # %l2.header
 ; X64-LFENCE-NEXT:    # Parent Loop BB3_2 Depth=1
 ; X64-LFENCE-NEXT:    # => This Inner Loop Header: Depth=2
diff --git a/llvm/test/CodeGen/X86/split-extend-vector-inreg.ll b/llvm/test/CodeGen/X86/split-extend-vector-inreg.ll
index 08088496b7428..8a6c2f851a6d6 100644
--- a/llvm/test/CodeGen/X86/split-extend-vector-inreg.ll
+++ b/llvm/test/CodeGen/X86/split-extend-vector-inreg.ll
@@ -9,7 +9,7 @@ define <4 x i64> @autogen_SD88863() {
 ; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
 ; CHECK-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[3],ymm1[3]
 ; CHECK-NEXT:    movb $1, %al
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %CF
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    testb %al, %al
diff --git a/llvm/test/CodeGen/X86/sse-domains.ll b/llvm/test/CodeGen/X86/sse-domains.ll
index 116efa69bb319..249425ec4dc9e 100644
--- a/llvm/test/CodeGen/X86/sse-domains.ll
+++ b/llvm/test/CodeGen/X86/sse-domains.ll
@@ -18,7 +18,7 @@ define void @f(ptr nocapture %p, i32 %n) nounwind uwtable ssp {
 ; CHECK-NEXT:    addq $16, %rdi
 ; CHECK-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-NEXT:    pmovsxbd {{.*#+}} xmm0 = [127,127,127,127]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_1: ## %while.body
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    pand %xmm0, %xmm1
diff --git a/llvm/test/CodeGen/X86/sse-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/sse-intrinsics-fast-isel.ll
index 86b8121f21cff..b1f9872f7b6eb 100644
--- a/llvm/test/CodeGen/X86/sse-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/sse-intrinsics-fast-isel.ll
@@ -96,7 +96,7 @@ define <4 x float> @test_mm_andnot_ps(<4 x float> %a0, <4 x float> %a1) nounwind
 ; AVX512-LABEL: test_mm_andnot_ps:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x25,0xc0,0x0f]
-; AVX512-NEXT:    # xmm0 = (~xmm0 & ~xmm0 & ~xmm0) | (~xmm0 & ~xmm0 & xmm0) | (~xmm0 & xmm0 & ~xmm0) | (~xmm0 & xmm0 & xmm0)
+; AVX512-NEXT:    # xmm0 = ~xmm0
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xdb,0xc1]
 ; AVX512-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
diff --git a/llvm/test/CodeGen/X86/sse2-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/sse2-intrinsics-fast-isel.ll
index 853bb6367fe4d..ca85afc352a57 100644
--- a/llvm/test/CodeGen/X86/sse2-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/sse2-intrinsics-fast-isel.ll
@@ -288,7 +288,7 @@ define <2 x double> @test_mm_andnot_pd(<2 x double> %a0, <2 x double> %a1) nounw
 ; AVX512-LABEL: test_mm_andnot_pd:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x25,0xc0,0x0f]
-; AVX512-NEXT:    # xmm0 = (~xmm0 & ~xmm0 & ~xmm0) | (~xmm0 & ~xmm0 & xmm0) | (~xmm0 & xmm0 & ~xmm0) | (~xmm0 & xmm0 & xmm0)
+; AVX512-NEXT:    # xmm0 = ~xmm0
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xdb,0xc1]
 ; AVX512-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %arg0 = bitcast <2 x double> %a0 to <4 x i32>
@@ -317,7 +317,7 @@ define <2 x i64> @test_mm_andnot_si128(<2 x i64> %a0, <2 x i64> %a1) nounwind {
 ; AVX512-LABEL: test_mm_andnot_si128:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x25,0xc0,0x0f]
-; AVX512-NEXT:    # xmm0 = (~xmm0 & ~xmm0 & ~xmm0) | (~xmm0 & ~xmm0 & xmm0) | (~xmm0 & xmm0 & ~xmm0) | (~xmm0 & xmm0 & xmm0)
+; AVX512-NEXT:    # xmm0 = ~xmm0
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xdb,0xc1]
 ; AVX512-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %not = xor <2 x i64> %a0, <i64 -1, i64 -1>
diff --git a/llvm/test/CodeGen/X86/stack-coloring-wineh.ll b/llvm/test/CodeGen/X86/stack-coloring-wineh.ll
index 79057e0ea78d3..aa2f9e1cd2b1e 100644
--- a/llvm/test/CodeGen/X86/stack-coloring-wineh.ll
+++ b/llvm/test/CodeGen/X86/stack-coloring-wineh.ll
@@ -42,7 +42,7 @@ define void @pr66984(ptr %arg) personality ptr @__CxxFrameHandler3 {
 ; I686-NEXT:    .scl 3;
 ; I686-NEXT:    .type 32;
 ; I686-NEXT:    .endef
-; I686-NEXT:    .p2align 4, 0x90
+; I686-NEXT:    .p2align 4
 ; I686-NEXT:  "?catch$2@?0?pr66984@4HA":
 ; I686-NEXT:  LBB0_2: # %bb17
 ; I686-NEXT:    pushl %ebp
@@ -58,7 +58,7 @@ define void @pr66984(ptr %arg) personality ptr @__CxxFrameHandler3 {
 ; I686-NEXT:    .scl 3;
 ; I686-NEXT:    .type 32;
 ; I686-NEXT:    .endef
-; I686-NEXT:    .p2align 4, 0x90
+; I686-NEXT:    .p2align 4
 ; I686-NEXT:  "?dtor$5@?0?pr66984@4HA":
 ; I686-NEXT:  LBB0_5: # %bb8
 ; I686-NEXT:    pushl %ebp
@@ -101,7 +101,7 @@ define void @pr66984(ptr %arg) personality ptr @__CxxFrameHandler3 {
 ; X86_64-NEXT:    .scl 3;
 ; X86_64-NEXT:    .type 32;
 ; X86_64-NEXT:    .endef
-; X86_64-NEXT:    .p2align 4, 0x90
+; X86_64-NEXT:    .p2align 4
 ; X86_64-NEXT:  "?catch$2@?0?pr66984@4HA":
 ; X86_64-NEXT:  .seh_proc "?catch$2@?0?pr66984@4HA"
 ; X86_64-NEXT:    .seh_handler __CxxFrameHandler3, @unwind, @except
@@ -127,7 +127,7 @@ define void @pr66984(ptr %arg) personality ptr @__CxxFrameHandler3 {
 ; X86_64-NEXT:    .scl 3;
 ; X86_64-NEXT:    .type 32;
 ; X86_64-NEXT:    .endef
-; X86_64-NEXT:    .p2align 4, 0x90
+; X86_64-NEXT:    .p2align 4
 ; X86_64-NEXT:  "?dtor$4@?0?pr66984@4HA":
 ; X86_64-NEXT:  .seh_proc "?dtor$4@?0?pr66984@4HA"
 ; X86_64-NEXT:  .LBB0_4: # %bb8
diff --git a/llvm/test/CodeGen/X86/stack-folding-int-avx512.ll b/llvm/test/CodeGen/X86/stack-folding-int-avx512.ll
index e8b0facf534b0..e826306fd533c 100644
--- a/llvm/test/CodeGen/X86/stack-folding-int-avx512.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-int-avx512.ll
@@ -7047,7 +7047,7 @@ define <16 x i32> @stack_fold_ternlogd(<16 x i32> %x0, <16 x i32> %x1, <16 x i32
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    vpternlogd $33, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
-; CHECK-NEXT:    # zmm0 = (~zmm0 & ~zmm1 & ~mem) | (zmm0 & ~zmm1 & mem)
+; CHECK-NEXT:    # zmm0 = ~(zmm1 | (zmm0 ^ mem))
 ; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33)
@@ -7063,7 +7063,7 @@ define <8 x i64> @stack_fold_ternlogq(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    vpternlogq $33, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
-; CHECK-NEXT:    # zmm0 = (~zmm0 & ~zmm1 & ~mem) | (zmm0 & ~zmm1 & mem)
+; CHECK-NEXT:    # zmm0 = ~(zmm1 | (zmm0 ^ mem))
 ; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   %2 = call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33)
diff --git a/llvm/test/CodeGen/X86/switch.ll b/llvm/test/CodeGen/X86/switch.ll
index b00044a1e4f79..629ba48fcae6b 100644
--- a/llvm/test/CodeGen/X86/switch.ll
+++ b/llvm/test/CodeGen/X86/switch.ll
@@ -1304,7 +1304,7 @@ return: ret void
 define void @phi_node_trouble(ptr %s) {
 ; CHECK-LABEL: phi_node_trouble:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB13_1: # %header
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    testq %rdi, %rdi
diff --git a/llvm/test/CodeGen/X86/tail-dup-merge-loop-headers.ll b/llvm/test/CodeGen/X86/tail-dup-merge-loop-headers.ll
index 8d84e887d3f27..72e4fe410e269 100644
--- a/llvm/test/CodeGen/X86/tail-dup-merge-loop-headers.ll
+++ b/llvm/test/CodeGen/X86/tail-dup-merge-loop-headers.ll
@@ -6,7 +6,7 @@ define void @tail_dup_merge_loops(i32 %a, ptr %b, ptr %c) local_unnamed_addr #0
 ; CHECK-LABEL: tail_dup_merge_loops:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    jmp .LBB0_1
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_3: # %inner_loop_exit
 ; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    incq %rsi
@@ -15,7 +15,7 @@ define void @tail_dup_merge_loops(i32 %a, ptr %b, ptr %c) local_unnamed_addr #0
 ; CHECK-NEXT:    # Child Loop BB0_4 Depth 2
 ; CHECK-NEXT:    testl %edi, %edi
 ; CHECK-NEXT:    je .LBB0_5
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  # %bb.2: # %inner_loop_top
 ; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    cmpb $0, (%rsi)
@@ -118,7 +118,7 @@ define i32 @loop_shared_header(ptr %exe, i32 %exesz, i32 %headsize, i32 %min, i3
 ; CHECK-NEXT:    movb $32, %cl
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    jmp .LBB1_4
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_15: # %merge_predecessor_split
 ; CHECK-NEXT:    # in Loop: Header=BB1_4 Depth=1
 ; CHECK-NEXT:    movb $32, %cl
@@ -127,7 +127,7 @@ define i32 @loop_shared_header(ptr %exe, i32 %exesz, i32 %headsize, i32 %min, i3
 ; CHECK-NEXT:    # Child Loop BB1_8 Depth 2
 ; CHECK-NEXT:    testl %r12d, %r12d
 ; CHECK-NEXT:    je .LBB1_5
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_8: # %shared_loop_header
 ; CHECK-NEXT:    # Parent Loop BB1_4 Depth=1
 ; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
diff --git a/llvm/test/CodeGen/X86/tail-dup-multiple-latch-loop.ll b/llvm/test/CodeGen/X86/tail-dup-multiple-latch-loop.ll
index fdcad3c1973e7..862f60c063804 100644
--- a/llvm/test/CodeGen/X86/tail-dup-multiple-latch-loop.ll
+++ b/llvm/test/CodeGen/X86/tail-dup-multiple-latch-loop.ll
@@ -16,7 +16,7 @@ define ptr @large_loop_switch(ptr %p) {
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    jmp ccc@PLT # TAILCALL
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # %sw.bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
@@ -26,7 +26,7 @@ define ptr @large_loop_switch(ptr %p) {
 ; CHECK-NEXT:    decl %ebx
 ; CHECK-NEXT:    movl %ebx, %ecx
 ; CHECK-NEXT:    jmpq *.LJTI0_0(,%rcx,8)
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_3: # %sw.bb3
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl $532, %edi # imm = 0x214
@@ -35,7 +35,7 @@ define ptr @large_loop_switch(ptr %p) {
 ; CHECK-NEXT:    decl %ebx
 ; CHECK-NEXT:    movl %ebx, %ecx
 ; CHECK-NEXT:    jmpq *.LJTI0_0(,%rcx,8)
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_4: # %sw.bb5
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl $533, %edi # imm = 0x215
@@ -44,7 +44,7 @@ define ptr @large_loop_switch(ptr %p) {
 ; CHECK-NEXT:    decl %ebx
 ; CHECK-NEXT:    movl %ebx, %ecx
 ; CHECK-NEXT:    jmpq *.LJTI0_0(,%rcx,8)
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_5: # %sw.bb7
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl $535, %edi # imm = 0x217
@@ -53,7 +53,7 @@ define ptr @large_loop_switch(ptr %p) {
 ; CHECK-NEXT:    decl %ebx
 ; CHECK-NEXT:    movl %ebx, %ecx
 ; CHECK-NEXT:    jmpq *.LJTI0_0(,%rcx,8)
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_6: # %sw.bb9
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl $536, %edi # imm = 0x218
@@ -62,7 +62,7 @@ define ptr @large_loop_switch(ptr %p) {
 ; CHECK-NEXT:    decl %ebx
 ; CHECK-NEXT:    movl %ebx, %ecx
 ; CHECK-NEXT:    jmpq *.LJTI0_0(,%rcx,8)
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_7: # %sw.bb11
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl $658, %edi # imm = 0x292
@@ -136,7 +136,7 @@ define i32 @interp_switch(ptr nocapture readonly %0, i32 %1) {
 ; CHECK-NEXT:  .LBB1_7: # in Loop: Header=BB1_1 Depth=1
 ; CHECK-NEXT:    addl $7, %eax
 ; CHECK-NEXT:    incq %rdi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movzbl (%rdi), %ecx
 ; CHECK-NEXT:    decl %ecx
diff --git a/llvm/test/CodeGen/X86/tail-dup-partial.ll b/llvm/test/CodeGen/X86/tail-dup-partial.ll
index 3b506fc6fce86..292d59d3450d3 100644
--- a/llvm/test/CodeGen/X86/tail-dup-partial.ll
+++ b/llvm/test/CodeGen/X86/tail-dup-partial.ll
@@ -8,7 +8,7 @@
 define void @partial_tail_dup(i1 %a1, i1 %a2, ptr %a4, ptr %a5, ptr %a6, i32 %a7) #0 align 2  !prof !1 {
 ; CHECK-LABEL: partial_tail_dup:
 ; CHECK:        # %bb.0: # %entry
-; CHECK-NEXT:   .p2align 4, 0x90
+; CHECK-NEXT:   .p2align 4
 ; CHECK-NEXT:   .LBB0_1: # %for.cond
 ; CHECK-NEXT:   # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:	  testb	$1, %dil
@@ -20,14 +20,14 @@ define void @partial_tail_dup(i1 %a1, i1 %a2, ptr %a4, ptr %a5, ptr %a6, i32 %a7
 ; CHECK-NEXT:	  testl	%r9d, %r9d
 ; CHECK-NEXT:	  je	.LBB0_1
 ; CHECK-NEXT:	  jmp	.LBB0_8
-; CHECK-NEXT:	  .p2align	4, 0x90
+; CHECK-NEXT:	  .p2align	4
 ; CHECK-NEXT:   .LBB0_6: # %dup2
 ; CHECK-NEXT:   # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:	  movl	$2, (%rcx)
 ; CHECK-NEXT:	  testl	%r9d, %r9d
 ; CHECK-NEXT:	  je	.LBB0_1
 ; CHECK-NEXT:	  jmp	.LBB0_8
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_3: # %if.end56
 ; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    testb $1, %sil
diff --git a/llvm/test/CodeGen/X86/tail-dup-repeat.ll b/llvm/test/CodeGen/X86/tail-dup-repeat.ll
index 5e0096a028d85..7a9eb0c3fb85f 100644
--- a/llvm/test/CodeGen/X86/tail-dup-repeat.ll
+++ b/llvm/test/CodeGen/X86/tail-dup-repeat.ll
@@ -11,7 +11,7 @@ define void @repeated_tail_dup(i1 %a1, i1 %a2, ptr %a4, ptr %a5, ptr %a6, i32 %a
 ; CHECK-LABEL: repeated_tail_dup:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    jmp .LBB0_1
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # %land.lhs.true
 ; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    movl $10, (%rdx)
@@ -34,7 +34,7 @@ define void @repeated_tail_dup(i1 %a1, i1 %a2, ptr %a4, ptr %a5, ptr %a6, i32 %a
 ; CHECK-NEXT:    testl %r9d, %r9d
 ; CHECK-NEXT:    je .LBB0_1
 ; CHECK-NEXT:    jmp .LBB0_8
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_5: # %if.end70
 ; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    movl $12, (%rdx)
diff --git a/llvm/test/CodeGen/X86/tailcall-cgp-dup.ll b/llvm/test/CodeGen/X86/tailcall-cgp-dup.ll
index 8a9ee60f341c2..f96ba351f53f7 100644
--- a/llvm/test/CodeGen/X86/tailcall-cgp-dup.ll
+++ b/llvm/test/CodeGen/X86/tailcall-cgp-dup.ll
@@ -32,7 +32,7 @@ define i32 @foo(i32 %x) nounwind ssp {
 ; CHECK-NEXT:  LBB0_8: ## %return
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    retq
-; CHECK-NEXT:    .p2align 2, 0x90
+; CHECK-NEXT:    .p2align 2
 ; CHECK-NEXT:    .data_region jt32
 ; CHECK-NEXT:  .set L0_0_set_2, LBB0_2-LJTI0_0
 ; CHECK-NEXT:  .set L0_0_set_3, LBB0_3-LJTI0_0
diff --git a/llvm/test/CodeGen/X86/tls-loads-control3.ll b/llvm/test/CodeGen/X86/tls-loads-control3.ll
index 4e521b1c696a4..92dccee296ad7 100644
--- a/llvm/test/CodeGen/X86/tls-loads-control3.ll
+++ b/llvm/test/CodeGen/X86/tls-loads-control3.ll
@@ -51,7 +51,7 @@ define i32 @_Z2f1i(i32 %c) local_unnamed_addr #0 {
 ; HOIST0-NEXT:    rex64
 ; HOIST0-NEXT:    callq __tls_get_addr@PLT
 ; HOIST0-NEXT:    movq %rax, %r15
-; HOIST0-NEXT:    .p2align 4, 0x90
+; HOIST0-NEXT:    .p2align 4
 ; HOIST0-NEXT:  .LBB0_2: # %while.body
 ; HOIST0-NEXT:    # =>This Inner Loop Header: Depth=1
 ; HOIST0-NEXT:    movl (%r15), %edi
@@ -86,7 +86,7 @@ define i32 @_Z2f1i(i32 %c) local_unnamed_addr #0 {
 ; HOIST2-NEXT:    je .LBB0_4
 ; HOIST2-NEXT:  # %bb.1:
 ; HOIST2-NEXT:    movl %edi, %ebx
-; HOIST2-NEXT:    .p2align 4, 0x90
+; HOIST2-NEXT:    .p2align 4
 ; HOIST2-NEXT:  .LBB0_2: # %while.body
 ; HOIST2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; HOIST2-NEXT:    data16
@@ -185,7 +185,7 @@ define i32 @_Z2f2i(i32 %c) local_unnamed_addr #0 {
 ; HOIST0-NEXT:    callq __tls_get_addr@PLT
 ; HOIST0-NEXT:    leaq _ZZ2f2iE2st.0@DTPOFF(%rax), %r15
 ; HOIST0-NEXT:    leaq _ZZ2f2iE2st.1@DTPOFF(%rax), %r12
-; HOIST0-NEXT:    .p2align 4, 0x90
+; HOIST0-NEXT:    .p2align 4
 ; HOIST0-NEXT:  .LBB1_2: # %while.body
 ; HOIST0-NEXT:    # =>This Inner Loop Header: Depth=1
 ; HOIST0-NEXT:    callq _Z5gfuncv@PLT
@@ -225,7 +225,7 @@ define i32 @_Z2f2i(i32 %c) local_unnamed_addr #0 {
 ; HOIST2-NEXT:    je .LBB1_3
 ; HOIST2-NEXT:  # %bb.1: # %while.body.preheader
 ; HOIST2-NEXT:    movl %edi, %ebx
-; HOIST2-NEXT:    .p2align 4, 0x90
+; HOIST2-NEXT:    .p2align 4
 ; HOIST2-NEXT:  .LBB1_2: # %while.body
 ; HOIST2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; HOIST2-NEXT:    callq _Z5gfuncv@PLT
diff --git a/llvm/test/CodeGen/X86/trunc-store.ll b/llvm/test/CodeGen/X86/trunc-store.ll
index fbdf966dc76ae..3ce06f7caa28e 100644
--- a/llvm/test/CodeGen/X86/trunc-store.ll
+++ b/llvm/test/CodeGen/X86/trunc-store.ll
@@ -29,7 +29,7 @@
 define void @fn1(i64 %a0) {
 ; CHECK-LABEL: fn1:
 ; CHECK:       # %bb.0: # %for.cond
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpq $8, %rdi
diff --git a/llvm/test/CodeGen/X86/twoaddr-coalesce.ll b/llvm/test/CodeGen/X86/twoaddr-coalesce.ll
index a7bdfbc7bb25f..fbedd92517da6 100644
--- a/llvm/test/CodeGen/X86/twoaddr-coalesce.ll
+++ b/llvm/test/CodeGen/X86/twoaddr-coalesce.ll
@@ -9,7 +9,7 @@ define i32 @foo() nounwind {
 ; CHECK:       # %bb.0: # %bb1.thread
 ; CHECK-NEXT:    pushl %ebx
 ; CHECK-NEXT:    xorl %ebx, %ebx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl %ebx, %eax
diff --git a/llvm/test/CodeGen/X86/twoaddr-lea.ll b/llvm/test/CodeGen/X86/twoaddr-lea.ll
index 2f6b943ca7275..f20b777531c5a 100644
--- a/llvm/test/CodeGen/X86/twoaddr-lea.ll
+++ b/llvm/test/CodeGen/X86/twoaddr-lea.ll
@@ -71,14 +71,14 @@ define void @ham() {
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %cl, %cl
 ; CHECK-NEXT:    je LBB3_2
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB3_6: ## %bb2
 ; CHECK-NEXT:    ## =>This Loop Header: Depth=1
 ; CHECK-NEXT:    ## Child Loop BB3_7 Depth 2
 ; CHECK-NEXT:    movl (%rdx), %edi
 ; CHECK-NEXT:    leal (%rdi,%rax), %r8d
 ; CHECK-NEXT:    movslq %r8d, %r8
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB3_7: ## %bb6
 ; CHECK-NEXT:    ## Parent Loop BB3_6 Depth=1
 ; CHECK-NEXT:    ## => This Inner Loop Header: Depth=2
@@ -94,7 +94,7 @@ define void @ham() {
 ; CHECK-NEXT:    jne LBB3_6
 ; CHECK-NEXT:  LBB3_2: ## %bb3.preheader
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB3_3: ## %bb3
 ; CHECK-NEXT:    ## =>This Loop Header: Depth=1
 ; CHECK-NEXT:    ## Child Loop BB3_4 Depth 2
@@ -102,7 +102,7 @@ define void @ham() {
 ; CHECK-NEXT:    addq $4, %rcx
 ; CHECK-NEXT:    movl %eax, %esi
 ; CHECK-NEXT:    subl %edx, %esi
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB3_4: ## %bb4
 ; CHECK-NEXT:    ## Parent Loop BB3_3 Depth=1
 ; CHECK-NEXT:    ## => This Inner Loop Header: Depth=2
diff --git a/llvm/test/CodeGen/X86/unaligned-load.ll b/llvm/test/CodeGen/X86/unaligned-load.ll
index c95cb3fe9d9c2..2310cb228f063 100644
--- a/llvm/test/CodeGen/X86/unaligned-load.ll
+++ b/llvm/test/CodeGen/X86/unaligned-load.ll
@@ -12,7 +12,7 @@ define void @func() nounwind ssp {
 ; I386-LABEL: func:
 ; I386:       ## %bb.0: ## %entry
 ; I386-NEXT:    subl $32, %esp
-; I386-NEXT:    .p2align 4, 0x90
+; I386-NEXT:    .p2align 4
 ; I386-NEXT:  LBB0_1: ## %bb
 ; I386-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; I386-NEXT:    movl $4673097, {{[0-9]+}}(%esp) ## imm = 0x474E49
@@ -31,7 +31,7 @@ define void @func() nounwind ssp {
 ; CORE2-NEXT:    movabsq $2325069237881678925, %rcx ## imm = 0x20444E2732202C4D
 ; CORE2-NEXT:    movabsq $4706902966564560965, %rdx ## imm = 0x4152474F52502045
 ; CORE2-NEXT:    movabsq $5642821575076104260, %rsi ## imm = 0x4E4F545359524844
-; CORE2-NEXT:    .p2align 4, 0x90
+; CORE2-NEXT:    .p2align 4
 ; CORE2-NEXT:  LBB0_1: ## %bb
 ; CORE2-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CORE2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
@@ -44,7 +44,7 @@ define void @func() nounwind ssp {
 ; COREI7:       ## %bb.0: ## %entry
 ; COREI7-NEXT:    movups _.str3+15(%rip), %xmm0
 ; COREI7-NEXT:    movups _.str3(%rip), %xmm1
-; COREI7-NEXT:    .p2align 4, 0x90
+; COREI7-NEXT:    .p2align 4
 ; COREI7-NEXT:  LBB0_1: ## %bb
 ; COREI7-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; COREI7-NEXT:    movups %xmm0, -{{[0-9]+}}(%rsp)
@@ -67,7 +67,7 @@ define void @func_aligned() nounwind ssp {
 ; I386:       ## %bb.0: ## %entry
 ; I386-NEXT:    subl $44, %esp
 ; I386-NEXT:    movaps {{.*#+}} xmm0 = [1498564676,1313821779,1380982853,1095911247]
-; I386-NEXT:    .p2align 4, 0x90
+; I386-NEXT:    .p2align 4
 ; I386-NEXT:  LBB1_1: ## %bb
 ; I386-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; I386-NEXT:    movaps %xmm0, (%esp)
@@ -83,7 +83,7 @@ define void @func_aligned() nounwind ssp {
 ; CORE2-NEXT:    movabsq $2325069237881678925, %rcx ## imm = 0x20444E2732202C4D
 ; CORE2-NEXT:    movabsq $4706902966564560965, %rdx ## imm = 0x4152474F52502045
 ; CORE2-NEXT:    movabsq $5642821575076104260, %rsi ## imm = 0x4E4F545359524844
-; CORE2-NEXT:    .p2align 4, 0x90
+; CORE2-NEXT:    .p2align 4
 ; CORE2-NEXT:  LBB1_1: ## %bb
 ; CORE2-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CORE2-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
@@ -96,7 +96,7 @@ define void @func_aligned() nounwind ssp {
 ; COREI7:       ## %bb.0: ## %entry
 ; COREI7-NEXT:    movups _.str3+15(%rip), %xmm0
 ; COREI7-NEXT:    movups _.str3(%rip), %xmm1
-; COREI7-NEXT:    .p2align 4, 0x90
+; COREI7-NEXT:    .p2align 4
 ; COREI7-NEXT:  LBB1_1: ## %bb
 ; COREI7-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; COREI7-NEXT:    movups %xmm0, -{{[0-9]+}}(%rsp)
diff --git a/llvm/test/CodeGen/X86/undef-label.ll b/llvm/test/CodeGen/X86/undef-label.ll
index 4cacaec85e7c0..a2d9bc571f582 100644
--- a/llvm/test/CodeGen/X86/undef-label.ll
+++ b/llvm/test/CodeGen/X86/undef-label.ll
@@ -15,7 +15,7 @@ define dso_local void @xyz() {
 ; CHECK-NEXT:    ucomisd %xmm1, %xmm0
 ; CHECK-NEXT:    jne .LBB0_1
 ; CHECK-NEXT:    jnp .LBB0_2
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %foo
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ucomisd %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/vec_setcc-2.ll b/llvm/test/CodeGen/X86/vec_setcc-2.ll
index edf726a1e37c4..5a71878ea4579 100644
--- a/llvm/test/CodeGen/X86/vec_setcc-2.ll
+++ b/llvm/test/CodeGen/X86/vec_setcc-2.ll
@@ -13,7 +13,7 @@ define void @loop_no_const_reload(ptr  %in, ptr %out, i32 %n) {
 ; SSE2-NEXT:    xorl %eax, %eax
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [25,25,25,25,25,25,25,25]
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  LBB0_2: ## %for.body
 ; SSE2-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movdqa (%rdi,%rax), %xmm2
@@ -33,7 +33,7 @@ define void @loop_no_const_reload(ptr  %in, ptr %out, i32 %n) {
 ; SSE41-NEXT:  ## %bb.1: ## %for.body.preheader
 ; SSE41-NEXT:    xorl %eax, %eax
 ; SSE41-NEXT:    pmovsxbw {{.*#+}} xmm0 = [25,25,25,25,25,25,25,25]
-; SSE41-NEXT:    .p2align 4, 0x90
+; SSE41-NEXT:    .p2align 4
 ; SSE41-NEXT:  LBB0_2: ## %for.body
 ; SSE41-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; SSE41-NEXT:    movdqa (%rdi,%rax), %xmm1
@@ -80,7 +80,7 @@ define void @loop_const_folding_underflow(ptr  %in, ptr %out, i32 %n) {
 ; SSE2-NEXT:    xorl %eax, %eax
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [32768,32768,32768,32768,32768,32768,32768,32768]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [32768,32794,32794,32794,32794,32794,32794,32794]
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  LBB1_2: ## %for.body
 ; SSE2-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movdqa (%rdi,%rax), %xmm2
@@ -102,7 +102,7 @@ define void @loop_const_folding_underflow(ptr  %in, ptr %out, i32 %n) {
 ; SSE41-NEXT:    xorl %eax, %eax
 ; SSE41-NEXT:    pmovsxbw {{.*#+}} xmm0 = [0,26,26,26,26,26,26,26]
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    .p2align 4, 0x90
+; SSE41-NEXT:    .p2align 4
 ; SSE41-NEXT:  LBB1_2: ## %for.body
 ; SSE41-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; SSE41-NEXT:    movdqa (%rdi,%rax), %xmm2
diff --git a/llvm/test/CodeGen/X86/vector-fshl-128.ll b/llvm/test/CodeGen/X86/vector-fshl-128.ll
index c7cff092c5a4f..28b3d7c7d717e 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-128.ll
@@ -1454,7 +1454,7 @@ define void @sink_splatvar(ptr %p, i32 %shift_amt) {
 ; SSE-NEXT:    movd %esi, %xmm0
 ; SSE-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; SSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE-NEXT:    .p2align 4, 0x90
+; SSE-NEXT:    .p2align 4
 ; SSE-NEXT:  .LBB8_1: # %loop
 ; SSE-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE-NEXT:    movdqu 1024(%rdi,%rax), %xmm1
@@ -1474,7 +1474,7 @@ define void @sink_splatvar(ptr %p, i32 %shift_amt) {
 ; AVX1-NEXT:    vmovd %esi, %xmm0
 ; AVX1-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB8_1: # %loop
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vmovdqu 1024(%rdi,%rax), %xmm1
@@ -1498,7 +1498,7 @@ define void @sink_splatvar(ptr %p, i32 %shift_amt) {
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [32,32,32,32]
 ; AVX2-NEXT:    vpsubd %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB8_1: # %loop
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    vmovdqu 1024(%rdi,%rax), %xmm2
@@ -1516,7 +1516,7 @@ define void @sink_splatvar(ptr %p, i32 %shift_amt) {
 ; AVX512F-NEXT:    vmovd %esi, %xmm0
 ; AVX512F-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; AVX512F-NEXT:    movq $-1024, %rax # imm = 0xFC00
-; AVX512F-NEXT:    .p2align 4, 0x90
+; AVX512F-NEXT:    .p2align 4
 ; AVX512F-NEXT:  .LBB8_1: # %loop
 ; AVX512F-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512F-NEXT:    vmovdqu 1024(%rdi,%rax), %xmm1
@@ -1532,7 +1532,7 @@ define void @sink_splatvar(ptr %p, i32 %shift_amt) {
 ; AVX512VL:       # %bb.0: # %entry
 ; AVX512VL-NEXT:    vpbroadcastd %esi, %xmm0
 ; AVX512VL-NEXT:    movq $-1024, %rax # imm = 0xFC00
-; AVX512VL-NEXT:    .p2align 4, 0x90
+; AVX512VL-NEXT:    .p2align 4
 ; AVX512VL-NEXT:  .LBB8_1: # %loop
 ; AVX512VL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512VL-NEXT:    vmovdqu 1024(%rdi,%rax), %xmm1
@@ -1548,7 +1548,7 @@ define void @sink_splatvar(ptr %p, i32 %shift_amt) {
 ; AVX512BW-NEXT:    vmovd %esi, %xmm0
 ; AVX512BW-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; AVX512BW-NEXT:    movq $-1024, %rax # imm = 0xFC00
-; AVX512BW-NEXT:    .p2align 4, 0x90
+; AVX512BW-NEXT:    .p2align 4
 ; AVX512BW-NEXT:  .LBB8_1: # %loop
 ; AVX512BW-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512BW-NEXT:    vmovdqu 1024(%rdi,%rax), %xmm1
@@ -1565,7 +1565,7 @@ define void @sink_splatvar(ptr %p, i32 %shift_amt) {
 ; AVX512VBMI2-NEXT:    vmovd %esi, %xmm0
 ; AVX512VBMI2-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; AVX512VBMI2-NEXT:    movq $-1024, %rax # imm = 0xFC00
-; AVX512VBMI2-NEXT:    .p2align 4, 0x90
+; AVX512VBMI2-NEXT:    .p2align 4
 ; AVX512VBMI2-NEXT:  .LBB8_1: # %loop
 ; AVX512VBMI2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512VBMI2-NEXT:    vmovdqu 1024(%rdi,%rax), %xmm1
@@ -1581,7 +1581,7 @@ define void @sink_splatvar(ptr %p, i32 %shift_amt) {
 ; AVX512VLBW:       # %bb.0: # %entry
 ; AVX512VLBW-NEXT:    vpbroadcastd %esi, %xmm0
 ; AVX512VLBW-NEXT:    movq $-1024, %rax # imm = 0xFC00
-; AVX512VLBW-NEXT:    .p2align 4, 0x90
+; AVX512VLBW-NEXT:    .p2align 4
 ; AVX512VLBW-NEXT:  .LBB8_1: # %loop
 ; AVX512VLBW-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512VLBW-NEXT:    vmovdqu 1024(%rdi,%rax), %xmm1
@@ -1596,7 +1596,7 @@ define void @sink_splatvar(ptr %p, i32 %shift_amt) {
 ; AVX512VLVBMI2:       # %bb.0: # %entry
 ; AVX512VLVBMI2-NEXT:    vpbroadcastd %esi, %xmm0
 ; AVX512VLVBMI2-NEXT:    movq $-1024, %rax # imm = 0xFC00
-; AVX512VLVBMI2-NEXT:    .p2align 4, 0x90
+; AVX512VLVBMI2-NEXT:    .p2align 4
 ; AVX512VLVBMI2-NEXT:  .LBB8_1: # %loop
 ; AVX512VLVBMI2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512VLVBMI2-NEXT:    vmovdqu 1024(%rdi,%rax), %xmm1
@@ -1612,7 +1612,7 @@ define void @sink_splatvar(ptr %p, i32 %shift_amt) {
 ; XOPAVX1-NEXT:    vmovd %esi, %xmm0
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    movq $-1024, %rax # imm = 0xFC00
-; XOPAVX1-NEXT:    .p2align 4, 0x90
+; XOPAVX1-NEXT:    .p2align 4
 ; XOPAVX1-NEXT:  .LBB8_1: # %loop
 ; XOPAVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; XOPAVX1-NEXT:    vprotd %xmm0, 1024(%rdi,%rax), %xmm1
@@ -1627,7 +1627,7 @@ define void @sink_splatvar(ptr %p, i32 %shift_amt) {
 ; XOPAVX2-NEXT:    vmovd %esi, %xmm0
 ; XOPAVX2-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; XOPAVX2-NEXT:    movq $-1024, %rax # imm = 0xFC00
-; XOPAVX2-NEXT:    .p2align 4, 0x90
+; XOPAVX2-NEXT:    .p2align 4
 ; XOPAVX2-NEXT:  .LBB8_1: # %loop
 ; XOPAVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; XOPAVX2-NEXT:    vprotd %xmm0, 1024(%rdi,%rax), %xmm1
@@ -1647,7 +1647,7 @@ define void @sink_splatvar(ptr %p, i32 %shift_amt) {
 ; X86-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
 ; X86-SSE2-NEXT:    xorl %edx, %edx
-; X86-SSE2-NEXT:    .p2align 4, 0x90
+; X86-SSE2-NEXT:    .p2align 4
 ; X86-SSE2-NEXT:  .LBB8_1: # %loop
 ; X86-SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-SSE2-NEXT:    movdqu (%eax,%ecx,4), %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-fshl-256.ll b/llvm/test/CodeGen/X86/vector-fshl-256.ll
index b26580541fe40..fca72cad474cd 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-256.ll
@@ -1279,7 +1279,7 @@ define void @fancierRotate2(ptr %arr, ptr %control, i32 %rot0, i32 %rot1) {
 ; AVX1-NEXT:    vpmovsxbq {{.*#+}} xmm3 = [31,0]
 ; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB8_1: # %loop
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero
@@ -1325,7 +1325,7 @@ define void @fancierRotate2(ptr %arr, ptr %control, i32 %rot0, i32 %rot1) {
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [31,31,31,31,31,31,31,31]
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm4 = [32,32,32,32,32,32,32,32]
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB8_1: # %loop
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm5 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
@@ -1352,7 +1352,7 @@ define void @fancierRotate2(ptr %arr, ptr %control, i32 %rot0, i32 %rot1) {
 ; AVX512F-NEXT:    vpbroadcastd %xmm1, %ymm1
 ; AVX512F-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    .p2align 4, 0x90
+; AVX512F-NEXT:    .p2align 4
 ; AVX512F-NEXT:  .LBB8_1: # %loop
 ; AVX512F-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512F-NEXT:    vpmovzxbd {{.*#+}} ymm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
@@ -1372,7 +1372,7 @@ define void @fancierRotate2(ptr %arr, ptr %control, i32 %rot0, i32 %rot1) {
 ; AVX512VL-NEXT:    vpbroadcastd %edx, %ymm0
 ; AVX512VL-NEXT:    vpbroadcastd %ecx, %ymm1
 ; AVX512VL-NEXT:    movq $-1024, %rax # imm = 0xFC00
-; AVX512VL-NEXT:    .p2align 4, 0x90
+; AVX512VL-NEXT:    .p2align 4
 ; AVX512VL-NEXT:  .LBB8_1: # %loop
 ; AVX512VL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
@@ -1395,7 +1395,7 @@ define void @fancierRotate2(ptr %arr, ptr %control, i32 %rot0, i32 %rot1) {
 ; AVX512BW-NEXT:    vpbroadcastd %xmm1, %ymm1
 ; AVX512BW-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-NEXT:    .p2align 4, 0x90
+; AVX512BW-NEXT:    .p2align 4
 ; AVX512BW-NEXT:  .LBB8_1: # %loop
 ; AVX512BW-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512BW-NEXT:    vpmovzxbd {{.*#+}} ymm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
@@ -1418,7 +1418,7 @@ define void @fancierRotate2(ptr %arr, ptr %control, i32 %rot0, i32 %rot1) {
 ; AVX512VBMI2-NEXT:    vpbroadcastd %xmm1, %ymm1
 ; AVX512VBMI2-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; AVX512VBMI2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    .p2align 4, 0x90
+; AVX512VBMI2-NEXT:    .p2align 4
 ; AVX512VBMI2-NEXT:  .LBB8_1: # %loop
 ; AVX512VBMI2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512VBMI2-NEXT:    vpmovzxbd {{.*#+}} ymm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
@@ -1438,7 +1438,7 @@ define void @fancierRotate2(ptr %arr, ptr %control, i32 %rot0, i32 %rot1) {
 ; AVX512VLBW-NEXT:    vpbroadcastd %edx, %ymm0
 ; AVX512VLBW-NEXT:    vpbroadcastd %ecx, %ymm1
 ; AVX512VLBW-NEXT:    movq $-1024, %rax # imm = 0xFC00
-; AVX512VLBW-NEXT:    .p2align 4, 0x90
+; AVX512VLBW-NEXT:    .p2align 4
 ; AVX512VLBW-NEXT:  .LBB8_1: # %loop
 ; AVX512VLBW-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX512VLBW-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
@@ -1458,7 +1458,7 @@ define void @fancierRotate2(ptr %arr, ptr %control, i32 %rot0, i32 %rot1) {
 ; AVX10-NEXT:    vpbroadcastd %edx, %ymm0
 ; AVX10-NEXT:    vpbroadcastd %ecx, %ymm1
 ; AVX10-NEXT:    movq $-1024, %rax # imm = 0xFC00
-; AVX10-NEXT:    .p2align 4, 0x90
+; AVX10-NEXT:    .p2align 4
 ; AVX10-NEXT:  .LBB8_1: # %loop
 ; AVX10-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX10-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
@@ -1485,7 +1485,7 @@ define void @fancierRotate2(ptr %arr, ptr %control, i32 %rot0, i32 %rot1) {
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; XOPAVX1-NEXT:    .p2align 4, 0x90
+; XOPAVX1-NEXT:    .p2align 4
 ; XOPAVX1-NEXT:  .LBB8_1: # %loop
 ; XOPAVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; XOPAVX1-NEXT:    vmovq {{.*#+}} xmm5 = mem[0],zero
@@ -1513,7 +1513,7 @@ define void @fancierRotate2(ptr %arr, ptr %control, i32 %rot0, i32 %rot1) {
 ; XOPAVX2-NEXT:    vpbroadcastd %xmm1, %ymm1
 ; XOPAVX2-NEXT:    movq $-1024, %rax # imm = 0xFC00
 ; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    .p2align 4, 0x90
+; XOPAVX2-NEXT:    .p2align 4
 ; XOPAVX2-NEXT:  .LBB8_1: # %loop
 ; XOPAVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; XOPAVX2-NEXT:    vpmovzxbd {{.*#+}} ymm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-5.ll
index fd9ba68d5707a..0b44571acb6b2 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-5.ll
@@ -3926,7 +3926,7 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpshufb %xmm7, %xmm13, %xmm7
 ; AVX512-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & zmm18 & zmm15) | (zmm7 & ~zmm18 & ~zmm15) | (zmm7 & ~zmm18 & zmm15) | (zmm7 & zmm18 & zmm15)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm18 & (zmm7 ^ zmm15))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm14, %zmm7, %zmm16
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm7 = ymm3[0],ymm9[1],ymm3[2,3],ymm9[4],ymm3[5],ymm9[6],ymm3[7,8],ymm9[9],ymm3[10,11],ymm9[12],ymm3[13],ymm9[14],ymm3[15]
 ; AVX512-NEXT:    vextracti128 $1, %ymm7, %xmm14
@@ -3959,7 +3959,7 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpblendd {{.*#+}} xmm13 = xmm12[0,1],xmm11[2],xmm12[3]
 ; AVX512-NEXT:    vpshufb %xmm14, %xmm13, %xmm13
 ; AVX512-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm13 = (~zmm13 & zmm18 & zmm0) | (zmm13 & ~zmm18 & ~zmm0) | (zmm13 & ~zmm18 & zmm0) | (zmm13 & zmm18 & zmm0)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm13 ^ (zmm18 & (zmm13 ^ zmm0))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm13, %zmm19
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm9[0],ymm3[1],ymm9[2],ymm3[3],ymm9[4,5],ymm3[6],ymm9[7,8],ymm3[9],ymm9[10],ymm3[11],ymm9[12,13],ymm3[14],ymm9[15]
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm7
@@ -4007,7 +4007,7 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vmovdqa %ymm2, %ymm9
 ; AVX512-NEXT:    vextracti128 $1, %ymm14, %xmm15
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm14 = xmm14[0],xmm15[1,2,3],xmm14[4,5],xmm15[6,7]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm13 = (~zmm13 & zmm0 & ~mem) | (zmm13 & ~zmm0 & mem) | (zmm13 & zmm0 & ~mem) | (zmm13 & zmm0 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm0 ^ (mem & (zmm13 ^ zmm0))
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm14[u,u,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
 ; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm13, %ymm14
@@ -4032,7 +4032,7 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vmovdqa64 %ymm21, %ymm2
 ; AVX512-NEXT:    vpshufb %xmm2, %xmm11, %xmm11
 ; AVX512-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm11 = (~zmm11 & zmm18 & zmm13) | (zmm11 & ~zmm18 & ~zmm13) | (zmm11 & ~zmm18 & zmm13) | (zmm11 & zmm18 & zmm13)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 ^ (zmm18 & (zmm11 ^ zmm13))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm11, %zmm7
 ; AVX512-NEXT:    vmovdqa64 %ymm20, %ymm2
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1,2],ymm3[3],ymm2[4],ymm3[5],ymm2[6,7],ymm3[8],ymm2[9,10],ymm3[11],ymm2[12],ymm3[13],ymm2[14,15]
@@ -4117,7 +4117,7 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm14 = [4,5,2,3,4,5,6,7,8,9,2,3,4,5,10,11,20,21,18,19,20,21,22,23,24,25,18,19,20,21,26,27]
 ; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm13, %ymm13
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm13 = (~zmm13 & zmm18 & zmm12) | (zmm13 & ~zmm18 & ~zmm12) | (zmm13 & ~zmm18 & zmm12) | (zmm13 & zmm18 & zmm12)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm13 ^ (zmm18 & (zmm13 ^ zmm12))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm13, %zmm16
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm12 = ymm11[0],ymm10[1],ymm11[2],ymm10[3],ymm11[4,5],ymm10[6],ymm11[7,8],ymm10[9],ymm11[10],ymm10[11],ymm11[12,13],ymm10[14],ymm11[15]
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm12, %xmm13
@@ -4133,7 +4133,7 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpermd %ymm7, %ymm13, %ymm15
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [0,1,6,7,4,5,6,7,8,9,0,1,6,7,8,9,16,17,22,23,20,21,22,23,24,25,16,17,22,23,24,25]
 ; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm15, %ymm15
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = (~zmm15 & zmm18 & zmm12) | (zmm15 & ~zmm18 & ~zmm12) | (zmm15 & ~zmm18 & zmm12) | (zmm15 & zmm18 & zmm12)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = zmm15 ^ (zmm18 & (zmm15 ^ zmm12))
 ; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm6
 ; AVX512-FCP-NEXT:    vpsrlq $48, %xmm2, %xmm12
 ; AVX512-FCP-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm12[0],xmm6[1],xmm12[1]
@@ -4177,7 +4177,7 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    # ymm15 = mem[0,1,0,1]
 ; AVX512-FCP-NEXT:    vpermd %ymm7, %ymm15, %ymm15
 ; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm15, %ymm14
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = (~zmm14 & zmm18 & zmm12) | (zmm14 & ~zmm18 & ~zmm12) | (zmm14 & ~zmm18 & zmm12) | (zmm14 & zmm18 & zmm12)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 ^ (zmm18 & (zmm14 ^ zmm12))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm14, %zmm14
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm10[0],ymm11[1,2],ymm10[3],ymm11[4],ymm10[5],ymm11[6,7],ymm10[8],ymm11[9,10],ymm10[11],ymm11[12],ymm10[13],ymm11[14,15]
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm6, %xmm12
@@ -4201,7 +4201,7 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpermd %ymm7, %ymm15, %ymm15
 ; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm15, %ymm13
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm13, %zmm12
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = (~zmm12 & zmm6 & mem) | (zmm12 & ~zmm6 & ~mem) | (zmm12 & zmm6 & ~mem) | (zmm12 & zmm6 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm12 ^ (mem & (zmm12 ^ zmm6))
 ; AVX512-FCP-NEXT:    vextracti64x4 $1, %zmm12, %ymm6
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm13 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm13, %xmm15
@@ -4298,7 +4298,7 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpshufb %xmm7, %xmm13, %xmm7
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & zmm18 & zmm15) | (zmm7 & ~zmm18 & ~zmm15) | (zmm7 & ~zmm18 & zmm15) | (zmm7 & zmm18 & zmm15)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm18 & (zmm7 ^ zmm15))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm14, %zmm7, %zmm16
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm7 = ymm3[0],ymm9[1],ymm3[2,3],ymm9[4],ymm3[5],ymm9[6],ymm3[7,8],ymm9[9],ymm3[10,11],ymm9[12],ymm3[13],ymm9[14],ymm3[15]
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm7, %xmm14
@@ -4331,7 +4331,7 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm13 = xmm12[0,1],xmm11[2],xmm12[3]
 ; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm13, %xmm13
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm13 = (~zmm13 & zmm18 & zmm0) | (zmm13 & ~zmm18 & ~zmm0) | (zmm13 & ~zmm18 & zmm0) | (zmm13 & zmm18 & zmm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm13 ^ (zmm18 & (zmm13 ^ zmm0))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm13, %zmm19
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm9[0],ymm3[1],ymm9[2],ymm3[3],ymm9[4,5],ymm3[6],ymm9[7,8],ymm3[9],ymm9[10],ymm3[11],ymm9[12,13],ymm3[14],ymm9[15]
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm7
@@ -4379,7 +4379,7 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vmovdqa %ymm2, %ymm9
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm14, %xmm15
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm14 = xmm14[0],xmm15[1,2,3],xmm14[4,5],xmm15[6,7]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm13 = (~zmm13 & zmm0 & ~mem) | (zmm13 & ~zmm0 & mem) | (zmm13 & zmm0 & ~mem) | (zmm13 & zmm0 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm0 ^ (mem & (zmm13 ^ zmm0))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm14[u,u,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm13, %ymm14
@@ -4404,7 +4404,7 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm21, %ymm2
 ; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm11, %xmm11
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm11 = (~zmm11 & zmm18 & zmm13) | (zmm11 & ~zmm18 & ~zmm13) | (zmm11 & ~zmm18 & zmm13) | (zmm11 & zmm18 & zmm13)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 ^ (zmm18 & (zmm11 ^ zmm13))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm11, %zmm7
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm20, %ymm2
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1,2],ymm3[3],ymm2[4],ymm3[5],ymm2[6,7],ymm3[8],ymm2[9,10],ymm3[11],ymm2[12],ymm3[13],ymm2[14,15]
@@ -4489,7 +4489,7 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm14 = [4,5,2,3,4,5,6,7,8,9,2,3,4,5,10,11,20,21,18,19,20,21,22,23,24,25,18,19,20,21,26,27]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm13, %ymm13
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm13 = (~zmm13 & zmm18 & zmm12) | (zmm13 & ~zmm18 & ~zmm12) | (zmm13 & ~zmm18 & zmm12) | (zmm13 & zmm18 & zmm12)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm13 ^ (zmm18 & (zmm13 ^ zmm12))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm13, %zmm16
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm12 = ymm11[0],ymm10[1],ymm11[2],ymm10[3],ymm11[4,5],ymm10[6],ymm11[7,8],ymm10[9],ymm11[10],ymm10[11],ymm11[12,13],ymm10[14],ymm11[15]
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm12, %xmm13
@@ -4505,7 +4505,7 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpermd %ymm7, %ymm13, %ymm15
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [0,1,6,7,4,5,6,7,8,9,0,1,6,7,8,9,16,17,22,23,20,21,22,23,24,25,16,17,22,23,24,25]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm15, %ymm15
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = (~zmm15 & zmm18 & zmm12) | (zmm15 & ~zmm18 & ~zmm12) | (zmm15 & ~zmm18 & zmm12) | (zmm15 & zmm18 & zmm12)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = zmm15 ^ (zmm18 & (zmm15 ^ zmm12))
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm6
 ; AVX512DQ-FCP-NEXT:    vpsrlq $48, %xmm2, %xmm12
 ; AVX512DQ-FCP-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm12[0],xmm6[1],xmm12[1]
@@ -4549,7 +4549,7 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    # ymm15 = mem[0,1,0,1]
 ; AVX512DQ-FCP-NEXT:    vpermd %ymm7, %ymm15, %ymm15
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm15, %ymm14
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = (~zmm14 & zmm18 & zmm12) | (zmm14 & ~zmm18 & ~zmm12) | (zmm14 & ~zmm18 & zmm12) | (zmm14 & zmm18 & zmm12)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 ^ (zmm18 & (zmm14 ^ zmm12))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm14, %zmm14
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm10[0],ymm11[1,2],ymm10[3],ymm11[4],ymm10[5],ymm11[6,7],ymm10[8],ymm11[9,10],ymm10[11],ymm11[12],ymm10[13],ymm11[14,15]
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm6, %xmm12
@@ -4573,7 +4573,7 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpermd %ymm7, %ymm15, %ymm15
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm15, %ymm13
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm13, %zmm12
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = (~zmm12 & zmm6 & mem) | (zmm12 & ~zmm6 & ~mem) | (zmm12 & zmm6 & ~mem) | (zmm12 & zmm6 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm12 ^ (mem & (zmm12 ^ zmm6))
 ; AVX512DQ-FCP-NEXT:    vextracti64x4 $1, %zmm12, %ymm6
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm13 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm13, %xmm15
@@ -8060,7 +8060,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm12[0],ymm15[1],ymm12[2,3],ymm15[4],ymm12[5],ymm15[6],ymm12[7,8],ymm15[9],ymm12[10,11],ymm15[12],ymm12[13],ymm15[14],ymm12[15]
 ; AVX512-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3],xmm2[4,5],xmm3[6,7]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & zmm0 & ~mem) | (zmm1 & ~zmm0 & mem) | (zmm1 & zmm0 & ~mem) | (zmm1 & zmm0 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm0 ^ (mem & (zmm1 ^ zmm0))
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm2[0,1,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
 ; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
@@ -8118,7 +8118,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm11[5,6,7]
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm27 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & zmm27 & zmm19) | (zmm10 & ~zmm27 & ~zmm19) | (zmm10 & ~zmm27 & zmm19) | (zmm10 & zmm27 & zmm19)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm27 & (zmm10 ^ zmm19))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm10, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm15[0,1],ymm12[2],ymm15[3],ymm12[4],ymm15[5,6],ymm12[7],ymm15[8,9],ymm12[10],ymm15[11],ymm12[12],ymm15[13,14],ymm12[15]
@@ -8146,7 +8146,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; AVX512-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm27, %zmm2 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm2 = (~zmm2 & zmm27 & mem) | (zmm2 & ~zmm27 & ~mem) | (zmm2 & ~zmm27 & mem) | (zmm2 & zmm27 & mem)
+; AVX512-NEXT:    # zmm2 = zmm2 ^ (zmm27 & (zmm2 ^ mem))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm4
@@ -8178,7 +8178,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm11[5,6,7]
 ; AVX512-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm27, %zmm2 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm2 = (~zmm2 & zmm27 & mem) | (zmm2 & ~zmm27 & ~mem) | (zmm2 & ~zmm27 & mem) | (zmm2 & zmm27 & mem)
+; AVX512-NEXT:    # zmm2 = zmm2 ^ (zmm27 & (zmm2 ^ mem))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm2, %zmm2
 ; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm1[1],ymm3[2,3],ymm1[4],ymm3[5],ymm1[6],ymm3[7,8],ymm1[9],ymm3[10,11],ymm1[12],ymm3[13],ymm1[14],ymm3[15]
@@ -8206,7 +8206,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vmovdqa64 %xmm6, %xmm23
 ; AVX512-NEXT:    vpshufb %xmm0, %xmm10, %xmm0
 ; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm27 & zmm28) | (zmm0 & ~zmm27 & ~zmm28) | (zmm0 & ~zmm27 & zmm28) | (zmm0 & zmm27 & zmm28)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm27 & (zmm0 ^ zmm28))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
@@ -8231,7 +8231,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vmovdqa64 %xmm8, %xmm28
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[0,1,6,7,4,5,6,7,8,9,6,7,0,1,10,11]
 ; AVX512-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm13 = (~zmm13 & zmm27 & zmm2) | (zmm13 & ~zmm27 & ~zmm2) | (zmm13 & ~zmm27 & zmm2) | (zmm13 & zmm27 & zmm2)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm13 ^ (zmm27 & (zmm13 ^ zmm2))
 ; AVX512-NEXT:    vmovdqa64 %ymm26, %ymm8
 ; AVX512-NEXT:    vmovdqa64 %ymm22, %ymm4
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm8[0],ymm4[1],ymm8[2],ymm4[3],ymm8[4,5],ymm4[6],ymm8[7,8],ymm4[9],ymm8[10],ymm4[11],ymm8[12,13],ymm4[14],ymm8[15]
@@ -8295,7 +8295,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpblendd {{.*#+}} xmm10 = xmm3[0],xmm1[1],xmm3[2,3]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[0,1,6,7,4,5,6,7,8,9,6,7,0,1,10,11]
 ; AVX512-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & zmm27 & zmm0) | (zmm10 & ~zmm27 & ~zmm0) | (zmm10 & ~zmm27 & zmm0) | (zmm10 & zmm27 & zmm0)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm27 & (zmm10 ^ zmm0))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm10, %zmm23
 ; AVX512-NEXT:    vmovdqa64 %ymm18, %ymm13
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm12[0],ymm13[1],ymm12[2],ymm13[3],ymm12[4,5],ymm13[6],ymm12[7,8],ymm13[9],ymm12[10],ymm13[11],ymm12[12,13],ymm13[14],ymm12[15]
@@ -8328,7 +8328,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,6,7,8,9,8,9,2,3,12,13]
 ; AVX512-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm8, %zmm8
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & zmm0 & mem) | (zmm8 & ~zmm0 & ~mem) | (zmm8 & zmm0 & ~mem) | (zmm8 & zmm0 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm0))
 ; AVX512-NEXT:    vmovdqa %ymm9, %ymm2
 ; AVX512-NEXT:    vmovdqa64 %ymm21, %ymm10
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm10[0],ymm9[1],ymm10[2,3],ymm9[4],ymm10[5],ymm9[6],ymm10[7,8],ymm9[9],ymm10[10,11],ymm9[12],ymm10[13],ymm9[14],ymm10[15]
@@ -8496,7 +8496,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm7, %ymm7
 ; AVX512-FCP-NEXT:    vmovdqa %ymm11, %ymm14
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & zmm8 & zmm4) | (zmm7 & ~zmm8 & ~zmm4) | (zmm7 & ~zmm8 & zmm4) | (zmm7 & zmm8 & zmm4)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm8 & (zmm7 ^ zmm4))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm8, %zmm16
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm7, %zmm4
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -8538,7 +8538,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vmovdqa64 128(%rdi), %ymm23
 ; AVX512-FCP-NEXT:    vpermd %ymm23, %ymm18, %ymm2
 ; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm2, %ymm2
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm16 & zmm1) | (zmm2 & ~zmm16 & ~zmm1) | (zmm2 & ~zmm16 & zmm1) | (zmm2 & zmm16 & zmm1)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm16 & (zmm2 ^ zmm1))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm6[0],ymm12[1],ymm6[2],ymm12[3],ymm6[4,5],ymm12[6],ymm6[7,8],ymm12[9],ymm6[10],ymm12[11],ymm6[12,13],ymm12[14],ymm6[15]
@@ -8583,7 +8583,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpermd %ymm23, %ymm25, %ymm4
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm14 = [0,1,6,7,4,5,6,7,8,9,0,1,6,7,8,9,16,17,22,23,20,21,22,23,24,25,16,17,22,23,24,25]
 ; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm4, %ymm4
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm16 & zmm10) | (zmm4 & ~zmm16 & ~zmm10) | (zmm4 & ~zmm16 & zmm10) | (zmm4 & zmm16 & zmm10)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm16 & (zmm4 ^ zmm10))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm16, %zmm10
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm4, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -8623,7 +8623,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Reload
 ; AVX512-FCP-NEXT:    vpermd %ymm17, %ymm25, %ymm2
 ; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm2, %ymm2
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm10 & zmm0) | (zmm2 & ~zmm10 & ~zmm0) | (zmm2 & ~zmm10 & zmm0) | (zmm2 & zmm10 & zmm0)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm10 & (zmm2 ^ zmm0))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm9[0],ymm12[1],ymm9[2,3],ymm12[4],ymm9[5],ymm12[6],ymm9[7,8],ymm12[9],ymm9[10,11],ymm12[12],ymm9[13],ymm12[14],ymm9[15]
@@ -8668,7 +8668,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [4,5,2,3,4,5,6,7,8,9,2,3,4,5,10,11,20,21,18,19,20,21,22,23,24,25,18,19,20,21,26,27]
 ; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm14, %ymm14
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm15 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = (~zmm14 & zmm15 & zmm0) | (zmm14 & ~zmm15 & ~zmm0) | (zmm14 & ~zmm15 & zmm0) | (zmm14 & zmm15 & zmm0)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 ^ (zmm15 & (zmm14 ^ zmm0))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm14, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm8[0],ymm4[1],ymm8[2,3],ymm4[4],ymm8[5],ymm4[6],ymm8[7,8],ymm4[9],ymm8[10,11],ymm4[12],ymm8[13],ymm4[14],ymm8[15]
@@ -8702,7 +8702,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7]
 ; AVX512-FCP-NEXT:    vpermd %ymm17, %ymm5, %ymm2
 ; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm15 & zmm1) | (zmm2 & ~zmm15 & ~zmm1) | (zmm2 & ~zmm15 & zmm1) | (zmm2 & zmm15 & zmm1)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm15 & (zmm2 ^ zmm1))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm28
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm31, %xmm15
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm20, %xmm7
@@ -8771,7 +8771,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm3, %xmm3
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm5[3,4,5,6,7]
 ; AVX512-FCP-NEXT:    vpmovsxdq {{.*#+}} zmm5 = [18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0,18446744073709486080,18446744073709551615]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm25 = (~zmm25 & zmm5 & zmm3) | (zmm25 & ~zmm5 & ~zmm3) | (zmm25 & ~zmm5 & zmm3) | (zmm25 & zmm5 & zmm3)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm25 = zmm25 ^ (zmm5 & (zmm25 ^ zmm3))
 ; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
 ; AVX512-FCP-NEXT:    vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    # ymm3 = ymm3[0],mem[1],ymm3[2,3],mem[4],ymm3[5],mem[6],ymm3[7,8],mem[9],ymm3[10,11],mem[12],ymm3[13],mem[14],ymm3[15]
@@ -8796,7 +8796,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpermd %ymm8, %ymm4, %ymm4
 ; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm4, %ymm4
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm7[0,1,2],ymm4[3,4,5,6,7]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm24 = (~zmm24 & zmm5 & zmm4) | (zmm24 & ~zmm5 & ~zmm4) | (zmm24 & ~zmm5 & zmm4) | (zmm24 & zmm5 & zmm4)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm24 = zmm24 ^ (zmm5 & (zmm24 ^ zmm4))
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm22, %xmm4
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm31, %xmm5
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3]
@@ -8994,7 +8994,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm12[0],ymm15[1],ymm12[2,3],ymm15[4],ymm12[5],ymm15[6],ymm12[7,8],ymm15[9],ymm12[10,11],ymm15[12],ymm12[13],ymm15[14],ymm12[15]
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3],xmm2[4,5],xmm3[6,7]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & zmm0 & ~mem) | (zmm1 & ~zmm0 & mem) | (zmm1 & zmm0 & ~mem) | (zmm1 & zmm0 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm0 ^ (mem & (zmm1 ^ zmm0))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm2[0,1,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
@@ -9052,7 +9052,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm11[5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm27 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & zmm27 & zmm19) | (zmm10 & ~zmm27 & ~zmm19) | (zmm10 & ~zmm27 & zmm19) | (zmm10 & zmm27 & zmm19)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm27 & (zmm10 ^ zmm19))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm10, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm15[0,1],ymm12[2],ymm15[3],ymm12[4],ymm15[5,6],ymm12[7],ymm15[8,9],ymm12[10],ymm15[11],ymm12[12],ymm15[13,14],ymm12[15]
@@ -9080,7 +9080,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; AVX512DQ-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm27, %zmm2 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm2 = (~zmm2 & zmm27 & mem) | (zmm2 & ~zmm27 & ~mem) | (zmm2 & ~zmm27 & mem) | (zmm2 & zmm27 & mem)
+; AVX512DQ-NEXT:    # zmm2 = zmm2 ^ (zmm27 & (zmm2 ^ mem))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm4
@@ -9112,7 +9112,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm11[5,6,7]
 ; AVX512DQ-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm27, %zmm2 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm2 = (~zmm2 & zmm27 & mem) | (zmm2 & ~zmm27 & ~mem) | (zmm2 & ~zmm27 & mem) | (zmm2 & zmm27 & mem)
+; AVX512DQ-NEXT:    # zmm2 = zmm2 ^ (zmm27 & (zmm2 ^ mem))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm2, %zmm2
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm1[1],ymm3[2,3],ymm1[4],ymm3[5],ymm1[6],ymm3[7,8],ymm1[9],ymm3[10,11],ymm1[12],ymm3[13],ymm1[14],ymm3[15]
@@ -9140,7 +9140,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm6, %xmm23
 ; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm10, %xmm0
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm27 & zmm28) | (zmm0 & ~zmm27 & ~zmm28) | (zmm0 & ~zmm27 & zmm28) | (zmm0 & zmm27 & zmm28)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm27 & (zmm0 ^ zmm28))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
@@ -9165,7 +9165,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm8, %xmm28
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[0,1,6,7,4,5,6,7,8,9,6,7,0,1,10,11]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm13 = (~zmm13 & zmm27 & zmm2) | (zmm13 & ~zmm27 & ~zmm2) | (zmm13 & ~zmm27 & zmm2) | (zmm13 & zmm27 & zmm2)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm13 ^ (zmm27 & (zmm13 ^ zmm2))
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm26, %ymm8
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm22, %ymm4
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm8[0],ymm4[1],ymm8[2],ymm4[3],ymm8[4,5],ymm4[6],ymm8[7,8],ymm4[9],ymm8[10],ymm4[11],ymm8[12,13],ymm4[14],ymm8[15]
@@ -9229,7 +9229,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm10 = xmm3[0],xmm1[1],xmm3[2,3]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[0,1,6,7,4,5,6,7,8,9,6,7,0,1,10,11]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & zmm27 & zmm0) | (zmm10 & ~zmm27 & ~zmm0) | (zmm10 & ~zmm27 & zmm0) | (zmm10 & zmm27 & zmm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm27 & (zmm10 ^ zmm0))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm10, %zmm23
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm18, %ymm13
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm12[0],ymm13[1],ymm12[2],ymm13[3],ymm12[4,5],ymm13[6],ymm12[7,8],ymm13[9],ymm12[10],ymm13[11],ymm12[12,13],ymm13[14],ymm12[15]
@@ -9262,7 +9262,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,6,7,8,9,8,9,2,3,12,13]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm8, %zmm8
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & zmm0 & mem) | (zmm8 & ~zmm0 & ~mem) | (zmm8 & zmm0 & ~mem) | (zmm8 & zmm0 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm0))
 ; AVX512DQ-NEXT:    vmovdqa %ymm9, %ymm2
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm21, %ymm10
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm10[0],ymm9[1],ymm10[2,3],ymm9[4],ymm10[5],ymm9[6],ymm10[7,8],ymm9[9],ymm10[10,11],ymm9[12],ymm10[13],ymm9[14],ymm10[15]
@@ -9430,7 +9430,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm7, %ymm7
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm11, %ymm14
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & zmm8 & zmm4) | (zmm7 & ~zmm8 & ~zmm4) | (zmm7 & ~zmm8 & zmm4) | (zmm7 & zmm8 & zmm4)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm8 & (zmm7 ^ zmm4))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm8, %zmm16
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm7, %zmm4
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -9472,7 +9472,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 128(%rdi), %ymm23
 ; AVX512DQ-FCP-NEXT:    vpermd %ymm23, %ymm18, %ymm2
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm2, %ymm2
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm16 & zmm1) | (zmm2 & ~zmm16 & ~zmm1) | (zmm2 & ~zmm16 & zmm1) | (zmm2 & zmm16 & zmm1)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm16 & (zmm2 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm6[0],ymm12[1],ymm6[2],ymm12[3],ymm6[4,5],ymm12[6],ymm6[7,8],ymm12[9],ymm6[10],ymm12[11],ymm6[12,13],ymm12[14],ymm6[15]
@@ -9517,7 +9517,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpermd %ymm23, %ymm25, %ymm4
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm14 = [0,1,6,7,4,5,6,7,8,9,0,1,6,7,8,9,16,17,22,23,20,21,22,23,24,25,16,17,22,23,24,25]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm4, %ymm4
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm16 & zmm10) | (zmm4 & ~zmm16 & ~zmm10) | (zmm4 & ~zmm16 & zmm10) | (zmm4 & zmm16 & zmm10)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm16 & (zmm4 ^ zmm10))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm16, %zmm10
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm4, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -9557,7 +9557,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpermd %ymm17, %ymm25, %ymm2
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm2, %ymm2
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm10 & zmm0) | (zmm2 & ~zmm10 & ~zmm0) | (zmm2 & ~zmm10 & zmm0) | (zmm2 & zmm10 & zmm0)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm10 & (zmm2 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm9[0],ymm12[1],ymm9[2,3],ymm12[4],ymm9[5],ymm12[6],ymm9[7,8],ymm12[9],ymm9[10,11],ymm12[12],ymm9[13],ymm12[14],ymm9[15]
@@ -9602,7 +9602,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [4,5,2,3,4,5,6,7,8,9,2,3,4,5,10,11,20,21,18,19,20,21,22,23,24,25,18,19,20,21,26,27]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm14, %ymm14
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm15 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = (~zmm14 & zmm15 & zmm0) | (zmm14 & ~zmm15 & ~zmm0) | (zmm14 & ~zmm15 & zmm0) | (zmm14 & zmm15 & zmm0)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 ^ (zmm15 & (zmm14 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm14, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm8[0],ymm4[1],ymm8[2,3],ymm4[4],ymm8[5],ymm4[6],ymm8[7,8],ymm4[9],ymm8[10,11],ymm4[12],ymm8[13],ymm4[14],ymm8[15]
@@ -9636,7 +9636,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpermd %ymm17, %ymm5, %ymm2
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm15 & zmm1) | (zmm2 & ~zmm15 & ~zmm1) | (zmm2 & ~zmm15 & zmm1) | (zmm2 & zmm15 & zmm1)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm15 & (zmm2 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm28
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm31, %xmm15
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm20, %xmm7
@@ -9705,7 +9705,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm3, %xmm3
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm5[3,4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpmovsxdq {{.*#+}} zmm5 = [18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0,18446744073709486080,18446744073709551615]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm25 = (~zmm25 & zmm5 & zmm3) | (zmm25 & ~zmm5 & ~zmm3) | (zmm25 & ~zmm5 & zmm3) | (zmm25 & zmm5 & zmm3)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm25 = zmm25 ^ (zmm5 & (zmm25 ^ zmm3))
 ; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # ymm3 = ymm3[0],mem[1],ymm3[2,3],mem[4],ymm3[5],mem[6],ymm3[7,8],mem[9],ymm3[10,11],mem[12],ymm3[13],mem[14],ymm3[15]
@@ -9730,7 +9730,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpermd %ymm8, %ymm4, %ymm4
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm4, %ymm4
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm7[0,1,2],ymm4[3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm24 = (~zmm24 & zmm5 & zmm4) | (zmm24 & ~zmm5 & ~zmm4) | (zmm24 & ~zmm5 & zmm4) | (zmm24 & zmm5 & zmm4)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm24 = zmm24 ^ (zmm5 & (zmm24 ^ zmm4))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm22, %xmm4
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm31, %xmm5
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-6.ll
index a37b8e33ceffe..f7218adf85667 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-6.ll
@@ -2497,7 +2497,7 @@ define void @load_i16_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm11 = xmm3[8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15]
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2,3],xmm10[4],xmm11[5,6,7]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm11 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm10 = (~ymm10 & ymm7 & ~ymm11) | (~ymm10 & ymm7 & ymm11) | (ymm10 & ~ymm7 & ymm11) | (ymm10 & ymm7 & ~ymm11) | (ymm10 & ymm7 & ymm11)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm10 = (ymm10 & ymm11) | ymm7
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7]
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm7 = xmm2[u,u,u,u,u,u,u,u,8,9,u,u,0,1,12,13]
@@ -2512,7 +2512,7 @@ define void @load_i16_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,5,5]
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1],xmm3[2,3],xmm4[4],xmm3[5,6,7]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm3 = (~ymm3 & ymm4 & ~ymm11) | (~ymm3 & ymm4 & ymm11) | (ymm3 & ~ymm4 & ymm11) | (ymm3 & ymm4 & ~ymm11) | (ymm3 & ymm4 & ymm11)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm3 = (ymm3 & ymm11) | ymm4
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,u,u,10,11,u,u,2,3,14,15]
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,3,3]
@@ -2608,7 +2608,7 @@ define void @load_i16_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm2[8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2,3],xmm10[4],xmm11[5,6,7]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = (~ymm10 & ymm7 & ~ymm11) | (~ymm10 & ymm7 & ymm11) | (ymm10 & ~ymm7 & ymm11) | (ymm10 & ymm7 & ~ymm11) | (ymm10 & ymm7 & ymm11)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = (ymm10 & ymm11) | ymm7
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm5[1],ymm0[2,3],ymm5[4],ymm0[5,6],ymm5[7]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [0,1,2,3,0,1,4,5,8,9,12,13,0,1,12,13]
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm0, %xmm7
@@ -2623,7 +2623,7 @@ define void @load_i16_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm2, %xmm2
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3],xmm3[4],xmm2[5,6,7]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm3 = (~ymm3 & ymm2 & ymm11) | (ymm3 & ~ymm2 & ~ymm11) | (ymm3 & ~ymm2 & ymm11) | (ymm3 & ymm2 & ~ymm11) | (ymm3 & ymm2 & ymm11)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm2 & ymm11)
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,2,3,2,3,6,7,10,11,14,15,2,3,14,15]
 ; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm7, %xmm6
 ; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
@@ -2721,7 +2721,7 @@ define void @load_i16_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm11 = xmm3[8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2,3],xmm10[4],xmm11[5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm11 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm10 = (~ymm10 & ymm7 & ~ymm11) | (~ymm10 & ymm7 & ymm11) | (ymm10 & ~ymm7 & ymm11) | (ymm10 & ymm7 & ~ymm11) | (ymm10 & ymm7 & ymm11)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm10 = (ymm10 & ymm11) | ymm7
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7]
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm7 = xmm2[u,u,u,u,u,u,u,u,8,9,u,u,0,1,12,13]
@@ -2736,7 +2736,7 @@ define void @load_i16_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,5,5]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1],xmm3[2,3],xmm4[4],xmm3[5,6,7]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm3 = (~ymm3 & ymm4 & ~ymm11) | (~ymm3 & ymm4 & ymm11) | (ymm3 & ~ymm4 & ymm11) | (ymm3 & ymm4 & ~ymm11) | (ymm3 & ymm4 & ymm11)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm3 = (ymm3 & ymm11) | ymm4
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,u,u,10,11,u,u,2,3,14,15]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,3,3]
@@ -2832,7 +2832,7 @@ define void @load_i16_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm2[8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2,3],xmm10[4],xmm11[5,6,7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = (~ymm10 & ymm7 & ~ymm11) | (~ymm10 & ymm7 & ymm11) | (ymm10 & ~ymm7 & ymm11) | (ymm10 & ymm7 & ~ymm11) | (ymm10 & ymm7 & ymm11)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = (ymm10 & ymm11) | ymm7
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm5[1],ymm0[2,3],ymm5[4],ymm0[5,6],ymm5[7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [0,1,2,3,0,1,4,5,8,9,12,13,0,1,12,13]
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm0, %xmm7
@@ -2847,7 +2847,7 @@ define void @load_i16_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm2, %xmm2
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3],xmm3[4],xmm2[5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm3 = (~ymm3 & ymm2 & ymm11) | (ymm3 & ~ymm2 & ~ymm11) | (ymm3 & ~ymm2 & ymm11) | (ymm3 & ymm2 & ~ymm11) | (ymm3 & ymm2 & ymm11)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm2 & ymm11)
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,2,3,2,3,6,7,10,11,14,15,2,3,14,15]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm7, %xmm6
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
@@ -5147,9 +5147,9 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm14[4,5,6,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm16
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & ~zmm0 & zmm17) | (zmm2 & ~zmm0 & zmm17) | (zmm2 & zmm0 & ~zmm17) | (zmm2 & zmm0 & zmm17)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm17 ^ (zmm0 & (zmm2 ^ zmm17))
 ; AVX512-NEXT:    vpmovsxdq {{.*#+}} zmm17 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm16 = (~zmm16 & zmm17 & zmm2) | (zmm16 & ~zmm17 & ~zmm2) | (zmm16 & ~zmm17 & zmm2) | (zmm16 & zmm17 & zmm2)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm16 = zmm16 ^ (zmm17 & (zmm16 ^ zmm2))
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm10[3,1,2,1,4,5,6,7]
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm11[0,1,3,3,4,5,6,7]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,7,7,7,7]
@@ -5178,9 +5178,9 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm1[0,1,2],ymm3[3,4,5,6,7],ymm1[8,9,10],ymm3[11,12,13,14,15]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,4,5]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & ~zmm0 & zmm2) | (zmm5 & ~zmm0 & zmm2) | (zmm5 & zmm0 & ~zmm2) | (zmm5 & zmm0 & zmm2)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm2 ^ (zmm0 & (zmm5 ^ zmm2))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm20
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm20 = (~zmm20 & zmm17 & zmm5) | (zmm20 & ~zmm17 & ~zmm5) | (zmm20 & ~zmm17 & zmm5) | (zmm20 & zmm17 & zmm5)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm20 ^ (zmm17 & (zmm20 ^ zmm5))
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm7 = [8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15]
 ; AVX512-NEXT:    vmovdqa64 %ymm31, %ymm0
 ; AVX512-NEXT:    vmovdqa64 %ymm21, %ymm1
@@ -5211,7 +5211,7 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm8 = xmm8[0],xmm10[1],xmm8[2,3],xmm10[4],xmm8[5,6,7]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm5[4,5,0,1,12,13,24,25,20,21],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm11 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm8 = (~ymm8 & ymm10 & ~ymm11) | (~ymm8 & ymm10 & ymm11) | (ymm8 & ~ymm10 & ymm11) | (ymm8 & ymm10 & ~ymm11) | (ymm8 & ymm10 & ymm11)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm8 = (ymm8 & ymm11) | ymm10
 ; AVX512-NEXT:    movw $31, %ax
 ; AVX512-NEXT:    kmovw %eax, %k1
 ; AVX512-NEXT:    vmovdqa32 %zmm8, %zmm2 {%k1}
@@ -5248,7 +5248,7 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,5,5]
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1],xmm3[2,3],xmm4[4],xmm3[5,6,7]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm5[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm3 = (~ymm3 & ymm4 & ~ymm11) | (~ymm3 & ymm4 & ymm11) | (ymm3 & ~ymm4 & ymm11) | (ymm3 & ymm4 & ~ymm11) | (ymm3 & ymm4 & ymm11)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm3 = (ymm3 & ymm11) | ymm4
 ; AVX512-NEXT:    vmovdqa32 %zmm3, %zmm0 {%k1}
 ; AVX512-NEXT:    vpshufb %xmm1, %xmm8, %xmm1
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm9[0,1,1,3,4,5,6,7]
@@ -5261,18 +5261,18 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
 ; AVX512-NEXT:    vmovdqu64 (%rsp), %zmm4 # 64-byte Reload
 ; AVX512-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm4 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm4 = (~zmm4 & ~zmm3 & mem) | (zmm4 & ~zmm3 & mem) | (zmm4 & zmm3 & ~mem) | (zmm4 & zmm3 & mem)
+; AVX512-NEXT:    # zmm4 = mem ^ (zmm3 & (zmm4 ^ mem))
 ; AVX512-NEXT:    movw $-2048, %ax # imm = 0xF800
 ; AVX512-NEXT:    kmovw %eax, %k1
 ; AVX512-NEXT:    vmovdqa32 %zmm30, %zmm4 {%k1}
 ; AVX512-NEXT:    vmovdqa64 %zmm4, (%rsi)
 ; AVX512-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm15 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm15 = (~zmm15 & ~zmm3 & mem) | (zmm15 & ~zmm3 & mem) | (zmm15 & zmm3 & ~mem) | (zmm15 & zmm3 & mem)
+; AVX512-NEXT:    # zmm15 = mem ^ (zmm3 & (zmm15 ^ mem))
 ; AVX512-NEXT:    vmovdqa32 %zmm18, %zmm15 {%k1}
 ; AVX512-NEXT:    vmovdqa64 %zmm15, (%rdx)
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & zmm17 & zmm2) | (zmm10 & ~zmm17 & ~zmm2) | (zmm10 & ~zmm17 & zmm2) | (zmm10 & zmm17 & zmm2)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & zmm17 & zmm0) | (zmm1 & ~zmm17 & ~zmm0) | (zmm1 & ~zmm17 & zmm0) | (zmm1 & zmm17 & zmm0)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm17 & (zmm10 ^ zmm2))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm17 & (zmm1 ^ zmm0))
 ; AVX512-NEXT:    vmovdqa64 %zmm16, (%rcx)
 ; AVX512-NEXT:    vmovdqa64 %zmm20, (%r8)
 ; AVX512-NEXT:    vmovdqa64 %zmm10, (%r9)
@@ -5428,9 +5428,9 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm13[4,5,6,7]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm16
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm11 = (~zmm11 & ~zmm0 & zmm17) | (zmm11 & ~zmm0 & zmm17) | (zmm11 & zmm0 & ~zmm17) | (zmm11 & zmm0 & zmm17)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm17 ^ (zmm0 & (zmm11 ^ zmm17))
 ; AVX512-FCP-NEXT:    vpmovsxdq {{.*#+}} zmm17 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm16 = (~zmm16 & zmm17 & zmm11) | (zmm16 & ~zmm17 & ~zmm11) | (zmm16 & ~zmm17 & zmm11) | (zmm16 & zmm17 & zmm11)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm16 = zmm16 ^ (zmm17 & (zmm16 ^ zmm11))
 ; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm10[3,1,2,1,4,5,6,7]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[u,u,2,3,6,7,u,u,14,15,14,15,14,15,14,15]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm9 = xmm10[0],xmm9[1,2],xmm10[3],xmm9[4,5,6,7]
@@ -5456,9 +5456,9 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1,2],ymm2[3,4,5,6,7],ymm1[8,9,10],ymm2[11,12,13,14,15]
 ; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,4,5]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & ~zmm0 & zmm7) | (zmm4 & ~zmm0 & zmm7) | (zmm4 & zmm0 & ~zmm7) | (zmm4 & zmm0 & zmm7)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm7 ^ (zmm0 & (zmm4 ^ zmm7))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm20
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm20 = (~zmm20 & zmm17 & zmm4) | (zmm20 & ~zmm17 & ~zmm4) | (zmm20 & ~zmm17 & zmm4) | (zmm20 & zmm17 & zmm4)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm20 ^ (zmm17 & (zmm20 ^ zmm4))
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15]
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm30, %ymm0
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm31, %ymm1
@@ -5488,7 +5488,7 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0],xmm10[1],xmm5[2,3],xmm10[4],xmm5[5,6,7]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[4,5,0,1,12,13,24,25,20,21],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ymm10 & ~ymm11) | (~ymm5 & ymm10 & ymm11) | (ymm5 & ~ymm10 & ymm11) | (ymm5 & ymm10 & ~ymm11) | (ymm5 & ymm10 & ymm11)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = (ymm5 & ymm11) | ymm10
 ; AVX512-FCP-NEXT:    movw $31, %ax
 ; AVX512-FCP-NEXT:    kmovw %eax, %k1
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm5, %zmm2 {%k1}
@@ -5511,7 +5511,7 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm7, %xmm7
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0],xmm9[1],xmm7[2,3],xmm9[4],xmm7[5,6,7]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ymm7 & ymm11) | (ymm0 & ~ymm7 & ~ymm11) | (ymm0 & ~ymm7 & ymm11) | (ymm0 & ymm7 & ~ymm11) | (ymm0 & ymm7 & ymm11)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm7 & ymm11)
 ; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm4, %xmm4
 ; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm1, %xmm1
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm4[1],xmm1[2,3],xmm4[4],xmm1[5,6,7]
@@ -5532,19 +5532,19 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm4 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm4 = (~zmm4 & ~zmm3 & mem) | (zmm4 & ~zmm3 & mem) | (zmm4 & zmm3 & ~mem) | (zmm4 & zmm3 & mem)
+; AVX512-FCP-NEXT:    # zmm4 = mem ^ (zmm3 & (zmm4 ^ mem))
 ; AVX512-FCP-NEXT:    movw $-2048, %ax # imm = 0xF800
 ; AVX512-FCP-NEXT:    kmovw %eax, %k1
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm19, %zmm4 {%k1}
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, (%rsi)
 ; AVX512-FCP-NEXT:    vmovdqu64 (%rsp), %zmm4 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm4 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm4 = (~zmm4 & ~zmm3 & mem) | (zmm4 & ~zmm3 & mem) | (zmm4 & zmm3 & ~mem) | (zmm4 & zmm3 & mem)
+; AVX512-FCP-NEXT:    # zmm4 = mem ^ (zmm3 & (zmm4 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm28, %zmm4 {%k1}
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, (%rdx)
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & zmm17 & zmm2) | (zmm8 & ~zmm17 & ~zmm2) | (zmm8 & ~zmm17 & zmm2) | (zmm8 & zmm17 & zmm2)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm17 & zmm1) | (zmm0 & ~zmm17 & ~zmm1) | (zmm0 & ~zmm17 & zmm1) | (zmm0 & zmm17 & zmm1)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (zmm17 & (zmm8 ^ zmm2))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm17 & (zmm0 ^ zmm1))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm16, (%rcx)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm20, (%r8)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm8, (%r9)
@@ -5616,7 +5616,7 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,6]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & ~zmm17 & zmm16) | (zmm10 & ~zmm17 & zmm16) | (zmm10 & zmm17 & ~zmm16) | (zmm10 & zmm17 & zmm16)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm16 ^ (zmm17 & (zmm10 ^ zmm16))
 ; AVX512DQ-NEXT:    movw $-2048, %ax # imm = 0xF800
 ; AVX512DQ-NEXT:    kmovw %eax, %k1
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm10 {%k1}
@@ -5639,7 +5639,7 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm5, %xmm0
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & ~zmm17 & zmm3) | (zmm2 & ~zmm17 & zmm3) | (zmm2 & zmm17 & ~zmm3) | (zmm2 & zmm17 & zmm3)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm3 ^ (zmm17 & (zmm2 ^ zmm3))
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm1 = ymm7[2,3,14,15,10,11,10,11,14,15,10,11,u,u,6,7,18,19,30,31,26,27,26,27,30,31,26,27,u,u,22,23]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
@@ -5706,9 +5706,9 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm18
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm9 = (~zmm9 & ~zmm0 & zmm16) | (zmm9 & ~zmm0 & zmm16) | (zmm9 & zmm0 & ~zmm16) | (zmm9 & zmm0 & zmm16)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm16 ^ (zmm0 & (zmm9 ^ zmm16))
 ; AVX512DQ-NEXT:    vpmovsxdq {{.*#+}} zmm17 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm18 = (~zmm18 & zmm17 & zmm9) | (zmm18 & ~zmm17 & ~zmm9) | (zmm18 & ~zmm17 & zmm9) | (zmm18 & zmm17 & zmm9)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm18 = zmm18 ^ (zmm17 & (zmm18 ^ zmm9))
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm15[3,1,2,1,4,5,6,7]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm8[0,1,3,3,4,5,6,7]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,7,7,7,7]
@@ -5735,9 +5735,9 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7],ymm2[8,9,10],ymm1[11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,4,5]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & ~zmm0 & zmm7) | (zmm4 & ~zmm0 & zmm7) | (zmm4 & zmm0 & ~zmm7) | (zmm4 & zmm0 & zmm7)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm7 ^ (zmm0 & (zmm4 ^ zmm7))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm16
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm16 = (~zmm16 & zmm17 & zmm4) | (zmm16 & ~zmm17 & ~zmm4) | (zmm16 & ~zmm17 & zmm4) | (zmm16 & zmm17 & zmm4)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm16 = zmm16 ^ (zmm17 & (zmm16 ^ zmm4))
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm6 = [8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15]
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm29, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm30, %ymm1
@@ -5767,7 +5767,7 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0],xmm8[1],xmm6[2,3],xmm8[4],xmm6[5,6,7]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[4,5,0,1,12,13,24,25,20,21],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm14 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ymm8 & ~ymm14) | (~ymm6 & ymm8 & ymm14) | (ymm6 & ~ymm8 & ymm14) | (ymm6 & ymm8 & ~ymm14) | (ymm6 & ymm8 & ymm14)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm6 = (ymm6 & ymm14) | ymm8
 ; AVX512DQ-NEXT:    movw $31, %ax
 ; AVX512DQ-NEXT:    kmovw %eax, %k1
 ; AVX512DQ-NEXT:    vinserti32x8 $0, %ymm6, %zmm0, %zmm4 {%k1}
@@ -5804,7 +5804,7 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,5,5]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm5[1],xmm2[2,3],xmm5[4],xmm2[5,6,7]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ymm3 & ~ymm14) | (~ymm2 & ymm3 & ymm14) | (ymm2 & ~ymm3 & ymm14) | (ymm2 & ymm3 & ~ymm14) | (ymm2 & ymm3 & ymm14)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm2 = (ymm2 & ymm14) | ymm3
 ; AVX512DQ-NEXT:    vinserti32x8 $0, %ymm2, %zmm0, %zmm0 {%k1}
 ; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm6, %xmm1
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm7[0,1,1,3,4,5,6,7]
@@ -5819,8 +5819,8 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512DQ-NEXT:    vmovaps %zmm2, (%rdx)
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & zmm17 & zmm4) | (zmm8 & ~zmm17 & ~zmm4) | (zmm8 & ~zmm17 & zmm4) | (zmm8 & zmm17 & zmm4)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & zmm17 & zmm0) | (zmm1 & ~zmm17 & ~zmm0) | (zmm1 & ~zmm17 & zmm0) | (zmm1 & zmm17 & zmm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (zmm17 & (zmm8 ^ zmm4))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm17 & (zmm1 ^ zmm0))
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm18, (%rcx)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm16, (%r8)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm8, (%r9)
@@ -5892,7 +5892,7 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,6]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & ~zmm17 & zmm16) | (zmm10 & ~zmm17 & zmm16) | (zmm10 & zmm17 & ~zmm16) | (zmm10 & zmm17 & zmm16)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm16 ^ (zmm17 & (zmm10 ^ zmm16))
 ; AVX512DQ-FCP-NEXT:    movw $-2048, %ax # imm = 0xF800
 ; AVX512DQ-FCP-NEXT:    kmovw %eax, %k1
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm10 {%k1}
@@ -5915,7 +5915,7 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm1, %xmm0
 ; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm4[0,1,2,3,5,5,5,5]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & ~zmm17 & zmm5) | (zmm2 & ~zmm17 & zmm5) | (zmm2 & zmm17 & ~zmm5) | (zmm2 & zmm17 & zmm5)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm5 ^ (zmm17 & (zmm2 ^ zmm5))
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm6[2,3,14,15,10,11,10,11,14,15,10,11,u,u,6,7,18,19,30,31,26,27,26,27,30,31,26,27,u,u,22,23]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
@@ -5979,9 +5979,9 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm18
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = (~zmm9 & ~zmm0 & zmm16) | (zmm9 & ~zmm0 & zmm16) | (zmm9 & zmm0 & ~zmm16) | (zmm9 & zmm0 & zmm16)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm16 ^ (zmm0 & (zmm9 ^ zmm16))
 ; AVX512DQ-FCP-NEXT:    vpmovsxdq {{.*#+}} zmm17 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm18 = (~zmm18 & zmm17 & zmm9) | (zmm18 & ~zmm17 & ~zmm9) | (zmm18 & ~zmm17 & zmm9) | (zmm18 & zmm17 & zmm9)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm18 = zmm18 ^ (zmm17 & (zmm18 ^ zmm9))
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm15[3,1,2,1,4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[u,u,2,3,6,7,u,u,14,15,14,15,14,15,14,15]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm8 = xmm9[0],xmm8[1,2],xmm9[3],xmm8[4,5,6,7]
@@ -6005,9 +6005,9 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1,2],ymm2[3,4,5,6,7],ymm1[8,9,10],ymm2[11,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,4,5]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & ~zmm0 & zmm7) | (zmm3 & ~zmm0 & zmm7) | (zmm3 & zmm0 & ~zmm7) | (zmm3 & zmm0 & zmm7)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm7 ^ (zmm0 & (zmm3 ^ zmm7))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm16
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm16 = (~zmm16 & zmm17 & zmm3) | (zmm16 & ~zmm17 & ~zmm3) | (zmm16 & ~zmm17 & zmm3) | (zmm16 & zmm17 & zmm3)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm16 = zmm16 ^ (zmm17 & (zmm16 ^ zmm3))
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [8,9,u,u,0,1,12,13,u,u,12,13,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm29, %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm30, %ymm1
@@ -6036,7 +6036,7 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0],xmm11[1],xmm5[2,3],xmm11[4],xmm5[5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[4,5,0,1,12,13,24,25,20,21],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ymm11 & ~ymm13) | (~ymm5 & ymm11 & ymm13) | (ymm5 & ~ymm11 & ymm13) | (ymm5 & ymm11 & ~ymm13) | (ymm5 & ymm11 & ymm13)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = (ymm5 & ymm13) | ymm11
 ; AVX512DQ-FCP-NEXT:    movw $31, %ax
 ; AVX512DQ-FCP-NEXT:    kmovw %eax, %k1
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $0, %ymm5, %zmm0, %zmm0 {%k1}
@@ -6060,7 +6060,7 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm8, %xmm8
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm8 = xmm8[0],xmm9[1],xmm8[2,3],xmm9[4],xmm8[5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[6,7,2,3,14,15,26,27,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ymm8 & ymm13) | (ymm2 & ~ymm8 & ~ymm13) | (ymm2 & ~ymm8 & ymm13) | (ymm2 & ymm8 & ~ymm13) | (ymm2 & ymm8 & ymm13)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm8 & ymm13)
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm1, %xmm1
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm3, %xmm3
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1],xmm3[2,3],xmm1[4],xmm3[5,6,7]
@@ -6083,8 +6083,8 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512DQ-FCP-NEXT:    vmovaps %zmm3, (%rdx)
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & zmm17 & zmm0) | (zmm7 & ~zmm17 & ~zmm0) | (zmm7 & ~zmm17 & zmm0) | (zmm7 & zmm17 & zmm0)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm17 & zmm1) | (zmm2 & ~zmm17 & ~zmm1) | (zmm2 & ~zmm17 & zmm1) | (zmm2 & zmm17 & zmm1)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm17 & (zmm7 ^ zmm0))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm17 & (zmm2 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm18, (%rcx)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm16, (%r8)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm7, (%r9)
@@ -10684,9 +10684,9 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm3
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm29 & zmm2) | (zmm1 & ~zmm29 & zmm2) | (zmm1 & zmm29 & ~zmm2) | (zmm1 & zmm29 & zmm2)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm29 & (zmm1 ^ zmm2))
 ; AVX512-NEXT:    vpmovsxdq {{.*#+}} zmm2 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & zmm2 & zmm1) | (zmm3 & ~zmm2 & ~zmm1) | (zmm3 & ~zmm2 & zmm1) | (zmm3 & zmm2 & zmm1)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm2 & (zmm3 ^ zmm1))
 ; AVX512-NEXT:    vmovdqa64 %zmm2, %zmm22
 ; AVX512-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
@@ -10747,8 +10747,8 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,5,4]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm14[4,5,6,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & ~zmm29 & zmm23) | (zmm2 & ~zmm29 & zmm23) | (zmm2 & zmm29 & ~zmm23) | (zmm2 & zmm29 & zmm23)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm22 & zmm2) | (zmm0 & ~zmm22 & ~zmm2) | (zmm0 & ~zmm22 & zmm2) | (zmm0 & zmm22 & zmm2)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm23 ^ (zmm29 & (zmm2 ^ zmm23))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm22 & (zmm0 ^ zmm2))
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm0
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,1,2,1,4,5,6,7]
@@ -10794,8 +10794,8 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm26, %zmm0, %zmm13
 ; AVX512-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm25
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & ~zmm29 & zmm28) | (zmm0 & ~zmm29 & zmm28) | (zmm0 & zmm29 & ~zmm28) | (zmm0 & zmm29 & zmm28)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm25 = (~zmm25 & zmm22 & zmm0) | (zmm25 & ~zmm22 & ~zmm0) | (zmm25 & ~zmm22 & zmm0) | (zmm25 & zmm22 & zmm0)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm28 ^ (zmm29 & (zmm0 ^ zmm28))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm25 = zmm25 ^ (zmm22 & (zmm25 ^ zmm0))
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm11[3,1,2,1,4,5,6,7]
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm12[0,1,3,3,4,5,6,7]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,7,7,7]
@@ -10823,8 +10823,8 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,4,5]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm28
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & ~zmm29 & zmm0) | (zmm2 & ~zmm29 & zmm0) | (zmm2 & zmm29 & ~zmm0) | (zmm2 & zmm29 & zmm0)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm28 = (~zmm28 & zmm22 & zmm2) | (zmm28 & ~zmm22 & ~zmm2) | (zmm28 & ~zmm22 & zmm2) | (zmm28 & zmm22 & zmm2)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm0 ^ (zmm29 & (zmm2 ^ zmm0))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm28 = zmm28 ^ (zmm22 & (zmm28 ^ zmm2))
 ; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX512-NEXT:    vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
 ; AVX512-NEXT:    # ymm2 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
@@ -10863,7 +10863,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,128,128,4,5,0,1,12,13,24,25,20,21,128,128,128,128,128,128,128,128,128,128,128,128]
 ; AVX512-NEXT:    vpshufb %ymm0, %ymm5, %ymm4
 ; AVX512-NEXT:    vmovdqa64 %ymm5, %ymm26
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm3 = (~ymm3 & ymm4 & ~ymm29) | (~ymm3 & ymm4 & ymm29) | (ymm3 & ~ymm4 & ymm29) | (ymm3 & ymm4 & ~ymm29) | (ymm3 & ymm4 & ymm29)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm3 = (ymm3 & ymm29) | ymm4
 ; AVX512-NEXT:    movw $31, %ax
 ; AVX512-NEXT:    kmovw %eax, %k1
 ; AVX512-NEXT:    vmovdqa32 %zmm3, %zmm30 {%k1}
@@ -10900,7 +10900,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm3[0],xmm4[1],xmm3[2,3],xmm4[4],xmm3[5,6,7]
 ; AVX512-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
 ; AVX512-NEXT:    vmovdqa64 %ymm5, %ymm17
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm4 = (~ymm4 & ymm0 & ~ymm29) | (~ymm4 & ymm0 & ymm29) | (ymm4 & ~ymm0 & ymm29) | (ymm4 & ymm0 & ~ymm29) | (ymm4 & ymm0 & ymm29)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm4 = (ymm4 & ymm29) | ymm0
 ; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX512-NEXT:    vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
 ; AVX512-NEXT:    # ymm5 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
@@ -10958,7 +10958,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm10 = [128,128,128,128,128,128,128,128,128,128,6,7,2,3,14,15,26,27,22,23,128,128,128,128,128,128,128,128,128,128,128,128]
 ; AVX512-NEXT:    vmovdqa64 %ymm26, %ymm0
 ; AVX512-NEXT:    vpshufb %ymm10, %ymm0, %ymm13
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm3 = (~ymm3 & ymm13 & ~ymm29) | (~ymm3 & ymm13 & ymm29) | (ymm3 & ~ymm13 & ymm29) | (ymm3 & ymm13 & ~ymm29) | (ymm3 & ymm13 & ymm29)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm3 = (ymm3 & ymm29) | ymm13
 ; AVX512-NEXT:    vmovdqa32 %zmm3, %zmm2 {%k1}
 ; AVX512-NEXT:    vmovdqa64 %xmm18, %xmm0
 ; AVX512-NEXT:    vpshufb %xmm6, %xmm0, %xmm3
@@ -10977,7 +10977,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm14 = xmm14[1,1,2,3]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,5,5,5,5]
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm10 = xmm10[0],xmm14[1],xmm10[2,3],xmm14[4],xmm10[5,6,7]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm10 = (~ymm10 & ymm3 & ~ymm29) | (~ymm10 & ymm3 & ymm29) | (ymm10 & ~ymm3 & ymm29) | (ymm10 & ymm3 & ~ymm29) | (ymm10 & ymm3 & ymm29)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm10 = (ymm10 & ymm29) | ymm3
 ; AVX512-NEXT:    vpshufb %xmm1, %xmm5, %xmm1
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm9[1,1,2,3]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,5,5,5]
@@ -11003,7 +11003,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
 ; AVX512-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm7 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm7 = (~zmm7 & ~zmm6 & mem) | (zmm7 & ~zmm6 & mem) | (zmm7 & zmm6 & ~mem) | (zmm7 & zmm6 & mem)
+; AVX512-NEXT:    # zmm7 = mem ^ (zmm6 & (zmm7 ^ mem))
 ; AVX512-NEXT:    movw $-2048, %ax # imm = 0xF800
 ; AVX512-NEXT:    kmovw %eax, %k1
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
@@ -11011,27 +11011,27 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vmovdqa64 %zmm7, (%rsi)
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
 ; AVX512-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm7 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm7 = (~zmm7 & ~zmm6 & mem) | (zmm7 & ~zmm6 & mem) | (zmm7 & zmm6 & ~mem) | (zmm7 & zmm6 & mem)
+; AVX512-NEXT:    # zmm7 = mem ^ (zmm6 & (zmm7 ^ mem))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
 ; AVX512-NEXT:    vmovdqa32 %zmm8, %zmm7 {%k1}
 ; AVX512-NEXT:    vmovdqa64 %zmm7, 64(%rsi)
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
 ; AVX512-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm7 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm7 = (~zmm7 & ~zmm6 & mem) | (zmm7 & ~zmm6 & mem) | (zmm7 & zmm6 & ~mem) | (zmm7 & zmm6 & mem)
+; AVX512-NEXT:    # zmm7 = mem ^ (zmm6 & (zmm7 ^ mem))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
 ; AVX512-NEXT:    vmovdqa32 %zmm8, %zmm7 {%k1}
 ; AVX512-NEXT:    vmovdqa64 %zmm7, 64(%rdx)
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
 ; AVX512-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm7 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm7 = (~zmm7 & ~zmm6 & mem) | (zmm7 & ~zmm6 & mem) | (zmm7 & zmm6 & ~mem) | (zmm7 & zmm6 & mem)
+; AVX512-NEXT:    # zmm7 = mem ^ (zmm6 & (zmm7 ^ mem))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
 ; AVX512-NEXT:    vmovdqa32 %zmm6, %zmm7 {%k1}
 ; AVX512-NEXT:    vmovdqa64 %zmm7, (%rdx)
 ; AVX512-NEXT:    vpmovsxdq {{.*#+}} zmm6 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & zmm6 & zmm30) | (zmm3 & ~zmm6 & ~zmm30) | (zmm3 & ~zmm6 & zmm30) | (zmm3 & zmm6 & zmm30)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm6 & zmm31) | (zmm5 & ~zmm6 & ~zmm31) | (zmm5 & ~zmm6 & zmm31) | (zmm5 & zmm6 & zmm31)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm6 & zmm2) | (zmm0 & ~zmm6 & ~zmm2) | (zmm0 & ~zmm6 & zmm2) | (zmm0 & zmm6 & zmm2)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm6 & zmm1) | (zmm4 & ~zmm6 & ~zmm1) | (zmm4 & ~zmm6 & zmm1) | (zmm4 & zmm6 & zmm1)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm6 & (zmm3 ^ zmm30))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm6 & (zmm5 ^ zmm31))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm6 & (zmm0 ^ zmm2))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm6 & (zmm4 ^ zmm1))
 ; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512-NEXT:    vmovaps %zmm1, 64(%rcx)
 ; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
@@ -11314,9 +11314,9 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3],ymm5[4,5,6,7]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm5
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & ~zmm29 & zmm3) | (zmm4 & ~zmm29 & zmm3) | (zmm4 & zmm29 & ~zmm3) | (zmm4 & zmm29 & zmm3)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm3 ^ (zmm29 & (zmm4 ^ zmm3))
 ; AVX512-FCP-NEXT:    vpmovsxdq {{.*#+}} zmm26 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm26 & zmm4) | (zmm5 & ~zmm26 & ~zmm4) | (zmm5 & ~zmm26 & zmm4) | (zmm5 & zmm26 & zmm4)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm26 & (zmm5 ^ zmm4))
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
 ; AVX512-FCP-NEXT:    vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
@@ -11372,8 +11372,8 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,5,4]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm13[4,5,6,7]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm25
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm29 & zmm11) | (zmm1 & ~zmm29 & zmm11) | (zmm1 & zmm29 & ~zmm11) | (zmm1 & zmm29 & zmm11)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm25 = (~zmm25 & zmm26 & zmm1) | (zmm25 & ~zmm26 & ~zmm1) | (zmm25 & ~zmm26 & zmm1) | (zmm25 & zmm26 & zmm1)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm11 ^ (zmm29 & (zmm1 ^ zmm11))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm25 = zmm25 ^ (zmm26 & (zmm25 ^ zmm1))
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [u,u,2,3,6,7,u,u,14,15,14,15,14,15,14,15]
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm24, %xmm0
 ; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm0, %xmm0
@@ -11412,8 +11412,8 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,7,4,5]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3],ymm13[4,5,6,7]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm0, %zmm23
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & ~zmm29 & zmm24) | (zmm0 & ~zmm29 & zmm24) | (zmm0 & zmm29 & ~zmm24) | (zmm0 & zmm29 & zmm24)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm23 = (~zmm23 & zmm26 & zmm0) | (zmm23 & ~zmm26 & ~zmm0) | (zmm23 & ~zmm26 & zmm0) | (zmm23 & zmm26 & zmm0)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm24 ^ (zmm29 & (zmm0 ^ zmm24))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm23 = zmm23 ^ (zmm26 & (zmm23 ^ zmm0))
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm22, %xmm0
 ; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm10, %xmm0
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm10 # 32-byte Folded Reload
@@ -11447,8 +11447,8 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,4,5]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm28
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & ~zmm29 & zmm0) | (zmm5 & ~zmm29 & zmm0) | (zmm5 & zmm29 & ~zmm0) | (zmm5 & zmm29 & zmm0)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm28 = (~zmm28 & zmm26 & zmm5) | (zmm28 & ~zmm26 & ~zmm5) | (zmm28 & ~zmm26 & zmm5) | (zmm28 & zmm26 & zmm5)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm0 ^ (zmm29 & (zmm5 ^ zmm0))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm28 = zmm28 ^ (zmm26 & (zmm28 ^ zmm5))
 ; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX512-FCP-NEXT:    vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm11 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    # ymm11 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
@@ -11485,7 +11485,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,128,128,4,5,0,1,12,13,24,25,20,21,128,128,128,128,128,128,128,128,128,128,128,128]
 ; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm4, %ymm3
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm4, %ymm17
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ymm3 & ~ymm29) | (~ymm2 & ymm3 & ymm29) | (ymm2 & ~ymm3 & ymm29) | (ymm2 & ymm3 & ~ymm29) | (ymm2 & ymm3 & ymm29)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = (ymm2 & ymm29) | ymm3
 ; AVX512-FCP-NEXT:    movw $31, %ax
 ; AVX512-FCP-NEXT:    kmovw %eax, %k1
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm2, %zmm30 {%k1}
@@ -11520,7 +11520,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm13[2,2,2,2,4,5,6,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1],xmm3[2,3],xmm4[4],xmm3[5,6,7]
 ; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm15, %ymm0
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm3 = (~ymm3 & ymm0 & ~ymm29) | (~ymm3 & ymm0 & ymm29) | (ymm3 & ~ymm0 & ymm29) | (ymm3 & ymm0 & ~ymm29) | (ymm3 & ymm0 & ymm29)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm3 = (ymm3 & ymm29) | ymm0
 ; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX512-FCP-NEXT:    vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    # ymm2 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
@@ -11563,7 +11563,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm17, %ymm0
 ; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm0, %ymm10
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm12, %ymm17
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm14 = (~ymm14 & ymm10 & ~ymm29) | (~ymm14 & ymm10 & ymm29) | (ymm14 & ~ymm10 & ymm29) | (ymm14 & ymm10 & ~ymm29) | (ymm14 & ymm10 & ymm29)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm14 = (ymm14 & ymm29) | ymm10
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm24, %xmm0
 ; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm0, %xmm10
 ; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm11, %xmm11
@@ -11592,7 +11592,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm13, %xmm13
 ; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm6, %xmm6
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0],xmm13[1],xmm6[2,3],xmm13[4],xmm6[5,6,7]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ymm14 & ~ymm29) | (~ymm6 & ymm14 & ymm29) | (ymm6 & ~ymm14 & ymm29) | (ymm6 & ymm14 & ~ymm29) | (ymm6 & ymm14 & ymm29)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = (ymm6 & ymm29) | ymm14
 ; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm7, %xmm7
 ; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm7[1],xmm2[2,3],xmm7[4],xmm2[5,6,7]
@@ -11615,7 +11615,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm5 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm5 = (~zmm5 & ~zmm3 & mem) | (zmm5 & ~zmm3 & mem) | (zmm5 & zmm3 & ~mem) | (zmm5 & zmm3 & mem)
+; AVX512-FCP-NEXT:    # zmm5 = mem ^ (zmm3 & (zmm5 ^ mem))
 ; AVX512-FCP-NEXT:    movw $-2048, %ax # imm = 0xF800
 ; AVX512-FCP-NEXT:    kmovw %eax, %k1
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
@@ -11623,26 +11623,26 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, (%rsi)
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm5 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm5 = (~zmm5 & ~zmm3 & mem) | (zmm5 & ~zmm3 & mem) | (zmm5 & zmm3 & ~mem) | (zmm5 & zmm3 & mem)
+; AVX512-FCP-NEXT:    # zmm5 = mem ^ (zmm3 & (zmm5 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm6, %zmm5 {%k1}
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, 64(%rsi)
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm5 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm5 = (~zmm5 & ~zmm3 & mem) | (zmm5 & ~zmm3 & mem) | (zmm5 & zmm3 & ~mem) | (zmm5 & zmm3 & mem)
+; AVX512-FCP-NEXT:    # zmm5 = mem ^ (zmm3 & (zmm5 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm6, %zmm5 {%k1}
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, 64(%rdx)
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm5 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm5 = (~zmm5 & ~zmm3 & mem) | (zmm5 & ~zmm3 & mem) | (zmm5 & zmm3 & ~mem) | (zmm5 & zmm3 & mem)
+; AVX512-FCP-NEXT:    # zmm5 = mem ^ (zmm3 & (zmm5 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm3, %zmm5 {%k1}
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, (%rdx)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm26 & zmm30) | (zmm4 & ~zmm26 & ~zmm30) | (zmm4 & ~zmm26 & zmm30) | (zmm4 & zmm26 & zmm30)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & zmm26 & zmm31) | (zmm7 & ~zmm26 & ~zmm31) | (zmm7 & ~zmm26 & zmm31) | (zmm7 & zmm26 & zmm31)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm11 = (~zmm11 & zmm26 & zmm0) | (zmm11 & ~zmm26 & ~zmm0) | (zmm11 & ~zmm26 & zmm0) | (zmm11 & zmm26 & zmm0)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & zmm26 & zmm2) | (zmm1 & ~zmm26 & ~zmm2) | (zmm1 & ~zmm26 & zmm2) | (zmm1 & zmm26 & zmm2)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm26 & (zmm4 ^ zmm30))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm26 & (zmm7 ^ zmm31))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 ^ (zmm26 & (zmm11 ^ zmm0))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm26 & (zmm1 ^ zmm2))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm25, 64(%rcx)
 ; AVX512-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vmovaps %zmm0, (%rcx)
@@ -11731,7 +11731,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,6]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & ~zmm16 & zmm0) | (zmm3 & ~zmm16 & zmm0) | (zmm3 & zmm16 & ~zmm0) | (zmm3 & zmm16 & zmm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm0 ^ (zmm16 & (zmm3 ^ zmm0))
 ; AVX512DQ-NEXT:    movw $-2048, %ax # imm = 0xF800
 ; AVX512DQ-NEXT:    kmovw %eax, %k1
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm1, %zmm0, %zmm3 {%k1}
@@ -11796,7 +11796,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm10 = ymm0[0,1,2],ymm10[3,4,5,6,7],ymm0[8,9,10],ymm10[11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,6]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm10[4,5,6,7]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & ~zmm16 & zmm17) | (zmm7 & ~zmm16 & zmm17) | (zmm7 & zmm16 & ~zmm17) | (zmm7 & zmm16 & zmm17)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm17 ^ (zmm16 & (zmm7 ^ zmm17))
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm7 {%k1}
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm10 = [2,3,14,15,10,11,6,7,2,3,14,15,12,13,14,15]
@@ -11824,7 +11824,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7],ymm2[8,9,10],ymm1[11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,5,7]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm6 = (~zmm6 & ~zmm16 & zmm0) | (zmm6 & ~zmm16 & zmm0) | (zmm6 & zmm16 & ~zmm0) | (zmm6 & zmm16 & zmm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm0 ^ (zmm16 & (zmm6 ^ zmm0))
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm1, %zmm0, %zmm6 {%k1}
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm0
@@ -11856,7 +11856,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1,2],ymm2[3,4,5,6,7],ymm1[8,9,10],ymm2[11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,5,7]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & ~zmm16 & zmm0) | (zmm4 & ~zmm16 & zmm0) | (zmm4 & zmm16 & ~zmm0) | (zmm4 & zmm16 & zmm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm0 ^ (zmm16 & (zmm4 ^ zmm0))
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm1, %zmm0, %zmm4 {%k1}
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
@@ -11927,9 +11927,9 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm3
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm21 & zmm2) | (zmm1 & ~zmm21 & zmm2) | (zmm1 & zmm21 & ~zmm2) | (zmm1 & zmm21 & zmm2)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm21 & (zmm1 ^ zmm2))
 ; AVX512DQ-NEXT:    vpmovsxdq {{.*#+}} zmm2 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & zmm2 & zmm1) | (zmm3 & ~zmm2 & ~zmm1) | (zmm3 & ~zmm2 & zmm1) | (zmm3 & zmm2 & zmm1)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm2 & (zmm3 ^ zmm1))
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm2, %zmm18
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
@@ -11990,8 +11990,8 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,5,4]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm14[4,5,6,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm28
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & ~zmm21 & zmm19) | (zmm2 & ~zmm21 & zmm19) | (zmm2 & zmm21 & ~zmm19) | (zmm2 & zmm21 & zmm19)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm28 = (~zmm28 & zmm18 & zmm2) | (zmm28 & ~zmm18 & ~zmm2) | (zmm28 & ~zmm18 & zmm2) | (zmm28 & zmm18 & zmm2)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm19 ^ (zmm21 & (zmm2 ^ zmm19))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm28 = zmm28 ^ (zmm18 & (zmm28 ^ zmm2))
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm0
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,1,2,1,4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm20, %xmm2
@@ -12028,8 +12028,8 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,4,7,4,5]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm13[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm27
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & ~zmm21 & zmm20) | (zmm0 & ~zmm21 & zmm20) | (zmm0 & zmm21 & ~zmm20) | (zmm0 & zmm21 & zmm20)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm27 = (~zmm27 & zmm18 & zmm0) | (zmm27 & ~zmm18 & ~zmm0) | (zmm27 & ~zmm18 & zmm0) | (zmm27 & zmm18 & zmm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm20 ^ (zmm21 & (zmm0 ^ zmm20))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm27 = zmm27 ^ (zmm18 & (zmm27 ^ zmm0))
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm11[3,1,2,1,4,5,6,7]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm12[0,1,3,3,4,5,6,7]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,7,7,7]
@@ -12057,8 +12057,8 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,4,5]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm20
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & ~zmm21 & zmm0) | (zmm2 & ~zmm21 & zmm0) | (zmm2 & zmm21 & ~zmm0) | (zmm2 & zmm21 & zmm0)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm20 = (~zmm20 & zmm18 & zmm2) | (zmm20 & ~zmm18 & ~zmm2) | (zmm20 & ~zmm18 & zmm2) | (zmm20 & zmm18 & zmm2)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm0 ^ (zmm21 & (zmm2 ^ zmm0))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm20 ^ (zmm18 & (zmm20 ^ zmm2))
 ; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX512DQ-NEXT:    vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    # ymm3 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
@@ -12096,7 +12096,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,128,128,4,5,0,1,12,13,24,25,20,21,128,128,128,128,128,128,128,128,128,128,128,128]
 ; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm5, %ymm4
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm5, %ymm31
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ymm4 & ~ymm22) | (~ymm2 & ymm4 & ymm22) | (ymm2 & ~ymm4 & ymm22) | (ymm2 & ymm4 & ~ymm22) | (ymm2 & ymm4 & ymm22)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm2 = (ymm2 & ymm22) | ymm4
 ; AVX512DQ-NEXT:    movw $31, %ax
 ; AVX512DQ-NEXT:    kmovw %eax, %k1
 ; AVX512DQ-NEXT:    vinserti32x8 $0, %ymm2, %zmm0, %zmm23 {%k1}
@@ -12132,7 +12132,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm12[2,2,2,2,4,5,6,7]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1],xmm2[2,3],xmm4[4],xmm2[5,6,7]
 ; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm14, %ymm0
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ymm0 & ~ymm22) | (~ymm2 & ymm0 & ymm22) | (ymm2 & ~ymm0 & ymm22) | (ymm2 & ymm0 & ~ymm22) | (ymm2 & ymm0 & ymm22)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm2 = (ymm2 & ymm22) | ymm0
 ; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX512DQ-NEXT:    vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm6 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    # ymm6 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
@@ -12190,7 +12190,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm11 = [128,128,128,128,128,128,128,128,128,128,6,7,2,3,14,15,26,27,22,23,128,128,128,128,128,128,128,128,128,128,128,128]
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm31, %ymm0
 ; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm0, %ymm13
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm9 = (~ymm9 & ymm13 & ~ymm22) | (~ymm9 & ymm13 & ymm22) | (ymm9 & ~ymm13 & ymm22) | (ymm9 & ymm13 & ~ymm22) | (ymm9 & ymm13 & ymm22)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm9 = (ymm9 & ymm22) | ymm13
 ; AVX512DQ-NEXT:    vinserti32x8 $0, %ymm9, %zmm0, %zmm26 {%k1}
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm16, %xmm0
 ; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm0, %xmm9
@@ -12209,7 +12209,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm12 = xmm12[1,1,2,3]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,5,5,5,5]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm11 = xmm11[0],xmm12[1],xmm11[2,3],xmm12[4],xmm11[5,6,7]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ymm9 & ~ymm22) | (~ymm11 & ymm9 & ymm22) | (ymm11 & ~ymm9 & ymm22) | (ymm11 & ymm9 & ~ymm22) | (ymm11 & ymm9 & ymm22)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm11 = (ymm11 & ymm22) | ymm9
 ; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm6, %xmm1
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm6 = xmm10[1,1,2,3]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,5,5,5,5]
@@ -12238,10 +12238,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512DQ-NEXT:    vmovaps %zmm3, (%rdx)
 ; AVX512DQ-NEXT:    vpmovsxdq {{.*#+}} zmm3 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm24 = (~zmm24 & zmm3 & zmm23) | (zmm24 & ~zmm3 & ~zmm23) | (zmm24 & ~zmm3 & zmm23) | (zmm24 & zmm3 & zmm23)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm21 = (~zmm21 & zmm3 & zmm25) | (zmm21 & ~zmm3 & ~zmm25) | (zmm21 & ~zmm3 & zmm25) | (zmm21 & zmm3 & zmm25)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm3 & zmm26) | (zmm0 & ~zmm3 & ~zmm26) | (zmm0 & ~zmm3 & zmm26) | (zmm0 & zmm3 & zmm26)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm3 & zmm1) | (zmm2 & ~zmm3 & ~zmm1) | (zmm2 & ~zmm3 & zmm1) | (zmm2 & zmm3 & zmm1)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm24 = zmm24 ^ (zmm3 & (zmm24 ^ zmm23))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm21 = zmm21 ^ (zmm3 & (zmm21 ^ zmm25))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm3 & (zmm0 ^ zmm26))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm3 & (zmm2 ^ zmm1))
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm28, 64(%rcx)
 ; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512DQ-NEXT:    vmovaps %zmm1, (%rcx)
@@ -12332,7 +12332,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,6]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & ~zmm16 & zmm0) | (zmm3 & ~zmm16 & zmm0) | (zmm3 & zmm16 & ~zmm0) | (zmm3 & zmm16 & zmm0)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm0 ^ (zmm16 & (zmm3 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    movw $-2048, %ax # imm = 0xF800
 ; AVX512DQ-FCP-NEXT:    kmovw %eax, %k1
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm1, %zmm0, %zmm3 {%k1}
@@ -12397,7 +12397,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm0[0,1,2],ymm10[3,4,5,6,7],ymm0[8,9,10],ymm10[11,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,6]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm10[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = (~zmm6 & ~zmm16 & zmm17) | (zmm6 & ~zmm16 & zmm17) | (zmm6 & zmm16 & ~zmm17) | (zmm6 & zmm16 & zmm17)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm17 ^ (zmm16 & (zmm6 ^ zmm17))
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm6 {%k1}
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm10 = [2,3,14,15,2,3,6,7,10,11,14,15,12,13,14,15]
@@ -12425,7 +12425,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7],ymm2[8,9,10],ymm1[11,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,5,7]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = (~zmm6 & ~zmm16 & zmm0) | (zmm6 & ~zmm16 & zmm0) | (zmm6 & zmm16 & ~zmm0) | (zmm6 & zmm16 & zmm0)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm0 ^ (zmm16 & (zmm6 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm1, %zmm0, %zmm6 {%k1}
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm25, %ymm0
@@ -12457,7 +12457,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1,2],ymm2[3,4,5,6,7],ymm1[8,9,10],ymm2[11,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,5,7]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & ~zmm16 & zmm0) | (zmm4 & ~zmm16 & zmm0) | (zmm4 & zmm16 & ~zmm0) | (zmm4 & zmm16 & zmm0)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm0 ^ (zmm16 & (zmm4 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm1, %zmm0, %zmm4 {%k1}
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
@@ -12528,9 +12528,9 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3],ymm5[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm5
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm20 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & ~zmm20 & zmm3) | (zmm4 & ~zmm20 & zmm3) | (zmm4 & zmm20 & ~zmm3) | (zmm4 & zmm20 & zmm3)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm3 ^ (zmm20 & (zmm4 ^ zmm3))
 ; AVX512DQ-FCP-NEXT:    vpmovsxdq {{.*#+}} zmm28 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm28 & zmm4) | (zmm5 & ~zmm28 & ~zmm4) | (zmm5 & ~zmm28 & zmm4) | (zmm5 & zmm28 & zmm4)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm28 & (zmm5 ^ zmm4))
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
@@ -12587,8 +12587,8 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,5,4]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm10[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm27
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm20 & zmm9) | (zmm1 & ~zmm20 & zmm9) | (zmm1 & zmm20 & ~zmm9) | (zmm1 & zmm20 & zmm9)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm27 = (~zmm27 & zmm28 & zmm1) | (zmm27 & ~zmm28 & ~zmm1) | (zmm27 & ~zmm28 & zmm1) | (zmm27 & zmm28 & zmm1)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm9 ^ (zmm20 & (zmm1 ^ zmm9))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm27 = zmm27 ^ (zmm28 & (zmm27 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [u,u,2,3,6,7,u,u,14,15,14,15,14,15,14,15]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm15, %xmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm1, %xmm26
@@ -12626,8 +12626,8 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,4,5]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm14[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm19
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm20 & zmm21) | (zmm1 & ~zmm20 & zmm21) | (zmm1 & zmm20 & ~zmm21) | (zmm1 & zmm20 & zmm21)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm19 = (~zmm19 & zmm28 & zmm1) | (zmm19 & ~zmm28 & ~zmm1) | (zmm19 & ~zmm28 & zmm1) | (zmm19 & zmm28 & zmm1)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm21 ^ (zmm20 & (zmm1 ^ zmm21))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm19 = zmm19 ^ (zmm28 & (zmm19 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm16, %xmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm26, %xmm1
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm0, %xmm0
@@ -12653,8 +12653,8 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,4,5]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm21
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm20 & zmm0) | (zmm1 & ~zmm20 & zmm0) | (zmm1 & zmm20 & ~zmm0) | (zmm1 & zmm20 & zmm0)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm21 = (~zmm21 & zmm28 & zmm1) | (zmm21 & ~zmm28 & ~zmm1) | (zmm21 & ~zmm28 & zmm1) | (zmm21 & zmm28 & zmm1)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm0 ^ (zmm20 & (zmm1 ^ zmm0))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm21 = zmm21 ^ (zmm28 & (zmm21 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm9 # 32-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # ymm9 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
@@ -12690,7 +12690,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} ymm20 = [65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,128,128,128,128,128,4,5,0,1,12,13,24,25,20,21,128,128,128,128,128,128,128,128,128,128,128,128]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm14, %ymm3
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ymm3 & ~ymm20) | (~ymm0 & ymm3 & ymm20) | (ymm0 & ~ymm3 & ymm20) | (ymm0 & ymm3 & ~ymm20) | (ymm0 & ymm3 & ymm20)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = (ymm0 & ymm20) | ymm3
 ; AVX512DQ-FCP-NEXT:    movw $31, %ax
 ; AVX512DQ-FCP-NEXT:    kmovw %eax, %k1
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $0, %ymm0, %zmm0, %zmm22 {%k1}
@@ -12727,7 +12727,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1],xmm3[2,3],xmm4[4],xmm3[5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm6, %ymm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm6, %ymm31
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm3 = (~ymm3 & ymm2 & ~ymm20) | (~ymm3 & ymm2 & ymm20) | (ymm3 & ~ymm2 & ymm20) | (ymm3 & ymm2 & ~ymm20) | (ymm3 & ymm2 & ymm20)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm3 = (ymm3 & ymm20) | ymm2
 ; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # ymm2 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
@@ -12769,7 +12769,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,128,128,6,7,2,3,14,15,26,27,22,23,128,128,128,128,128,128,128,128,128,128,128,128]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm14, %ymm14
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm26
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = (~ymm10 & ymm14 & ~ymm20) | (~ymm10 & ymm14 & ymm20) | (ymm10 & ~ymm14 & ymm20) | (ymm10 & ymm14 & ~ymm20) | (ymm10 & ymm14 & ymm20)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = (ymm10 & ymm20) | ymm14
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm11, %xmm14
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm9, %xmm9
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm14 = xmm9[0],xmm14[1],xmm9[2,3],xmm14[4],xmm9[5,6,7]
@@ -12800,7 +12800,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm11, %xmm11
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm5, %xmm5
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0],xmm11[1],xmm5[2,3],xmm11[4],xmm5[5,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ymm15 & ~ymm20) | (~ymm5 & ymm15 & ymm20) | (ymm5 & ~ymm15 & ymm20) | (ymm5 & ymm15 & ~ymm20) | (ymm5 & ymm15 & ymm20)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = (ymm5 & ymm20) | ymm15
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm8, %xmm8
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm2, %xmm2
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm8[1],xmm2[2,3],xmm8[4],xmm2[5,6,7]
@@ -12825,10 +12825,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vmovaps %zmm3, 64(%rdx)
 ; AVX512DQ-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512DQ-FCP-NEXT:    vmovaps %zmm3, (%rdx)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm23 = (~zmm23 & zmm28 & zmm22) | (zmm23 & ~zmm28 & ~zmm22) | (zmm23 & ~zmm28 & zmm22) | (zmm23 & zmm28 & zmm22)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm13 = (~zmm13 & zmm28 & zmm24) | (zmm13 & ~zmm28 & ~zmm24) | (zmm13 & ~zmm28 & zmm24) | (zmm13 & zmm28 & zmm24)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & zmm28 & zmm0) | (zmm10 & ~zmm28 & ~zmm0) | (zmm10 & ~zmm28 & zmm0) | (zmm10 & zmm28 & zmm0)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & zmm28 & zmm2) | (zmm1 & ~zmm28 & ~zmm2) | (zmm1 & ~zmm28 & zmm2) | (zmm1 & zmm28 & zmm2)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm23 = zmm23 ^ (zmm28 & (zmm23 ^ zmm22))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm13 ^ (zmm28 & (zmm13 ^ zmm24))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm28 & (zmm10 ^ zmm0))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm28 & (zmm1 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm27, 64(%rcx)
 ; AVX512DQ-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FCP-NEXT:    vmovaps %zmm0, (%rcx)
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll
index 9c2fb7704d1d4..26e19566f048c 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll
@@ -6979,7 +6979,7 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm7[4],xmm3[5],xmm7[6],xmm3[7]
 ; AVX512-NEXT:    vmovdqa64 %ymm28, %ymm7
 ; AVX512-NEXT:    vpshufb %ymm7, %ymm3, %ymm3
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm3 = (~ymm3 & ymm0 & ~mem) | (ymm3 & ~ymm0 & ~mem) | (ymm3 & ~ymm0 & mem) | (ymm3 & ymm0 & ~mem) | (ymm3 & ymm0 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm3 = ymm3 | (ymm0 & ~mem)
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm13[0,1,2],ymm8[3],ymm13[4,5],ymm8[6],ymm13[7]
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm7
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,4]
@@ -7128,18 +7128,18 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm11 = (~zmm11 & zmm4 & zmm21) | (zmm11 & ~zmm4 & ~zmm21) | (zmm11 & ~zmm4 & zmm21) | (zmm11 & zmm4 & zmm21)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 ^ (zmm4 & (zmm11 ^ zmm21))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm22 = (~zmm22 & zmm5 & ~mem) | (zmm22 & ~zmm5 & mem) | (zmm22 & zmm5 & ~mem) | (zmm22 & zmm5 & mem)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm12 = (~zmm12 & zmm4 & zmm22) | (zmm12 & ~zmm4 & ~zmm22) | (zmm12 & ~zmm4 & zmm22) | (zmm12 & zmm4 & zmm22)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm22 = zmm5 ^ (mem & (zmm22 ^ zmm5))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm12 ^ (zmm4 & (zmm12 ^ zmm22))
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm23 = (~zmm23 & zmm4 & zmm19) | (zmm23 & ~zmm4 & ~zmm19) | (zmm23 & ~zmm4 & zmm19) | (zmm23 & zmm4 & zmm19)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm23 = zmm23 ^ (zmm4 & (zmm23 ^ zmm19))
 ; AVX512-NEXT:    movw $-512, %ax # imm = 0xFE00
 ; AVX512-NEXT:    kmovw %eax, %k1
 ; AVX512-NEXT:    vmovdqa32 %zmm14, %zmm23 {%k1}
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm27 = (~zmm27 & zmm4 & zmm20) | (zmm27 & ~zmm4 & ~zmm20) | (zmm27 & ~zmm4 & zmm20) | (zmm27 & zmm4 & zmm20)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm27 = zmm27 ^ (zmm4 & (zmm27 ^ zmm20))
 ; AVX512-NEXT:    vmovdqa32 %zmm25, %zmm27 {%k1}
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm4 & zmm18) | (zmm1 & ~zmm4 & zmm18) | (zmm1 & zmm4 & ~zmm18) | (zmm1 & zmm4 & zmm18)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm18 ^ (zmm4 & (zmm1 ^ zmm18))
 ; AVX512-NEXT:    vmovdqa32 %zmm10, %zmm1 {%k1}
 ; AVX512-NEXT:    vmovdqa64 %zmm11, (%rsi)
 ; AVX512-NEXT:    vmovdqa64 %zmm12, (%rdx)
@@ -7147,11 +7147,11 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vmovdqa64 %zmm27, (%r8)
 ; AVX512-NEXT:    vmovdqa64 %zmm1, (%r9)
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & ~zmm4 & zmm17) | (zmm2 & ~zmm4 & zmm17) | (zmm2 & zmm4 & ~zmm17) | (zmm2 & zmm4 & zmm17)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm17 ^ (zmm4 & (zmm2 ^ zmm17))
 ; AVX512-NEXT:    vmovdqa32 %zmm26, %zmm2 {%k1}
 ; AVX512-NEXT:    vmovdqa64 %zmm2, (%rax)
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & ~zmm4 & zmm16) | (zmm3 & ~zmm4 & zmm16) | (zmm3 & zmm4 & ~zmm16) | (zmm3 & zmm4 & zmm16)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm16 ^ (zmm4 & (zmm3 ^ zmm16))
 ; AVX512-NEXT:    vmovdqa32 %zmm0, %zmm3 {%k1}
 ; AVX512-NEXT:    vmovdqa64 %zmm3, (%rax)
 ; AVX512-NEXT:    vzeroupper
@@ -7344,7 +7344,7 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm4, %ymm4
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm4[2,3,4,5,6,7]
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm30 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm16 = (~zmm16 & zmm30 & zmm0) | (zmm16 & ~zmm30 & ~zmm0) | (zmm16 & ~zmm30 & zmm0) | (zmm16 & zmm30 & zmm0)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm16 = zmm16 ^ (zmm30 & (zmm16 ^ zmm0))
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2],ymm13[3],ymm3[4,5],ymm13[6],ymm3[7]
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm0, %xmm4
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm4[0],xmm0[1],xmm4[2,3,4,5],xmm0[6],xmm4[7]
@@ -7429,17 +7429,17 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm2
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm26 = (~zmm26 & zmm3 & zmm23) | (zmm26 & ~zmm3 & ~zmm23) | (zmm26 & ~zmm3 & zmm23) | (zmm26 & zmm3 & zmm23)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm24 = (~zmm24 & zmm20 & ~mem) | (zmm24 & ~zmm20 & mem) | (zmm24 & zmm20 & ~mem) | (zmm24 & zmm20 & mem)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm27 = (~zmm27 & zmm3 & zmm24) | (zmm27 & ~zmm3 & ~zmm24) | (zmm27 & ~zmm3 & zmm24) | (zmm27 & zmm3 & zmm24)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = (~zmm15 & zmm30 & zmm21) | (zmm15 & ~zmm30 & ~zmm21) | (zmm15 & ~zmm30 & zmm21) | (zmm15 & zmm30 & zmm21)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm26 = zmm26 ^ (zmm3 & (zmm26 ^ zmm23))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm24 = zmm20 ^ (mem & (zmm24 ^ zmm20))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm27 = zmm27 ^ (zmm3 & (zmm27 ^ zmm24))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = zmm15 ^ (zmm30 & (zmm15 ^ zmm21))
 ; AVX512-FCP-NEXT:    movw $-512, %ax # imm = 0xFE00
 ; AVX512-FCP-NEXT:    kmovw %eax, %k1
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm28, %zmm15 {%k1}
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm19 = (~zmm19 & zmm30 & zmm18) | (zmm19 & ~zmm30 & ~zmm18) | (zmm19 & ~zmm30 & zmm18) | (zmm19 & zmm30 & zmm18)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm19 = zmm19 ^ (zmm30 & (zmm19 ^ zmm18))
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm12, %zmm19 {%k1}
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm9, %zmm16 {%k1}
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm11 = (~zmm11 & ~zmm30 & zmm25) | (zmm11 & ~zmm30 & zmm25) | (zmm11 & zmm30 & ~zmm25) | (zmm11 & zmm30 & zmm25)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm25 ^ (zmm30 & (zmm11 ^ zmm25))
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm1, %zmm11 {%k1}
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm26, (%rsi)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm27, (%rdx)
@@ -7448,7 +7448,7 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm16, (%r9)
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm11, (%rax)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & ~zmm30 & zmm10) | (zmm0 & ~zmm30 & zmm10) | (zmm0 & zmm30 & ~zmm10) | (zmm0 & zmm30 & zmm10)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm10 ^ (zmm30 & (zmm0 ^ zmm10))
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm2, %zmm0 {%k1}
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, (%rax)
@@ -7639,7 +7639,7 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4],xmm2[5],xmm3[6],xmm2[7]
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm25, %ymm3
 ; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ymm0 & ~mem) | (ymm2 & ~ymm0 & ~mem) | (ymm2 & ~ymm0 & mem) | (ymm2 & ymm0 & ~mem) | (ymm2 & ymm0 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm0 & ~mem)
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm13[0,1,2],ymm5[3],ymm13[4,5],ymm5[6],ymm13[7]
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,4]
@@ -7649,7 +7649,7 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm25 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm22 = (~zmm22 & zmm25 & zmm19) | (zmm22 & ~zmm25 & ~zmm19) | (zmm22 & ~zmm25 & zmm19) | (zmm22 & zmm25 & zmm19)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm22 = zmm22 ^ (zmm25 & (zmm22 ^ zmm19))
 ; AVX512DQ-NEXT:    movw $-512, %ax # imm = 0xFE00
 ; AVX512DQ-NEXT:    kmovw %eax, %k1
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm22 {%k1}
@@ -7682,7 +7682,7 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7],ymm3[8,9,10],ymm2[11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm19 = (~zmm19 & zmm25 & zmm17) | (zmm19 & ~zmm25 & ~zmm17) | (zmm19 & ~zmm25 & zmm17) | (zmm19 & zmm25 & zmm17)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm19 = zmm19 ^ (zmm25 & (zmm19 ^ zmm17))
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm19 {%k1}
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1],ymm10[2],ymm4[3,4,5],ymm10[6],ymm4[7]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
@@ -7714,7 +7714,7 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[0,1,2,3,4,5,2,3,0,1,14,15,12,13],zero,zero
 ; AVX512DQ-NEXT:    vpor %ymm3, %ymm12, %ymm3
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm11 = (~zmm11 & ~zmm25 & zmm20) | (zmm11 & ~zmm25 & zmm20) | (zmm11 & zmm25 & ~zmm20) | (zmm11 & zmm25 & zmm20)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm20 ^ (zmm25 & (zmm11 ^ zmm20))
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm2, %zmm0, %zmm11 {%k1}
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm13[0],ymm5[1],ymm13[2,3],ymm5[4],ymm13[5,6,7]
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm2, %xmm3
@@ -7768,7 +7768,7 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm15[0],xmm12[1],xmm15[1],xmm12[2],xmm15[2],xmm12[3],xmm15[3]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,2,3,0,1,14,15,12,13,26,27,26,27,26,27,26,27,18,19,16,17,30,31,28,29]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm12[0,1],ymm2[2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & ~zmm25 & zmm24) | (zmm2 & ~zmm25 & zmm24) | (zmm2 & zmm25 & ~zmm24) | (zmm2 & zmm25 & zmm24)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm24 ^ (zmm25 & (zmm2 ^ zmm24))
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm17, %zmm0, %zmm2 {%k1}
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm10[0],ymm3[1],ymm10[2,3],ymm3[4],ymm10[5,6,7]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
@@ -7799,13 +7799,13 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm4 = ymm1[0,1,2],ymm4[3,4,5,6,7],ymm1[8,9,10],ymm4[11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm4[4,5,6,7]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & ~zmm25 & zmm27) | (zmm3 & ~zmm25 & zmm27) | (zmm3 & zmm25 & ~zmm27) | (zmm3 & zmm25 & zmm27)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm27 ^ (zmm25 & (zmm3 ^ zmm27))
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm1, %zmm0, %zmm3 {%k1}
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & zmm1 & zmm18) | (zmm8 & ~zmm1 & ~zmm18) | (zmm8 & ~zmm1 & zmm18) | (zmm8 & zmm1 & zmm18)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (zmm1 & (zmm8 ^ zmm18))
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm21 = (~zmm21 & zmm0 & ~mem) | (zmm21 & ~zmm0 & mem) | (zmm21 & zmm0 & ~mem) | (zmm21 & zmm0 & mem)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm9 = (~zmm9 & zmm1 & zmm21) | (zmm9 & ~zmm1 & ~zmm21) | (zmm9 & ~zmm1 & zmm21) | (zmm9 & zmm1 & zmm21)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm21 = zmm0 ^ (mem & (zmm21 ^ zmm0))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 ^ (zmm1 & (zmm9 ^ zmm21))
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm8, (%rsi)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm9, (%rdx)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm22, (%rcx)
@@ -7939,7 +7939,7 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
 ; AVX512DQ-FCP-NEXT:    vpermd %zmm30, %zmm14, %zmm3
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm16 = (~zmm16 & zmm19 & zmm9) | (zmm16 & ~zmm19 & ~zmm9) | (zmm16 & ~zmm19 & zmm9) | (zmm16 & zmm19 & zmm9)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm16 = zmm16 ^ (zmm19 & (zmm16 ^ zmm9))
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
 ; AVX512DQ-FCP-NEXT:    movw $-512, %ax # imm = 0xFE00
 ; AVX512DQ-FCP-NEXT:    kmovw %eax, %k1
@@ -7977,7 +7977,7 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[0,1,2,3,2,3,0,1,14,15,12,13,10,11],zero,zero
 ; AVX512DQ-FCP-NEXT:    vpor %ymm14, %ymm12, %ymm12
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3],ymm9[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm20 = (~zmm20 & zmm19 & zmm2) | (zmm20 & ~zmm19 & ~zmm2) | (zmm20 & ~zmm19 & zmm2) | (zmm20 & zmm19 & zmm2)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm20 ^ (zmm19 & (zmm20 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm9, %zmm0, %zmm20 {%k1}
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm9 = ymm6[0,1],ymm5[2],ymm6[3,4],ymm5[5],ymm6[6,7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [8,9,4,5,4,5,6,7,8,9,10,11,12,13,14,15]
@@ -8002,7 +8002,7 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpsrlq $48, %xmm15, %xmm11
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
 ; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm10, %zmm0, %zmm10
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & zmm19 & zmm12) | (zmm10 & ~zmm19 & ~zmm12) | (zmm10 & ~zmm19 & zmm12) | (zmm10 & zmm19 & zmm12)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm19 & (zmm10 ^ zmm12))
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2],ymm3[3],ymm4[4,5],ymm3[6],ymm4[7]
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm0, %xmm11
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm11[0],xmm0[1],xmm11[2,3,4,5],xmm0[6],xmm11[7]
@@ -8065,7 +8065,7 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[0,1,2,3,6,7,4,5,2,3,0,1,14,15],zero,zero
 ; AVX512DQ-FCP-NEXT:    vpor %ymm13, %ymm9, %ymm9
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm11 = (~zmm11 & ~zmm19 & zmm12) | (zmm11 & ~zmm19 & zmm12) | (zmm11 & zmm19 & ~zmm12) | (zmm11 & zmm19 & zmm12)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm12 ^ (zmm19 & (zmm11 ^ zmm12))
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm11 {%k1}
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [1,4,8,11,15,0,0,0]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm27, %zmm0, %zmm9
@@ -8094,12 +8094,12 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3,4,5,6,7],ymm3[8,9,10],ymm1[11,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & ~zmm19 & zmm2) | (zmm0 & ~zmm19 & zmm2) | (zmm0 & zmm19 & ~zmm2) | (zmm0 & zmm19 & zmm2)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm2 ^ (zmm19 & (zmm0 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm1, %zmm0, %zmm0 {%k1}
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = (~zmm9 & zmm1 & zmm23) | (zmm9 & ~zmm1 & ~zmm23) | (zmm9 & ~zmm1 & zmm23) | (zmm9 & zmm1 & zmm23)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm24 = (~zmm24 & zmm21 & ~mem) | (zmm24 & ~zmm21 & mem) | (zmm24 & zmm21 & ~mem) | (zmm24 & zmm21 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm1 & zmm24) | (zmm5 & ~zmm1 & ~zmm24) | (zmm5 & ~zmm1 & zmm24) | (zmm5 & zmm1 & zmm24)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 ^ (zmm1 & (zmm9 ^ zmm23))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm24 = zmm21 ^ (mem & (zmm24 ^ zmm21))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm1 & (zmm5 ^ zmm24))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm9, (%rsi)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, (%rdx)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm16, (%rcx)
@@ -14527,7 +14527,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm12 = [0,1,2,3,0,1,14,15,12,13,10,11,8,9,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u,u,u]
 ; AVX512-NEXT:    vpshufb %ymm12, %ymm11, %ymm11
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} ymm20 = [65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ~ymm20 & ymm0) | (ymm11 & ~ymm20 & ~ymm0) | (ymm11 & ~ymm20 & ymm0) | (ymm11 & ymm20 & ~ymm0) | (ymm11 & ymm20 & ymm0)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm0 & ~ymm20)
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2],ymm15[3],ymm3[4,5],ymm15[6],ymm3[7]
 ; AVX512-NEXT:    vmovdqa %ymm15, %ymm13
 ; AVX512-NEXT:    vextracti32x4 $1, %ymm0, %xmm25
@@ -14546,7 +14546,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm11
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm11[4],xmm1[5],xmm11[6],xmm1[7]
 ; AVX512-NEXT:    vpshufb %ymm12, %ymm1, %ymm1
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ~ymm20 & ymm0) | (ymm1 & ~ymm20 & ~ymm0) | (ymm1 & ~ymm20 & ymm0) | (ymm1 & ymm20 & ~ymm0) | (ymm1 & ymm20 & ymm0)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm0 & ~ymm20)
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1,2],ymm9[3],ymm8[4,5],ymm9[6],ymm8[7]
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm11
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,4]
@@ -14700,7 +14700,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3,4,5,6,7]
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & zmm19 & zmm0) | (zmm3 & ~zmm19 & ~zmm0) | (zmm3 & ~zmm19 & zmm0) | (zmm3 & zmm19 & zmm0)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm19 & (zmm3 ^ zmm0))
 ; AVX512-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2],ymm5[3],ymm4[4,5],ymm5[6],ymm4[7]
 ; AVX512-NEXT:    vmovdqa %ymm4, %ymm14
@@ -14812,7 +14812,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm30 = (~zmm30 & zmm19 & zmm0) | (zmm30 & ~zmm19 & ~zmm0) | (zmm30 & ~zmm19 & zmm0) | (zmm30 & zmm19 & zmm0)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm30 = zmm30 ^ (zmm19 & (zmm30 ^ zmm0))
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0],ymm8[1],ymm10[2,3],ymm8[4],ymm10[5,6,7]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6,7,8],ymm1[9],ymm0[10,11,12,13,14,15]
@@ -14868,7 +14868,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm8[4],xmm9[4],xmm8[5],xmm9[5],xmm8[6],xmm9[6],xmm8[7],xmm9[7]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1],ymm6[2,3,4,5,6,7]
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm13 = (~zmm13 & zmm19 & zmm6) | (zmm13 & ~zmm19 & ~zmm6) | (zmm13 & ~zmm19 & zmm6) | (zmm13 & zmm19 & zmm6)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm13 ^ (zmm19 & (zmm13 ^ zmm6))
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm6 = ymm12[0],ymm14[1],ymm12[2,3,4],ymm14[5],ymm12[6,7]
 ; AVX512-NEXT:    vextracti128 $1, %ymm6, %xmm8
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm6 = xmm8[0],xmm6[1],xmm8[2],xmm6[3],xmm8[4,5,6,7]
@@ -14887,17 +14887,17 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpmovsxdq {{.*#+}} zmm9 = [0,0,18446744073709486080,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615]
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
 ; AVX512-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm10 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm10 = (~zmm10 & ~zmm9 & mem) | (zmm10 & ~zmm9 & mem) | (zmm10 & zmm9 & ~mem) | (zmm10 & zmm9 & mem)
+; AVX512-NEXT:    # zmm10 = mem ^ (zmm9 & (zmm10 ^ mem))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
 ; AVX512-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm12 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm12 = (~zmm12 & ~zmm9 & mem) | (zmm12 & ~zmm9 & mem) | (zmm12 & zmm9 & ~mem) | (zmm12 & zmm9 & mem)
+; AVX512-NEXT:    # zmm12 = mem ^ (zmm9 & (zmm12 ^ mem))
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0]
 ; AVX512-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm1 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm1 = (~zmm1 & zmm9 & mem) | (zmm1 & ~zmm9 & ~mem) | (zmm1 & ~zmm9 & mem) | (zmm1 & zmm9 & mem)
+; AVX512-NEXT:    # zmm1 = zmm1 ^ (zmm9 & (zmm1 ^ mem))
 ; AVX512-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm3 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm3 = (~zmm3 & zmm9 & mem) | (zmm3 & ~zmm9 & ~mem) | (zmm3 & ~zmm9 & mem) | (zmm3 & zmm9 & mem)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm9 & zmm10) | (zmm2 & ~zmm9 & ~zmm10) | (zmm2 & ~zmm9 & zmm10) | (zmm2 & zmm9 & zmm10)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm9 & zmm12) | (zmm4 & ~zmm9 & ~zmm12) | (zmm4 & ~zmm9 & zmm12) | (zmm4 & zmm9 & zmm12)
+; AVX512-NEXT:    # zmm3 = zmm3 ^ (zmm9 & (zmm3 ^ mem))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm9 & (zmm2 ^ zmm10))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm9 & (zmm4 ^ zmm12))
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,1,3,1]
 ; AVX512-NEXT:    vpshufb %ymm5, %ymm8, %ymm5
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0],ymm7[1,2],ymm5[3,4,5,6,7]
@@ -14916,26 +14916,26 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm5
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
 ; AVX512-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm19, %zmm12 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm12 = (~zmm12 & zmm19 & mem) | (zmm12 & ~zmm19 & ~mem) | (zmm12 & ~zmm19 & mem) | (zmm12 & zmm19 & mem)
+; AVX512-NEXT:    # zmm12 = zmm12 ^ (zmm19 & (zmm12 ^ mem))
 ; AVX512-NEXT:    movw $-512, %ax # imm = 0xFE00
 ; AVX512-NEXT:    kmovw %eax, %k1
 ; AVX512-NEXT:    vmovdqa32 %zmm6, %zmm12 {%k1}
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
 ; AVX512-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm19, %zmm6 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm6 = (~zmm6 & zmm19 & mem) | (zmm6 & ~zmm19 & ~mem) | (zmm6 & ~zmm19 & mem) | (zmm6 & zmm19 & mem)
+; AVX512-NEXT:    # zmm6 = zmm6 ^ (zmm19 & (zmm6 ^ mem))
 ; AVX512-NEXT:    vmovdqa32 %zmm7, %zmm6 {%k1}
 ; AVX512-NEXT:    vmovdqa64 %zmm6, %zmm7
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
 ; AVX512-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm19, %zmm6 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm6 = (~zmm6 & zmm19 & mem) | (zmm6 & ~zmm19 & ~mem) | (zmm6 & ~zmm19 & mem) | (zmm6 & zmm19 & mem)
+; AVX512-NEXT:    # zmm6 = zmm6 ^ (zmm19 & (zmm6 ^ mem))
 ; AVX512-NEXT:    vmovdqa32 %zmm8, %zmm6 {%k1}
 ; AVX512-NEXT:    vmovdqa64 %zmm6, %zmm8
 ; AVX512-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm19, %zmm25 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm25 = (~zmm25 & zmm19 & mem) | (zmm25 & ~zmm19 & ~mem) | (zmm25 & ~zmm19 & mem) | (zmm25 & zmm19 & mem)
+; AVX512-NEXT:    # zmm25 = zmm25 ^ (zmm19 & (zmm25 ^ mem))
 ; AVX512-NEXT:    vmovdqa32 %zmm9, %zmm25 {%k1}
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
 ; AVX512-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm19, %zmm6 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm6 = (~zmm6 & ~zmm19 & mem) | (zmm6 & ~zmm19 & mem) | (zmm6 & zmm19 & ~mem) | (zmm6 & zmm19 & mem)
+; AVX512-NEXT:    # zmm6 = mem ^ (zmm19 & (zmm6 ^ mem))
 ; AVX512-NEXT:    vmovdqa32 %zmm10, %zmm6 {%k1}
 ; AVX512-NEXT:    vmovdqa64 %zmm3, (%rsi)
 ; AVX512-NEXT:    vmovdqa64 %zmm1, 64(%rsi)
@@ -14950,13 +14950,13 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vmovdqa32 %zmm14, %zmm2 {%k1}
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm19, %zmm1 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm1 = (~zmm1 & ~zmm19 & mem) | (zmm1 & ~zmm19 & mem) | (zmm1 & zmm19 & ~mem) | (zmm1 & zmm19 & mem)
+; AVX512-NEXT:    # zmm1 = mem ^ (zmm19 & (zmm1 ^ mem))
 ; AVX512-NEXT:    vmovdqa32 %zmm15, %zmm1 {%k1}
 ; AVX512-NEXT:    vmovdqa64 %zmm2, 64(%r9)
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-NEXT:    vmovdqa64 %zmm1, (%rax)
 ; AVX512-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm19, %zmm11 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm11 = (~zmm11 & ~zmm19 & mem) | (zmm11 & ~zmm19 & mem) | (zmm11 & zmm19 & ~mem) | (zmm11 & zmm19 & mem)
+; AVX512-NEXT:    # zmm11 = mem ^ (zmm19 & (zmm11 ^ mem))
 ; AVX512-NEXT:    vmovdqa64 %zmm30, %zmm1
 ; AVX512-NEXT:    vmovdqa32 %zmm23, %zmm1 {%k1}
 ; AVX512-NEXT:    vmovdqa32 %zmm0, %zmm11 {%k1}
@@ -15444,7 +15444,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm3, %ymm3
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm3[2,3,4,5,6,7]
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm27 = (~zmm27 & zmm29 & zmm1) | (zmm27 & ~zmm29 & ~zmm1) | (zmm27 & ~zmm29 & zmm1) | (zmm27 & zmm29 & zmm1)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm27 = zmm27 ^ (zmm29 & (zmm27 ^ zmm1))
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm15[0,1,2],ymm7[3],ymm15[4,5],ymm7[6],ymm15[7]
 ; AVX512-FCP-NEXT:    vmovdqa %ymm7, %ymm12
@@ -15480,7 +15480,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpermd %zmm25, %zmm5, %zmm3
 ; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm3, %ymm3
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm3[2,3,4,5,6,7]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm26 = (~zmm26 & zmm29 & zmm1) | (zmm26 & ~zmm29 & ~zmm1) | (zmm26 & ~zmm29 & zmm1) | (zmm26 & zmm29 & zmm1)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm26 = zmm26 ^ (zmm29 & (zmm26 ^ zmm1))
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm21, %ymm5
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm19, %ymm4
@@ -15630,24 +15630,24 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpmovsxdq {{.*#+}} zmm9 = [0,0,18446744073709486080,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615]
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm12 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm12 = (~zmm12 & ~zmm9 & mem) | (zmm12 & ~zmm9 & mem) | (zmm12 & zmm9 & ~mem) | (zmm12 & zmm9 & mem)
+; AVX512-FCP-NEXT:    # zmm12 = mem ^ (zmm9 & (zmm12 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm13 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm13 = (~zmm13 & ~zmm9 & mem) | (zmm13 & ~zmm9 & mem) | (zmm13 & zmm9 & ~mem) | (zmm13 & zmm9 & mem)
+; AVX512-FCP-NEXT:    # zmm13 = mem ^ (zmm9 & (zmm13 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0]
 ; AVX512-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm17 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm17 = (~zmm17 & zmm9 & mem) | (zmm17 & ~zmm9 & ~mem) | (zmm17 & ~zmm9 & mem) | (zmm17 & zmm9 & mem)
+; AVX512-FCP-NEXT:    # zmm17 = zmm17 ^ (zmm9 & (zmm17 ^ mem))
 ; AVX512-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm21 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm21 = (~zmm21 & zmm9 & mem) | (zmm21 & ~zmm9 & ~mem) | (zmm21 & ~zmm9 & mem) | (zmm21 & zmm9 & mem)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm9 & zmm12) | (zmm5 & ~zmm9 & ~zmm12) | (zmm5 & ~zmm9 & zmm12) | (zmm5 & zmm9 & zmm12)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm11 = (~zmm11 & zmm9 & zmm13) | (zmm11 & ~zmm9 & ~zmm13) | (zmm11 & ~zmm9 & zmm13) | (zmm11 & zmm9 & zmm13)
+; AVX512-FCP-NEXT:    # zmm21 = zmm21 ^ (zmm9 & (zmm21 ^ mem))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm9 & (zmm5 ^ zmm12))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 ^ (zmm9 & (zmm11 ^ zmm13))
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0],ymm10[1,2],ymm8[3,4,5,6,7]
 ; AVX512-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm8[0,1,2],ymm3[3,4,5,6,7],ymm8[8,9,10],ymm3[11,12,13,14,15]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm8[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm19 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm19 = (~zmm19 & zmm29 & mem) | (zmm19 & ~zmm29 & ~mem) | (zmm19 & ~zmm29 & mem) | (zmm19 & zmm29 & mem)
+; AVX512-FCP-NEXT:    # zmm19 = zmm19 ^ (zmm29 & (zmm19 ^ mem))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm8 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm9 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm10 # 32-byte Folded Reload
@@ -15663,22 +15663,22 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm8, %zmm19 {%k1}
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm2 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm2 = (~zmm2 & zmm29 & mem) | (zmm2 & ~zmm29 & ~mem) | (zmm2 & ~zmm29 & mem) | (zmm2 & zmm29 & mem)
+; AVX512-FCP-NEXT:    # zmm2 = zmm2 ^ (zmm29 & (zmm2 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm9, %zmm2 {%k1}
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm8 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm8 = (~zmm8 & zmm29 & mem) | (zmm8 & ~zmm29 & ~mem) | (zmm8 & ~zmm29 & mem) | (zmm8 & zmm29 & mem)
+; AVX512-FCP-NEXT:    # zmm8 = zmm8 ^ (zmm29 & (zmm8 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm10, %zmm8 {%k1}
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm6 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm6 = (~zmm6 & zmm29 & mem) | (zmm6 & ~zmm29 & ~mem) | (zmm6 & ~zmm29 & mem) | (zmm6 & zmm29 & mem)
+; AVX512-FCP-NEXT:    # zmm6 = zmm6 ^ (zmm29 & (zmm6 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm12, %zmm6 {%k1}
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm13, %zmm9 {%k1}
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm15, %zmm10 {%k1}
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm1 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm1 = (~zmm1 & ~zmm29 & mem) | (zmm1 & ~zmm29 & mem) | (zmm1 & zmm29 & ~mem) | (zmm1 & zmm29 & mem)
+; AVX512-FCP-NEXT:    # zmm1 = mem ^ (zmm29 & (zmm1 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm18, %zmm1 {%k1}
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm21, (%rsi)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm17, 64(%rsi)
@@ -15694,16 +15694,16 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 64(%rax)
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm1 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm1 = (~zmm1 & ~zmm29 & mem) | (zmm1 & ~zmm29 & mem) | (zmm1 & zmm29 & ~mem) | (zmm1 & zmm29 & mem)
+; AVX512-FCP-NEXT:    # zmm1 = mem ^ (zmm29 & (zmm1 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm16, %zmm1 {%k1}
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, (%rax)
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm4 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm4 = (~zmm4 & ~zmm29 & mem) | (zmm4 & ~zmm29 & mem) | (zmm4 & zmm29 & ~mem) | (zmm4 & zmm29 & mem)
+; AVX512-FCP-NEXT:    # zmm4 = mem ^ (zmm29 & (zmm4 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm3, %zmm4 {%k1}
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, 64(%rax)
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm0 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm0 = (~zmm0 & ~zmm29 & mem) | (zmm0 & ~zmm29 & mem) | (zmm0 & zmm29 & ~mem) | (zmm0 & zmm29 & mem)
+; AVX512-FCP-NEXT:    # zmm0 = mem ^ (zmm29 & (zmm0 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa32 %zmm7, %zmm0 {%k1}
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, (%rax)
 ; AVX512-FCP-NEXT:    addq $1800, %rsp # imm = 0x708
@@ -16175,7 +16175,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm12 = [0,1,2,3,0,1,14,15,12,13,10,11,8,9,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm10, %ymm10
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} ymm19 = [65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm10 = (~ymm10 & ~ymm19 & ymm0) | (ymm10 & ~ymm19 & ~ymm0) | (ymm10 & ~ymm19 & ymm0) | (ymm10 & ymm19 & ~ymm0) | (ymm10 & ymm19 & ymm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm10 | (ymm0 & ~ymm19)
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm23, %ymm2
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2],ymm2[3],ymm4[4,5],ymm2[6],ymm4[7]
 ; AVX512DQ-NEXT:    vextracti32x4 $1, %ymm0, %xmm28
@@ -16188,7 +16188,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
 ; AVX512DQ-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm10 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm10 = (~zmm10 & zmm17 & mem) | (zmm10 & ~zmm17 & ~mem) | (zmm10 & ~zmm17 & mem) | (zmm10 & zmm17 & mem)
+; AVX512DQ-NEXT:    # zmm10 = zmm10 ^ (zmm17 & (zmm10 ^ mem))
 ; AVX512DQ-NEXT:    movw $-512, %ax # imm = 0xFE00
 ; AVX512DQ-NEXT:    kmovw %eax, %k1
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm10 {%k1}
@@ -16202,7 +16202,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm11, %xmm15
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm11 = xmm11[0,1,2,3],xmm15[4],xmm11[5],xmm15[6],xmm11[7]
 ; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm11, %ymm11
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ~ymm19 & ymm0) | (ymm11 & ~ymm19 & ~ymm0) | (ymm11 & ~ymm19 & ymm0) | (ymm11 & ymm19 & ~ymm0) | (ymm11 & ymm19 & ymm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm0 & ~ymm19)
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1,2],ymm9[3],ymm8[4,5],ymm9[6],ymm8[7]
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm12
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,4]
@@ -16213,7 +16213,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3,4,5],ymm0[6,7]
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
 ; AVX512DQ-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm11 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm11 = (~zmm11 & zmm17 & mem) | (zmm11 & ~zmm17 & ~mem) | (zmm11 & ~zmm17 & mem) | (zmm11 & zmm17 & mem)
+; AVX512DQ-NEXT:    # zmm11 = zmm11 ^ (zmm17 & (zmm11 ^ mem))
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm11 {%k1}
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm20, %ymm0
@@ -16252,7 +16252,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm11 = ymm13[0,1,2],ymm11[3,4,5,6,7],ymm13[8,9,10],ymm11[11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512DQ-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm27 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm27 = (~zmm27 & zmm17 & mem) | (zmm27 & ~zmm17 & ~mem) | (zmm27 & ~zmm17 & mem) | (zmm27 & zmm17 & mem)
+; AVX512DQ-NEXT:    # zmm27 = zmm27 ^ (zmm17 & (zmm27 ^ mem))
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm27 {%k1}
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm18, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm25, %ymm2
@@ -16292,7 +16292,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm11 = ymm11[0,1,2],ymm12[3,4,5,6,7],ymm11[8,9,10],ymm12[11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512DQ-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm29 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm29 = (~zmm29 & zmm17 & mem) | (zmm29 & ~zmm17 & ~mem) | (zmm29 & ~zmm17 & mem) | (zmm29 & zmm17 & mem)
+; AVX512DQ-NEXT:    # zmm29 = zmm29 ^ (zmm17 & (zmm29 ^ mem))
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm29 {%k1}
 ; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX512DQ-NEXT:    vpblendd $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
@@ -16341,7 +16341,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm17, %zmm14
 ; AVX512DQ-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm7 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm7 = (~zmm7 & ~zmm17 & mem) | (zmm7 & ~zmm17 & mem) | (zmm7 & zmm17 & ~mem) | (zmm7 & zmm17 & mem)
+; AVX512DQ-NEXT:    # zmm7 = mem ^ (zmm17 & (zmm7 ^ mem))
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm7 {%k1}
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm22 # 32-byte Reload
@@ -16364,7 +16364,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1],ymm0[2,3,4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm12 = (~zmm12 & zmm14 & zmm0) | (zmm12 & ~zmm14 & ~zmm0) | (zmm12 & ~zmm14 & zmm0) | (zmm12 & zmm14 & zmm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm12 ^ (zmm14 & (zmm12 ^ zmm0))
 ; AVX512DQ-NEXT:    vmovdqa %ymm6, %ymm9
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm20, %ymm7
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm7[0,1,2],ymm6[3],ymm7[4,5],ymm6[6],ymm7[7]
@@ -16435,7 +16435,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm23 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
 ; AVX512DQ-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm12 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm12 = (~zmm12 & ~zmm24 & mem) | (zmm12 & ~zmm24 & mem) | (zmm12 & zmm24 & ~mem) | (zmm12 & zmm24 & mem)
+; AVX512DQ-NEXT:    # zmm12 = mem ^ (zmm24 & (zmm12 ^ mem))
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm12 {%k1}
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm14[0,1,2],ymm3[3],ymm14[4,5],ymm3[6],ymm14[7]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
@@ -16474,7 +16474,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & zmm24 & zmm1) | (zmm7 & ~zmm24 & ~zmm1) | (zmm7 & ~zmm24 & zmm1) | (zmm7 & zmm24 & zmm1)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm24 & (zmm7 ^ zmm1))
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm7 {%k1}
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm26, %ymm0
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm4[1],ymm0[2,3],ymm4[4],ymm0[5,6,7]
@@ -16515,7 +16515,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3,4,5,6,7],ymm3[8,9,10],ymm1[11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512DQ-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm11 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm11 = (~zmm11 & ~zmm24 & mem) | (zmm11 & ~zmm24 & mem) | (zmm11 & zmm24 & ~mem) | (zmm11 & zmm24 & mem)
+; AVX512DQ-NEXT:    # zmm11 = mem ^ (zmm24 & (zmm11 ^ mem))
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm1, %zmm0, %zmm11 {%k1}
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm6[0],ymm5[1],ymm6[2,3],ymm5[4],ymm6[5,6,7]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
@@ -16531,7 +16531,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1],ymm1[2,3,4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm6 = (~zmm6 & zmm24 & zmm1) | (zmm6 & ~zmm24 & ~zmm1) | (zmm6 & ~zmm24 & zmm1) | (zmm6 & zmm24 & zmm1)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm24 & (zmm6 ^ zmm1))
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm30, %ymm1
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm14[0],ymm1[1],ymm14[2,3,4],ymm1[5],ymm14[6,7]
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm1, %xmm5
@@ -16557,17 +16557,17 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpmovsxdq {{.*#+}} zmm0 = [0,0,18446744073709486080,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615]
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512DQ-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm1 = (~zmm1 & ~zmm0 & mem) | (zmm1 & ~zmm0 & mem) | (zmm1 & zmm0 & ~mem) | (zmm1 & zmm0 & mem)
+; AVX512DQ-NEXT:    # zmm1 = mem ^ (zmm0 & (zmm1 ^ mem))
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
 ; AVX512DQ-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm4 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm4 = (~zmm4 & ~zmm0 & mem) | (zmm4 & ~zmm0 & mem) | (zmm4 & zmm0 & ~mem) | (zmm4 & zmm0 & mem)
+; AVX512DQ-NEXT:    # zmm4 = mem ^ (zmm0 & (zmm4 ^ mem))
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0]
 ; AVX512DQ-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm28 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm28 = (~zmm28 & zmm0 & mem) | (zmm28 & ~zmm0 & ~mem) | (zmm28 & ~zmm0 & mem) | (zmm28 & zmm0 & mem)
+; AVX512DQ-NEXT:    # zmm28 = zmm28 ^ (zmm0 & (zmm28 ^ mem))
 ; AVX512DQ-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm23 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm23 = (~zmm23 & zmm0 & mem) | (zmm23 & ~zmm0 & ~mem) | (zmm23 & ~zmm0 & mem) | (zmm23 & zmm0 & mem)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm0 & zmm1) | (zmm2 & ~zmm0 & ~zmm1) | (zmm2 & ~zmm0 & zmm1) | (zmm2 & zmm0 & zmm1)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & zmm0 & zmm4) | (zmm3 & ~zmm0 & ~zmm4) | (zmm3 & ~zmm0 & zmm4) | (zmm3 & zmm0 & zmm4)
+; AVX512DQ-NEXT:    # zmm23 = zmm23 ^ (zmm0 & (zmm23 ^ mem))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm0 & (zmm2 ^ zmm1))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm0 & (zmm3 ^ zmm4))
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm23, (%rsi)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm28, 64(%rsi)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm3, 64(%rdx)
@@ -16980,7 +16980,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm25 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm6 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm6 = (~zmm6 & zmm25 & mem) | (zmm6 & ~zmm25 & ~mem) | (zmm6 & ~zmm25 & mem) | (zmm6 & zmm25 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm6 = zmm6 ^ (zmm25 & (zmm6 ^ mem))
 ; AVX512DQ-FCP-NEXT:    movw $-512, %ax # imm = 0xFE00
 ; AVX512DQ-FCP-NEXT:    kmovw %eax, %k1
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm1, %zmm0, %zmm6 {%k1}
@@ -16998,7 +16998,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm1 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm1 = (~zmm1 & zmm25 & mem) | (zmm1 & ~zmm25 & ~mem) | (zmm1 & ~zmm25 & mem) | (zmm1 & zmm25 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm1 = zmm1 ^ (zmm25 & (zmm1 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm1 {%k1}
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpermd %ymm22, %ymm7, %ymm0
@@ -17036,7 +17036,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpor %ymm6, %ymm2, %ymm2
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm23 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm23 = (~zmm23 & zmm25 & mem) | (zmm23 & ~zmm25 & ~mem) | (zmm23 & ~zmm25 & mem) | (zmm23 & zmm25 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm23 = zmm23 ^ (zmm25 & (zmm23 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm23 {%k1}
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 864(%rdi), %ymm1
@@ -17059,7 +17059,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm29 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm29 = (~zmm29 & zmm25 & mem) | (zmm29 & ~zmm25 & ~mem) | (zmm29 & ~zmm25 & mem) | (zmm29 & zmm25 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm29 = zmm29 ^ (zmm25 & (zmm29 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm29 {%k1}
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
@@ -17078,7 +17078,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm3, %ymm3
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm1, %ymm5
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm3[2,3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm28 = (~zmm28 & zmm25 & zmm0) | (zmm28 & ~zmm25 & ~zmm0) | (zmm28 & ~zmm25 & zmm0) | (zmm28 & zmm25 & zmm0)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm28 = zmm28 ^ (zmm25 & (zmm28 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm11[0,1,2],ymm8[3],ymm11[4,5],ymm8[6],ymm11[7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm11, %ymm12
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm8, %ymm19
@@ -17115,7 +17115,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm2, %ymm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm5, %ymm29
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2,3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm26 = (~zmm26 & zmm25 & zmm0) | (zmm26 & ~zmm25 & ~zmm0) | (zmm26 & ~zmm25 & zmm0) | (zmm26 & zmm25 & zmm0)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm26 = zmm26 ^ (zmm25 & (zmm26 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm31, %ymm5
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm13[0,1,2],ymm5[3],ymm13[4,5],ymm5[6],ymm13[7]
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm0, %xmm2
@@ -17170,7 +17170,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpor %ymm1, %ymm10, %ymm1
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm0 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm0 = (~zmm0 & ~zmm25 & mem) | (zmm0 & ~zmm25 & mem) | (zmm0 & zmm25 & ~mem) | (zmm0 & zmm25 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm0 = mem ^ (zmm25 & (zmm0 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm1, %zmm0, %zmm0 {%k1}
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm20, %ymm10
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm14[0,1],ymm10[2,3],ymm14[4,5],ymm10[6,7]
@@ -17199,7 +17199,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm9 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm9 = (~zmm9 & ~zmm25 & mem) | (zmm9 & ~zmm25 & mem) | (zmm9 & zmm25 & ~mem) | (zmm9 & zmm25 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm9 = mem ^ (zmm25 & (zmm9 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm1, %zmm0, %zmm9 {%k1}
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm23, %ymm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm31, %ymm2
@@ -17236,7 +17236,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm6[0,1,2],ymm1[3,4,5,6,7],ymm6[8,9,10],ymm1[11,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm12 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm12 = (~zmm12 & ~zmm25 & mem) | (zmm12 & ~zmm25 & mem) | (zmm12 & zmm25 & ~mem) | (zmm12 & zmm25 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm12 = mem ^ (zmm25 & (zmm12 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm1, %zmm0, %zmm12 {%k1}
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm14[0,1,2],ymm10[3],ymm14[4,5],ymm10[6],ymm14[7]
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm6
@@ -17268,22 +17268,22 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm2[0,1,2],ymm3[3,4,5,6,7],ymm2[8,9,10],ymm3[11,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm1 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm1 = (~zmm1 & ~zmm25 & mem) | (zmm1 & ~zmm25 & mem) | (zmm1 & zmm25 & ~mem) | (zmm1 & zmm25 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm1 = mem ^ (zmm25 & (zmm1 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vinserti32x8 $1, %ymm2, %zmm0, %zmm1 {%k1}
 ; AVX512DQ-FCP-NEXT:    vpmovsxdq {{.*#+}} zmm2 = [0,0,18446744073709486080,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615]
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm3 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm3 = (~zmm3 & ~zmm2 & mem) | (zmm3 & ~zmm2 & mem) | (zmm3 & zmm2 & ~mem) | (zmm3 & zmm2 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm3 = mem ^ (zmm2 & (zmm3 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm4 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm4 = (~zmm4 & ~zmm2 & mem) | (zmm4 & ~zmm2 & mem) | (zmm4 & zmm2 & ~mem) | (zmm4 & zmm2 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm4 = mem ^ (zmm2 & (zmm4 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0]
 ; AVX512DQ-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm5 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm5 = (~zmm5 & zmm2 & mem) | (zmm5 & ~zmm2 & ~mem) | (zmm5 & ~zmm2 & mem) | (zmm5 & zmm2 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm5 = zmm5 ^ (zmm2 & (zmm5 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm6 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm6 = (~zmm6 & zmm2 & mem) | (zmm6 & ~zmm2 & ~mem) | (zmm6 & ~zmm2 & mem) | (zmm6 & zmm2 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & zmm2 & zmm3) | (zmm10 & ~zmm2 & ~zmm3) | (zmm10 & ~zmm2 & zmm3) | (zmm10 & zmm2 & zmm3)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & zmm2 & zmm4) | (zmm8 & ~zmm2 & ~zmm4) | (zmm8 & ~zmm2 & zmm4) | (zmm8 & zmm2 & zmm4)
+; AVX512DQ-FCP-NEXT:    # zmm6 = zmm6 ^ (zmm2 & (zmm6 ^ mem))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm2 & (zmm10 ^ zmm3))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (zmm2 & (zmm8 ^ zmm4))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, (%rsi)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, 64(%rsi)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm8, 64(%rdx)
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll
index 44684603e301d..2ca665d7981a9 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll
@@ -1979,7 +1979,7 @@ define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vmovdqa (%rdi), %ymm3
 ; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm4
 ; AVX512-NEXT:    vmovdqa %ymm0, %ymm5
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ~ymm3 & ymm4) | (~ymm5 & ymm3 & ymm4) | (ymm5 & ymm3 & ~ymm4) | (ymm5 & ymm3 & ymm4)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm4 ^ (ymm5 & (ymm3 ^ ymm4))
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm1 = xmm5[0,6,12],zero,zero,zero,xmm5[4,10],zero,zero,zero,xmm5[u,u,u,u,u]
 ; AVX512-NEXT:    vextracti128 $1, %ymm5, %xmm6
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm6[2,8,14],zero,zero,xmm6[0,6,12,u,u,u,u,u]
@@ -1990,34 +1990,34 @@ define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm9 = xmm1[u,u,u,u,u,u,u,u,u,u,u,2,8,14],zero,zero
 ; AVX512-NEXT:    vpor %xmm8, %xmm9, %xmm8
 ; AVX512-NEXT:    vpmovsxdq {{.*#+}} xmm9 = [18446744073709551615,16777215]
-; AVX512-NEXT:    vpternlogq {{.*#+}} xmm8 = (~xmm8 & xmm9 & xmm7) | (xmm8 & ~xmm9 & ~xmm7) | (xmm8 & ~xmm9 & xmm7) | (xmm8 & xmm9 & xmm7)
+; AVX512-NEXT:    vpternlogq {{.*#+}} xmm8 = xmm8 ^ (xmm9 & (xmm8 ^ xmm7))
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[1,7,13],zero,zero,zero,xmm5[5,11],zero,zero,zero,xmm5[u,u,u,u,u]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm6[3,9,15],zero,zero,xmm6[1,7,13,u,u,u,u,u]
 ; AVX512-NEXT:    vpor %xmm5, %xmm6, %xmm5
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm6 = xmm2[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm2[5,11]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm7 = xmm1[u,u,u,u,u,u,u,u,u,u,u,3,9,15],zero,zero
 ; AVX512-NEXT:    vpor %xmm6, %xmm7, %xmm6
-; AVX512-NEXT:    vpternlogq {{.*#+}} xmm6 = (~xmm6 & xmm9 & xmm5) | (xmm6 & ~xmm9 & ~xmm5) | (xmm6 & ~xmm9 & xmm5) | (xmm6 & xmm9 & xmm5)
+; AVX512-NEXT:    vpternlogq {{.*#+}} xmm6 = xmm6 ^ (xmm9 & (xmm6 ^ xmm5))
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm5 = xmm1[u,u,u,u,u,u,u,u,u,u,u,4,10],zero,zero,zero
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm7 = xmm2[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm2[0,6,12]
 ; AVX512-NEXT:    vpor %xmm5, %xmm7, %xmm5
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm7 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm7 = (~ymm7 & ~ymm4 & ymm3) | (~ymm7 & ymm4 & ymm3) | (ymm7 & ymm4 & ~ymm3) | (ymm7 & ymm4 & ymm3)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm7 = ymm3 ^ (ymm7 & (ymm4 ^ ymm3))
 ; AVX512-NEXT:    vextracti128 $1, %ymm7, %xmm10
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm11 = zero,zero,zero,xmm10[4,10],zero,zero,zero,xmm10[2,8,14,u,u,u,u,u]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm12 = xmm7[2,8,14],zero,zero,xmm7[0,6,12],zero,zero,zero,xmm7[u,u,u,u,u]
 ; AVX512-NEXT:    vpor %xmm11, %xmm12, %xmm11
-; AVX512-NEXT:    vpternlogq {{.*#+}} xmm11 = (~xmm11 & ~xmm9 & xmm5) | (xmm11 & ~xmm9 & xmm5) | (xmm11 & xmm9 & ~xmm5) | (xmm11 & xmm9 & xmm5)
+; AVX512-NEXT:    vpternlogq {{.*#+}} xmm11 = xmm5 ^ (xmm9 & (xmm11 ^ xmm5))
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm5 = xmm1[u,u,u,u,u,u,u,u,u,u,u,5,11],zero,zero,zero
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm12 = xmm2[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm2[1,7,13]
 ; AVX512-NEXT:    vpor %xmm5, %xmm12, %xmm5
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,zero,xmm10[5,11],zero,zero,zero,xmm10[3,9,15,u,u,u,u,u]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[3,9,15],zero,zero,xmm7[1,7,13],zero,zero,zero,xmm7[u,u,u,u,u]
 ; AVX512-NEXT:    vpor %xmm7, %xmm10, %xmm7
-; AVX512-NEXT:    vpternlogq {{.*#+}} xmm7 = (~xmm7 & ~xmm9 & xmm5) | (xmm7 & ~xmm9 & xmm5) | (xmm7 & xmm9 & ~xmm5) | (xmm7 & xmm9 & xmm5)
+; AVX512-NEXT:    vpternlogq {{.*#+}} xmm7 = xmm5 ^ (xmm9 & (xmm7 ^ xmm5))
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm5 = [128,128,0,6,12,128,128,128,4,10,128,128,128,2,8,14]
 ; AVX512-NEXT:    vpshufb %xmm5, %xmm2, %xmm9
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ~ymm4 & ymm3) | (~ymm0 & ymm4 & ymm3) | (ymm0 & ymm4 & ~ymm3) | (ymm0 & ymm4 & ymm3)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm3 ^ (ymm0 & (ymm4 ^ ymm3))
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; AVX512-NEXT:    vpshufb %xmm5, %xmm3, %xmm4
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm9[5,6,7]
@@ -2051,7 +2051,7 @@ define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm3
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm4
 ; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm5
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ~ymm3 & ymm4) | (~ymm5 & ymm3 & ymm4) | (ymm5 & ymm3 & ~ymm4) | (ymm5 & ymm3 & ymm4)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm4 ^ (ymm5 & (ymm3 ^ ymm4))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm5[0,6,12],zero,zero,zero,xmm5[4,10],zero,zero,zero,xmm5[u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm5, %xmm6
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm6[2,8,14],zero,zero,xmm6[0,6,12,u,u,u,u,u]
@@ -2062,34 +2062,34 @@ define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm9 = xmm1[u,u,u,u,u,u,u,u,u,u,u,2,8,14],zero,zero
 ; AVX512-FCP-NEXT:    vpor %xmm8, %xmm9, %xmm8
 ; AVX512-FCP-NEXT:    vpmovsxdq {{.*#+}} xmm9 = [18446744073709551615,16777215]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} xmm8 = (~xmm8 & xmm9 & xmm7) | (xmm8 & ~xmm9 & ~xmm7) | (xmm8 & ~xmm9 & xmm7) | (xmm8 & xmm9 & xmm7)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} xmm8 = xmm8 ^ (xmm9 & (xmm8 ^ xmm7))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[1,7,13],zero,zero,zero,xmm5[5,11],zero,zero,zero,xmm5[u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm6[3,9,15],zero,zero,xmm6[1,7,13,u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpor %xmm5, %xmm6, %xmm5
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm2[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm2[5,11]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm7 = xmm1[u,u,u,u,u,u,u,u,u,u,u,3,9,15],zero,zero
 ; AVX512-FCP-NEXT:    vpor %xmm6, %xmm7, %xmm6
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} xmm6 = (~xmm6 & xmm9 & xmm5) | (xmm6 & ~xmm9 & ~xmm5) | (xmm6 & ~xmm9 & xmm5) | (xmm6 & xmm9 & xmm5)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} xmm6 = xmm6 ^ (xmm9 & (xmm6 ^ xmm5))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm1[u,u,u,u,u,u,u,u,u,u,u,4,10],zero,zero,zero
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm7 = xmm2[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm2[0,6,12]
 ; AVX512-FCP-NEXT:    vpor %xmm5, %xmm7, %xmm5
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm7 = (~ymm7 & ~ymm4 & ymm3) | (~ymm7 & ymm4 & ymm3) | (ymm7 & ymm4 & ~ymm3) | (ymm7 & ymm4 & ymm3)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm7 = ymm3 ^ (ymm7 & (ymm4 ^ ymm3))
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm7, %xmm10
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = zero,zero,zero,xmm10[4,10],zero,zero,zero,xmm10[2,8,14,u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = xmm7[2,8,14],zero,zero,xmm7[0,6,12],zero,zero,zero,xmm7[u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpor %xmm11, %xmm12, %xmm11
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} xmm11 = (~xmm11 & ~xmm9 & xmm5) | (xmm11 & ~xmm9 & xmm5) | (xmm11 & xmm9 & ~xmm5) | (xmm11 & xmm9 & xmm5)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} xmm11 = xmm5 ^ (xmm9 & (xmm11 ^ xmm5))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm1[u,u,u,u,u,u,u,u,u,u,u,5,11],zero,zero,zero
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = xmm2[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm2[1,7,13]
 ; AVX512-FCP-NEXT:    vpor %xmm5, %xmm12, %xmm5
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,zero,xmm10[5,11],zero,zero,zero,xmm10[3,9,15,u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[3,9,15],zero,zero,xmm7[1,7,13],zero,zero,zero,xmm7[u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpor %xmm7, %xmm10, %xmm7
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} xmm7 = (~xmm7 & ~xmm9 & xmm5) | (xmm7 & ~xmm9 & xmm5) | (xmm7 & xmm9 & ~xmm5) | (xmm7 & xmm9 & xmm5)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} xmm7 = xmm5 ^ (xmm9 & (xmm7 ^ xmm5))
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [128,128,0,6,12,128,128,128,4,10,128,128,128,2,8,14]
 ; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm2, %xmm9
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ~ymm4 & ymm3) | (~ymm0 & ymm4 & ymm3) | (ymm0 & ymm4 & ~ymm3) | (ymm0 & ymm4 & ymm3)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm3 ^ (ymm0 & (ymm4 ^ ymm3))
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm3, %xmm4
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm9[5,6,7]
@@ -2123,7 +2123,7 @@ define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm3
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm4
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm5
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ~ymm3 & ymm4) | (~ymm5 & ymm3 & ymm4) | (ymm5 & ymm3 & ~ymm4) | (ymm5 & ymm3 & ymm4)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm4 ^ (ymm5 & (ymm3 ^ ymm4))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm1 = xmm5[0,6,12],zero,zero,zero,xmm5[4,10],zero,zero,zero,xmm5[u,u,u,u,u]
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm5, %xmm6
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm6[2,8,14],zero,zero,xmm6[0,6,12,u,u,u,u,u]
@@ -2134,34 +2134,34 @@ define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm9 = xmm1[u,u,u,u,u,u,u,u,u,u,u,2,8,14],zero,zero
 ; AVX512DQ-NEXT:    vpor %xmm8, %xmm9, %xmm8
 ; AVX512DQ-NEXT:    vpmovsxdq {{.*#+}} xmm9 = [18446744073709551615,16777215]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} xmm8 = (~xmm8 & xmm9 & xmm7) | (xmm8 & ~xmm9 & ~xmm7) | (xmm8 & ~xmm9 & xmm7) | (xmm8 & xmm9 & xmm7)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} xmm8 = xmm8 ^ (xmm9 & (xmm8 ^ xmm7))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[1,7,13],zero,zero,zero,xmm5[5,11],zero,zero,zero,xmm5[u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm6[3,9,15],zero,zero,xmm6[1,7,13,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpor %xmm5, %xmm6, %xmm5
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm6 = xmm2[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm2[5,11]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm7 = xmm1[u,u,u,u,u,u,u,u,u,u,u,3,9,15],zero,zero
 ; AVX512DQ-NEXT:    vpor %xmm6, %xmm7, %xmm6
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} xmm6 = (~xmm6 & xmm9 & xmm5) | (xmm6 & ~xmm9 & ~xmm5) | (xmm6 & ~xmm9 & xmm5) | (xmm6 & xmm9 & xmm5)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} xmm6 = xmm6 ^ (xmm9 & (xmm6 ^ xmm5))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm5 = xmm1[u,u,u,u,u,u,u,u,u,u,u,4,10],zero,zero,zero
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm7 = xmm2[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm2[0,6,12]
 ; AVX512DQ-NEXT:    vpor %xmm5, %xmm7, %xmm5
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm7 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm7 = (~ymm7 & ~ymm4 & ymm3) | (~ymm7 & ymm4 & ymm3) | (ymm7 & ymm4 & ~ymm3) | (ymm7 & ymm4 & ymm3)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm7 = ymm3 ^ (ymm7 & (ymm4 ^ ymm3))
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm7, %xmm10
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm11 = zero,zero,zero,xmm10[4,10],zero,zero,zero,xmm10[2,8,14,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm12 = xmm7[2,8,14],zero,zero,xmm7[0,6,12],zero,zero,zero,xmm7[u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpor %xmm11, %xmm12, %xmm11
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} xmm11 = (~xmm11 & ~xmm9 & xmm5) | (xmm11 & ~xmm9 & xmm5) | (xmm11 & xmm9 & ~xmm5) | (xmm11 & xmm9 & xmm5)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} xmm11 = xmm5 ^ (xmm9 & (xmm11 ^ xmm5))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm5 = xmm1[u,u,u,u,u,u,u,u,u,u,u,5,11],zero,zero,zero
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm12 = xmm2[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm2[1,7,13]
 ; AVX512DQ-NEXT:    vpor %xmm5, %xmm12, %xmm5
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,zero,xmm10[5,11],zero,zero,zero,xmm10[3,9,15,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[3,9,15],zero,zero,xmm7[1,7,13],zero,zero,zero,xmm7[u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpor %xmm7, %xmm10, %xmm7
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} xmm7 = (~xmm7 & ~xmm9 & xmm5) | (xmm7 & ~xmm9 & xmm5) | (xmm7 & xmm9 & ~xmm5) | (xmm7 & xmm9 & xmm5)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} xmm7 = xmm5 ^ (xmm9 & (xmm7 ^ xmm5))
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm5 = [128,128,0,6,12,128,128,128,4,10,128,128,128,2,8,14]
 ; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm2, %xmm9
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ~ymm4 & ymm3) | (~ymm0 & ymm4 & ymm3) | (ymm0 & ymm4 & ~ymm3) | (ymm0 & ymm4 & ymm3)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm3 ^ (ymm0 & (ymm4 ^ ymm3))
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm3, %xmm4
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm9[5,6,7]
@@ -2195,7 +2195,7 @@ define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm3
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm4
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm5
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ~ymm3 & ymm4) | (~ymm5 & ymm3 & ymm4) | (ymm5 & ymm3 & ~ymm4) | (ymm5 & ymm3 & ymm4)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm4 ^ (ymm5 & (ymm3 ^ ymm4))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm5[0,6,12],zero,zero,zero,xmm5[4,10],zero,zero,zero,xmm5[u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm5, %xmm6
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm6[2,8,14],zero,zero,xmm6[0,6,12,u,u,u,u,u]
@@ -2206,34 +2206,34 @@ define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm9 = xmm1[u,u,u,u,u,u,u,u,u,u,u,2,8,14],zero,zero
 ; AVX512DQ-FCP-NEXT:    vpor %xmm8, %xmm9, %xmm8
 ; AVX512DQ-FCP-NEXT:    vpmovsxdq {{.*#+}} xmm9 = [18446744073709551615,16777215]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} xmm8 = (~xmm8 & xmm9 & xmm7) | (xmm8 & ~xmm9 & ~xmm7) | (xmm8 & ~xmm9 & xmm7) | (xmm8 & xmm9 & xmm7)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} xmm8 = xmm8 ^ (xmm9 & (xmm8 ^ xmm7))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[1,7,13],zero,zero,zero,xmm5[5,11],zero,zero,zero,xmm5[u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm6[3,9,15],zero,zero,xmm6[1,7,13,u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpor %xmm5, %xmm6, %xmm5
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm2[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm2[5,11]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm7 = xmm1[u,u,u,u,u,u,u,u,u,u,u,3,9,15],zero,zero
 ; AVX512DQ-FCP-NEXT:    vpor %xmm6, %xmm7, %xmm6
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} xmm6 = (~xmm6 & xmm9 & xmm5) | (xmm6 & ~xmm9 & ~xmm5) | (xmm6 & ~xmm9 & xmm5) | (xmm6 & xmm9 & xmm5)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} xmm6 = xmm6 ^ (xmm9 & (xmm6 ^ xmm5))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm1[u,u,u,u,u,u,u,u,u,u,u,4,10],zero,zero,zero
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm7 = xmm2[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm2[0,6,12]
 ; AVX512DQ-FCP-NEXT:    vpor %xmm5, %xmm7, %xmm5
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm7 = (~ymm7 & ~ymm4 & ymm3) | (~ymm7 & ymm4 & ymm3) | (ymm7 & ymm4 & ~ymm3) | (ymm7 & ymm4 & ymm3)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm7 = ymm3 ^ (ymm7 & (ymm4 ^ ymm3))
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm7, %xmm10
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = zero,zero,zero,xmm10[4,10],zero,zero,zero,xmm10[2,8,14,u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = xmm7[2,8,14],zero,zero,xmm7[0,6,12],zero,zero,zero,xmm7[u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpor %xmm11, %xmm12, %xmm11
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} xmm11 = (~xmm11 & ~xmm9 & xmm5) | (xmm11 & ~xmm9 & xmm5) | (xmm11 & xmm9 & ~xmm5) | (xmm11 & xmm9 & xmm5)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} xmm11 = xmm5 ^ (xmm9 & (xmm11 ^ xmm5))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm1[u,u,u,u,u,u,u,u,u,u,u,5,11],zero,zero,zero
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = xmm2[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm2[1,7,13]
 ; AVX512DQ-FCP-NEXT:    vpor %xmm5, %xmm12, %xmm5
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,zero,xmm10[5,11],zero,zero,zero,xmm10[3,9,15,u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[3,9,15],zero,zero,xmm7[1,7,13],zero,zero,zero,xmm7[u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpor %xmm7, %xmm10, %xmm7
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} xmm7 = (~xmm7 & ~xmm9 & xmm5) | (xmm7 & ~xmm9 & xmm5) | (xmm7 & xmm9 & ~xmm5) | (xmm7 & xmm9 & xmm5)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} xmm7 = xmm5 ^ (xmm9 & (xmm7 ^ xmm5))
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [128,128,0,6,12,128,128,128,4,10,128,128,128,2,8,14]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm2, %xmm9
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ~ymm4 & ymm3) | (~ymm0 & ymm4 & ymm3) | (ymm0 & ymm4 & ~ymm3) | (ymm0 & ymm4 & ymm3)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm3 ^ (ymm0 & (ymm4 ^ ymm3))
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm3, %xmm4
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm9[5,6,7]
@@ -3775,7 +3775,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vmovdqa 64(%rdi), %ymm1
 ; AVX512-NEXT:    vmovdqa 128(%rdi), %ymm6
 ; AVX512-NEXT:    vmovdqa %ymm0, %ymm7
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm7 = (~ymm7 & ~ymm17 & ymm3) | (~ymm7 & ymm17 & ymm3) | (ymm7 & ymm17 & ~ymm3) | (ymm7 & ymm17 & ymm3)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm7 = ymm3 ^ (ymm7 & (ymm17 ^ ymm3))
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm4 = xmm7[0,6,12],zero,zero,zero,xmm7[4,10],zero,zero,zero,xmm7[u,u,u,u,u]
 ; AVX512-NEXT:    vextracti128 $1, %ymm7, %xmm8
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm8[2,8,14],zero,zero,xmm8[0,6,12,u,u,u,u,u]
@@ -3784,13 +3784,13 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vinserti128 $1, 96(%rdi), %ymm1, %ymm1
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,0]
 ; AVX512-NEXT:    vmovdqa %ymm9, %ymm10
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm10 = (~ymm10 & ~ymm1 & ymm5) | (~ymm10 & ymm1 & ymm5) | (ymm10 & ymm1 & ~ymm5) | (ymm10 & ymm1 & ymm5)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm5 ^ (ymm10 & (ymm1 ^ ymm5))
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ymm4 & ymm16) | (ymm11 & ~ymm4 & ~ymm16) | (ymm11 & ~ymm4 & ymm16) | (ymm11 & ymm4 & ~ymm16) | (ymm11 & ymm4 & ymm16)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm4 & ymm16)
 ; AVX512-NEXT:    vmovdqa 160(%rdi), %ymm13
 ; AVX512-NEXT:    vmovdqa %ymm0, %ymm14
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm14 = (~ymm14 & ~ymm13 & ymm6) | (~ymm14 & ymm13 & ymm6) | (ymm14 & ymm13 & ~ymm6) | (ymm14 & ymm13 & ymm6)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm14 = ymm6 ^ (ymm14 & (ymm13 ^ ymm6))
 ; AVX512-NEXT:    vextracti128 $1, %ymm14, %xmm15
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm4 = xmm15[u,u,u,u,u,u],zero,zero,xmm15[0,6,12],zero,zero,zero,xmm15[4,10]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm12 = xmm14[u,u,u,u,u,u,4,10],zero,zero,zero,xmm14[2,8,14],zero,zero
@@ -3803,7 +3803,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm8 = zero,zero,zero,xmm8[3,9,15],zero,zero,xmm8[1,7,13,u,u,u,u,u]
 ; AVX512-NEXT:    vpor %xmm7, %xmm8, %xmm7
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm8 = (~ymm8 & ymm7 & ymm16) | (ymm8 & ~ymm7 & ~ymm16) | (ymm8 & ~ymm7 & ymm16) | (ymm8 & ymm7 & ~ymm16) | (ymm8 & ymm7 & ymm16)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm7 & ymm16)
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm7 = xmm15[u,u,u,u,u,u],zero,zero,xmm15[1,7,13],zero,zero,zero,xmm15[5,11]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm10 = xmm14[u,u,u,u,u,u,5,11],zero,zero,zero,xmm14[3,9,15],zero,zero
 ; AVX512-NEXT:    vpor %xmm7, %xmm10, %xmm7
@@ -3812,50 +3812,50 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm8 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
 ; AVX512-NEXT:    vmovdqa %ymm8, %ymm10
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm10 = (~ymm10 & ~ymm3 & ymm17) | (~ymm10 & ymm3 & ymm17) | (ymm10 & ymm3 & ~ymm17) | (ymm10 & ymm3 & ymm17)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm17 ^ (ymm10 & (ymm3 ^ ymm17))
 ; AVX512-NEXT:    vextracti128 $1, %ymm10, %xmm11
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm12 = zero,zero,zero,xmm11[4,10],zero,zero,zero,xmm11[2,8,14,u,u,u,u,u]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm14 = xmm10[2,8,14],zero,zero,xmm10[0,6,12],zero,zero,zero,xmm10[u,u,u,u,u]
 ; AVX512-NEXT:    vpor %xmm12, %xmm14, %xmm12
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm9 = (~ymm9 & ~ymm5 & ymm1) | (~ymm9 & ymm5 & ymm1) | (ymm9 & ymm5 & ~ymm1) | (ymm9 & ymm5 & ymm1)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm9 = ymm1 ^ (ymm9 & (ymm5 ^ ymm1))
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[4,10,0,6,12,18,24,30,20,26,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm14 = (~ymm14 & ymm12 & ymm16) | (ymm14 & ~ymm12 & ~ymm16) | (ymm14 & ~ymm12 & ymm16) | (ymm14 & ymm12 & ~ymm16) | (ymm14 & ymm12 & ymm16)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm12 & ymm16)
 ; AVX512-NEXT:    vmovdqa %ymm0, %ymm12
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm12 = (~ymm12 & ~ymm6 & ymm13) | (~ymm12 & ymm6 & ymm13) | (ymm12 & ymm6 & ~ymm13) | (ymm12 & ymm6 & ymm13)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm12 = ymm13 ^ (ymm12 & (ymm6 ^ ymm13))
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm15 = xmm12[u,u,u,u,u,0,6,12],zero,zero,zero,xmm12[4,10],zero,zero,zero
 ; AVX512-NEXT:    vextracti128 $1, %ymm12, %xmm2
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm4 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[2,8,14],zero,zero,xmm2[0,6,12]
 ; AVX512-NEXT:    vpor %xmm4, %xmm15, %xmm4
 ; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512-NEXT:    vpmovsxwd {{.*#+}} ymm15 = [4294967295,4294967295,4294967295,4294967295,4294967295,255,0,0]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm4 = (~ymm4 & ymm15 & ymm14) | (ymm4 & ~ymm15 & ~ymm14) | (ymm4 & ~ymm15 & ymm14) | (ymm4 & ymm15 & ymm14)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm4 = ymm4 ^ (ymm15 & (ymm4 ^ ymm14))
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm11 = zero,zero,zero,xmm11[5,11],zero,zero,zero,xmm11[3,9,15,u,u,u,u,u]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[3,9,15],zero,zero,xmm10[1,7,13],zero,zero,zero,xmm10[u,u,u,u,u]
 ; AVX512-NEXT:    vpor %xmm11, %xmm10, %xmm10
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[5,11,1,7,13,19,25,31,21,27,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm9 = (~ymm9 & ymm10 & ymm16) | (ymm9 & ~ymm10 & ~ymm16) | (ymm9 & ~ymm10 & ymm16) | (ymm9 & ymm10 & ~ymm16) | (ymm9 & ymm10 & ymm16)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm10 & ymm16)
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm10 = xmm12[u,u,u,u,u,1,7,13],zero,zero,zero,xmm12[5,11],zero,zero,zero
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[3,9,15],zero,zero,xmm2[1,7,13]
 ; AVX512-NEXT:    vpor %xmm2, %xmm10, %xmm2
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ymm15 & ymm9) | (ymm2 & ~ymm15 & ~ymm9) | (ymm2 & ~ymm15 & ymm9) | (ymm2 & ymm15 & ymm9)
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm8 = (~ymm8 & ~ymm13 & ymm6) | (~ymm8 & ymm13 & ymm6) | (ymm8 & ymm13 & ~ymm6) | (ymm8 & ymm13 & ymm6)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm2 ^ (ymm15 & (ymm2 ^ ymm9))
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm8 = ymm6 ^ (ymm8 & (ymm13 ^ ymm6))
 ; AVX512-NEXT:    vextracti128 $1, %ymm8, %xmm6
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm9 = xmm6[u,u,u,u,u],zero,zero,zero,xmm6[4,10],zero,zero,zero,xmm6[2,8,14]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm10 = xmm8[u,u,u,u,u,2,8,14],zero,zero,xmm8[0,6,12],zero,zero,zero
 ; AVX512-NEXT:    vpor %xmm9, %xmm10, %xmm9
 ; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ~ymm3 & ymm17) | (~ymm0 & ymm3 & ymm17) | (ymm0 & ymm3 & ~ymm17) | (ymm0 & ymm3 & ymm17)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm17 ^ (ymm0 & (ymm3 ^ ymm17))
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,xmm3[0,6,12],zero,zero,zero,xmm3[4,10,u,u,u,u,u,u]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm11 = xmm0[4,10],zero,zero,zero,xmm0[2,8,14],zero,zero,xmm0[u,u,u,u,u,u]
 ; AVX512-NEXT:    vpor %xmm10, %xmm11, %xmm10
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ymm5 & mem) | (ymm1 & ~ymm5 & ~mem) | (ymm1 & ymm5 & ~mem) | (ymm1 & ymm5 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm5))
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm5 = ymm1[u,u,u,u,u,u,u,u,u,u,0,6,12,2,8,14,20,26,16,22,28,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm10 = xmm10[0,1,2,3,4],xmm5[5,6,7]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3],ymm5[4,5,6,7]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ~ymm15 & ymm9) | (ymm5 & ~ymm15 & ymm9) | (ymm5 & ymm15 & ~ymm9) | (ymm5 & ymm15 & ymm9)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm9 ^ (ymm15 & (ymm5 ^ ymm9))
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u],zero,zero,zero,xmm6[5,11],zero,zero,zero,xmm6[3,9,15]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[u,u,u,u,u,3,9,15],zero,zero,xmm8[1,7,13],zero,zero,zero
 ; AVX512-NEXT:    vpor %xmm6, %xmm8, %xmm6
@@ -3866,7 +3866,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,1,7,13,3,9,15,21,27,17,23,29,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm1[5,6,7]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ~ymm15 & ymm6) | (ymm0 & ~ymm15 & ymm6) | (ymm0 & ymm15 & ~ymm6) | (ymm0 & ymm15 & ymm6)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm6 ^ (ymm15 & (ymm0 ^ ymm6))
 ; AVX512-NEXT:    vmovdqa64 %ymm18, (%rsi)
 ; AVX512-NEXT:    vmovdqa %ymm7, (%rdx)
 ; AVX512-NEXT:    vmovdqa %ymm4, (%rcx)
@@ -3885,7 +3885,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %ymm1
 ; AVX512-FCP-NEXT:    vmovdqa 128(%rdi), %ymm6
 ; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm7
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm7 = (~ymm7 & ~ymm17 & ymm3) | (~ymm7 & ymm17 & ymm3) | (ymm7 & ymm17 & ~ymm3) | (ymm7 & ymm17 & ymm3)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm7 = ymm3 ^ (ymm7 & (ymm17 ^ ymm3))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm7[0,6,12],zero,zero,zero,xmm7[4,10],zero,zero,zero,xmm7[u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm7, %xmm8
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm8[2,8,14],zero,zero,xmm8[0,6,12,u,u,u,u,u]
@@ -3894,13 +3894,13 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vinserti128 $1, 96(%rdi), %ymm1, %ymm1
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,0]
 ; AVX512-FCP-NEXT:    vmovdqa %ymm9, %ymm10
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = (~ymm10 & ~ymm1 & ymm5) | (~ymm10 & ymm1 & ymm5) | (ymm10 & ymm1 & ~ymm5) | (ymm10 & ymm1 & ymm5)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm5 ^ (ymm10 & (ymm1 ^ ymm5))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ymm4 & ymm16) | (ymm11 & ~ymm4 & ~ymm16) | (ymm11 & ~ymm4 & ymm16) | (ymm11 & ymm4 & ~ymm16) | (ymm11 & ymm4 & ymm16)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm4 & ymm16)
 ; AVX512-FCP-NEXT:    vmovdqa 160(%rdi), %ymm13
 ; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm14
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm14 = (~ymm14 & ~ymm13 & ymm6) | (~ymm14 & ymm13 & ymm6) | (ymm14 & ymm13 & ~ymm6) | (ymm14 & ymm13 & ymm6)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm14 = ymm6 ^ (ymm14 & (ymm13 ^ ymm6))
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm14, %xmm15
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm15[u,u,u,u,u,u],zero,zero,xmm15[0,6,12],zero,zero,zero,xmm15[4,10]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = xmm14[u,u,u,u,u,u,4,10],zero,zero,zero,xmm14[2,8,14],zero,zero
@@ -3913,7 +3913,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm8 = zero,zero,zero,xmm8[3,9,15],zero,zero,xmm8[1,7,13,u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpor %xmm7, %xmm8, %xmm7
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm8 = (~ymm8 & ymm7 & ymm16) | (ymm8 & ~ymm7 & ~ymm16) | (ymm8 & ~ymm7 & ymm16) | (ymm8 & ymm7 & ~ymm16) | (ymm8 & ymm7 & ymm16)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm7 & ymm16)
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm7 = xmm15[u,u,u,u,u,u],zero,zero,xmm15[1,7,13],zero,zero,zero,xmm15[5,11]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm14[u,u,u,u,u,u,5,11],zero,zero,zero,xmm14[3,9,15],zero,zero
 ; AVX512-FCP-NEXT:    vpor %xmm7, %xmm10, %xmm7
@@ -3922,50 +3922,50 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm8 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
 ; AVX512-FCP-NEXT:    vmovdqa %ymm8, %ymm10
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = (~ymm10 & ~ymm3 & ymm17) | (~ymm10 & ymm3 & ymm17) | (ymm10 & ymm3 & ~ymm17) | (ymm10 & ymm3 & ymm17)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm17 ^ (ymm10 & (ymm3 ^ ymm17))
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm10, %xmm11
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = zero,zero,zero,xmm11[4,10],zero,zero,zero,xmm11[2,8,14,u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm10[2,8,14],zero,zero,xmm10[0,6,12],zero,zero,zero,xmm10[u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpor %xmm12, %xmm14, %xmm12
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm9 = (~ymm9 & ~ymm5 & ymm1) | (~ymm9 & ymm5 & ymm1) | (ymm9 & ymm5 & ~ymm1) | (ymm9 & ymm5 & ymm1)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm9 = ymm1 ^ (ymm9 & (ymm5 ^ ymm1))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[4,10,0,6,12,18,24,30,20,26,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm14 = (~ymm14 & ymm12 & ymm16) | (ymm14 & ~ymm12 & ~ymm16) | (ymm14 & ~ymm12 & ymm16) | (ymm14 & ymm12 & ~ymm16) | (ymm14 & ymm12 & ymm16)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm12 & ymm16)
 ; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm12
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm12 = (~ymm12 & ~ymm6 & ymm13) | (~ymm12 & ymm6 & ymm13) | (ymm12 & ymm6 & ~ymm13) | (ymm12 & ymm6 & ymm13)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm12 = ymm13 ^ (ymm12 & (ymm6 ^ ymm13))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm12[u,u,u,u,u,0,6,12],zero,zero,zero,xmm12[4,10],zero,zero,zero
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm12, %xmm2
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[2,8,14],zero,zero,xmm2[0,6,12]
 ; AVX512-FCP-NEXT:    vpor %xmm4, %xmm15, %xmm4
 ; AVX512-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512-FCP-NEXT:    vpmovsxwd {{.*#+}} ymm15 = [4294967295,4294967295,4294967295,4294967295,4294967295,255,0,0]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = (~ymm4 & ymm15 & ymm14) | (ymm4 & ~ymm15 & ~ymm14) | (ymm4 & ~ymm15 & ymm14) | (ymm4 & ymm15 & ymm14)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = ymm4 ^ (ymm15 & (ymm4 ^ ymm14))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = zero,zero,zero,xmm11[5,11],zero,zero,zero,xmm11[3,9,15,u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[3,9,15],zero,zero,xmm10[1,7,13],zero,zero,zero,xmm10[u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpor %xmm11, %xmm10, %xmm10
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[5,11,1,7,13,19,25,31,21,27,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm9 = (~ymm9 & ymm10 & ymm16) | (ymm9 & ~ymm10 & ~ymm16) | (ymm9 & ~ymm10 & ymm16) | (ymm9 & ymm10 & ~ymm16) | (ymm9 & ymm10 & ymm16)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm10 & ymm16)
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm12[u,u,u,u,u,1,7,13],zero,zero,zero,xmm12[5,11],zero,zero,zero
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[3,9,15],zero,zero,xmm2[1,7,13]
 ; AVX512-FCP-NEXT:    vpor %xmm2, %xmm10, %xmm2
 ; AVX512-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ymm15 & ymm9) | (ymm2 & ~ymm15 & ~ymm9) | (ymm2 & ~ymm15 & ymm9) | (ymm2 & ymm15 & ymm9)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm8 = (~ymm8 & ~ymm13 & ymm6) | (~ymm8 & ymm13 & ymm6) | (ymm8 & ymm13 & ~ymm6) | (ymm8 & ymm13 & ymm6)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm2 ^ (ymm15 & (ymm2 ^ ymm9))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm8 = ymm6 ^ (ymm8 & (ymm13 ^ ymm6))
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm8, %xmm6
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm9 = xmm6[u,u,u,u,u],zero,zero,zero,xmm6[4,10],zero,zero,zero,xmm6[2,8,14]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm8[u,u,u,u,u,2,8,14],zero,zero,xmm8[0,6,12],zero,zero,zero
 ; AVX512-FCP-NEXT:    vpor %xmm9, %xmm10, %xmm9
 ; AVX512-FCP-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ~ymm3 & ymm17) | (~ymm0 & ymm3 & ymm17) | (ymm0 & ymm3 & ~ymm17) | (ymm0 & ymm3 & ymm17)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm17 ^ (ymm0 & (ymm3 ^ ymm17))
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,xmm3[0,6,12],zero,zero,zero,xmm3[4,10,u,u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm0[4,10],zero,zero,zero,xmm0[2,8,14],zero,zero,xmm0[u,u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpor %xmm10, %xmm11, %xmm10
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ymm5 & mem) | (ymm1 & ~ymm5 & ~mem) | (ymm1 & ymm5 & ~mem) | (ymm1 & ymm5 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm5))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm1[u,u,u,u,u,u,u,u,u,u,0,6,12,2,8,14,20,26,16,22,28,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm10 = xmm10[0,1,2,3,4],xmm5[5,6,7]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3],ymm5[4,5,6,7]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ~ymm15 & ymm9) | (ymm5 & ~ymm15 & ymm9) | (ymm5 & ymm15 & ~ymm9) | (ymm5 & ymm15 & ymm9)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm9 ^ (ymm15 & (ymm5 ^ ymm9))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u],zero,zero,zero,xmm6[5,11],zero,zero,zero,xmm6[3,9,15]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[u,u,u,u,u,3,9,15],zero,zero,xmm8[1,7,13],zero,zero,zero
 ; AVX512-FCP-NEXT:    vpor %xmm6, %xmm8, %xmm6
@@ -3976,7 +3976,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,1,7,13,3,9,15,21,27,17,23,29,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm1[5,6,7]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ~ymm15 & ymm6) | (ymm0 & ~ymm15 & ymm6) | (ymm0 & ymm15 & ~ymm6) | (ymm0 & ymm15 & ymm6)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm6 ^ (ymm15 & (ymm0 ^ ymm6))
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm18, (%rsi)
 ; AVX512-FCP-NEXT:    vmovdqa %ymm7, (%rdx)
 ; AVX512-FCP-NEXT:    vmovdqa %ymm4, (%rcx)
@@ -3995,7 +3995,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %ymm1
 ; AVX512DQ-NEXT:    vmovdqa 128(%rdi), %ymm6
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm7
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm7 = (~ymm7 & ~ymm17 & ymm3) | (~ymm7 & ymm17 & ymm3) | (ymm7 & ymm17 & ~ymm3) | (ymm7 & ymm17 & ymm3)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm7 = ymm3 ^ (ymm7 & (ymm17 ^ ymm3))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm4 = xmm7[0,6,12],zero,zero,zero,xmm7[4,10],zero,zero,zero,xmm7[u,u,u,u,u]
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm7, %xmm8
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm8[2,8,14],zero,zero,xmm8[0,6,12,u,u,u,u,u]
@@ -4004,13 +4004,13 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vinserti128 $1, 96(%rdi), %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,0]
 ; AVX512DQ-NEXT:    vmovdqa %ymm9, %ymm10
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm10 = (~ymm10 & ~ymm1 & ymm5) | (~ymm10 & ymm1 & ymm5) | (ymm10 & ymm1 & ~ymm5) | (ymm10 & ymm1 & ymm5)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm5 ^ (ymm10 & (ymm1 ^ ymm5))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ymm4 & ymm16) | (ymm11 & ~ymm4 & ~ymm16) | (ymm11 & ~ymm4 & ymm16) | (ymm11 & ymm4 & ~ymm16) | (ymm11 & ymm4 & ymm16)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm4 & ymm16)
 ; AVX512DQ-NEXT:    vmovdqa 160(%rdi), %ymm13
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm14
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm14 = (~ymm14 & ~ymm13 & ymm6) | (~ymm14 & ymm13 & ymm6) | (ymm14 & ymm13 & ~ymm6) | (ymm14 & ymm13 & ymm6)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm14 = ymm6 ^ (ymm14 & (ymm13 ^ ymm6))
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm14, %xmm15
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm4 = xmm15[u,u,u,u,u,u],zero,zero,xmm15[0,6,12],zero,zero,zero,xmm15[4,10]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm12 = xmm14[u,u,u,u,u,u,4,10],zero,zero,zero,xmm14[2,8,14],zero,zero
@@ -4023,7 +4023,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm8 = zero,zero,zero,xmm8[3,9,15],zero,zero,xmm8[1,7,13,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpor %xmm7, %xmm8, %xmm7
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm8 = (~ymm8 & ymm7 & ymm16) | (ymm8 & ~ymm7 & ~ymm16) | (ymm8 & ~ymm7 & ymm16) | (ymm8 & ymm7 & ~ymm16) | (ymm8 & ymm7 & ymm16)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm7 & ymm16)
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm7 = xmm15[u,u,u,u,u,u],zero,zero,xmm15[1,7,13],zero,zero,zero,xmm15[5,11]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm10 = xmm14[u,u,u,u,u,u,5,11],zero,zero,zero,xmm14[3,9,15],zero,zero
 ; AVX512DQ-NEXT:    vpor %xmm7, %xmm10, %xmm7
@@ -4032,50 +4032,50 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm8 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
 ; AVX512DQ-NEXT:    vmovdqa %ymm8, %ymm10
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm10 = (~ymm10 & ~ymm3 & ymm17) | (~ymm10 & ymm3 & ymm17) | (ymm10 & ymm3 & ~ymm17) | (ymm10 & ymm3 & ymm17)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm17 ^ (ymm10 & (ymm3 ^ ymm17))
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm10, %xmm11
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm12 = zero,zero,zero,xmm11[4,10],zero,zero,zero,xmm11[2,8,14,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm14 = xmm10[2,8,14],zero,zero,xmm10[0,6,12],zero,zero,zero,xmm10[u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpor %xmm12, %xmm14, %xmm12
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm9 = (~ymm9 & ~ymm5 & ymm1) | (~ymm9 & ymm5 & ymm1) | (ymm9 & ymm5 & ~ymm1) | (ymm9 & ymm5 & ymm1)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm9 = ymm1 ^ (ymm9 & (ymm5 ^ ymm1))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[4,10,0,6,12,18,24,30,20,26,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm14 = (~ymm14 & ymm12 & ymm16) | (ymm14 & ~ymm12 & ~ymm16) | (ymm14 & ~ymm12 & ymm16) | (ymm14 & ymm12 & ~ymm16) | (ymm14 & ymm12 & ymm16)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm12 & ymm16)
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm12
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm12 = (~ymm12 & ~ymm6 & ymm13) | (~ymm12 & ymm6 & ymm13) | (ymm12 & ymm6 & ~ymm13) | (ymm12 & ymm6 & ymm13)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm12 = ymm13 ^ (ymm12 & (ymm6 ^ ymm13))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm15 = xmm12[u,u,u,u,u,0,6,12],zero,zero,zero,xmm12[4,10],zero,zero,zero
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm12, %xmm2
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm4 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[2,8,14],zero,zero,xmm2[0,6,12]
 ; AVX512DQ-NEXT:    vpor %xmm4, %xmm15, %xmm4
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQ-NEXT:    vpmovsxwd {{.*#+}} ymm15 = [4294967295,4294967295,4294967295,4294967295,4294967295,255,0,0]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm4 = (~ymm4 & ymm15 & ymm14) | (ymm4 & ~ymm15 & ~ymm14) | (ymm4 & ~ymm15 & ymm14) | (ymm4 & ymm15 & ymm14)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm4 = ymm4 ^ (ymm15 & (ymm4 ^ ymm14))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm11 = zero,zero,zero,xmm11[5,11],zero,zero,zero,xmm11[3,9,15,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[3,9,15],zero,zero,xmm10[1,7,13],zero,zero,zero,xmm10[u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpor %xmm11, %xmm10, %xmm10
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[5,11,1,7,13,19,25,31,21,27,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm9 = (~ymm9 & ymm10 & ymm16) | (ymm9 & ~ymm10 & ~ymm16) | (ymm9 & ~ymm10 & ymm16) | (ymm9 & ymm10 & ~ymm16) | (ymm9 & ymm10 & ymm16)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm10 & ymm16)
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm10 = xmm12[u,u,u,u,u,1,7,13],zero,zero,zero,xmm12[5,11],zero,zero,zero
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[3,9,15],zero,zero,xmm2[1,7,13]
 ; AVX512DQ-NEXT:    vpor %xmm2, %xmm10, %xmm2
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ymm15 & ymm9) | (ymm2 & ~ymm15 & ~ymm9) | (ymm2 & ~ymm15 & ymm9) | (ymm2 & ymm15 & ymm9)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm8 = (~ymm8 & ~ymm13 & ymm6) | (~ymm8 & ymm13 & ymm6) | (ymm8 & ymm13 & ~ymm6) | (ymm8 & ymm13 & ymm6)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm2 ^ (ymm15 & (ymm2 ^ ymm9))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm8 = ymm6 ^ (ymm8 & (ymm13 ^ ymm6))
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm8, %xmm6
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm9 = xmm6[u,u,u,u,u],zero,zero,zero,xmm6[4,10],zero,zero,zero,xmm6[2,8,14]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm10 = xmm8[u,u,u,u,u,2,8,14],zero,zero,xmm8[0,6,12],zero,zero,zero
 ; AVX512DQ-NEXT:    vpor %xmm9, %xmm10, %xmm9
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ~ymm3 & ymm17) | (~ymm0 & ymm3 & ymm17) | (ymm0 & ymm3 & ~ymm17) | (ymm0 & ymm3 & ymm17)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm17 ^ (ymm0 & (ymm3 ^ ymm17))
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,xmm3[0,6,12],zero,zero,zero,xmm3[4,10,u,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm11 = xmm0[4,10],zero,zero,zero,xmm0[2,8,14],zero,zero,xmm0[u,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpor %xmm10, %xmm11, %xmm10
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ymm5 & mem) | (ymm1 & ~ymm5 & ~mem) | (ymm1 & ymm5 & ~mem) | (ymm1 & ymm5 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm5))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm5 = ymm1[u,u,u,u,u,u,u,u,u,u,0,6,12,2,8,14,20,26,16,22,28,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm10 = xmm10[0,1,2,3,4],xmm5[5,6,7]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ~ymm15 & ymm9) | (ymm5 & ~ymm15 & ymm9) | (ymm5 & ymm15 & ~ymm9) | (ymm5 & ymm15 & ymm9)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm9 ^ (ymm15 & (ymm5 ^ ymm9))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u],zero,zero,zero,xmm6[5,11],zero,zero,zero,xmm6[3,9,15]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[u,u,u,u,u,3,9,15],zero,zero,xmm8[1,7,13],zero,zero,zero
 ; AVX512DQ-NEXT:    vpor %xmm6, %xmm8, %xmm6
@@ -4086,7 +4086,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,1,7,13,3,9,15,21,27,17,23,29,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm1[5,6,7]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ~ymm15 & ymm6) | (ymm0 & ~ymm15 & ymm6) | (ymm0 & ymm15 & ~ymm6) | (ymm0 & ymm15 & ymm6)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm6 ^ (ymm15 & (ymm0 ^ ymm6))
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm18, (%rsi)
 ; AVX512DQ-NEXT:    vmovdqa %ymm7, (%rdx)
 ; AVX512DQ-NEXT:    vmovdqa %ymm4, (%rcx)
@@ -4105,7 +4105,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %ymm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa 128(%rdi), %ymm6
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm7
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm7 = (~ymm7 & ~ymm17 & ymm3) | (~ymm7 & ymm17 & ymm3) | (ymm7 & ymm17 & ~ymm3) | (ymm7 & ymm17 & ymm3)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm7 = ymm3 ^ (ymm7 & (ymm17 ^ ymm3))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm7[0,6,12],zero,zero,zero,xmm7[4,10],zero,zero,zero,xmm7[u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm7, %xmm8
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm8[2,8,14],zero,zero,xmm8[0,6,12,u,u,u,u,u]
@@ -4114,13 +4114,13 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, 96(%rdi), %ymm1, %ymm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,0]
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm9, %ymm10
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = (~ymm10 & ~ymm1 & ymm5) | (~ymm10 & ymm1 & ymm5) | (ymm10 & ymm1 & ~ymm5) | (ymm10 & ymm1 & ymm5)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm5 ^ (ymm10 & (ymm1 ^ ymm5))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ymm4 & ymm16) | (ymm11 & ~ymm4 & ~ymm16) | (ymm11 & ~ymm4 & ymm16) | (ymm11 & ymm4 & ~ymm16) | (ymm11 & ymm4 & ymm16)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm4 & ymm16)
 ; AVX512DQ-FCP-NEXT:    vmovdqa 160(%rdi), %ymm13
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm14
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm14 = (~ymm14 & ~ymm13 & ymm6) | (~ymm14 & ymm13 & ymm6) | (ymm14 & ymm13 & ~ymm6) | (ymm14 & ymm13 & ymm6)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm14 = ymm6 ^ (ymm14 & (ymm13 ^ ymm6))
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm14, %xmm15
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm15[u,u,u,u,u,u],zero,zero,xmm15[0,6,12],zero,zero,zero,xmm15[4,10]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = xmm14[u,u,u,u,u,u,4,10],zero,zero,zero,xmm14[2,8,14],zero,zero
@@ -4133,7 +4133,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm8 = zero,zero,zero,xmm8[3,9,15],zero,zero,xmm8[1,7,13,u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpor %xmm7, %xmm8, %xmm7
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm8 = (~ymm8 & ymm7 & ymm16) | (ymm8 & ~ymm7 & ~ymm16) | (ymm8 & ~ymm7 & ymm16) | (ymm8 & ymm7 & ~ymm16) | (ymm8 & ymm7 & ymm16)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm8 = ymm8 | (ymm7 & ymm16)
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm7 = xmm15[u,u,u,u,u,u],zero,zero,xmm15[1,7,13],zero,zero,zero,xmm15[5,11]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm14[u,u,u,u,u,u,5,11],zero,zero,zero,xmm14[3,9,15],zero,zero
 ; AVX512DQ-FCP-NEXT:    vpor %xmm7, %xmm10, %xmm7
@@ -4142,50 +4142,50 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm8 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm8, %ymm10
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = (~ymm10 & ~ymm3 & ymm17) | (~ymm10 & ymm3 & ymm17) | (ymm10 & ymm3 & ~ymm17) | (ymm10 & ymm3 & ymm17)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm17 ^ (ymm10 & (ymm3 ^ ymm17))
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm10, %xmm11
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = zero,zero,zero,xmm11[4,10],zero,zero,zero,xmm11[2,8,14,u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm14 = xmm10[2,8,14],zero,zero,xmm10[0,6,12],zero,zero,zero,xmm10[u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpor %xmm12, %xmm14, %xmm12
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm9 = (~ymm9 & ~ymm5 & ymm1) | (~ymm9 & ymm5 & ymm1) | (ymm9 & ymm5 & ~ymm1) | (ymm9 & ymm5 & ymm1)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm9 = ymm1 ^ (ymm9 & (ymm5 ^ ymm1))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[4,10,0,6,12,18,24,30,20,26,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm14 = (~ymm14 & ymm12 & ymm16) | (ymm14 & ~ymm12 & ~ymm16) | (ymm14 & ~ymm12 & ymm16) | (ymm14 & ymm12 & ~ymm16) | (ymm14 & ymm12 & ymm16)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm14 = ymm14 | (ymm12 & ymm16)
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm12
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm12 = (~ymm12 & ~ymm6 & ymm13) | (~ymm12 & ymm6 & ymm13) | (ymm12 & ymm6 & ~ymm13) | (ymm12 & ymm6 & ymm13)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm12 = ymm13 ^ (ymm12 & (ymm6 ^ ymm13))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm15 = xmm12[u,u,u,u,u,0,6,12],zero,zero,zero,xmm12[4,10],zero,zero,zero
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm12, %xmm2
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[2,8,14],zero,zero,xmm2[0,6,12]
 ; AVX512DQ-FCP-NEXT:    vpor %xmm4, %xmm15, %xmm4
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQ-FCP-NEXT:    vpmovsxwd {{.*#+}} ymm15 = [4294967295,4294967295,4294967295,4294967295,4294967295,255,0,0]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = (~ymm4 & ymm15 & ymm14) | (ymm4 & ~ymm15 & ~ymm14) | (ymm4 & ~ymm15 & ymm14) | (ymm4 & ymm15 & ymm14)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = ymm4 ^ (ymm15 & (ymm4 ^ ymm14))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = zero,zero,zero,xmm11[5,11],zero,zero,zero,xmm11[3,9,15,u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[3,9,15],zero,zero,xmm10[1,7,13],zero,zero,zero,xmm10[u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpor %xmm11, %xmm10, %xmm10
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[5,11,1,7,13,19,25,31,21,27,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm9 = (~ymm9 & ymm10 & ymm16) | (ymm9 & ~ymm10 & ~ymm16) | (ymm9 & ~ymm10 & ymm16) | (ymm9 & ymm10 & ~ymm16) | (ymm9 & ymm10 & ymm16)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm10 & ymm16)
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm12[u,u,u,u,u,1,7,13],zero,zero,zero,xmm12[5,11],zero,zero,zero
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u],zero,zero,zero,xmm2[3,9,15],zero,zero,xmm2[1,7,13]
 ; AVX512DQ-FCP-NEXT:    vpor %xmm2, %xmm10, %xmm2
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ymm15 & ymm9) | (ymm2 & ~ymm15 & ~ymm9) | (ymm2 & ~ymm15 & ymm9) | (ymm2 & ymm15 & ymm9)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm8 = (~ymm8 & ~ymm13 & ymm6) | (~ymm8 & ymm13 & ymm6) | (ymm8 & ymm13 & ~ymm6) | (ymm8 & ymm13 & ymm6)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm2 ^ (ymm15 & (ymm2 ^ ymm9))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm8 = ymm6 ^ (ymm8 & (ymm13 ^ ymm6))
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm8, %xmm6
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm9 = xmm6[u,u,u,u,u],zero,zero,zero,xmm6[4,10],zero,zero,zero,xmm6[2,8,14]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm8[u,u,u,u,u,2,8,14],zero,zero,xmm8[0,6,12],zero,zero,zero
 ; AVX512DQ-FCP-NEXT:    vpor %xmm9, %xmm10, %xmm9
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ~ymm3 & ymm17) | (~ymm0 & ymm3 & ymm17) | (ymm0 & ymm3 & ~ymm17) | (ymm0 & ymm3 & ymm17)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm17 ^ (ymm0 & (ymm3 ^ ymm17))
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,xmm3[0,6,12],zero,zero,zero,xmm3[4,10,u,u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm11 = xmm0[4,10],zero,zero,zero,xmm0[2,8,14],zero,zero,xmm0[u,u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpor %xmm10, %xmm11, %xmm10
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ymm5 & mem) | (ymm1 & ~ymm5 & ~mem) | (ymm1 & ymm5 & ~mem) | (ymm1 & ymm5 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm5))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm1[u,u,u,u,u,u,u,u,u,u,0,6,12,2,8,14,20,26,16,22,28,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm10 = xmm10[0,1,2,3,4],xmm5[5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ~ymm15 & ymm9) | (ymm5 & ~ymm15 & ymm9) | (ymm5 & ymm15 & ~ymm9) | (ymm5 & ymm15 & ymm9)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm9 ^ (ymm15 & (ymm5 ^ ymm9))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u],zero,zero,zero,xmm6[5,11],zero,zero,zero,xmm6[3,9,15]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[u,u,u,u,u,3,9,15],zero,zero,xmm8[1,7,13],zero,zero,zero
 ; AVX512DQ-FCP-NEXT:    vpor %xmm6, %xmm8, %xmm6
@@ -4196,7 +4196,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,1,7,13,3,9,15,21,27,17,23,29,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm1[5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ~ymm15 & ymm6) | (ymm0 & ~ymm15 & ymm6) | (ymm0 & ymm15 & ~ymm6) | (ymm0 & ymm15 & ymm6)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm6 ^ (ymm15 & (ymm0 ^ ymm6))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm18, (%rsi)
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm7, (%rdx)
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm4, (%rcx)
@@ -7385,7 +7385,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vmovdqa64 224(%rdi), %ymm25
 ; AVX512-NEXT:    vmovdqa64 192(%rdi), %ymm26
 ; AVX512-NEXT:    vmovdqa %ymm12, %ymm0
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ~ymm26 & ymm25) | (~ymm0 & ymm26 & ymm25) | (ymm0 & ymm26 & ~ymm25) | (ymm0 & ymm26 & ymm25)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm25 ^ (ymm0 & (ymm26 ^ ymm25))
 ; AVX512-NEXT:    vpshufb %xmm1, %xmm0, %xmm3
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm5 = [128,128,128,2,8,14,128,128,0,6,12,u,u,u,u,u]
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm4
@@ -7396,7 +7396,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vmovdqa64 128(%rdi), %ymm24
 ; AVX512-NEXT:    vmovdqa64 160(%rdi), %ymm18
 ; AVX512-NEXT:    vmovdqa %ymm12, %ymm6
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ~ymm18 & ymm24) | (~ymm6 & ymm18 & ymm24) | (ymm6 & ymm18 & ~ymm24) | (ymm6 & ymm18 & ymm24)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm24 ^ (ymm6 & (ymm18 ^ ymm24))
 ; AVX512-NEXT:    vextracti128 $1, %ymm6, %xmm7
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [u,u,u,u,u,u,128,128,0,6,12,128,128,128,4,10]
 ; AVX512-NEXT:    vpshufb %xmm3, %xmm7, %xmm10
@@ -7407,7 +7407,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vinserti32x4 $2, %xmm9, %zmm10, %zmm2
 ; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vmovdqa %ymm12, %ymm9
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm9 = (~ymm9 & ~ymm30 & ymm31) | (~ymm9 & ymm30 & ymm31) | (ymm9 & ymm30 & ~ymm31) | (ymm9 & ymm30 & ymm31)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm9 = ymm31 ^ (ymm9 & (ymm30 ^ ymm31))
 ; AVX512-NEXT:    vpshufb %xmm1, %xmm9, %xmm1
 ; AVX512-NEXT:    vextracti128 $1, %ymm9, %xmm13
 ; AVX512-NEXT:    vpshufb %xmm5, %xmm13, %xmm5
@@ -7415,7 +7415,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vmovdqa64 320(%rdi), %ymm29
 ; AVX512-NEXT:    vmovdqa64 352(%rdi), %ymm22
 ; AVX512-NEXT:    vmovdqa %ymm12, %ymm1
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ~ymm22 & ymm29) | (~ymm1 & ymm22 & ymm29) | (ymm1 & ymm22 & ~ymm29) | (ymm1 & ymm22 & ymm29)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm29 ^ (ymm1 & (ymm22 ^ ymm29))
 ; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm5
 ; AVX512-NEXT:    vpshufb %xmm3, %xmm5, %xmm3
 ; AVX512-NEXT:    vpshufb %xmm8, %xmm1, %xmm8
@@ -7441,7 +7441,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = [128,128,128,4,10,128,128,128,2,8,14,u,u,u,u,u]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
 ; AVX512-NEXT:    vmovdqa %ymm9, %ymm4
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm4 = (~ymm4 & ~ymm25 & ymm26) | (~ymm4 & ymm25 & ymm26) | (ymm4 & ymm25 & ~ymm26) | (ymm4 & ymm25 & ymm26)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm4 = ymm26 ^ (ymm4 & (ymm25 ^ ymm26))
 ; AVX512-NEXT:    vextracti128 $1, %ymm4, %xmm15
 ; AVX512-NEXT:    vpshufb %xmm0, %xmm15, %xmm1
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm6 = [2,8,14,128,128,0,6,12,128,128,128,u,u,u,u,u]
@@ -7449,7 +7449,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vpor %xmm1, %xmm5, %xmm1
 ; AVX512-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-NEXT:    vmovdqa %ymm12, %ymm5
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ~ymm24 & ymm18) | (~ymm5 & ymm24 & ymm18) | (ymm5 & ymm24 & ~ymm18) | (ymm5 & ymm24 & ymm18)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm18 ^ (ymm5 & (ymm24 ^ ymm18))
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm8 = [u,u,u,u,u,0,6,12,128,128,128,4,10,128,128,128]
 ; AVX512-NEXT:    vpshufb %xmm8, %xmm5, %xmm7
 ; AVX512-NEXT:    vextracti128 $1, %ymm5, %xmm1
@@ -7458,13 +7458,13 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vpor %xmm7, %xmm13, %xmm2
 ; AVX512-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-NEXT:    vmovdqa %ymm9, %ymm13
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm13 = (~ymm13 & ~ymm31 & ymm30) | (~ymm13 & ymm31 & ymm30) | (ymm13 & ymm31 & ~ymm30) | (ymm13 & ymm31 & ymm30)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm13 = ymm30 ^ (ymm13 & (ymm31 ^ ymm30))
 ; AVX512-NEXT:    vextracti128 $1, %ymm13, %xmm14
 ; AVX512-NEXT:    vpshufb %xmm0, %xmm14, %xmm0
 ; AVX512-NEXT:    vpshufb %xmm6, %xmm13, %xmm6
 ; AVX512-NEXT:    vporq %xmm0, %xmm6, %xmm16
 ; AVX512-NEXT:    vmovdqa %ymm12, %ymm11
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ~ymm29 & ymm22) | (~ymm11 & ymm29 & ymm22) | (ymm11 & ymm29 & ~ymm22) | (ymm11 & ymm29 & ymm22)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm22 ^ (ymm11 & (ymm29 ^ ymm22))
 ; AVX512-NEXT:    vpshufb %xmm8, %xmm11, %xmm8
 ; AVX512-NEXT:    vextracti128 $1, %ymm11, %xmm7
 ; AVX512-NEXT:    vpshufb %xmm10, %xmm7, %xmm10
@@ -7485,7 +7485,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vinserti32x4 $1, 288(%rdi), %ymm1, %ymm20
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm5 = [65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,0]
 ; AVX512-NEXT:    vmovdqa %ymm5, %ymm1
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ~ymm20 & ymm19) | (~ymm1 & ymm20 & ymm19) | (ymm1 & ymm20 & ~ymm19) | (ymm1 & ymm20 & ymm19)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm19 ^ (ymm1 & (ymm20 ^ ymm19))
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm15 = ymm1[u,u,u,u,u,u,u,u,u,u,u,2,8,14,4,10,16,22,28,18,24,30,u,u,u,u,u,u,u,u,u,u]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm15[0,1,2],ymm3[3,4,5,6,7],ymm15[8,9,10],ymm3[11,12,13,14,15]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm15 = ymm15[0,1,2,3],ymm3[4,5,6,7]
@@ -7494,26 +7494,26 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vinserti32x4 $1, 96(%rdi), %ymm23, %ymm23
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm15, %zmm0, %zmm15
 ; AVX512-NEXT:    vmovdqa %ymm5, %ymm2
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ~ymm23 & ymm6) | (~ymm2 & ymm23 & ymm6) | (ymm2 & ymm23 & ~ymm6) | (ymm2 & ymm23 & ymm6)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm6 ^ (ymm2 & (ymm23 ^ ymm6))
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ymm17 & ymm4) | (ymm0 & ~ymm17 & ~ymm4) | (ymm0 & ~ymm17 & ymm4) | (ymm0 & ymm17 & ~ymm4) | (ymm0 & ymm17 & ymm4)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm17 & ymm4)
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
 ; AVX512-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm0 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm0 = (~zmm0 & ~zmm17 & mem) | (zmm0 & ~zmm17 & mem) | (zmm0 & zmm17 & ~mem) | (zmm0 & zmm17 & mem)
+; AVX512-NEXT:    # zmm0 = mem ^ (zmm17 & (zmm0 ^ mem))
 ; AVX512-NEXT:    vpmovsxdq {{.*#+}} zmm3 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,16777215,0,0]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm15 = (~zmm15 & zmm3 & zmm0) | (zmm15 & ~zmm3 & ~zmm0) | (zmm15 & ~zmm3 & zmm0) | (zmm15 & zmm3 & zmm0)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm15 = zmm15 ^ (zmm3 & (zmm15 ^ zmm0))
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,3,9,15,5,11,17,23,29,19,25,31,u,u,u,u,u,u,u,u,u,u]
 ; AVX512-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm1
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ymm21 & ymm4) | (ymm1 & ~ymm21 & ~ymm4) | (ymm1 & ~ymm21 & ymm4) | (ymm1 & ymm21 & ~ymm4) | (ymm1 & ymm21 & ymm4)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm21 & ymm4)
 ; AVX512-NEXT:    vinserti32x4 $1, %xmm28, %ymm0, %ymm2
 ; AVX512-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 16-byte Folded Reload
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm17 & zmm2) | (zmm1 & ~zmm17 & zmm2) | (zmm1 & zmm17 & ~zmm2) | (zmm1 & zmm17 & zmm2)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm17 & (zmm1 ^ zmm2))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm17
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm17 = (~zmm17 & zmm3 & zmm1) | (zmm17 & ~zmm3 & ~zmm1) | (zmm17 & ~zmm3 & zmm1) | (zmm17 & zmm3 & zmm1)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm17 = zmm17 ^ (zmm3 & (zmm17 ^ zmm1))
 ; AVX512-NEXT:    vpshufb %xmm10, %xmm14, %xmm0
 ; AVX512-NEXT:    vpshufb %xmm8, %xmm13, %xmm1
 ; AVX512-NEXT:    vporq %xmm0, %xmm1, %xmm21
@@ -7521,7 +7521,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm1 = xmm7[u,u,u,u,u],zero,zero,zero,xmm7[3,9,15],zero,zero,xmm7[1,7,13]
 ; AVX512-NEXT:    vporq %xmm0, %xmm1, %xmm28
 ; AVX512-NEXT:    vmovdqa64 %ymm25, %ymm11
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ~ymm12 & ymm26) | (ymm11 & ~ymm12 & ymm26) | (ymm11 & ymm12 & ~ymm26) | (ymm11 & ymm12 & ymm26)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm26 ^ (ymm12 & (ymm11 ^ ymm26))
 ; AVX512-NEXT:    vextracti128 $1, %ymm11, %xmm0
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,xmm0[0,6,12],zero,zero,zero,xmm0[4,10,u,u,u,u,u,u]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [4,10,128,128,128,2,8,14,128,128,u,u,u,u,u,u]
@@ -7529,17 +7529,17 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vmovdqa64 %xmm3, %xmm25
 ; AVX512-NEXT:    vporq %xmm1, %xmm2, %xmm26
 ; AVX512-NEXT:    vmovdqa64 %ymm18, %ymm14
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm14 = (~ymm14 & ~ymm9 & ymm24) | (ymm14 & ~ymm9 & ymm24) | (ymm14 & ymm9 & ~ymm24) | (ymm14 & ymm9 & ymm24)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm14 = ymm24 ^ (ymm9 & (ymm14 ^ ymm24))
 ; AVX512-NEXT:    vextracti128 $1, %ymm14, %xmm10
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [u,u,u,u,u,128,128,128,4,10,128,128,128,2,8,14]
 ; AVX512-NEXT:    vpshufb %xmm1, %xmm10, %xmm2
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [u,u,u,u,u,2,8,14,128,128,0,6,12,128,128,128]
 ; AVX512-NEXT:    vpshufb %xmm3, %xmm14, %xmm4
 ; AVX512-NEXT:    vporq %xmm2, %xmm4, %xmm27
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm12 = (~ymm12 & ~ymm31 & ymm30) | (~ymm12 & ymm31 & ymm30) | (ymm12 & ymm31 & ~ymm30) | (ymm12 & ymm31 & ymm30)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm12 = ymm30 ^ (ymm12 & (ymm31 ^ ymm30))
 ; AVX512-NEXT:    vmovdqa %ymm5, %ymm4
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm4 = (~ymm4 & ~ymm6 & ymm23) | (~ymm4 & ymm6 & ymm23) | (ymm4 & ymm6 & ~ymm23) | (ymm4 & ymm6 & ymm23)
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm9 = (~ymm9 & ~ymm22 & ymm29) | (~ymm9 & ymm22 & ymm29) | (ymm9 & ymm22 & ~ymm29) | (ymm9 & ymm22 & ymm29)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm4 = ymm23 ^ (ymm4 & (ymm6 ^ ymm23))
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm9 = ymm29 ^ (ymm9 & (ymm22 ^ ymm29))
 ; AVX512-NEXT:    vextracti128 $1, %ymm9, %xmm8
 ; AVX512-NEXT:    vpshufb %xmm1, %xmm8, %xmm1
 ; AVX512-NEXT:    vpshufb %xmm3, %xmm9, %xmm2
@@ -7558,14 +7558,14 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm1 = [128,128,128,128,128,128,128,128,128,128,128,4,10,0,6,12,18,24,30,20,26,128,128,128,128,128,128,128,128,128,128,128]
 ; AVX512-NEXT:    vpshufb %ymm1, %ymm4, %ymm11
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} ymm18 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm16 = (~ymm16 & ymm11 & ~ymm18) | (~ymm16 & ymm11 & ymm18) | (ymm16 & ~ymm11 & ymm18) | (ymm16 & ymm11 & ~ymm18) | (ymm16 & ymm11 & ymm18)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm16 = (ymm16 & ymm18) | ymm11
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm11 = [128,128,128,128,128,128,128,128,128,128,128,5,11,1,7,13,19,25,31,21,27,128,128,128,128,128,128,128,128,128,128,128]
 ; AVX512-NEXT:    vpshufb %ymm11, %ymm4, %ymm4
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm21 = (~ymm21 & ymm4 & ~ymm18) | (~ymm21 & ymm4 & ymm18) | (ymm21 & ~ymm4 & ymm18) | (ymm21 & ymm4 & ~ymm18) | (ymm21 & ymm4 & ymm18)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm21 = (ymm21 & ymm18) | ymm4
 ; AVX512-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 16-byte Folded Reload
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ~ymm19 & ymm20) | (~ymm5 & ymm19 & ymm20) | (ymm5 & ymm19 & ~ymm20) | (ymm5 & ymm19 & ymm20)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm20 ^ (ymm5 & (ymm19 ^ ymm20))
 ; AVX512-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ymm4 & ymm18) | (ymm1 & ~ymm4 & ~ymm18) | (ymm1 & ~ymm4 & ymm18) | (ymm1 & ymm4 & ~ymm18) | (ymm1 & ymm4 & ymm18)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm18)
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm4
 ; AVX512-NEXT:    vpshufb %ymm11, %ymm5, %ymm5
 ; AVX512-NEXT:    vextracti128 $1, %ymm12, %xmm1
@@ -7577,24 +7577,24 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vpshufb %xmm0, %xmm9, %xmm0
 ; AVX512-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ~ymm2 & ymm23) | (ymm6 & ~ymm2 & ymm23) | (ymm6 & ymm2 & ~ymm23) | (ymm6 & ymm2 & ymm23)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm23 ^ (ymm2 & (ymm6 ^ ymm23))
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm8 = ymm6[u,u,u,u,u,u,u,u,u,u,0,6,12,2,8,14,20,26,16,22,28,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm9 = xmm11[0,1,2,3,4],xmm8[5,6,7]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
 ; AVX512-NEXT:    vinserti32x4 $1, %xmm28, %ymm0, %ymm9
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ymm9 & ymm18) | (ymm5 & ~ymm9 & ~ymm18) | (ymm5 & ~ymm9 & ymm18) | (ymm5 & ymm9 & ~ymm18) | (ymm5 & ymm9 & ymm18)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm9 & ymm18)
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm5
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ~ymm19 & ymm20) | (~ymm2 & ymm19 & ymm20) | (ymm2 & ymm19 & ~ymm20) | (ymm2 & ymm19 & ymm20)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm20 ^ (ymm2 & (ymm19 ^ ymm20))
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX512-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm11 = ymm2[u,u,u,u,u,u,u,u,u,u,0,6,12,2,8,14,20,26,16,22,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ~ymm9 & ymm7) | (ymm11 & ~ymm9 & ~ymm7) | (ymm11 & ~ymm9 & ymm7) | (ymm11 & ymm9 & ~ymm7) | (ymm11 & ymm9 & ymm7)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm7 & ~ymm9)
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm7
 ; AVX512-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm11
 ; AVX512-NEXT:    vinserti32x4 $2, %xmm26, %zmm11, %zmm11
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & ~zmm9 & zmm11) | (zmm8 & ~zmm9 & zmm11) | (zmm8 & zmm9 & ~zmm11) | (zmm8 & zmm9 & zmm11)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm11 ^ (zmm9 & (zmm8 ^ zmm11))
 ; AVX512-NEXT:    vpmovsxdq {{.*#+}} zmm11 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & zmm11 & zmm8) | (zmm7 & ~zmm11 & ~zmm8) | (zmm7 & ~zmm11 & zmm8) | (zmm7 & zmm11 & zmm8)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm11 & (zmm7 ^ zmm8))
 ; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm8
 ; AVX512-NEXT:    vpshufb %xmm8, %xmm1, %xmm1
 ; AVX512-NEXT:    vpshufb %xmm13, %xmm12, %xmm8
@@ -7604,22 +7604,22 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm6[4,5,6,7]
 ; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,1,7,13,3,9,15,21,27,17,23,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ~ymm9 & ymm0) | (ymm2 & ~ymm9 & ~ymm0) | (ymm2 & ~ymm9 & ymm0) | (ymm2 & ymm9 & ~ymm0) | (ymm2 & ymm9 & ymm0)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm0 & ~ymm9)
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm2
 ; AVX512-NEXT:    vinserti32x4 $2, %xmm3, %zmm2, %zmm2
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm9 & zmm2) | (zmm1 & ~zmm9 & zmm2) | (zmm1 & zmm9 & ~zmm2) | (zmm1 & zmm9 & zmm2)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm11 & zmm1) | (zmm0 & ~zmm11 & ~zmm1) | (zmm0 & ~zmm11 & zmm1) | (zmm0 & zmm11 & zmm1)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm9 & (zmm1 ^ zmm2))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm11 & (zmm0 ^ zmm1))
 ; AVX512-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 16-byte Folded Reload
 ; AVX512-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 16-byte Folded Reload
 ; AVX512-NEXT:    vpmovsxwd {{.*#+}} zmm2 = [0,0,0,0,0,4294967040,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm16 = (~zmm16 & zmm2 & zmm1) | (zmm16 & ~zmm2 & ~zmm1) | (zmm16 & ~zmm2 & zmm1) | (zmm16 & zmm2 & zmm1)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm16 = zmm16 ^ (zmm2 & (zmm16 ^ zmm1))
 ; AVX512-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 16-byte Folded Reload
 ; AVX512-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 16-byte Folded Reload
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm21 = (~zmm21 & zmm2 & zmm1) | (zmm21 & ~zmm2 & ~zmm1) | (zmm21 & ~zmm2 & zmm1) | (zmm21 & zmm2 & zmm1)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm21 = zmm21 ^ (zmm2 & (zmm21 ^ zmm1))
 ; AVX512-NEXT:    vpmovsxdq {{.*#+}} zmm1 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,16777215,0,0]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm1 & zmm16) | (zmm4 & ~zmm1 & ~zmm16) | (zmm4 & ~zmm1 & zmm16) | (zmm4 & zmm1 & zmm16)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm1 & zmm21) | (zmm5 & ~zmm1 & ~zmm21) | (zmm5 & ~zmm1 & zmm21) | (zmm5 & zmm1 & zmm21)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm1 & (zmm4 ^ zmm16))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm1 & (zmm5 ^ zmm21))
 ; AVX512-NEXT:    vmovdqa64 %zmm15, (%rsi)
 ; AVX512-NEXT:    vmovdqa64 %zmm17, (%rdx)
 ; AVX512-NEXT:    vmovdqa64 %zmm4, (%rcx)
@@ -7639,7 +7639,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vmovdqa64 224(%rdi), %ymm25
 ; AVX512-FCP-NEXT:    vmovdqa64 192(%rdi), %ymm26
 ; AVX512-FCP-NEXT:    vmovdqa %ymm12, %ymm0
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ~ymm26 & ymm25) | (~ymm0 & ymm26 & ymm25) | (ymm0 & ymm26 & ~ymm25) | (ymm0 & ymm26 & ymm25)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm25 ^ (ymm0 & (ymm26 ^ ymm25))
 ; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm0, %xmm3
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [128,128,128,2,8,14,128,128,0,6,12,u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm0, %xmm4
@@ -7650,7 +7650,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vmovdqa64 128(%rdi), %ymm24
 ; AVX512-FCP-NEXT:    vmovdqa64 160(%rdi), %ymm18
 ; AVX512-FCP-NEXT:    vmovdqa %ymm12, %ymm6
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ~ymm18 & ymm24) | (~ymm6 & ymm18 & ymm24) | (ymm6 & ymm18 & ~ymm24) | (ymm6 & ymm18 & ymm24)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm24 ^ (ymm6 & (ymm18 ^ ymm24))
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm6, %xmm7
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [u,u,u,u,u,u,128,128,0,6,12,128,128,128,4,10]
 ; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm7, %xmm10
@@ -7661,7 +7661,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm9, %zmm10, %zmm2
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa %ymm12, %ymm9
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm9 = (~ymm9 & ~ymm30 & ymm31) | (~ymm9 & ymm30 & ymm31) | (ymm9 & ymm30 & ~ymm31) | (ymm9 & ymm30 & ymm31)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm9 = ymm31 ^ (ymm9 & (ymm30 ^ ymm31))
 ; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm9, %xmm1
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm9, %xmm13
 ; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm13, %xmm5
@@ -7669,7 +7669,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vmovdqa64 320(%rdi), %ymm29
 ; AVX512-FCP-NEXT:    vmovdqa64 352(%rdi), %ymm22
 ; AVX512-FCP-NEXT:    vmovdqa %ymm12, %ymm1
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ~ymm22 & ymm29) | (~ymm1 & ymm22 & ymm29) | (ymm1 & ymm22 & ~ymm29) | (ymm1 & ymm22 & ymm29)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm29 ^ (ymm1 & (ymm22 ^ ymm29))
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm5
 ; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm5, %xmm3
 ; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm1, %xmm8
@@ -7695,7 +7695,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm0 = [128,128,128,4,10,128,128,128,2,8,14,u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
 ; AVX512-FCP-NEXT:    vmovdqa %ymm9, %ymm4
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = (~ymm4 & ~ymm25 & ymm26) | (~ymm4 & ymm25 & ymm26) | (ymm4 & ymm25 & ~ymm26) | (ymm4 & ymm25 & ymm26)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = ymm26 ^ (ymm4 & (ymm25 ^ ymm26))
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm4, %xmm15
 ; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm15, %xmm1
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [2,8,14,128,128,0,6,12,128,128,128,u,u,u,u,u]
@@ -7703,7 +7703,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vpor %xmm1, %xmm5, %xmm1
 ; AVX512-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa %ymm12, %ymm5
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ~ymm24 & ymm18) | (~ymm5 & ymm24 & ymm18) | (ymm5 & ymm24 & ~ymm18) | (ymm5 & ymm24 & ymm18)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm18 ^ (ymm5 & (ymm24 ^ ymm18))
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [u,u,u,u,u,0,6,12,128,128,128,4,10,128,128,128]
 ; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm5, %xmm7
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm5, %xmm1
@@ -7712,13 +7712,13 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vpor %xmm7, %xmm13, %xmm2
 ; AVX512-FCP-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa %ymm9, %ymm13
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm13 = (~ymm13 & ~ymm31 & ymm30) | (~ymm13 & ymm31 & ymm30) | (ymm13 & ymm31 & ~ymm30) | (ymm13 & ymm31 & ymm30)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm13 = ymm30 ^ (ymm13 & (ymm31 ^ ymm30))
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm13, %xmm14
 ; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm14, %xmm0
 ; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm13, %xmm6
 ; AVX512-FCP-NEXT:    vporq %xmm0, %xmm6, %xmm16
 ; AVX512-FCP-NEXT:    vmovdqa %ymm12, %ymm11
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ~ymm29 & ymm22) | (~ymm11 & ymm29 & ymm22) | (ymm11 & ymm29 & ~ymm22) | (ymm11 & ymm29 & ymm22)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm22 ^ (ymm11 & (ymm29 ^ ymm22))
 ; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm11, %xmm8
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm11, %xmm7
 ; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm7, %xmm10
@@ -7739,7 +7739,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vinserti32x4 $1, 288(%rdi), %ymm1, %ymm20
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm5 = [65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,0]
 ; AVX512-FCP-NEXT:    vmovdqa %ymm5, %ymm1
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ~ymm20 & ymm19) | (~ymm1 & ymm20 & ymm19) | (ymm1 & ymm20 & ~ymm19) | (ymm1 & ymm20 & ymm19)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm19 ^ (ymm1 & (ymm20 ^ ymm19))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm15 = ymm1[u,u,u,u,u,u,u,u,u,u,u,2,8,14,4,10,16,22,28,18,24,30,u,u,u,u,u,u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm15[0,1,2],ymm3[3,4,5,6,7],ymm15[8,9,10],ymm3[11,12,13,14,15]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm15 = ymm15[0,1,2,3],ymm3[4,5,6,7]
@@ -7748,26 +7748,26 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vinserti32x4 $1, 96(%rdi), %ymm23, %ymm23
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm15, %zmm0, %zmm15
 ; AVX512-FCP-NEXT:    vmovdqa %ymm5, %ymm2
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ~ymm23 & ymm6) | (~ymm2 & ymm23 & ymm6) | (ymm2 & ymm23 & ~ymm6) | (ymm2 & ymm23 & ymm6)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm6 ^ (ymm2 & (ymm23 ^ ymm6))
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ymm17 & ymm4) | (ymm0 & ~ymm17 & ~ymm4) | (ymm0 & ~ymm17 & ymm4) | (ymm0 & ymm17 & ~ymm4) | (ymm0 & ymm17 & ymm4)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm17 & ymm4)
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm0 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm0 = (~zmm0 & ~zmm17 & mem) | (zmm0 & ~zmm17 & mem) | (zmm0 & zmm17 & ~mem) | (zmm0 & zmm17 & mem)
+; AVX512-FCP-NEXT:    # zmm0 = mem ^ (zmm17 & (zmm0 ^ mem))
 ; AVX512-FCP-NEXT:    vpmovsxdq {{.*#+}} zmm3 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,16777215,0,0]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = (~zmm15 & zmm3 & zmm0) | (zmm15 & ~zmm3 & ~zmm0) | (zmm15 & ~zmm3 & zmm0) | (zmm15 & zmm3 & zmm0)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = zmm15 ^ (zmm3 & (zmm15 ^ zmm0))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,3,9,15,5,11,17,23,29,19,25,31,u,u,u,u,u,u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm1
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ymm21 & ymm4) | (ymm1 & ~ymm21 & ~ymm4) | (ymm1 & ~ymm21 & ymm4) | (ymm1 & ymm21 & ~ymm4) | (ymm1 & ymm21 & ymm4)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm21 & ymm4)
 ; AVX512-FCP-NEXT:    vinserti32x4 $1, %xmm28, %ymm0, %ymm2
 ; AVX512-FCP-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 16-byte Folded Reload
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm17 & zmm2) | (zmm1 & ~zmm17 & zmm2) | (zmm1 & zmm17 & ~zmm2) | (zmm1 & zmm17 & zmm2)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm17 & (zmm1 ^ zmm2))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm17
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm17 = (~zmm17 & zmm3 & zmm1) | (zmm17 & ~zmm3 & ~zmm1) | (zmm17 & ~zmm3 & zmm1) | (zmm17 & zmm3 & zmm1)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm17 = zmm17 ^ (zmm3 & (zmm17 ^ zmm1))
 ; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm14, %xmm0
 ; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm13, %xmm1
 ; AVX512-FCP-NEXT:    vporq %xmm0, %xmm1, %xmm21
@@ -7775,7 +7775,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm7[u,u,u,u,u],zero,zero,zero,xmm7[3,9,15],zero,zero,xmm7[1,7,13]
 ; AVX512-FCP-NEXT:    vporq %xmm0, %xmm1, %xmm28
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm25, %ymm11
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ~ymm12 & ymm26) | (ymm11 & ~ymm12 & ymm26) | (ymm11 & ymm12 & ~ymm26) | (ymm11 & ymm12 & ymm26)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm26 ^ (ymm12 & (ymm11 ^ ymm26))
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm11, %xmm0
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,xmm0[0,6,12],zero,zero,zero,xmm0[4,10,u,u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [4,10,128,128,128,2,8,14,128,128,u,u,u,u,u,u]
@@ -7783,17 +7783,17 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm3, %xmm25
 ; AVX512-FCP-NEXT:    vporq %xmm1, %xmm2, %xmm26
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm18, %ymm14
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm14 = (~ymm14 & ~ymm9 & ymm24) | (ymm14 & ~ymm9 & ymm24) | (ymm14 & ymm9 & ~ymm24) | (ymm14 & ymm9 & ymm24)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm14 = ymm24 ^ (ymm9 & (ymm14 ^ ymm24))
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm14, %xmm10
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [u,u,u,u,u,128,128,128,4,10,128,128,128,2,8,14]
 ; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm10, %xmm2
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [u,u,u,u,u,2,8,14,128,128,0,6,12,128,128,128]
 ; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm14, %xmm4
 ; AVX512-FCP-NEXT:    vporq %xmm2, %xmm4, %xmm27
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm12 = (~ymm12 & ~ymm31 & ymm30) | (~ymm12 & ymm31 & ymm30) | (ymm12 & ymm31 & ~ymm30) | (ymm12 & ymm31 & ymm30)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm12 = ymm30 ^ (ymm12 & (ymm31 ^ ymm30))
 ; AVX512-FCP-NEXT:    vmovdqa %ymm5, %ymm4
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = (~ymm4 & ~ymm6 & ymm23) | (~ymm4 & ymm6 & ymm23) | (ymm4 & ymm6 & ~ymm23) | (ymm4 & ymm6 & ymm23)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm9 = (~ymm9 & ~ymm22 & ymm29) | (~ymm9 & ymm22 & ymm29) | (ymm9 & ymm22 & ~ymm29) | (ymm9 & ymm22 & ymm29)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = ymm23 ^ (ymm4 & (ymm6 ^ ymm23))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm9 = ymm29 ^ (ymm9 & (ymm22 ^ ymm29))
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm9, %xmm8
 ; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm8, %xmm1
 ; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm9, %xmm2
@@ -7812,14 +7812,14 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [128,128,128,128,128,128,128,128,128,128,128,4,10,0,6,12,18,24,30,20,26,128,128,128,128,128,128,128,128,128,128,128]
 ; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm4, %ymm11
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} ymm18 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm16 = (~ymm16 & ymm11 & ~ymm18) | (~ymm16 & ymm11 & ymm18) | (ymm16 & ~ymm11 & ymm18) | (ymm16 & ymm11 & ~ymm18) | (ymm16 & ymm11 & ymm18)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm16 = (ymm16 & ymm18) | ymm11
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [128,128,128,128,128,128,128,128,128,128,128,5,11,1,7,13,19,25,31,21,27,128,128,128,128,128,128,128,128,128,128,128]
 ; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm4, %ymm4
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm21 = (~ymm21 & ymm4 & ~ymm18) | (~ymm21 & ymm4 & ymm18) | (ymm21 & ~ymm4 & ymm18) | (ymm21 & ymm4 & ~ymm18) | (ymm21 & ymm4 & ymm18)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm21 = (ymm21 & ymm18) | ymm4
 ; AVX512-FCP-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 16-byte Folded Reload
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ~ymm19 & ymm20) | (~ymm5 & ymm19 & ymm20) | (ymm5 & ymm19 & ~ymm20) | (ymm5 & ymm19 & ymm20)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm20 ^ (ymm5 & (ymm19 ^ ymm20))
 ; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ymm4 & ymm18) | (ymm1 & ~ymm4 & ~ymm18) | (ymm1 & ~ymm4 & ymm18) | (ymm1 & ymm4 & ~ymm18) | (ymm1 & ymm4 & ymm18)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm18)
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm4
 ; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm5, %ymm5
 ; AVX512-FCP-NEXT:    vextracti128 $1, %ymm12, %xmm1
@@ -7831,24 +7831,24 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm9, %xmm0
 ; AVX512-FCP-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ~ymm2 & ymm23) | (ymm6 & ~ymm2 & ymm23) | (ymm6 & ymm2 & ~ymm23) | (ymm6 & ymm2 & ymm23)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm23 ^ (ymm2 & (ymm6 ^ ymm23))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = ymm6[u,u,u,u,u,u,u,u,u,u,0,6,12,2,8,14,20,26,16,22,28,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm9 = xmm11[0,1,2,3,4],xmm8[5,6,7]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
 ; AVX512-FCP-NEXT:    vinserti32x4 $1, %xmm28, %ymm0, %ymm9
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ymm9 & ymm18) | (ymm5 & ~ymm9 & ~ymm18) | (ymm5 & ~ymm9 & ymm18) | (ymm5 & ymm9 & ~ymm18) | (ymm5 & ymm9 & ymm18)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm9 & ymm18)
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm5
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ~ymm19 & ymm20) | (~ymm2 & ymm19 & ymm20) | (ymm2 & ymm19 & ~ymm20) | (ymm2 & ymm19 & ymm20)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm20 ^ (ymm2 & (ymm19 ^ ymm20))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX512-FCP-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm2[u,u,u,u,u,u,u,u,u,u,0,6,12,2,8,14,20,26,16,22,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ~ymm9 & ymm7) | (ymm11 & ~ymm9 & ~ymm7) | (ymm11 & ~ymm9 & ymm7) | (ymm11 & ymm9 & ~ymm7) | (ymm11 & ymm9 & ymm7)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm7 & ~ymm9)
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm7
 ; AVX512-FCP-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm11
 ; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm26, %zmm11, %zmm11
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & ~zmm9 & zmm11) | (zmm8 & ~zmm9 & zmm11) | (zmm8 & zmm9 & ~zmm11) | (zmm8 & zmm9 & zmm11)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm11 ^ (zmm9 & (zmm8 ^ zmm11))
 ; AVX512-FCP-NEXT:    vpmovsxdq {{.*#+}} zmm11 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & zmm11 & zmm8) | (zmm7 & ~zmm11 & ~zmm8) | (zmm7 & ~zmm11 & zmm8) | (zmm7 & zmm11 & zmm8)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm11 & (zmm7 ^ zmm8))
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm22, %xmm8
 ; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm1, %xmm1
 ; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm12, %xmm8
@@ -7858,22 +7858,22 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm6[4,5,6,7]
 ; AVX512-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,1,7,13,3,9,15,21,27,17,23,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ~ymm9 & ymm0) | (ymm2 & ~ymm9 & ~ymm0) | (ymm2 & ~ymm9 & ymm0) | (ymm2 & ymm9 & ~ymm0) | (ymm2 & ymm9 & ymm0)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm0 & ~ymm9)
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512-FCP-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm2
 ; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm3, %zmm2, %zmm2
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm9 & zmm2) | (zmm1 & ~zmm9 & zmm2) | (zmm1 & zmm9 & ~zmm2) | (zmm1 & zmm9 & zmm2)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm11 & zmm1) | (zmm0 & ~zmm11 & ~zmm1) | (zmm0 & ~zmm11 & zmm1) | (zmm0 & zmm11 & zmm1)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm9 & (zmm1 ^ zmm2))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm11 & (zmm0 ^ zmm1))
 ; AVX512-FCP-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 16-byte Folded Reload
 ; AVX512-FCP-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 16-byte Folded Reload
 ; AVX512-FCP-NEXT:    vpmovsxwd {{.*#+}} zmm2 = [0,0,0,0,0,4294967040,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm16 = (~zmm16 & zmm2 & zmm1) | (zmm16 & ~zmm2 & ~zmm1) | (zmm16 & ~zmm2 & zmm1) | (zmm16 & zmm2 & zmm1)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm16 = zmm16 ^ (zmm2 & (zmm16 ^ zmm1))
 ; AVX512-FCP-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 16-byte Folded Reload
 ; AVX512-FCP-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 16-byte Folded Reload
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm21 = (~zmm21 & zmm2 & zmm1) | (zmm21 & ~zmm2 & ~zmm1) | (zmm21 & ~zmm2 & zmm1) | (zmm21 & zmm2 & zmm1)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm21 = zmm21 ^ (zmm2 & (zmm21 ^ zmm1))
 ; AVX512-FCP-NEXT:    vpmovsxdq {{.*#+}} zmm1 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,16777215,0,0]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm1 & zmm16) | (zmm4 & ~zmm1 & ~zmm16) | (zmm4 & ~zmm1 & zmm16) | (zmm4 & zmm1 & zmm16)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm1 & zmm21) | (zmm5 & ~zmm1 & ~zmm21) | (zmm5 & ~zmm1 & zmm21) | (zmm5 & zmm1 & zmm21)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm1 & (zmm4 ^ zmm16))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm1 & (zmm5 ^ zmm21))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm15, (%rsi)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm17, (%rdx)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, (%rcx)
@@ -7893,7 +7893,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vmovdqa64 224(%rdi), %ymm25
 ; AVX512DQ-NEXT:    vmovdqa64 192(%rdi), %ymm26
 ; AVX512DQ-NEXT:    vmovdqa %ymm12, %ymm0
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ~ymm26 & ymm25) | (~ymm0 & ymm26 & ymm25) | (ymm0 & ymm26 & ~ymm25) | (ymm0 & ymm26 & ymm25)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm25 ^ (ymm0 & (ymm26 ^ ymm25))
 ; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm0, %xmm3
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm5 = [128,128,128,2,8,14,128,128,0,6,12,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm4
@@ -7904,7 +7904,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vmovdqa64 128(%rdi), %ymm24
 ; AVX512DQ-NEXT:    vmovdqa64 160(%rdi), %ymm18
 ; AVX512DQ-NEXT:    vmovdqa %ymm12, %ymm6
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ~ymm18 & ymm24) | (~ymm6 & ymm18 & ymm24) | (ymm6 & ymm18 & ~ymm24) | (ymm6 & ymm18 & ymm24)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm24 ^ (ymm6 & (ymm18 ^ ymm24))
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm6, %xmm7
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm3 = [u,u,u,u,u,u,128,128,0,6,12,128,128,128,4,10]
 ; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm7, %xmm10
@@ -7915,7 +7915,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm9, %zmm10, %zmm2
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa %ymm12, %ymm9
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm9 = (~ymm9 & ~ymm30 & ymm31) | (~ymm9 & ymm30 & ymm31) | (ymm9 & ymm30 & ~ymm31) | (ymm9 & ymm30 & ymm31)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm9 = ymm31 ^ (ymm9 & (ymm30 ^ ymm31))
 ; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm9, %xmm1
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm9, %xmm13
 ; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm13, %xmm5
@@ -7923,7 +7923,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vmovdqa64 320(%rdi), %ymm29
 ; AVX512DQ-NEXT:    vmovdqa64 352(%rdi), %ymm22
 ; AVX512DQ-NEXT:    vmovdqa %ymm12, %ymm1
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ~ymm22 & ymm29) | (~ymm1 & ymm22 & ymm29) | (ymm1 & ymm22 & ~ymm29) | (ymm1 & ymm22 & ymm29)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm29 ^ (ymm1 & (ymm22 ^ ymm29))
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm1, %xmm5
 ; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm5, %xmm3
 ; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm1, %xmm8
@@ -7949,7 +7949,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm0 = [128,128,128,4,10,128,128,128,2,8,14,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
 ; AVX512DQ-NEXT:    vmovdqa %ymm9, %ymm4
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm4 = (~ymm4 & ~ymm25 & ymm26) | (~ymm4 & ymm25 & ymm26) | (ymm4 & ymm25 & ~ymm26) | (ymm4 & ymm25 & ymm26)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm4 = ymm26 ^ (ymm4 & (ymm25 ^ ymm26))
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm4, %xmm15
 ; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm15, %xmm1
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm6 = [2,8,14,128,128,0,6,12,128,128,128,u,u,u,u,u]
@@ -7957,7 +7957,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vpor %xmm1, %xmm5, %xmm1
 ; AVX512DQ-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa %ymm12, %ymm5
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ~ymm24 & ymm18) | (~ymm5 & ymm24 & ymm18) | (ymm5 & ymm24 & ~ymm18) | (ymm5 & ymm24 & ymm18)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm18 ^ (ymm5 & (ymm24 ^ ymm18))
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm8 = [u,u,u,u,u,0,6,12,128,128,128,4,10,128,128,128]
 ; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm5, %xmm7
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm5, %xmm1
@@ -7966,13 +7966,13 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vpor %xmm7, %xmm13, %xmm2
 ; AVX512DQ-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa %ymm9, %ymm13
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm13 = (~ymm13 & ~ymm31 & ymm30) | (~ymm13 & ymm31 & ymm30) | (ymm13 & ymm31 & ~ymm30) | (ymm13 & ymm31 & ymm30)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm13 = ymm30 ^ (ymm13 & (ymm31 ^ ymm30))
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm13, %xmm14
 ; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm14, %xmm0
 ; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm13, %xmm6
 ; AVX512DQ-NEXT:    vporq %xmm0, %xmm6, %xmm16
 ; AVX512DQ-NEXT:    vmovdqa %ymm12, %ymm11
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ~ymm29 & ymm22) | (~ymm11 & ymm29 & ymm22) | (ymm11 & ymm29 & ~ymm22) | (ymm11 & ymm29 & ymm22)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm22 ^ (ymm11 & (ymm29 ^ ymm22))
 ; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm11, %xmm8
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm11, %xmm7
 ; AVX512DQ-NEXT:    vpshufb %xmm10, %xmm7, %xmm10
@@ -7993,7 +7993,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vinserti32x4 $1, 288(%rdi), %ymm1, %ymm20
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm5 = [65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,0]
 ; AVX512DQ-NEXT:    vmovdqa %ymm5, %ymm1
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ~ymm20 & ymm19) | (~ymm1 & ymm20 & ymm19) | (ymm1 & ymm20 & ~ymm19) | (ymm1 & ymm20 & ymm19)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm19 ^ (ymm1 & (ymm20 ^ ymm19))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm15 = ymm1[u,u,u,u,u,u,u,u,u,u,u,2,8,14,4,10,16,22,28,18,24,30,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm15[0,1,2],ymm3[3,4,5,6,7],ymm15[8,9,10],ymm3[11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm15 = ymm15[0,1,2,3],ymm3[4,5,6,7]
@@ -8002,26 +8002,26 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vinserti32x4 $1, 96(%rdi), %ymm23, %ymm23
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm15, %zmm0, %zmm15
 ; AVX512DQ-NEXT:    vmovdqa %ymm5, %ymm2
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ~ymm23 & ymm6) | (~ymm2 & ymm23 & ymm6) | (ymm2 & ymm23 & ~ymm6) | (ymm2 & ymm23 & ymm6)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm6 ^ (ymm2 & (ymm23 ^ ymm6))
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ymm17 & ymm4) | (ymm0 & ~ymm17 & ~ymm4) | (ymm0 & ~ymm17 & ymm4) | (ymm0 & ymm17 & ~ymm4) | (ymm0 & ymm17 & ymm4)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm17 & ymm4)
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
 ; AVX512DQ-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm0 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm0 = (~zmm0 & ~zmm17 & mem) | (zmm0 & ~zmm17 & mem) | (zmm0 & zmm17 & ~mem) | (zmm0 & zmm17 & mem)
+; AVX512DQ-NEXT:    # zmm0 = mem ^ (zmm17 & (zmm0 ^ mem))
 ; AVX512DQ-NEXT:    vpmovsxdq {{.*#+}} zmm3 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,16777215,0,0]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm15 = (~zmm15 & zmm3 & zmm0) | (zmm15 & ~zmm3 & ~zmm0) | (zmm15 & ~zmm3 & zmm0) | (zmm15 & zmm3 & zmm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm15 = zmm15 ^ (zmm3 & (zmm15 ^ zmm0))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,3,9,15,5,11,17,23,29,19,25,31,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm1
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ymm21 & ymm4) | (ymm1 & ~ymm21 & ~ymm4) | (ymm1 & ~ymm21 & ymm4) | (ymm1 & ymm21 & ~ymm4) | (ymm1 & ymm21 & ymm4)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm21 & ymm4)
 ; AVX512DQ-NEXT:    vinserti32x4 $1, %xmm28, %ymm0, %ymm2
 ; AVX512DQ-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm17 & zmm2) | (zmm1 & ~zmm17 & zmm2) | (zmm1 & zmm17 & ~zmm2) | (zmm1 & zmm17 & zmm2)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm17 & (zmm1 ^ zmm2))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm17
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm17 = (~zmm17 & zmm3 & zmm1) | (zmm17 & ~zmm3 & ~zmm1) | (zmm17 & ~zmm3 & zmm1) | (zmm17 & zmm3 & zmm1)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm17 = zmm17 ^ (zmm3 & (zmm17 ^ zmm1))
 ; AVX512DQ-NEXT:    vpshufb %xmm10, %xmm14, %xmm0
 ; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm13, %xmm1
 ; AVX512DQ-NEXT:    vporq %xmm0, %xmm1, %xmm21
@@ -8029,7 +8029,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm1 = xmm7[u,u,u,u,u],zero,zero,zero,xmm7[3,9,15],zero,zero,xmm7[1,7,13]
 ; AVX512DQ-NEXT:    vporq %xmm0, %xmm1, %xmm28
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm25, %ymm11
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ~ymm12 & ymm26) | (ymm11 & ~ymm12 & ymm26) | (ymm11 & ymm12 & ~ymm26) | (ymm11 & ymm12 & ymm26)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm26 ^ (ymm12 & (ymm11 ^ ymm26))
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm11, %xmm0
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,xmm0[0,6,12],zero,zero,zero,xmm0[4,10,u,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm3 = [4,10,128,128,128,2,8,14,128,128,u,u,u,u,u,u]
@@ -8037,17 +8037,17 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm25
 ; AVX512DQ-NEXT:    vporq %xmm1, %xmm2, %xmm26
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm18, %ymm14
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm14 = (~ymm14 & ~ymm9 & ymm24) | (ymm14 & ~ymm9 & ymm24) | (ymm14 & ymm9 & ~ymm24) | (ymm14 & ymm9 & ymm24)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm14 = ymm24 ^ (ymm9 & (ymm14 ^ ymm24))
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm14, %xmm10
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm1 = [u,u,u,u,u,128,128,128,4,10,128,128,128,2,8,14]
 ; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm10, %xmm2
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm3 = [u,u,u,u,u,2,8,14,128,128,0,6,12,128,128,128]
 ; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm14, %xmm4
 ; AVX512DQ-NEXT:    vporq %xmm2, %xmm4, %xmm27
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm12 = (~ymm12 & ~ymm31 & ymm30) | (~ymm12 & ymm31 & ymm30) | (ymm12 & ymm31 & ~ymm30) | (ymm12 & ymm31 & ymm30)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm12 = ymm30 ^ (ymm12 & (ymm31 ^ ymm30))
 ; AVX512DQ-NEXT:    vmovdqa %ymm5, %ymm4
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm4 = (~ymm4 & ~ymm6 & ymm23) | (~ymm4 & ymm6 & ymm23) | (ymm4 & ymm6 & ~ymm23) | (ymm4 & ymm6 & ymm23)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm9 = (~ymm9 & ~ymm22 & ymm29) | (~ymm9 & ymm22 & ymm29) | (ymm9 & ymm22 & ~ymm29) | (ymm9 & ymm22 & ymm29)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm4 = ymm23 ^ (ymm4 & (ymm6 ^ ymm23))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm9 = ymm29 ^ (ymm9 & (ymm22 ^ ymm29))
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm9, %xmm8
 ; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm8, %xmm1
 ; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm9, %xmm2
@@ -8066,14 +8066,14 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm1 = [128,128,128,128,128,128,128,128,128,128,128,4,10,0,6,12,18,24,30,20,26,128,128,128,128,128,128,128,128,128,128,128]
 ; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm4, %ymm11
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} ymm18 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm16 = (~ymm16 & ymm11 & ~ymm18) | (~ymm16 & ymm11 & ymm18) | (ymm16 & ~ymm11 & ymm18) | (ymm16 & ymm11 & ~ymm18) | (ymm16 & ymm11 & ymm18)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm16 = (ymm16 & ymm18) | ymm11
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm11 = [128,128,128,128,128,128,128,128,128,128,128,5,11,1,7,13,19,25,31,21,27,128,128,128,128,128,128,128,128,128,128,128]
 ; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm4, %ymm4
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm21 = (~ymm21 & ymm4 & ~ymm18) | (~ymm21 & ymm4 & ymm18) | (ymm21 & ~ymm4 & ymm18) | (ymm21 & ymm4 & ~ymm18) | (ymm21 & ymm4 & ymm18)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm21 = (ymm21 & ymm18) | ymm4
 ; AVX512DQ-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ~ymm19 & ymm20) | (~ymm5 & ymm19 & ymm20) | (ymm5 & ymm19 & ~ymm20) | (ymm5 & ymm19 & ymm20)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm20 ^ (ymm5 & (ymm19 ^ ymm20))
 ; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ymm4 & ymm18) | (ymm1 & ~ymm4 & ~ymm18) | (ymm1 & ~ymm4 & ymm18) | (ymm1 & ymm4 & ~ymm18) | (ymm1 & ymm4 & ymm18)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm18)
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm4
 ; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm5, %ymm5
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm12, %xmm1
@@ -8085,24 +8085,24 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm9, %xmm0
 ; AVX512DQ-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ~ymm2 & ymm23) | (ymm6 & ~ymm2 & ymm23) | (ymm6 & ymm2 & ~ymm23) | (ymm6 & ymm2 & ymm23)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm23 ^ (ymm2 & (ymm6 ^ ymm23))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm8 = ymm6[u,u,u,u,u,u,u,u,u,u,0,6,12,2,8,14,20,26,16,22,28,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm9 = xmm11[0,1,2,3,4],xmm8[5,6,7]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
 ; AVX512DQ-NEXT:    vinserti32x4 $1, %xmm28, %ymm0, %ymm9
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ymm9 & ymm18) | (ymm5 & ~ymm9 & ~ymm18) | (ymm5 & ~ymm9 & ymm18) | (ymm5 & ymm9 & ~ymm18) | (ymm5 & ymm9 & ymm18)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm9 & ymm18)
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm5
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ~ymm19 & ymm20) | (~ymm2 & ymm19 & ymm20) | (ymm2 & ymm19 & ~ymm20) | (ymm2 & ymm19 & ymm20)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm20 ^ (ymm2 & (ymm19 ^ ymm20))
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm11 = ymm2[u,u,u,u,u,u,u,u,u,u,0,6,12,2,8,14,20,26,16,22,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ~ymm9 & ymm7) | (ymm11 & ~ymm9 & ~ymm7) | (ymm11 & ~ymm9 & ymm7) | (ymm11 & ymm9 & ~ymm7) | (ymm11 & ymm9 & ymm7)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm7 & ~ymm9)
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm7
 ; AVX512DQ-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm11
 ; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm26, %zmm11, %zmm11
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & ~zmm9 & zmm11) | (zmm8 & ~zmm9 & zmm11) | (zmm8 & zmm9 & ~zmm11) | (zmm8 & zmm9 & zmm11)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm11 ^ (zmm9 & (zmm8 ^ zmm11))
 ; AVX512DQ-NEXT:    vpmovsxdq {{.*#+}} zmm11 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & zmm11 & zmm8) | (zmm7 & ~zmm11 & ~zmm8) | (zmm7 & ~zmm11 & zmm8) | (zmm7 & zmm11 & zmm8)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm11 & (zmm7 ^ zmm8))
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm8
 ; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm12, %xmm8
@@ -8112,22 +8112,22 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm6[4,5,6,7]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,1,7,13,3,9,15,21,27,17,23,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ~ymm9 & ymm0) | (ymm2 & ~ymm9 & ~ymm0) | (ymm2 & ~ymm9 & ymm0) | (ymm2 & ymm9 & ~ymm0) | (ymm2 & ymm9 & ymm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm0 & ~ymm9)
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm2
 ; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm3, %zmm2, %zmm2
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm9 & zmm2) | (zmm1 & ~zmm9 & zmm2) | (zmm1 & zmm9 & ~zmm2) | (zmm1 & zmm9 & zmm2)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm11 & zmm1) | (zmm0 & ~zmm11 & ~zmm1) | (zmm0 & ~zmm11 & zmm1) | (zmm0 & zmm11 & zmm1)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm9 & (zmm1 ^ zmm2))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm11 & (zmm0 ^ zmm1))
 ; AVX512DQ-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 16-byte Folded Reload
 ; AVX512DQ-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 16-byte Folded Reload
 ; AVX512DQ-NEXT:    vpmovsxwd {{.*#+}} zmm2 = [0,0,0,0,0,4294967040,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm16 = (~zmm16 & zmm2 & zmm1) | (zmm16 & ~zmm2 & ~zmm1) | (zmm16 & ~zmm2 & zmm1) | (zmm16 & zmm2 & zmm1)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm16 = zmm16 ^ (zmm2 & (zmm16 ^ zmm1))
 ; AVX512DQ-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 16-byte Folded Reload
 ; AVX512DQ-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 16-byte Folded Reload
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm21 = (~zmm21 & zmm2 & zmm1) | (zmm21 & ~zmm2 & ~zmm1) | (zmm21 & ~zmm2 & zmm1) | (zmm21 & zmm2 & zmm1)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm21 = zmm21 ^ (zmm2 & (zmm21 ^ zmm1))
 ; AVX512DQ-NEXT:    vpmovsxdq {{.*#+}} zmm1 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,16777215,0,0]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm1 & zmm16) | (zmm4 & ~zmm1 & ~zmm16) | (zmm4 & ~zmm1 & zmm16) | (zmm4 & zmm1 & zmm16)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm1 & zmm21) | (zmm5 & ~zmm1 & ~zmm21) | (zmm5 & ~zmm1 & zmm21) | (zmm5 & zmm1 & zmm21)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm1 & (zmm4 ^ zmm16))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm1 & (zmm5 ^ zmm21))
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm15, (%rsi)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm17, (%rdx)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm4, (%rcx)
@@ -8147,7 +8147,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 224(%rdi), %ymm25
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 192(%rdi), %ymm26
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm12, %ymm0
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ~ymm26 & ymm25) | (~ymm0 & ymm26 & ymm25) | (ymm0 & ymm26 & ~ymm25) | (ymm0 & ymm26 & ymm25)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm25 ^ (ymm0 & (ymm26 ^ ymm25))
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm0, %xmm3
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [128,128,128,2,8,14,128,128,0,6,12,u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm0, %xmm4
@@ -8158,7 +8158,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 128(%rdi), %ymm24
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 160(%rdi), %ymm18
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm12, %ymm6
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ~ymm18 & ymm24) | (~ymm6 & ymm18 & ymm24) | (ymm6 & ymm18 & ~ymm24) | (ymm6 & ymm18 & ymm24)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm24 ^ (ymm6 & (ymm18 ^ ymm24))
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm6, %xmm7
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [u,u,u,u,u,u,128,128,0,6,12,128,128,128,4,10]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm7, %xmm10
@@ -8169,7 +8169,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm9, %zmm10, %zmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm12, %ymm9
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm9 = (~ymm9 & ~ymm30 & ymm31) | (~ymm9 & ymm30 & ymm31) | (ymm9 & ymm30 & ~ymm31) | (ymm9 & ymm30 & ymm31)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm9 = ymm31 ^ (ymm9 & (ymm30 ^ ymm31))
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm9, %xmm1
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm9, %xmm13
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm13, %xmm5
@@ -8177,7 +8177,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 320(%rdi), %ymm29
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 352(%rdi), %ymm22
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm12, %ymm1
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ~ymm22 & ymm29) | (~ymm1 & ymm22 & ymm29) | (ymm1 & ymm22 & ~ymm29) | (ymm1 & ymm22 & ymm29)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm29 ^ (ymm1 & (ymm22 ^ ymm29))
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm5
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm5, %xmm3
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm1, %xmm8
@@ -8203,7 +8203,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm0 = [128,128,128,4,10,128,128,128,2,8,14,u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm9, %ymm4
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = (~ymm4 & ~ymm25 & ymm26) | (~ymm4 & ymm25 & ymm26) | (ymm4 & ymm25 & ~ymm26) | (ymm4 & ymm25 & ymm26)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = ymm26 ^ (ymm4 & (ymm25 ^ ymm26))
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm4, %xmm15
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm15, %xmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [2,8,14,128,128,0,6,12,128,128,128,u,u,u,u,u]
@@ -8211,7 +8211,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vpor %xmm1, %xmm5, %xmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm12, %ymm5
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ~ymm24 & ymm18) | (~ymm5 & ymm24 & ymm18) | (ymm5 & ymm24 & ~ymm18) | (ymm5 & ymm24 & ymm18)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm18 ^ (ymm5 & (ymm24 ^ ymm18))
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [u,u,u,u,u,0,6,12,128,128,128,4,10,128,128,128]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm5, %xmm7
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm5, %xmm1
@@ -8220,13 +8220,13 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vpor %xmm7, %xmm13, %xmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm9, %ymm13
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm13 = (~ymm13 & ~ymm31 & ymm30) | (~ymm13 & ymm31 & ymm30) | (ymm13 & ymm31 & ~ymm30) | (ymm13 & ymm31 & ymm30)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm13 = ymm30 ^ (ymm13 & (ymm31 ^ ymm30))
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm13, %xmm14
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm14, %xmm0
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm13, %xmm6
 ; AVX512DQ-FCP-NEXT:    vporq %xmm0, %xmm6, %xmm16
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm12, %ymm11
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ~ymm29 & ymm22) | (~ymm11 & ymm29 & ymm22) | (ymm11 & ymm29 & ~ymm22) | (ymm11 & ymm29 & ymm22)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm22 ^ (ymm11 & (ymm29 ^ ymm22))
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm11, %xmm8
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm11, %xmm7
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm7, %xmm10
@@ -8247,7 +8247,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vinserti32x4 $1, 288(%rdi), %ymm1, %ymm20
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm5 = [65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,0]
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm5, %ymm1
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ~ymm20 & ymm19) | (~ymm1 & ymm20 & ymm19) | (ymm1 & ymm20 & ~ymm19) | (ymm1 & ymm20 & ymm19)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm19 ^ (ymm1 & (ymm20 ^ ymm19))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm15 = ymm1[u,u,u,u,u,u,u,u,u,u,u,2,8,14,4,10,16,22,28,18,24,30,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm15[0,1,2],ymm3[3,4,5,6,7],ymm15[8,9,10],ymm3[11,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm15 = ymm15[0,1,2,3],ymm3[4,5,6,7]
@@ -8256,26 +8256,26 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vinserti32x4 $1, 96(%rdi), %ymm23, %ymm23
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm15, %zmm0, %zmm15
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm5, %ymm2
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ~ymm23 & ymm6) | (~ymm2 & ymm23 & ymm6) | (ymm2 & ymm23 & ~ymm6) | (ymm2 & ymm23 & ymm6)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm6 ^ (ymm2 & (ymm23 ^ ymm6))
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ymm17 & ymm4) | (ymm0 & ~ymm17 & ~ymm4) | (ymm0 & ~ymm17 & ymm4) | (ymm0 & ymm17 & ~ymm4) | (ymm0 & ymm17 & ymm4)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm17 & ymm4)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
 ; AVX512DQ-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm0 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm0 = (~zmm0 & ~zmm17 & mem) | (zmm0 & ~zmm17 & mem) | (zmm0 & zmm17 & ~mem) | (zmm0 & zmm17 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm0 = mem ^ (zmm17 & (zmm0 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vpmovsxdq {{.*#+}} zmm3 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,16777215,0,0]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = (~zmm15 & zmm3 & zmm0) | (zmm15 & ~zmm3 & ~zmm0) | (zmm15 & ~zmm3 & zmm0) | (zmm15 & zmm3 & zmm0)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = zmm15 ^ (zmm3 & (zmm15 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,3,9,15,5,11,17,23,29,19,25,31,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm1
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[3,9,15,5,11,17,23,29,19,25,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ymm21 & ymm4) | (ymm1 & ~ymm21 & ~ymm4) | (ymm1 & ~ymm21 & ymm4) | (ymm1 & ymm21 & ~ymm4) | (ymm1 & ymm21 & ymm4)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm21 & ymm4)
 ; AVX512DQ-FCP-NEXT:    vinserti32x4 $1, %xmm28, %ymm0, %ymm2
 ; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 16-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm17 & zmm2) | (zmm1 & ~zmm17 & zmm2) | (zmm1 & zmm17 & ~zmm2) | (zmm1 & zmm17 & zmm2)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm17 & (zmm1 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm17
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm17 = (~zmm17 & zmm3 & zmm1) | (zmm17 & ~zmm3 & ~zmm1) | (zmm17 & ~zmm3 & zmm1) | (zmm17 & zmm3 & zmm1)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm17 = zmm17 ^ (zmm3 & (zmm17 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm14, %xmm0
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm13, %xmm1
 ; AVX512DQ-FCP-NEXT:    vporq %xmm0, %xmm1, %xmm21
@@ -8283,7 +8283,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm7[u,u,u,u,u],zero,zero,zero,xmm7[3,9,15],zero,zero,xmm7[1,7,13]
 ; AVX512DQ-FCP-NEXT:    vporq %xmm0, %xmm1, %xmm28
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm25, %ymm11
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ~ymm12 & ymm26) | (ymm11 & ~ymm12 & ymm26) | (ymm11 & ymm12 & ~ymm26) | (ymm11 & ymm12 & ymm26)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm26 ^ (ymm12 & (ymm11 ^ ymm26))
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm11, %xmm0
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,xmm0[0,6,12],zero,zero,zero,xmm0[4,10,u,u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [4,10,128,128,128,2,8,14,128,128,u,u,u,u,u,u]
@@ -8291,17 +8291,17 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm3, %xmm25
 ; AVX512DQ-FCP-NEXT:    vporq %xmm1, %xmm2, %xmm26
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm18, %ymm14
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm14 = (~ymm14 & ~ymm9 & ymm24) | (ymm14 & ~ymm9 & ymm24) | (ymm14 & ymm9 & ~ymm24) | (ymm14 & ymm9 & ymm24)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm14 = ymm24 ^ (ymm9 & (ymm14 ^ ymm24))
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm14, %xmm10
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [u,u,u,u,u,128,128,128,4,10,128,128,128,2,8,14]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm10, %xmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [u,u,u,u,u,2,8,14,128,128,0,6,12,128,128,128]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm14, %xmm4
 ; AVX512DQ-FCP-NEXT:    vporq %xmm2, %xmm4, %xmm27
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm12 = (~ymm12 & ~ymm31 & ymm30) | (~ymm12 & ymm31 & ymm30) | (ymm12 & ymm31 & ~ymm30) | (ymm12 & ymm31 & ymm30)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm12 = ymm30 ^ (ymm12 & (ymm31 ^ ymm30))
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm5, %ymm4
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = (~ymm4 & ~ymm6 & ymm23) | (~ymm4 & ymm6 & ymm23) | (ymm4 & ymm6 & ~ymm23) | (ymm4 & ymm6 & ymm23)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm9 = (~ymm9 & ~ymm22 & ymm29) | (~ymm9 & ymm22 & ymm29) | (ymm9 & ymm22 & ~ymm29) | (ymm9 & ymm22 & ymm29)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = ymm23 ^ (ymm4 & (ymm6 ^ ymm23))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm9 = ymm29 ^ (ymm9 & (ymm22 ^ ymm29))
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm9, %xmm8
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm8, %xmm1
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm9, %xmm2
@@ -8320,14 +8320,14 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [128,128,128,128,128,128,128,128,128,128,128,4,10,0,6,12,18,24,30,20,26,128,128,128,128,128,128,128,128,128,128,128]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm4, %ymm11
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} ymm18 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm16 = (~ymm16 & ymm11 & ~ymm18) | (~ymm16 & ymm11 & ymm18) | (ymm16 & ~ymm11 & ymm18) | (ymm16 & ymm11 & ~ymm18) | (ymm16 & ymm11 & ymm18)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm16 = (ymm16 & ymm18) | ymm11
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [128,128,128,128,128,128,128,128,128,128,128,5,11,1,7,13,19,25,31,21,27,128,128,128,128,128,128,128,128,128,128,128]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm4, %ymm4
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm21 = (~ymm21 & ymm4 & ~ymm18) | (~ymm21 & ymm4 & ymm18) | (ymm21 & ~ymm4 & ymm18) | (ymm21 & ymm4 & ~ymm18) | (ymm21 & ymm4 & ymm18)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm21 = (ymm21 & ymm18) | ymm4
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 16-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ~ymm19 & ymm20) | (~ymm5 & ymm19 & ymm20) | (ymm5 & ymm19 & ~ymm20) | (ymm5 & ymm19 & ymm20)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm20 ^ (ymm5 & (ymm19 ^ ymm20))
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ymm4 & ymm18) | (ymm1 & ~ymm4 & ~ymm18) | (ymm1 & ~ymm4 & ymm18) | (ymm1 & ymm4 & ~ymm18) | (ymm1 & ymm4 & ymm18)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm18)
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm4
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm5, %ymm5
 ; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm12, %xmm1
@@ -8339,24 +8339,24 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm9, %xmm0
 ; AVX512DQ-FCP-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ~ymm2 & ymm23) | (ymm6 & ~ymm2 & ymm23) | (ymm6 & ymm2 & ~ymm23) | (ymm6 & ymm2 & ymm23)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm23 ^ (ymm2 & (ymm6 ^ ymm23))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = ymm6[u,u,u,u,u,u,u,u,u,u,0,6,12,2,8,14,20,26,16,22,28,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm9 = xmm11[0,1,2,3,4],xmm8[5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vinserti32x4 $1, %xmm28, %ymm0, %ymm9
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = (~ymm5 & ymm9 & ymm18) | (ymm5 & ~ymm9 & ~ymm18) | (ymm5 & ~ymm9 & ymm18) | (ymm5 & ymm9 & ~ymm18) | (ymm5 & ymm9 & ymm18)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm9 & ymm18)
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm5
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ~ymm19 & ymm20) | (~ymm2 & ymm19 & ymm20) | (ymm2 & ymm19 & ~ymm20) | (ymm2 & ymm19 & ymm20)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm20 ^ (ymm2 & (ymm19 ^ ymm20))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm2[u,u,u,u,u,u,u,u,u,u,0,6,12,2,8,14,20,26,16,22,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ~ymm9 & ymm7) | (ymm11 & ~ymm9 & ~ymm7) | (ymm11 & ~ymm9 & ymm7) | (ymm11 & ymm9 & ~ymm7) | (ymm11 & ymm9 & ymm7)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm7 & ~ymm9)
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm7
 ; AVX512DQ-FCP-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm11
 ; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm26, %zmm11, %zmm11
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & ~zmm9 & zmm11) | (zmm8 & ~zmm9 & zmm11) | (zmm8 & zmm9 & ~zmm11) | (zmm8 & zmm9 & zmm11)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm11 ^ (zmm9 & (zmm8 ^ zmm11))
 ; AVX512DQ-FCP-NEXT:    vpmovsxdq {{.*#+}} zmm11 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & zmm11 & zmm8) | (zmm7 & ~zmm11 & ~zmm8) | (zmm7 & ~zmm11 & zmm8) | (zmm7 & zmm11 & zmm8)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm11 & (zmm7 ^ zmm8))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm22, %xmm8
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm1, %xmm1
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm12, %xmm8
@@ -8366,22 +8366,22 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm6[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,1,7,13,3,9,15,21,27,17,23,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ~ymm9 & ymm0) | (ymm2 & ~ymm9 & ~ymm0) | (ymm2 & ~ymm9 & ymm0) | (ymm2 & ymm9 & ~ymm0) | (ymm2 & ymm9 & ymm0)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm2 | (ymm0 & ~ymm9)
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm2
 ; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm3, %zmm2, %zmm2
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm9 & zmm2) | (zmm1 & ~zmm9 & zmm2) | (zmm1 & zmm9 & ~zmm2) | (zmm1 & zmm9 & zmm2)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm11 & zmm1) | (zmm0 & ~zmm11 & ~zmm1) | (zmm0 & ~zmm11 & zmm1) | (zmm0 & zmm11 & zmm1)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm9 & (zmm1 ^ zmm2))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm11 & (zmm0 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 16-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 16-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vpmovsxwd {{.*#+}} zmm2 = [0,0,0,0,0,4294967040,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295,4294967295]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm16 = (~zmm16 & zmm2 & zmm1) | (zmm16 & ~zmm2 & ~zmm1) | (zmm16 & ~zmm2 & zmm1) | (zmm16 & zmm2 & zmm1)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm16 = zmm16 ^ (zmm2 & (zmm16 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 16-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 16-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm21 = (~zmm21 & zmm2 & zmm1) | (zmm21 & ~zmm2 & ~zmm1) | (zmm21 & ~zmm2 & zmm1) | (zmm21 & zmm2 & zmm1)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm21 = zmm21 ^ (zmm2 & (zmm21 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vpmovsxdq {{.*#+}} zmm1 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,16777215,0,0]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm1 & zmm16) | (zmm4 & ~zmm1 & ~zmm16) | (zmm4 & ~zmm1 & zmm16) | (zmm4 & zmm1 & zmm16)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm1 & zmm21) | (zmm5 & ~zmm1 & ~zmm21) | (zmm5 & ~zmm1 & zmm21) | (zmm5 & zmm1 & zmm21)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm1 & (zmm4 ^ zmm16))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm1 & (zmm5 ^ zmm21))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm15, (%rsi)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm17, (%rdx)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, (%rcx)
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll
index cd481a30cb211..aa449b36c2eb5 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll
@@ -444,7 +444,7 @@ define void @store_i16_stride5_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,3,0,1]
 ; AVX512-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm5[0,1],ymm2[2,3],ymm5[4,5,6],ymm2[7],ymm5[8,9],ymm2[10,11],ymm5[12,13,14],ymm2[15]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ymm3 & ~mem) | (ymm2 & ~ymm3 & mem) | (ymm2 & ymm3 & ~mem) | (ymm2 & ymm3 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm3 ^ (mem & (ymm2 ^ ymm3))
 ; AVX512-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,3,1,3,4,5,6,7]
@@ -472,7 +472,7 @@ define void @store_i16_stride5_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,3,0,1]
 ; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm5[0,1],ymm2[2,3],ymm5[4,5,6],ymm2[7],ymm5[8,9],ymm2[10,11],ymm5[12,13,14],ymm2[15]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ymm3 & ~mem) | (ymm2 & ~ymm3 & mem) | (ymm2 & ymm3 & ~mem) | (ymm2 & ymm3 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm3 ^ (mem & (ymm2 ^ ymm3))
 ; AVX512-FCP-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,6,7,14,15,u,u,8,9,10,11,12,13,14,15]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
@@ -499,7 +499,7 @@ define void @store_i16_stride5_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,3,0,1]
 ; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm5[0,1],ymm2[2,3],ymm5[4,5,6],ymm2[7],ymm5[8,9],ymm2[10,11],ymm5[12,13,14],ymm2[15]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ymm3 & ~mem) | (ymm2 & ~ymm3 & mem) | (ymm2 & ymm3 & ~mem) | (ymm2 & ymm3 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm3 ^ (mem & (ymm2 ^ ymm3))
 ; AVX512DQ-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,3,1,3,4,5,6,7]
@@ -527,7 +527,7 @@ define void @store_i16_stride5_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,3,0,1]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm5[0,1],ymm2[2,3],ymm5[4,5,6],ymm2[7],ymm5[8,9],ymm2[10,11],ymm5[12,13,14],ymm2[15]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ymm3 & ~mem) | (ymm2 & ~ymm3 & mem) | (ymm2 & ymm3 & ~mem) | (ymm2 & ymm3 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm3 ^ (mem & (ymm2 ^ ymm3))
 ; AVX512DQ-FCP-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,6,7,14,15,u,u,8,9,10,11,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
@@ -936,7 +936,7 @@ define void @store_i16_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpbroadcastq (%r8), %ymm6
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,1,1,1]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm6, %zmm4
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm4 = (~zmm4 & zmm5 & mem) | (zmm4 & ~zmm5 & ~mem) | (zmm4 & zmm5 & ~mem) | (zmm4 & zmm5 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm5))
 ; AVX512-NEXT:    vpsrlq $48, %xmm3, %xmm3
 ; AVX512-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm3[1]
 ; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
@@ -975,7 +975,7 @@ define void @store_i16_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpbroadcastq (%r8), %ymm6
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,1,1,1]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm6, %zmm4
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = (~zmm4 & zmm5 & mem) | (zmm4 & ~zmm5 & ~mem) | (zmm4 & zmm5 & ~mem) | (zmm4 & zmm5 & mem)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm5))
 ; AVX512-FCP-NEXT:    vpsrlq $48, %xmm1, %xmm1
 ; AVX512-FCP-NEXT:    vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1]
 ; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
@@ -1020,7 +1020,7 @@ define void @store_i16_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpbroadcastq (%r8), %ymm6
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,1,1,1]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm6, %zmm4
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm4 = (~zmm4 & zmm5 & mem) | (zmm4 & ~zmm5 & ~mem) | (zmm4 & zmm5 & ~mem) | (zmm4 & zmm5 & mem)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm5))
 ; AVX512DQ-NEXT:    vpsrlq $48, %xmm3, %xmm3
 ; AVX512DQ-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm3[1]
 ; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
@@ -1059,7 +1059,7 @@ define void @store_i16_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq (%r8), %ymm6
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,1,1,1]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm6, %zmm4
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = (~zmm4 & zmm5 & mem) | (zmm4 & ~zmm5 & ~mem) | (zmm4 & zmm5 & ~mem) | (zmm4 & zmm5 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm5))
 ; AVX512DQ-FCP-NEXT:    vpsrlq $48, %xmm1, %xmm1
 ; AVX512DQ-FCP-NEXT:    vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1]
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
@@ -1737,7 +1737,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm10 = xmm9[10,11,u,u,6,7,u,u,8,9,8,9,u,u,8,9]
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm8 = xmm10[0],xmm8[1],xmm10[2],xmm8[3],xmm10[4,5],xmm8[6],xmm10[7]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,1,0,0]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm8 = (~ymm8 & ymm5 & mem) | (ymm8 & ~ymm5 & ~mem) | (ymm8 & ymm5 & ~mem) | (ymm8 & ymm5 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm8 = ymm8 ^ (mem & (ymm8 ^ ymm5))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm5
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[4,5,2,3,2,3,0,1,10,11,8,9,4,5,6,7]
@@ -1748,12 +1748,12 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,4,5,6]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm8 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ~ymm8 & ymm7) | (ymm6 & ~ymm8 & ymm7) | (ymm6 & ymm8 & ~ymm7) | (ymm6 & ymm8 & ymm7)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm7 ^ (ymm8 & (ymm6 ^ ymm7))
 ; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm6[0,1,2,3],zmm5[4,5,6,7]
 ; AVX512-NEXT:    vpbroadcastq (%r8), %ymm6
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm4[0,1,1,1]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm6 = (~zmm6 & zmm5 & mem) | (zmm6 & ~zmm5 & ~mem) | (zmm6 & zmm5 & ~mem) | (zmm6 & zmm5 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm5 = ymm1[u,u,u,u,u,u,u,u,14,15,u,u,u,u,u,u,u,u,16,17,u,u,u,u,u,u,u,u,18,19,u,u]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm0[1,1,2,2]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm5 = ymm7[0],ymm5[1],ymm7[2,3],ymm5[4],ymm7[5],ymm5[6],ymm7[7,8],ymm5[9],ymm7[10,11],ymm5[12],ymm7[13],ymm5[14],ymm7[15]
@@ -1761,7 +1761,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm9 = ymm2[3,0,3,0,7,4,7,4]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm7 = ymm9[0],ymm7[1],ymm9[2],ymm7[3],ymm9[4,5],ymm7[6],ymm9[7,8],ymm7[9],ymm9[10],ymm7[11],ymm9[12,13],ymm7[14],ymm9[15]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm7 = (~ymm7 & ~ymm9 & ymm5) | (ymm7 & ~ymm9 & ymm5) | (ymm7 & ymm9 & ~ymm5) | (ymm7 & ymm9 & ymm5)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm7 = ymm5 ^ (ymm9 & (ymm7 ^ ymm5))
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm0[0,1,2,1,4,5,6,5]
 ; AVX512-NEXT:    vprolq $16, %ymm1, %ymm10
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm5 = ymm10[0,1],ymm5[2],ymm10[3],ymm5[4],ymm10[5,6],ymm5[7],ymm10[8,9],ymm5[10],ymm10[11],ymm5[12],ymm10[13,14],ymm5[15]
@@ -1770,14 +1770,14 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm11 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,u,u,22,23,22,23,u,u,20,21,u,u,24,25]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm10 = ymm11[0],ymm10[1],ymm11[2,3],ymm10[4],ymm11[5],ymm10[6],ymm11[7,8],ymm10[9],ymm11[10,11],ymm10[12],ymm11[13],ymm10[14],ymm11[15]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,3,2,3]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm10 = (~ymm10 & ~ymm8 & ymm5) | (ymm10 & ~ymm8 & ymm5) | (ymm10 & ymm8 & ~ymm5) | (ymm10 & ymm8 & ymm5)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm5 ^ (ymm8 & (ymm10 ^ ymm5))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm0, %zmm5
 ; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm7[0,1,2,3],zmm5[4,5,6,7]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm7 = [65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
 ; AVX512-NEXT:    vpandnq 16(%r8){1to4}, %ymm7, %ymm7
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,ymm4[12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[16,17],zero,zero,zero,zero,zero,zero
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm4, %zmm4
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm5 & mem) | (zmm4 & ~zmm5 & ~mem) | (zmm4 & ~zmm5 & mem) | (zmm4 & zmm5 & ~mem) | (zmm4 & zmm5 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm5 & mem)
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[3,2,3,3,7,6,7,7]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[2,3,2,3,6,7,6,7]
@@ -1788,9 +1788,9 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[2,3,2,2,6,7,6,6]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,2]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ~ymm9 & ymm2) | (ymm0 & ~ymm9 & ymm2) | (ymm0 & ymm9 & ~ymm2) | (ymm0 & ymm9 & ymm2)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm9 & (ymm0 ^ ymm2))
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ~ymm0 & mem) | (~ymm1 & ymm0 & mem) | (ymm1 & ymm0 & ~mem) | (ymm1 & ymm0 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm1 = mem ^ (ymm1 & (ymm0 ^ mem))
 ; AVX512-NEXT:    vmovdqa %ymm1, 128(%r9)
 ; AVX512-NEXT:    vmovdqa64 %zmm4, 64(%r9)
 ; AVX512-NEXT:    vmovdqa64 %zmm6, (%r9)
@@ -1815,7 +1815,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm9[10,11,u,u,6,7,u,u,8,9,8,9,u,u,8,9]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm8 = xmm10[0],xmm8[1],xmm10[2],xmm8[3],xmm10[4,5],xmm8[6],xmm10[7]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,1,0,0]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm8 = (~ymm8 & ymm5 & mem) | (ymm8 & ~ymm5 & ~mem) | (ymm8 & ymm5 & ~mem) | (ymm8 & ymm5 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm8 = ymm8 ^ (mem & (ymm8 ^ ymm5))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm5
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[4,5,2,3,2,3,0,1,10,11,8,9,4,5,6,7]
@@ -1825,12 +1825,12 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[0,1,2,3,8,9,10,11,4,5,4,5,6,7,12,13]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm8 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ~ymm8 & ymm7) | (ymm6 & ~ymm8 & ymm7) | (ymm6 & ymm8 & ~ymm7) | (ymm6 & ymm8 & ymm7)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm7 ^ (ymm8 & (ymm6 ^ ymm7))
 ; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm6[0,1,2,3],zmm5[4,5,6,7]
 ; AVX512-FCP-NEXT:    vpbroadcastq (%r8), %ymm6
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm4[0,1,1,1]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = (~zmm6 & zmm5 & mem) | (zmm6 & ~zmm5 & ~mem) | (zmm6 & zmm5 & ~mem) | (zmm6 & zmm5 & mem)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm1[u,u,u,u,u,u,u,u,14,15,u,u,u,u,u,u,u,u,16,17,u,u,u,u,u,u,u,u,18,19,u,u]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm0[1,1,2,2]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm7[0],ymm5[1],ymm7[2,3],ymm5[4],ymm7[5],ymm5[6],ymm7[7,8],ymm5[9],ymm7[10,11],ymm5[12],ymm7[13],ymm5[14],ymm7[15]
@@ -1838,7 +1838,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm9 = ymm2[3,0,3,0,7,4,7,4]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm7 = ymm9[0],ymm7[1],ymm9[2],ymm7[3],ymm9[4,5],ymm7[6],ymm9[7,8],ymm7[9],ymm9[10],ymm7[11],ymm9[12,13],ymm7[14],ymm9[15]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm7 = (~ymm7 & ~ymm9 & ymm5) | (ymm7 & ~ymm9 & ymm5) | (ymm7 & ymm9 & ~ymm5) | (ymm7 & ymm9 & ymm5)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm7 = ymm5 ^ (ymm9 & (ymm7 ^ ymm5))
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm0[0,1,2,1,4,5,6,5]
 ; AVX512-FCP-NEXT:    vprolq $16, %ymm1, %ymm10
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm10[0,1],ymm5[2],ymm10[3],ymm5[4],ymm10[5,6],ymm5[7],ymm10[8,9],ymm5[10],ymm10[11],ymm5[12],ymm10[13,14],ymm5[15]
@@ -1847,14 +1847,14 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,u,u,22,23,22,23,u,u,20,21,u,u,24,25]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm11[0],ymm10[1],ymm11[2,3],ymm10[4],ymm11[5],ymm10[6],ymm11[7,8],ymm10[9],ymm11[10,11],ymm10[12],ymm11[13],ymm10[14],ymm11[15]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,3,2,3]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = (~ymm10 & ~ymm8 & ymm5) | (ymm10 & ~ymm8 & ymm5) | (ymm10 & ymm8 & ~ymm5) | (ymm10 & ymm8 & ymm5)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm5 ^ (ymm8 & (ymm10 ^ ymm5))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm0, %zmm5
 ; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm7[0,1,2,3],zmm5[4,5,6,7]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
 ; AVX512-FCP-NEXT:    vpandnq 16(%r8){1to4}, %ymm7, %ymm7
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,ymm4[12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[16,17],zero,zero,zero,zero,zero,zero
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm4, %zmm4
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm5 & mem) | (zmm4 & ~zmm5 & ~mem) | (zmm4 & ~zmm5 & mem) | (zmm4 & zmm5 & ~mem) | (zmm4 & zmm5 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm5 & mem)
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[3,2,3,3,7,6,7,7]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,u,u,30,31,u,u,u,u,28,29,30,31,30,31]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3,4],ymm3[5,6,7,8],ymm2[9],ymm3[10],ymm2[11,12],ymm3[13,14,15]
@@ -1863,9 +1863,9 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,30,31,u,u,26,27,u,u,30,31,28,29,u,u,28,29]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,2]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ~ymm9 & ymm2) | (ymm0 & ~ymm9 & ymm2) | (ymm0 & ymm9 & ~ymm2) | (ymm0 & ymm9 & ymm2)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm9 & (ymm0 ^ ymm2))
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ~ymm0 & mem) | (~ymm1 & ymm0 & mem) | (ymm1 & ymm0 & ~mem) | (ymm1 & ymm0 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = mem ^ (ymm1 & (ymm0 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa %ymm1, 128(%r9)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, 64(%r9)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, (%r9)
@@ -1890,7 +1890,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm10 = xmm9[10,11,u,u,6,7,u,u,8,9,8,9,u,u,8,9]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm8 = xmm10[0],xmm8[1],xmm10[2],xmm8[3],xmm10[4,5],xmm8[6],xmm10[7]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,1,0,0]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm8 = (~ymm8 & ymm5 & mem) | (ymm8 & ~ymm5 & ~mem) | (ymm8 & ymm5 & ~mem) | (ymm8 & ymm5 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm8 = ymm8 ^ (mem & (ymm8 ^ ymm5))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm5
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[4,5,2,3,2,3,0,1,10,11,8,9,4,5,6,7]
@@ -1901,12 +1901,12 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,4,5,6]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm8 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ~ymm8 & ymm7) | (ymm6 & ~ymm8 & ymm7) | (ymm6 & ymm8 & ~ymm7) | (ymm6 & ymm8 & ymm7)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm7 ^ (ymm8 & (ymm6 ^ ymm7))
 ; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm6[0,1,2,3],zmm5[4,5,6,7]
 ; AVX512DQ-NEXT:    vpbroadcastq (%r8), %ymm6
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm4[0,1,1,1]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm6 = (~zmm6 & zmm5 & mem) | (zmm6 & ~zmm5 & ~mem) | (zmm6 & zmm5 & ~mem) | (zmm6 & zmm5 & mem)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm5 = ymm1[u,u,u,u,u,u,u,u,14,15,u,u,u,u,u,u,u,u,16,17,u,u,u,u,u,u,u,u,18,19,u,u]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm0[1,1,2,2]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm5 = ymm7[0],ymm5[1],ymm7[2,3],ymm5[4],ymm7[5],ymm5[6],ymm7[7,8],ymm5[9],ymm7[10,11],ymm5[12],ymm7[13],ymm5[14],ymm7[15]
@@ -1914,7 +1914,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm9 = ymm2[3,0,3,0,7,4,7,4]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm7 = ymm9[0],ymm7[1],ymm9[2],ymm7[3],ymm9[4,5],ymm7[6],ymm9[7,8],ymm7[9],ymm9[10],ymm7[11],ymm9[12,13],ymm7[14],ymm9[15]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm7 = (~ymm7 & ~ymm9 & ymm5) | (ymm7 & ~ymm9 & ymm5) | (ymm7 & ymm9 & ~ymm5) | (ymm7 & ymm9 & ymm5)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm7 = ymm5 ^ (ymm9 & (ymm7 ^ ymm5))
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm5 = ymm0[0,1,2,1,4,5,6,5]
 ; AVX512DQ-NEXT:    vprolq $16, %ymm1, %ymm10
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm5 = ymm10[0,1],ymm5[2],ymm10[3],ymm5[4],ymm10[5,6],ymm5[7],ymm10[8,9],ymm5[10],ymm10[11],ymm5[12],ymm10[13,14],ymm5[15]
@@ -1923,14 +1923,14 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm11 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,u,u,22,23,22,23,u,u,20,21,u,u,24,25]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm10 = ymm11[0],ymm10[1],ymm11[2,3],ymm10[4],ymm11[5],ymm10[6],ymm11[7,8],ymm10[9],ymm11[10,11],ymm10[12],ymm11[13],ymm10[14],ymm11[15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,3,2,3]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm10 = (~ymm10 & ~ymm8 & ymm5) | (ymm10 & ~ymm8 & ymm5) | (ymm10 & ymm8 & ~ymm5) | (ymm10 & ymm8 & ymm5)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm5 ^ (ymm8 & (ymm10 ^ ymm5))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm0, %zmm5
 ; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm7[0,1,2,3],zmm5[4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm7 = [65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
 ; AVX512DQ-NEXT:    vpandnq 16(%r8){1to4}, %ymm7, %ymm7
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,ymm4[12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[16,17],zero,zero,zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm4, %zmm4
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm5 & mem) | (zmm4 & ~zmm5 & ~mem) | (zmm4 & ~zmm5 & mem) | (zmm4 & zmm5 & ~mem) | (zmm4 & zmm5 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm5 & mem)
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[3,2,3,3,7,6,7,7]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[2,3,2,3,6,7,6,7]
@@ -1941,9 +1941,9 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[2,3,2,2,6,7,6,6]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,2]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ~ymm9 & ymm2) | (ymm0 & ~ymm9 & ymm2) | (ymm0 & ymm9 & ~ymm2) | (ymm0 & ymm9 & ymm2)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm9 & (ymm0 ^ ymm2))
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ~ymm0 & mem) | (~ymm1 & ymm0 & mem) | (ymm1 & ymm0 & ~mem) | (ymm1 & ymm0 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm1 = mem ^ (ymm1 & (ymm0 ^ mem))
 ; AVX512DQ-NEXT:    vmovdqa %ymm1, 128(%r9)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 64(%r9)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm6, (%r9)
@@ -1968,7 +1968,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm10 = xmm9[10,11,u,u,6,7,u,u,8,9,8,9,u,u,8,9]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm8 = xmm10[0],xmm8[1],xmm10[2],xmm8[3],xmm10[4,5],xmm8[6],xmm10[7]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,1,0,0]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm8 = (~ymm8 & ymm5 & mem) | (ymm8 & ~ymm5 & ~mem) | (ymm8 & ymm5 & ~mem) | (ymm8 & ymm5 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm8 = ymm8 ^ (mem & (ymm8 ^ ymm5))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm5
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[4,5,2,3,2,3,0,1,10,11,8,9,4,5,6,7]
@@ -1978,12 +1978,12 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[0,1,2,3,8,9,10,11,4,5,4,5,6,7,12,13]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm8 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ~ymm8 & ymm7) | (ymm6 & ~ymm8 & ymm7) | (ymm6 & ymm8 & ~ymm7) | (ymm6 & ymm8 & ymm7)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm7 ^ (ymm8 & (ymm6 ^ ymm7))
 ; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm6[0,1,2,3],zmm5[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq (%r8), %ymm6
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm4[0,1,1,1]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = (~zmm6 & zmm5 & mem) | (zmm6 & ~zmm5 & ~mem) | (zmm6 & zmm5 & ~mem) | (zmm6 & zmm5 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm1[u,u,u,u,u,u,u,u,14,15,u,u,u,u,u,u,u,u,16,17,u,u,u,u,u,u,u,u,18,19,u,u]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm0[1,1,2,2]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm7[0],ymm5[1],ymm7[2,3],ymm5[4],ymm7[5],ymm5[6],ymm7[7,8],ymm5[9],ymm7[10,11],ymm5[12],ymm7[13],ymm5[14],ymm7[15]
@@ -1991,7 +1991,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm9 = ymm2[3,0,3,0,7,4,7,4]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm7 = ymm9[0],ymm7[1],ymm9[2],ymm7[3],ymm9[4,5],ymm7[6],ymm9[7,8],ymm7[9],ymm9[10],ymm7[11],ymm9[12,13],ymm7[14],ymm9[15]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm7 = (~ymm7 & ~ymm9 & ymm5) | (ymm7 & ~ymm9 & ymm5) | (ymm7 & ymm9 & ~ymm5) | (ymm7 & ymm9 & ymm5)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm7 = ymm5 ^ (ymm9 & (ymm7 ^ ymm5))
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm0[0,1,2,1,4,5,6,5]
 ; AVX512DQ-FCP-NEXT:    vprolq $16, %ymm1, %ymm10
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm10[0,1],ymm5[2],ymm10[3],ymm5[4],ymm10[5,6],ymm5[7],ymm10[8,9],ymm5[10],ymm10[11],ymm5[12],ymm10[13,14],ymm5[15]
@@ -2000,14 +2000,14 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,u,u,22,23,22,23,u,u,20,21,u,u,24,25]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm11[0],ymm10[1],ymm11[2,3],ymm10[4],ymm11[5],ymm10[6],ymm11[7,8],ymm10[9],ymm11[10,11],ymm10[12],ymm11[13],ymm10[14],ymm11[15]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,3,2,3]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = (~ymm10 & ~ymm8 & ymm5) | (ymm10 & ~ymm8 & ymm5) | (ymm10 & ymm8 & ~ymm5) | (ymm10 & ymm8 & ymm5)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm5 ^ (ymm8 & (ymm10 ^ ymm5))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm0, %zmm5
 ; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm7[0,1,2,3],zmm5[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
 ; AVX512DQ-FCP-NEXT:    vpandnq 16(%r8){1to4}, %ymm7, %ymm7
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,ymm4[12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[16,17],zero,zero,zero,zero,zero,zero
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm4, %zmm4
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm5 & mem) | (zmm4 & ~zmm5 & ~mem) | (zmm4 & ~zmm5 & mem) | (zmm4 & zmm5 & ~mem) | (zmm4 & zmm5 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm5 & mem)
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[3,2,3,3,7,6,7,7]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,u,u,30,31,u,u,u,u,28,29,30,31,30,31]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3,4],ymm3[5,6,7,8],ymm2[9],ymm3[10],ymm2[11,12],ymm3[13,14,15]
@@ -2016,9 +2016,9 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,30,31,u,u,26,27,u,u,30,31,28,29,u,u,28,29]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,2]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ~ymm9 & ymm2) | (ymm0 & ~ymm9 & ymm2) | (ymm0 & ymm9 & ~ymm2) | (ymm0 & ymm9 & ymm2)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm9 & (ymm0 ^ ymm2))
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ~ymm0 & mem) | (~ymm1 & ymm0 & mem) | (ymm1 & ymm0 & ~mem) | (ymm1 & ymm0 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = mem ^ (ymm1 & (ymm0 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm1, 128(%r9)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, 64(%r9)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, (%r9)
@@ -3383,11 +3383,11 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,3,2,2]
 ; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm2[0,1,0,1]
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & ~zmm16 & zmm0) | (zmm3 & ~zmm16 & zmm0) | (zmm3 & zmm16 & ~zmm0) | (zmm3 & zmm16 & zmm0)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm0 ^ (zmm16 & (zmm3 ^ zmm0))
 ; AVX512-NEXT:    vpbroadcastq 24(%r8), %ymm0
 ; AVX512-NEXT:    vpbroadcastq 32(%r8), %ymm2
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm19
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm19 = (~zmm19 & zmm3 & mem) | (zmm19 & ~zmm3 & ~mem) | (zmm19 & zmm3 & ~mem) | (zmm19 & zmm3 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm19 = zmm19 ^ (mem & (zmm19 ^ zmm3))
 ; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm3
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm2 = [u,u,0,1,u,u,u,u,14,15,u,u,2,3,u,u,u,u,16,17,u,u,u,u,30,31,u,u,18,19,u,u]
 ; AVX512-NEXT:    vpshufb %ymm2, %ymm3, %ymm0
@@ -3413,7 +3413,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm9 = xmm10[0],xmm9[1],xmm10[2],xmm9[3],xmm10[4,5],xmm9[6],xmm10[7]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,0,0]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm9, %zmm2
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm11 & mem) | (zmm2 & ~zmm11 & ~mem) | (zmm2 & zmm11 & ~mem) | (zmm2 & zmm11 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm11))
 ; AVX512-NEXT:    vmovdqa (%r8), %ymm9
 ; AVX512-NEXT:    vmovdqa 32(%r8), %ymm10
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm11 = [128,128,128,128,12,13,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128]
@@ -3422,7 +3422,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,1,1,1]
 ; AVX512-NEXT:    vpandnq %ymm10, %ymm21, %ymm10
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm10, %zmm10
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & zmm2 & zmm21) | (zmm10 & ~zmm2 & ~zmm21) | (zmm10 & ~zmm2 & zmm21) | (zmm10 & zmm2 & ~zmm21) | (zmm10 & zmm2 & zmm21)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 | (zmm2 & zmm21)
 ; AVX512-NEXT:    vmovdqa (%rdx), %xmm2
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm12[0],xmm2[0],xmm12[1],xmm2[1],xmm12[2],xmm2[2],xmm12[3],xmm2[3]
 ; AVX512-NEXT:    vpshufb %xmm13, %xmm4, %xmm4
@@ -3441,11 +3441,11 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vinserti32x4 $2, %xmm4, %zmm7, %zmm4
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm4 = zmm4[0,1,0,1,4,5,4,5]
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & ~zmm7 & zmm2) | (zmm4 & ~zmm7 & zmm2) | (zmm4 & zmm7 & ~zmm2) | (zmm4 & zmm7 & zmm2)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm2 ^ (zmm7 & (zmm4 ^ zmm2))
 ; AVX512-NEXT:    vpbroadcastq (%r8), %ymm2
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm9[0,1,1,1]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm2, %zmm2
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm2 = (~zmm2 & zmm4 & mem) | (zmm2 & ~zmm4 & ~mem) | (zmm2 & zmm4 & ~mem) | (zmm2 & zmm4 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm4))
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm18[0,1,2,1,4,5,6,5]
 ; AVX512-NEXT:    vprolq $16, %ymm3, %ymm8
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm4 = ymm8[0,1],ymm4[2],ymm8[3],ymm4[4],ymm8[5,6],ymm4[7],ymm8[8,9],ymm4[10],ymm8[11],ymm4[12],ymm8[13,14],ymm4[15]
@@ -3468,12 +3468,12 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm6[1],ymm0[2],ymm6[3,4],ymm0[5,6,7,8],ymm6[9],ymm0[10],ymm6[11,12],ymm0[13,14,15]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,3,2]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm8, %zmm0
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & ~zmm7 & zmm4) | (zmm0 & ~zmm7 & zmm4) | (zmm0 & zmm7 & ~zmm4) | (zmm0 & zmm7 & zmm4)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm4 ^ (zmm7 & (zmm0 ^ zmm4))
 ; AVX512-NEXT:    vpbroadcastq 48(%r8), %ymm4
 ; AVX512-NEXT:    vpbroadcastq 56(%r8), %ymm6
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm4
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm4 = (~zmm4 & zmm6 & zmm0) | (zmm4 & ~zmm6 & ~zmm0) | (zmm4 & ~zmm6 & zmm0) | (zmm4 & zmm6 & zmm0)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm6 & (zmm4 ^ zmm0))
 ; AVX512-NEXT:    vmovdqa64 %ymm22, %ymm0
 ; AVX512-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm20[1,1,2,2]
@@ -3492,11 +3492,11 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5],ymm3[6],ymm1[7,8],ymm3[9],ymm1[10,11],ymm3[12],ymm1[13],ymm3[14],ymm1[15]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm5, %zmm1
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm16 & zmm0) | (zmm1 & ~zmm16 & zmm0) | (zmm1 & zmm16 & ~zmm0) | (zmm1 & zmm16 & zmm0)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm0 ^ (zmm16 & (zmm1 ^ zmm0))
 ; AVX512-NEXT:    vpandnq 16(%r8){1to4}, %ymm6, %ymm0
 ; AVX512-NEXT:    vpshufb %ymm11, %ymm9, %ymm3
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm3, %zmm0
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm1 & mem) | (zmm0 & ~zmm1 & ~mem) | (zmm0 & ~zmm1 & mem) | (zmm0 & zmm1 & ~mem) | (zmm0 & zmm1 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm1 & mem)
 ; AVX512-NEXT:    vmovdqa64 %zmm0, 64(%r9)
 ; AVX512-NEXT:    vmovdqa64 %zmm4, 256(%r9)
 ; AVX512-NEXT:    vmovdqa64 %zmm2, (%r9)
@@ -3540,7 +3540,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm13[1],xmm2[2],xmm13[3],xmm2[4,5],xmm13[6],xmm2[7]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,0,0]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & zmm0 & mem) | (zmm1 & ~zmm0 & ~mem) | (zmm1 & zmm0 & ~mem) | (zmm1 & zmm0 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm0))
 ; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm0
 ; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm2
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [128,128,128,128,12,13,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128]
@@ -3550,7 +3550,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,1,1]
 ; AVX512-FCP-NEXT:    vpandnq %ymm2, %ymm16, %ymm2
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm18
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm18 = (~zmm18 & zmm1 & zmm16) | (zmm18 & ~zmm1 & ~zmm16) | (zmm18 & ~zmm1 & zmm16) | (zmm18 & zmm1 & ~zmm16) | (zmm18 & zmm1 & zmm16)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm18 = zmm18 | (zmm1 & zmm16)
 ; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm1
 ; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm1, %xmm3
 ; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm15
@@ -3574,11 +3574,11 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[0,1,0,1,4,5,4,5]
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm16 & zmm14) | (zmm1 & ~zmm16 & zmm14) | (zmm1 & zmm16 & ~zmm14) | (zmm1 & zmm16 & zmm14)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm14 ^ (zmm16 & (zmm1 ^ zmm14))
 ; AVX512-FCP-NEXT:    vpbroadcastq (%r8), %ymm3
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm0[0,1,1,1]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm3, %zmm10
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm10 = (~zmm10 & zmm1 & mem) | (zmm10 & ~zmm1 & ~mem) | (zmm10 & zmm1 & ~mem) | (zmm10 & zmm1 & mem)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm1))
 ; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm15
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm19, %ymm1
 ; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm15, %ymm1
@@ -3602,13 +3602,13 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,3,2,3]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm13, %zmm2
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & ~zmm19 & zmm3) | (zmm2 & ~zmm19 & zmm3) | (zmm2 & zmm19 & ~zmm3) | (zmm2 & zmm19 & zmm3)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm3 ^ (zmm19 & (zmm2 ^ zmm3))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0]
 ; AVX512-FCP-NEXT:    vpandnq 16(%r8){1to4}, %ymm3, %ymm13
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm25, %ymm12
 ; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm0, %ymm0
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm2 & mem) | (zmm0 & ~zmm2 & ~mem) | (zmm0 & ~zmm2 & mem) | (zmm0 & zmm2 & ~mem) | (zmm0 & zmm2 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm2 & mem)
 ; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [30,31,28,29,26,27,30,31,30,31,28,29,30,31,28,29,30,31,28,29,26,27,30,31,30,31,28,29,30,31,28,29]
 ; AVX512-FCP-NEXT:    # ymm13 = mem[0,1,0,1]
 ; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm5, %ymm2
@@ -3628,11 +3628,11 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm8[1],ymm1[2,3],ymm8[4],ymm1[5],ymm8[6],ymm1[7,8],ymm8[9],ymm1[10,11],ymm8[12],ymm1[13],ymm8[14],ymm1[15]
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm8 = [2,3,2,3,10,10,11,10]
 ; AVX512-FCP-NEXT:    vpermi2q %zmm5, %zmm1, %zmm8
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & ~zmm16 & zmm7) | (zmm8 & ~zmm16 & zmm7) | (zmm8 & zmm16 & ~zmm7) | (zmm8 & zmm16 & zmm7)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm7 ^ (zmm16 & (zmm8 ^ zmm7))
 ; AVX512-FCP-NEXT:    vpbroadcastq 48(%r8), %ymm1
 ; AVX512-FCP-NEXT:    vpbroadcastq 56(%r8), %ymm5
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm1, %zmm1
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = (~zmm1 & zmm3 & zmm8) | (zmm1 & ~zmm3 & ~zmm8) | (zmm1 & ~zmm3 & zmm8) | (zmm1 & zmm3 & zmm8)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm3 & (zmm1 ^ zmm8))
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm11[0],xmm6[0],xmm11[1],xmm6[1],xmm11[2],xmm6[2],xmm11[3],xmm6[3]
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm21, %xmm5
 ; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm3, %xmm3
@@ -3651,11 +3651,11 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4,5],ymm5[6],ymm3[7,8],ymm5[9],ymm3[10],ymm5[11],ymm3[12,13],ymm5[14],ymm3[15]
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [2,3,2,2,8,9,8,9]
 ; AVX512-FCP-NEXT:    vpermi2q %zmm2, %zmm3, %zmm5
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & ~zmm19 & zmm4) | (zmm5 & ~zmm19 & zmm4) | (zmm5 & zmm19 & ~zmm4) | (zmm5 & zmm19 & zmm4)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm4 ^ (zmm19 & (zmm5 ^ zmm4))
 ; AVX512-FCP-NEXT:    vpbroadcastq 24(%r8), %ymm2
 ; AVX512-FCP-NEXT:    vpbroadcastq 32(%r8), %ymm3
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = (~zmm2 & zmm5 & mem) | (zmm2 & ~zmm5 & ~mem) | (zmm2 & zmm5 & ~mem) | (zmm2 & zmm5 & mem)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm5))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, 128(%r9)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 256(%r9)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 64(%r9)
@@ -3697,11 +3697,11 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,3,2,2]
 ; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm2[0,1,0,1]
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & ~zmm16 & zmm0) | (zmm3 & ~zmm16 & zmm0) | (zmm3 & zmm16 & ~zmm0) | (zmm3 & zmm16 & zmm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm0 ^ (zmm16 & (zmm3 ^ zmm0))
 ; AVX512DQ-NEXT:    vpbroadcastq 24(%r8), %ymm0
 ; AVX512DQ-NEXT:    vpbroadcastq 32(%r8), %ymm2
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm19
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm19 = (~zmm19 & zmm3 & mem) | (zmm19 & ~zmm3 & ~mem) | (zmm19 & zmm3 & ~mem) | (zmm19 & zmm3 & mem)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm19 = zmm19 ^ (mem & (zmm19 ^ zmm3))
 ; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %ymm3
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [u,u,0,1,u,u,u,u,14,15,u,u,2,3,u,u,u,u,16,17,u,u,u,u,30,31,u,u,18,19,u,u]
 ; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm3, %ymm0
@@ -3727,7 +3727,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm9 = xmm10[0],xmm9[1],xmm10[2],xmm9[3],xmm10[4,5],xmm9[6],xmm10[7]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,0,0]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm9, %zmm2
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm11 & mem) | (zmm2 & ~zmm11 & ~mem) | (zmm2 & zmm11 & ~mem) | (zmm2 & zmm11 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm11))
 ; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm9
 ; AVX512DQ-NEXT:    vmovdqa 32(%r8), %ymm10
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm11 = [128,128,128,128,12,13,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128]
@@ -3736,7 +3736,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,1,1,1]
 ; AVX512DQ-NEXT:    vpandnq %ymm10, %ymm21, %ymm10
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm10, %zmm10
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & zmm2 & zmm21) | (zmm10 & ~zmm2 & ~zmm21) | (zmm10 & ~zmm2 & zmm21) | (zmm10 & zmm2 & ~zmm21) | (zmm10 & zmm2 & zmm21)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 | (zmm2 & zmm21)
 ; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm2
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm12[0],xmm2[0],xmm12[1],xmm2[1],xmm12[2],xmm2[2],xmm12[3],xmm2[3]
 ; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm4, %xmm4
@@ -3755,11 +3755,11 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm4, %zmm7, %zmm4
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm4 = zmm4[0,1,0,1,4,5,4,5]
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & ~zmm7 & zmm2) | (zmm4 & ~zmm7 & zmm2) | (zmm4 & zmm7 & ~zmm2) | (zmm4 & zmm7 & zmm2)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm2 ^ (zmm7 & (zmm4 ^ zmm2))
 ; AVX512DQ-NEXT:    vpbroadcastq (%r8), %ymm2
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm9[0,1,1,1]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm2, %zmm2
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm2 = (~zmm2 & zmm4 & mem) | (zmm2 & ~zmm4 & ~mem) | (zmm2 & zmm4 & ~mem) | (zmm2 & zmm4 & mem)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm4))
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm18[0,1,2,1,4,5,6,5]
 ; AVX512DQ-NEXT:    vprolq $16, %ymm3, %ymm8
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm4 = ymm8[0,1],ymm4[2],ymm8[3],ymm4[4],ymm8[5,6],ymm4[7],ymm8[8,9],ymm4[10],ymm8[11],ymm4[12],ymm8[13,14],ymm4[15]
@@ -3782,12 +3782,12 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm6[1],ymm0[2],ymm6[3,4],ymm0[5,6,7,8],ymm6[9],ymm0[10],ymm6[11,12],ymm0[13,14,15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,3,2]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm8, %zmm0
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & ~zmm7 & zmm4) | (zmm0 & ~zmm7 & zmm4) | (zmm0 & zmm7 & ~zmm4) | (zmm0 & zmm7 & zmm4)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm4 ^ (zmm7 & (zmm0 ^ zmm4))
 ; AVX512DQ-NEXT:    vpbroadcastq 48(%r8), %ymm4
 ; AVX512DQ-NEXT:    vpbroadcastq 56(%r8), %ymm6
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm4
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm4 = (~zmm4 & zmm6 & zmm0) | (zmm4 & ~zmm6 & ~zmm0) | (zmm4 & ~zmm6 & zmm0) | (zmm4 & zmm6 & zmm0)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm6 & (zmm4 ^ zmm0))
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm22, %ymm0
 ; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm20[1,1,2,2]
@@ -3806,11 +3806,11 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5],ymm3[6],ymm1[7,8],ymm3[9],ymm1[10,11],ymm3[12],ymm1[13],ymm3[14],ymm1[15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm5, %zmm1
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm16 & zmm0) | (zmm1 & ~zmm16 & zmm0) | (zmm1 & zmm16 & ~zmm0) | (zmm1 & zmm16 & zmm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm0 ^ (zmm16 & (zmm1 ^ zmm0))
 ; AVX512DQ-NEXT:    vpandnq 16(%r8){1to4}, %ymm6, %ymm0
 ; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm9, %ymm3
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm3, %zmm0
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm1 & mem) | (zmm0 & ~zmm1 & ~mem) | (zmm0 & ~zmm1 & mem) | (zmm0 & zmm1 & ~mem) | (zmm0 & zmm1 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm1 & mem)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 64(%r9)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 256(%r9)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm2, (%r9)
@@ -3854,7 +3854,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm13[1],xmm2[2],xmm13[3],xmm2[4,5],xmm13[6],xmm2[7]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,0,0]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & zmm0 & mem) | (zmm1 & ~zmm0 & ~mem) | (zmm1 & zmm0 & ~mem) | (zmm1 & zmm0 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [128,128,128,128,12,13,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128]
@@ -3864,7 +3864,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,1,1]
 ; AVX512DQ-FCP-NEXT:    vpandnq %ymm2, %ymm16, %ymm2
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm18
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm18 = (~zmm18 & zmm1 & zmm16) | (zmm18 & ~zmm1 & ~zmm16) | (zmm18 & ~zmm1 & zmm16) | (zmm18 & zmm1 & ~zmm16) | (zmm18 & zmm1 & zmm16)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm18 = zmm18 | (zmm1 & zmm16)
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm1
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm1, %xmm3
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm15
@@ -3888,11 +3888,11 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[0,1,0,1,4,5,4,5]
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm16 & zmm14) | (zmm1 & ~zmm16 & zmm14) | (zmm1 & zmm16 & ~zmm14) | (zmm1 & zmm16 & zmm14)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm14 ^ (zmm16 & (zmm1 ^ zmm14))
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq (%r8), %ymm3
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm0[0,1,1,1]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm3, %zmm10
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm10 = (~zmm10 & zmm1 & mem) | (zmm10 & ~zmm1 & ~mem) | (zmm10 & zmm1 & ~mem) | (zmm10 & zmm1 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm15
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm19, %ymm1
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm15, %ymm1
@@ -3916,13 +3916,13 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,3,2,3]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm13, %zmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & ~zmm19 & zmm3) | (zmm2 & ~zmm19 & zmm3) | (zmm2 & zmm19 & ~zmm3) | (zmm2 & zmm19 & zmm3)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm3 ^ (zmm19 & (zmm2 ^ zmm3))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0]
 ; AVX512DQ-FCP-NEXT:    vpandnq 16(%r8){1to4}, %ymm3, %ymm13
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm25, %ymm12
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm0, %ymm0
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm2 & mem) | (zmm0 & ~zmm2 & ~mem) | (zmm0 & ~zmm2 & mem) | (zmm0 & zmm2 & ~mem) | (zmm0 & zmm2 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm2 & mem)
 ; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [30,31,28,29,26,27,30,31,30,31,28,29,30,31,28,29,30,31,28,29,26,27,30,31,30,31,28,29,30,31,28,29]
 ; AVX512DQ-FCP-NEXT:    # ymm13 = mem[0,1,0,1]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm5, %ymm2
@@ -3942,11 +3942,11 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm8[1],ymm1[2,3],ymm8[4],ymm1[5],ymm8[6],ymm1[7,8],ymm8[9],ymm1[10,11],ymm8[12],ymm1[13],ymm8[14],ymm1[15]
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm8 = [2,3,2,3,10,10,11,10]
 ; AVX512DQ-FCP-NEXT:    vpermi2q %zmm5, %zmm1, %zmm8
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & ~zmm16 & zmm7) | (zmm8 & ~zmm16 & zmm7) | (zmm8 & zmm16 & ~zmm7) | (zmm8 & zmm16 & zmm7)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm7 ^ (zmm16 & (zmm8 ^ zmm7))
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 48(%r8), %ymm1
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 56(%r8), %ymm5
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm1, %zmm1
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = (~zmm1 & zmm3 & zmm8) | (zmm1 & ~zmm3 & ~zmm8) | (zmm1 & ~zmm3 & zmm8) | (zmm1 & zmm3 & zmm8)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm3 & (zmm1 ^ zmm8))
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm11[0],xmm6[0],xmm11[1],xmm6[1],xmm11[2],xmm6[2],xmm11[3],xmm6[3]
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm21, %xmm5
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm3, %xmm3
@@ -3965,11 +3965,11 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4,5],ymm5[6],ymm3[7,8],ymm5[9],ymm3[10],ymm5[11],ymm3[12,13],ymm5[14],ymm3[15]
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [2,3,2,2,8,9,8,9]
 ; AVX512DQ-FCP-NEXT:    vpermi2q %zmm2, %zmm3, %zmm5
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & ~zmm19 & zmm4) | (zmm5 & ~zmm19 & zmm4) | (zmm5 & zmm19 & ~zmm4) | (zmm5 & zmm19 & zmm4)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm4 ^ (zmm19 & (zmm5 ^ zmm4))
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 24(%r8), %ymm2
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 32(%r8), %ymm3
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = (~zmm2 & zmm5 & mem) | (zmm2 & ~zmm5 & ~mem) | (zmm2 & zmm5 & ~mem) | (zmm2 & zmm5 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm5))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, 128(%r9)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 256(%r9)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 64(%r9)
@@ -6984,23 +6984,23 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpshufb %ymm7, %ymm2, %ymm4
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535]
 ; AVX512-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm19, %zmm31 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm31 = (~zmm31 & ~zmm19 & mem) | (zmm31 & ~zmm19 & mem) | (zmm31 & zmm19 & ~mem) | (zmm31 & zmm19 & mem)
+; AVX512-NEXT:    # zmm31 = mem ^ (zmm19 & (zmm31 ^ mem))
 ; AVX512-NEXT:    vpbroadcastq 88(%r8), %ymm1
 ; AVX512-NEXT:    vpbroadcastq 96(%r8), %ymm2
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm1 = (~zmm1 & zmm18 & zmm31) | (zmm1 & ~zmm18 & ~zmm31) | (zmm1 & ~zmm18 & zmm31) | (zmm1 & zmm18 & zmm31)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm27 = (~zmm27 & ~zmm19 & zmm25) | (zmm27 & ~zmm19 & zmm25) | (zmm27 & zmm19 & ~zmm25) | (zmm27 & zmm19 & zmm25)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm18 & (zmm1 ^ zmm31))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm27 = zmm25 ^ (zmm19 & (zmm27 ^ zmm25))
 ; AVX512-NEXT:    vpbroadcastq 24(%r8), %ymm2
 ; AVX512-NEXT:    vpbroadcastq 32(%r8), %ymm25
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm25, %zmm2, %zmm2
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm2 = (~zmm2 & zmm18 & zmm27) | (zmm2 & ~zmm18 & ~zmm27) | (zmm2 & ~zmm18 & zmm27) | (zmm2 & zmm18 & zmm27)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (zmm18 & (zmm2 ^ zmm27))
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0]
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
 ; AVX512-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm31 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm31 = (~zmm31 & ~zmm18 & mem) | (zmm31 & ~zmm18 & mem) | (zmm31 & zmm18 & ~mem) | (zmm31 & zmm18 & mem)
+; AVX512-NEXT:    # zmm31 = mem ^ (zmm18 & (zmm31 ^ mem))
 ; AVX512-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm24 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm24 = (~zmm24 & ~zmm18 & mem) | (zmm24 & ~zmm18 & mem) | (zmm24 & zmm18 & ~mem) | (zmm24 & zmm18 & mem)
+; AVX512-NEXT:    # zmm24 = mem ^ (zmm18 & (zmm24 ^ mem))
 ; AVX512-NEXT:    vpermq $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm18 # 32-byte Folded Reload
 ; AVX512-NEXT:    # ymm18 = mem[0,1,0,1]
 ; AVX512-NEXT:    vpermq $4, {{[-0-9]+}}(%r{{[sb]}}p), %ymm25 # 32-byte Folded Reload
@@ -7028,43 +7028,43 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm29, %zmm4, %zmm4
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535]
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm23 = (~zmm23 & zmm31 & zmm21) | (zmm23 & ~zmm31 & ~zmm21) | (zmm23 & ~zmm31 & zmm21) | (zmm23 & zmm31 & ~zmm21) | (zmm23 & zmm31 & zmm21)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm26 = (~zmm26 & zmm24 & zmm21) | (zmm26 & ~zmm24 & ~zmm21) | (zmm26 & ~zmm24 & zmm21) | (zmm26 & zmm24 & ~zmm21) | (zmm26 & zmm24 & zmm21)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm23 = zmm23 | (zmm31 & zmm21)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm26 = zmm26 | (zmm24 & zmm21)
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm25, %zmm18, %zmm18
 ; AVX512-NEXT:    vpermq $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Folded Reload
 ; AVX512-NEXT:    # zmm21 = mem[0,1,0,1,4,5,4,5]
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm21 = (~zmm21 & ~zmm24 & zmm18) | (zmm21 & ~zmm24 & zmm18) | (zmm21 & zmm24 & ~zmm18) | (zmm21 & zmm24 & zmm18)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm21 = zmm18 ^ (zmm24 & (zmm21 ^ zmm18))
 ; AVX512-NEXT:    vpbroadcastq 64(%r8), %ymm18
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm18, %zmm8
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm8 = (~zmm8 & zmm18 & zmm21) | (zmm8 & ~zmm18 & ~zmm21) | (zmm8 & ~zmm18 & zmm21) | (zmm8 & zmm18 & zmm21)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm8 ^ (zmm18 & (zmm8 ^ zmm21))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm28, %zmm27, %zmm21
 ; AVX512-NEXT:    vpermq $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Folded Reload
 ; AVX512-NEXT:    # zmm22 = mem[0,1,0,1,4,5,4,5]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm22 = (~zmm22 & ~zmm24 & zmm21) | (zmm22 & ~zmm24 & zmm21) | (zmm22 & zmm24 & ~zmm21) | (zmm22 & zmm24 & zmm21)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm22 = zmm21 ^ (zmm24 & (zmm22 ^ zmm21))
 ; AVX512-NEXT:    vpbroadcastq (%r8), %ymm21
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm21, %zmm10
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm10 = (~zmm10 & zmm18 & zmm22) | (zmm10 & ~zmm18 & ~zmm22) | (zmm10 & ~zmm18 & zmm22) | (zmm10 & zmm18 & zmm22)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm10 = zmm10 ^ (zmm18 & (zmm10 ^ zmm22))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm13, %zmm14, %zmm13
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm12, %zmm11
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm11 = (~zmm11 & ~zmm24 & zmm13) | (zmm11 & ~zmm24 & zmm13) | (zmm11 & zmm24 & ~zmm13) | (zmm11 & zmm24 & zmm13)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm13 ^ (zmm24 & (zmm11 ^ zmm13))
 ; AVX512-NEXT:    vpbroadcastq 112(%r8), %ymm12
 ; AVX512-NEXT:    vpbroadcastq 120(%r8), %ymm13
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm12 = (~zmm12 & zmm16 & zmm11) | (zmm12 & ~zmm16 & ~zmm11) | (zmm12 & ~zmm16 & zmm11) | (zmm12 & zmm16 & zmm11)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm12 = zmm12 ^ (zmm16 & (zmm12 ^ zmm11))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm15, %zmm7
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm6, %zmm5
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & ~zmm24 & zmm7) | (zmm5 & ~zmm24 & zmm7) | (zmm5 & zmm24 & ~zmm7) | (zmm5 & zmm24 & zmm7)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm7 ^ (zmm24 & (zmm5 ^ zmm7))
 ; AVX512-NEXT:    vpbroadcastq 48(%r8), %ymm6
 ; AVX512-NEXT:    vpbroadcastq 56(%r8), %ymm7
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm6 = (~zmm6 & zmm16 & zmm5) | (zmm6 & ~zmm16 & ~zmm5) | (zmm6 & ~zmm16 & zmm5) | (zmm6 & zmm16 & zmm5)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm9 = (~zmm9 & ~zmm19 & zmm30) | (zmm9 & ~zmm19 & zmm30) | (zmm9 & zmm19 & ~zmm30) | (zmm9 & zmm19 & zmm30)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & ~zmm19 & zmm20) | (zmm0 & ~zmm19 & zmm20) | (zmm0 & zmm19 & ~zmm20) | (zmm0 & zmm19 & zmm20)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm16 & (zmm6 ^ zmm5))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm30 ^ (zmm19 & (zmm9 ^ zmm30))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm20 ^ (zmm19 & (zmm0 ^ zmm20))
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm17 = (~zmm17 & zmm9 & zmm5) | (zmm17 & ~zmm9 & ~zmm5) | (zmm17 & ~zmm9 & zmm5) | (zmm17 & zmm9 & ~zmm5) | (zmm17 & zmm9 & zmm5)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm0 & zmm5) | (zmm4 & ~zmm0 & ~zmm5) | (zmm4 & ~zmm0 & zmm5) | (zmm4 & zmm0 & ~zmm5) | (zmm4 & zmm0 & zmm5)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm9 & zmm5)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm0 & zmm5)
 ; AVX512-NEXT:    vmovdqa64 %zmm4, 384(%r9)
 ; AVX512-NEXT:    vmovdqa64 %zmm17, 64(%r9)
 ; AVX512-NEXT:    vmovdqa64 %zmm6, 256(%r9)
@@ -7187,7 +7187,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpermt2q %zmm5, %zmm6, %zmm1
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm20 = zmm2[0,1,0,1,4,5,4,5]
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm31 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm20 = (~zmm20 & ~zmm31 & zmm1) | (zmm20 & ~zmm31 & zmm1) | (zmm20 & zmm31 & ~zmm1) | (zmm20 & zmm31 & zmm1)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm1 ^ (zmm31 & (zmm20 ^ zmm1))
 ; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm1
 ; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
 ; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm2
@@ -7197,7 +7197,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm1, %xmm1
 ; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm6, %zmm1
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm21 = zmm4[0,1,0,1,4,5,4,5]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm21 = (~zmm21 & ~zmm31 & zmm1) | (zmm21 & ~zmm31 & zmm1) | (zmm21 & zmm31 & ~zmm1) | (zmm21 & zmm31 & zmm1)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm21 = zmm1 ^ (zmm31 & (zmm21 ^ zmm1))
 ; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm2
 ; AVX512-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa64 (%rdi), %ymm28
@@ -7246,11 +7246,11 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2,3],ymm3[4],ymm2[5],ymm3[6],ymm2[7,8],ymm3[9],ymm2[10,11],ymm3[12],ymm2[13],ymm3[14],ymm2[15]
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [2,3,2,3,10,10,11,10]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm18, %zmm2
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & ~zmm31 & zmm1) | (zmm2 & ~zmm31 & zmm1) | (zmm2 & zmm31 & ~zmm1) | (zmm2 & zmm31 & zmm1)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm1 ^ (zmm31 & (zmm2 ^ zmm1))
 ; AVX512-FCP-NEXT:    vpbroadcastq 112(%r8), %ymm0
 ; AVX512-FCP-NEXT:    vpbroadcastq 120(%r8), %ymm1
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm14
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm14 = (~zmm14 & zmm12 & zmm2) | (zmm14 & ~zmm12 & ~zmm2) | (zmm14 & ~zmm12 & zmm2) | (zmm14 & zmm12 & zmm2)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm14 = zmm14 ^ (zmm12 & (zmm14 ^ zmm2))
 ; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %ymm8
 ; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm8, %ymm0
 ; AVX512-FCP-NEXT:    vmovdqa64 64(%rdi), %ymm23
@@ -7291,11 +7291,11 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm15 = ymm29[1,1,1,2,5,5,5,6]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm15 = ymm7[0],ymm15[1],ymm7[2,3],ymm15[4],ymm7[5],ymm15[6],ymm7[7,8],ymm15[9],ymm7[10,11],ymm15[12],ymm7[13],ymm15[14],ymm7[15]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm13, %zmm18, %zmm15
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = (~zmm15 & ~zmm31 & zmm11) | (zmm15 & ~zmm31 & zmm11) | (zmm15 & zmm31 & ~zmm11) | (zmm15 & zmm31 & zmm11)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = zmm11 ^ (zmm31 & (zmm15 ^ zmm11))
 ; AVX512-FCP-NEXT:    vpbroadcastq 48(%r8), %ymm7
 ; AVX512-FCP-NEXT:    vpbroadcastq 56(%r8), %ymm11
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm7, %zmm7
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = (~zmm7 & zmm12 & zmm15) | (zmm7 & ~zmm12 & ~zmm15) | (zmm7 & ~zmm12 & zmm15) | (zmm7 & zmm12 & zmm15)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (zmm12 & (zmm7 ^ zmm15))
 ; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
 ; AVX512-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm11, %xmm11 # 16-byte Folded Reload
 ; AVX512-FCP-NEXT:    # xmm11 = xmm11[0],mem[0],xmm11[1],mem[1],xmm11[2],mem[2],xmm11[3],mem[3]
@@ -7318,7 +7318,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm12 = [2,3,2,2,8,9,8,9]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm12, %zmm8
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & ~zmm1 & zmm0) | (zmm8 & ~zmm1 & zmm0) | (zmm8 & zmm1 & ~zmm0) | (zmm8 & zmm1 & zmm0)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm0 ^ (zmm1 & (zmm8 ^ zmm0))
 ; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; AVX512-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
 ; AVX512-FCP-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
@@ -7332,7 +7332,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpbroadcastq 96(%r8), %ymm5
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = (~zmm0 & zmm5 & zmm8) | (zmm0 & ~zmm5 & ~zmm8) | (zmm0 & ~zmm5 & zmm8) | (zmm0 & zmm5 & zmm8)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm5 & (zmm0 ^ zmm8))
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm8
 ; AVX512-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm8 # 16-byte Folded Reload
 ; AVX512-FCP-NEXT:    # xmm8 = xmm8[0],mem[0],xmm8[1],mem[1],xmm8[2],mem[2],xmm8[3],mem[3]
@@ -7343,34 +7343,34 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm11 = ymm28[2,3,2,3,6,7,6,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0],ymm11[1],ymm6[2],ymm11[3],ymm6[4,5],ymm11[6],ymm6[7,8],ymm11[9],ymm6[10],ymm11[11],ymm6[12,13],ymm11[14],ymm6[15]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm8, %zmm12, %zmm6
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = (~zmm6 & ~zmm1 & zmm4) | (zmm6 & ~zmm1 & zmm4) | (zmm6 & zmm1 & ~zmm4) | (zmm6 & zmm1 & zmm4)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm4 ^ (zmm1 & (zmm6 ^ zmm4))
 ; AVX512-FCP-NEXT:    vpbroadcastq 24(%r8), %ymm4
 ; AVX512-FCP-NEXT:    vpbroadcastq 32(%r8), %ymm8
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm4, %zmm4
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = (~zmm4 & zmm5 & zmm6) | (zmm4 & ~zmm5 & ~zmm6) | (zmm4 & ~zmm5 & zmm6) | (zmm4 & zmm5 & zmm6)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm5 & (zmm4 ^ zmm6))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0]
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm3 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm3 = (~zmm3 & ~zmm5 & mem) | (zmm3 & ~zmm5 & mem) | (zmm3 & zmm5 & ~mem) | (zmm3 & zmm5 & mem)
+; AVX512-FCP-NEXT:    # zmm3 = mem ^ (zmm5 & (zmm3 ^ mem))
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm16 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm16 = (~zmm16 & ~zmm5 & mem) | (zmm16 & ~zmm5 & mem) | (zmm16 & zmm5 & ~mem) | (zmm16 & zmm5 & mem)
+; AVX512-FCP-NEXT:    # zmm16 = mem ^ (zmm5 & (zmm16 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & zmm3 & zmm5) | (zmm8 & ~zmm3 & ~zmm5) | (zmm8 & ~zmm3 & zmm5) | (zmm8 & zmm3 & ~zmm5) | (zmm8 & zmm3 & zmm5)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm17 = (~zmm17 & zmm16 & zmm5) | (zmm17 & ~zmm16 & ~zmm5) | (zmm17 & ~zmm16 & zmm5) | (zmm17 & zmm16 & ~zmm5) | (zmm17 & zmm16 & zmm5)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 | (zmm3 & zmm5)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm16 & zmm5)
 ; AVX512-FCP-NEXT:    vpbroadcastq 64(%r8), %ymm5
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm5, %zmm2
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = (~zmm2 & zmm5 & zmm20) | (zmm2 & ~zmm5 & ~zmm20) | (zmm2 & ~zmm5 & zmm20) | (zmm2 & zmm5 & zmm20)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (zmm5 & (zmm2 ^ zmm20))
 ; AVX512-FCP-NEXT:    vpbroadcastq (%r8), %ymm6
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm6, %zmm6
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = (~zmm6 & zmm5 & zmm21) | (zmm6 & ~zmm5 & ~zmm21) | (zmm6 & ~zmm5 & zmm21) | (zmm6 & zmm5 & zmm21)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm5 & (zmm6 ^ zmm21))
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm25 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm25 = (~zmm25 & ~zmm1 & mem) | (zmm25 & ~zmm1 & mem) | (zmm25 & zmm1 & ~mem) | (zmm25 & zmm1 & mem)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = (~zmm9 & ~zmm1 & zmm19) | (zmm9 & ~zmm1 & zmm19) | (zmm9 & zmm1 & ~zmm19) | (zmm9 & zmm1 & zmm19)
+; AVX512-FCP-NEXT:    # zmm25 = mem ^ (zmm1 & (zmm25 ^ mem))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm19 ^ (zmm1 & (zmm9 ^ zmm19))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm27 = (~zmm27 & zmm25 & zmm1) | (zmm27 & ~zmm25 & ~zmm1) | (zmm27 & ~zmm25 & zmm1) | (zmm27 & zmm25 & ~zmm1) | (zmm27 & zmm25 & zmm1)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm24 = (~zmm24 & zmm9 & zmm1) | (zmm24 & ~zmm9 & ~zmm1) | (zmm24 & ~zmm9 & zmm1) | (zmm24 & zmm9 & ~zmm1) | (zmm24 & zmm9 & zmm1)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm27 = zmm27 | (zmm25 & zmm1)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm24 = zmm24 | (zmm9 & zmm1)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, 128(%r9)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm7, 256(%r9)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 448(%r9)
@@ -7617,23 +7617,23 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm2, %ymm4
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535]
 ; AVX512DQ-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm19, %zmm31 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm31 = (~zmm31 & ~zmm19 & mem) | (zmm31 & ~zmm19 & mem) | (zmm31 & zmm19 & ~mem) | (zmm31 & zmm19 & mem)
+; AVX512DQ-NEXT:    # zmm31 = mem ^ (zmm19 & (zmm31 ^ mem))
 ; AVX512DQ-NEXT:    vpbroadcastq 88(%r8), %ymm1
 ; AVX512DQ-NEXT:    vpbroadcastq 96(%r8), %ymm2
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm1 = (~zmm1 & zmm18 & zmm31) | (zmm1 & ~zmm18 & ~zmm31) | (zmm1 & ~zmm18 & zmm31) | (zmm1 & zmm18 & zmm31)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm27 = (~zmm27 & ~zmm19 & zmm25) | (zmm27 & ~zmm19 & zmm25) | (zmm27 & zmm19 & ~zmm25) | (zmm27 & zmm19 & zmm25)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm18 & (zmm1 ^ zmm31))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm27 = zmm25 ^ (zmm19 & (zmm27 ^ zmm25))
 ; AVX512DQ-NEXT:    vpbroadcastq 24(%r8), %ymm2
 ; AVX512DQ-NEXT:    vpbroadcastq 32(%r8), %ymm25
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm25, %zmm2, %zmm2
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm2 = (~zmm2 & zmm18 & zmm27) | (zmm2 & ~zmm18 & ~zmm27) | (zmm2 & ~zmm18 & zmm27) | (zmm2 & zmm18 & zmm27)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (zmm18 & (zmm2 ^ zmm27))
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0]
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
 ; AVX512DQ-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm31 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm31 = (~zmm31 & ~zmm18 & mem) | (zmm31 & ~zmm18 & mem) | (zmm31 & zmm18 & ~mem) | (zmm31 & zmm18 & mem)
+; AVX512DQ-NEXT:    # zmm31 = mem ^ (zmm18 & (zmm31 ^ mem))
 ; AVX512DQ-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm24 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm24 = (~zmm24 & ~zmm18 & mem) | (zmm24 & ~zmm18 & mem) | (zmm24 & zmm18 & ~mem) | (zmm24 & zmm18 & mem)
+; AVX512DQ-NEXT:    # zmm24 = mem ^ (zmm18 & (zmm24 ^ mem))
 ; AVX512DQ-NEXT:    vpermq $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm18 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    # ymm18 = mem[0,1,0,1]
 ; AVX512DQ-NEXT:    vpermq $4, {{[-0-9]+}}(%r{{[sb]}}p), %ymm25 # 32-byte Folded Reload
@@ -7661,43 +7661,43 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm29, %zmm4, %zmm4
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535]
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm23 = (~zmm23 & zmm31 & zmm21) | (zmm23 & ~zmm31 & ~zmm21) | (zmm23 & ~zmm31 & zmm21) | (zmm23 & zmm31 & ~zmm21) | (zmm23 & zmm31 & zmm21)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm26 = (~zmm26 & zmm24 & zmm21) | (zmm26 & ~zmm24 & ~zmm21) | (zmm26 & ~zmm24 & zmm21) | (zmm26 & zmm24 & ~zmm21) | (zmm26 & zmm24 & zmm21)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm23 = zmm23 | (zmm31 & zmm21)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm26 = zmm26 | (zmm24 & zmm21)
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm25, %zmm18, %zmm18
 ; AVX512DQ-NEXT:    vpermq $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Folded Reload
 ; AVX512DQ-NEXT:    # zmm21 = mem[0,1,0,1,4,5,4,5]
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm21 = (~zmm21 & ~zmm24 & zmm18) | (zmm21 & ~zmm24 & zmm18) | (zmm21 & zmm24 & ~zmm18) | (zmm21 & zmm24 & zmm18)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm21 = zmm18 ^ (zmm24 & (zmm21 ^ zmm18))
 ; AVX512DQ-NEXT:    vpbroadcastq 64(%r8), %ymm18
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm18, %zmm8
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm8 = (~zmm8 & zmm18 & zmm21) | (zmm8 & ~zmm18 & ~zmm21) | (zmm8 & ~zmm18 & zmm21) | (zmm8 & zmm18 & zmm21)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm8 ^ (zmm18 & (zmm8 ^ zmm21))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm28, %zmm27, %zmm21
 ; AVX512DQ-NEXT:    vpermq $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Folded Reload
 ; AVX512DQ-NEXT:    # zmm22 = mem[0,1,0,1,4,5,4,5]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm22 = (~zmm22 & ~zmm24 & zmm21) | (zmm22 & ~zmm24 & zmm21) | (zmm22 & zmm24 & ~zmm21) | (zmm22 & zmm24 & zmm21)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm22 = zmm21 ^ (zmm24 & (zmm22 ^ zmm21))
 ; AVX512DQ-NEXT:    vpbroadcastq (%r8), %ymm21
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm21, %zmm10
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm10 = (~zmm10 & zmm18 & zmm22) | (zmm10 & ~zmm18 & ~zmm22) | (zmm10 & ~zmm18 & zmm22) | (zmm10 & zmm18 & zmm22)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm10 = zmm10 ^ (zmm18 & (zmm10 ^ zmm22))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm13, %zmm14, %zmm13
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm12, %zmm11
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm11 = (~zmm11 & ~zmm24 & zmm13) | (zmm11 & ~zmm24 & zmm13) | (zmm11 & zmm24 & ~zmm13) | (zmm11 & zmm24 & zmm13)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm13 ^ (zmm24 & (zmm11 ^ zmm13))
 ; AVX512DQ-NEXT:    vpbroadcastq 112(%r8), %ymm12
 ; AVX512DQ-NEXT:    vpbroadcastq 120(%r8), %ymm13
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm12 = (~zmm12 & zmm16 & zmm11) | (zmm12 & ~zmm16 & ~zmm11) | (zmm12 & ~zmm16 & zmm11) | (zmm12 & zmm16 & zmm11)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm12 = zmm12 ^ (zmm16 & (zmm12 ^ zmm11))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm15, %zmm7
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm6, %zmm5
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & ~zmm24 & zmm7) | (zmm5 & ~zmm24 & zmm7) | (zmm5 & zmm24 & ~zmm7) | (zmm5 & zmm24 & zmm7)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm7 ^ (zmm24 & (zmm5 ^ zmm7))
 ; AVX512DQ-NEXT:    vpbroadcastq 48(%r8), %ymm6
 ; AVX512DQ-NEXT:    vpbroadcastq 56(%r8), %ymm7
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm6 = (~zmm6 & zmm16 & zmm5) | (zmm6 & ~zmm16 & ~zmm5) | (zmm6 & ~zmm16 & zmm5) | (zmm6 & zmm16 & zmm5)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm9 = (~zmm9 & ~zmm19 & zmm30) | (zmm9 & ~zmm19 & zmm30) | (zmm9 & zmm19 & ~zmm30) | (zmm9 & zmm19 & zmm30)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & ~zmm19 & zmm20) | (zmm0 & ~zmm19 & zmm20) | (zmm0 & zmm19 & ~zmm20) | (zmm0 & zmm19 & zmm20)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm16 & (zmm6 ^ zmm5))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm30 ^ (zmm19 & (zmm9 ^ zmm30))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm20 ^ (zmm19 & (zmm0 ^ zmm20))
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm17 = (~zmm17 & zmm9 & zmm5) | (zmm17 & ~zmm9 & ~zmm5) | (zmm17 & ~zmm9 & zmm5) | (zmm17 & zmm9 & ~zmm5) | (zmm17 & zmm9 & zmm5)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm0 & zmm5) | (zmm4 & ~zmm0 & ~zmm5) | (zmm4 & ~zmm0 & zmm5) | (zmm4 & zmm0 & ~zmm5) | (zmm4 & zmm0 & zmm5)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm9 & zmm5)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm0 & zmm5)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 384(%r9)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm17, 64(%r9)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm6, 256(%r9)
@@ -7820,7 +7820,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm5, %zmm6, %zmm1
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm20 = zmm2[0,1,0,1,4,5,4,5]
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm31 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm20 = (~zmm20 & ~zmm31 & zmm1) | (zmm20 & ~zmm31 & zmm1) | (zmm20 & zmm31 & ~zmm1) | (zmm20 & zmm31 & zmm1)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm1 ^ (zmm31 & (zmm20 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm1
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm2
@@ -7830,7 +7830,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm1, %xmm1
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm6, %zmm1
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm21 = zmm4[0,1,0,1,4,5,4,5]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm21 = (~zmm21 & ~zmm31 & zmm1) | (zmm21 & ~zmm31 & zmm1) | (zmm21 & zmm31 & ~zmm1) | (zmm21 & zmm31 & zmm1)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm21 = zmm1 ^ (zmm31 & (zmm21 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm2
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 (%rdi), %ymm28
@@ -7879,11 +7879,11 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2,3],ymm3[4],ymm2[5],ymm3[6],ymm2[7,8],ymm3[9],ymm2[10,11],ymm3[12],ymm2[13],ymm3[14],ymm2[15]
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [2,3,2,3,10,10,11,10]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm18, %zmm2
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & ~zmm31 & zmm1) | (zmm2 & ~zmm31 & zmm1) | (zmm2 & zmm31 & ~zmm1) | (zmm2 & zmm31 & zmm1)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm1 ^ (zmm31 & (zmm2 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 112(%r8), %ymm0
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 120(%r8), %ymm1
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm14
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm14 = (~zmm14 & zmm12 & zmm2) | (zmm14 & ~zmm12 & ~zmm2) | (zmm14 & ~zmm12 & zmm2) | (zmm14 & zmm12 & zmm2)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm14 = zmm14 ^ (zmm12 & (zmm14 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rsi), %ymm8
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm8, %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 64(%rdi), %ymm23
@@ -7924,11 +7924,11 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm15 = ymm29[1,1,1,2,5,5,5,6]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm15 = ymm7[0],ymm15[1],ymm7[2,3],ymm15[4],ymm7[5],ymm15[6],ymm7[7,8],ymm15[9],ymm7[10,11],ymm15[12],ymm7[13],ymm15[14],ymm7[15]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm13, %zmm18, %zmm15
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = (~zmm15 & ~zmm31 & zmm11) | (zmm15 & ~zmm31 & zmm11) | (zmm15 & zmm31 & ~zmm11) | (zmm15 & zmm31 & zmm11)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = zmm11 ^ (zmm31 & (zmm15 ^ zmm11))
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 48(%r8), %ymm7
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 56(%r8), %ymm11
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm7, %zmm7
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = (~zmm7 & zmm12 & zmm15) | (zmm7 & ~zmm12 & ~zmm15) | (zmm7 & ~zmm12 & zmm15) | (zmm7 & zmm12 & zmm15)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (zmm12 & (zmm7 ^ zmm15))
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm11, %xmm11 # 16-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # xmm11 = xmm11[0],mem[0],xmm11[1],mem[1],xmm11[2],mem[2],xmm11[3],mem[3]
@@ -7951,7 +7951,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm12 = [2,3,2,2,8,9,8,9]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm12, %zmm8
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & ~zmm1 & zmm0) | (zmm8 & ~zmm1 & zmm0) | (zmm8 & zmm1 & ~zmm0) | (zmm8 & zmm1 & zmm0)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm0 ^ (zmm1 & (zmm8 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
@@ -7965,7 +7965,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 96(%r8), %ymm5
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = (~zmm0 & zmm5 & zmm8) | (zmm0 & ~zmm5 & ~zmm8) | (zmm0 & ~zmm5 & zmm8) | (zmm0 & zmm5 & zmm8)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm5 & (zmm0 ^ zmm8))
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm8
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm8 # 16-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # xmm8 = xmm8[0],mem[0],xmm8[1],mem[1],xmm8[2],mem[2],xmm8[3],mem[3]
@@ -7976,34 +7976,34 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm11 = ymm28[2,3,2,3,6,7,6,7]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0],ymm11[1],ymm6[2],ymm11[3],ymm6[4,5],ymm11[6],ymm6[7,8],ymm11[9],ymm6[10],ymm11[11],ymm6[12,13],ymm11[14],ymm6[15]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm8, %zmm12, %zmm6
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = (~zmm6 & ~zmm1 & zmm4) | (zmm6 & ~zmm1 & zmm4) | (zmm6 & zmm1 & ~zmm4) | (zmm6 & zmm1 & zmm4)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm4 ^ (zmm1 & (zmm6 ^ zmm4))
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 24(%r8), %ymm4
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 32(%r8), %ymm8
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm4, %zmm4
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = (~zmm4 & zmm5 & zmm6) | (zmm4 & ~zmm5 & ~zmm6) | (zmm4 & ~zmm5 & zmm6) | (zmm4 & zmm5 & zmm6)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm5 & (zmm4 ^ zmm6))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0]
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm3 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm3 = (~zmm3 & ~zmm5 & mem) | (zmm3 & ~zmm5 & mem) | (zmm3 & zmm5 & ~mem) | (zmm3 & zmm5 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm3 = mem ^ (zmm5 & (zmm3 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm16 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm16 = (~zmm16 & ~zmm5 & mem) | (zmm16 & ~zmm5 & mem) | (zmm16 & zmm5 & ~mem) | (zmm16 & zmm5 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm16 = mem ^ (zmm5 & (zmm16 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & zmm3 & zmm5) | (zmm8 & ~zmm3 & ~zmm5) | (zmm8 & ~zmm3 & zmm5) | (zmm8 & zmm3 & ~zmm5) | (zmm8 & zmm3 & zmm5)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm17 = (~zmm17 & zmm16 & zmm5) | (zmm17 & ~zmm16 & ~zmm5) | (zmm17 & ~zmm16 & zmm5) | (zmm17 & zmm16 & ~zmm5) | (zmm17 & zmm16 & zmm5)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 | (zmm3 & zmm5)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm16 & zmm5)
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 64(%r8), %ymm5
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm5, %zmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = (~zmm2 & zmm5 & zmm20) | (zmm2 & ~zmm5 & ~zmm20) | (zmm2 & ~zmm5 & zmm20) | (zmm2 & zmm5 & zmm20)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (zmm5 & (zmm2 ^ zmm20))
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq (%r8), %ymm6
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm6, %zmm6
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = (~zmm6 & zmm5 & zmm21) | (zmm6 & ~zmm5 & ~zmm21) | (zmm6 & ~zmm5 & zmm21) | (zmm6 & zmm5 & zmm21)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm5 & (zmm6 ^ zmm21))
 ; AVX512DQ-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm25 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm25 = (~zmm25 & ~zmm1 & mem) | (zmm25 & ~zmm1 & mem) | (zmm25 & zmm1 & ~mem) | (zmm25 & zmm1 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = (~zmm9 & ~zmm1 & zmm19) | (zmm9 & ~zmm1 & zmm19) | (zmm9 & zmm1 & ~zmm19) | (zmm9 & zmm1 & zmm19)
+; AVX512DQ-FCP-NEXT:    # zmm25 = mem ^ (zmm1 & (zmm25 ^ mem))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm19 ^ (zmm1 & (zmm9 ^ zmm19))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm27 = (~zmm27 & zmm25 & zmm1) | (zmm27 & ~zmm25 & ~zmm1) | (zmm27 & ~zmm25 & zmm1) | (zmm27 & zmm25 & ~zmm1) | (zmm27 & zmm25 & zmm1)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm24 = (~zmm24 & zmm9 & zmm1) | (zmm24 & ~zmm9 & ~zmm1) | (zmm24 & ~zmm9 & zmm1) | (zmm24 & zmm9 & ~zmm1) | (zmm24 & zmm9 & zmm1)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm27 = zmm27 | (zmm25 & zmm1)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm24 = zmm24 | (zmm9 & zmm1)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, 128(%r9)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm7, 256(%r9)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 448(%r9)
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll
index c93d7b7a720c3..93262fb657d49 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll
@@ -4128,19 +4128,19 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm8, %zmm8
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm8 = (~zmm8 & zmm9 & zmm16) | (zmm8 & ~zmm9 & ~zmm16) | (zmm8 & ~zmm9 & zmm16) | (zmm8 & zmm9 & zmm16)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm8 ^ (zmm9 & (zmm8 ^ zmm16))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm14, %zmm13, %zmm13
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm13 = (~zmm13 & zmm9 & zmm17) | (zmm13 & ~zmm9 & ~zmm17) | (zmm13 & ~zmm9 & zmm17) | (zmm13 & zmm9 & zmm17)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm13 = zmm13 ^ (zmm9 & (zmm13 ^ zmm17))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm5, %zmm0
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = (~zmm0 & zmm5 & zmm2) | (zmm0 & ~zmm5 & ~zmm2) | (zmm0 & ~zmm5 & zmm2) | (zmm0 & zmm5 & zmm2)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm5 & (zmm0 ^ zmm2))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm7, %zmm2
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm2 = (~zmm2 & zmm5 & zmm4) | (zmm2 & ~zmm5 & ~zmm4) | (zmm2 & ~zmm5 & zmm4) | (zmm2 & zmm5 & zmm4)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (zmm5 & (zmm2 ^ zmm4))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm11, %zmm4
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm4 = (~zmm4 & zmm5 & zmm12) | (zmm4 & ~zmm5 & ~zmm12) | (zmm4 & ~zmm5 & zmm12) | (zmm4 & zmm5 & zmm12)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm5 & (zmm4 ^ zmm12))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm15, %zmm3
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm3 = (~zmm3 & zmm5 & zmm1) | (zmm3 & ~zmm5 & ~zmm1) | (zmm3 & ~zmm5 & zmm1) | (zmm3 & zmm5 & zmm1)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (zmm5 & (zmm3 ^ zmm1))
 ; AVX512-NEXT:    vmovdqa64 %zmm3, (%rax)
 ; AVX512-NEXT:    vmovdqa64 %zmm4, 192(%rax)
 ; AVX512-NEXT:    vmovdqa64 %zmm2, 256(%rax)
@@ -4337,23 +4337,23 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm5, %zmm0
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm18, %zmm1
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = (~zmm0 & zmm2 & zmm1) | (zmm0 & ~zmm2 & ~zmm1) | (zmm0 & ~zmm2 & zmm1) | (zmm0 & zmm2 & zmm1)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm2 & (zmm0 ^ zmm1))
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 256(%rax)
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm22, %zmm0
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm8 = (~zmm8 & zmm2 & zmm0) | (zmm8 & ~zmm2 & ~zmm0) | (zmm8 & ~zmm2 & zmm0) | (zmm8 & zmm2 & zmm0)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm8 ^ (zmm2 & (zmm8 ^ zmm0))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm8, 64(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm15 = (~zmm15 & zmm0 & zmm21) | (zmm15 & ~zmm0 & ~zmm21) | (zmm15 & ~zmm0 & zmm21) | (zmm15 & zmm0 & zmm21)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm15 = zmm15 ^ (zmm0 & (zmm15 ^ zmm21))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm15, (%rax)
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm14 = (~zmm14 & zmm0 & zmm19) | (zmm14 & ~zmm0 & ~zmm19) | (zmm14 & ~zmm0 & zmm19) | (zmm14 & zmm0 & zmm19)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm14 = zmm14 ^ (zmm0 & (zmm14 ^ zmm19))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm14, 192(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = (~zmm1 & zmm0 & zmm17) | (zmm1 & ~zmm0 & ~zmm17) | (zmm1 & ~zmm0 & zmm17) | (zmm1 & zmm0 & zmm17)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm0 & (zmm1 ^ zmm17))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 128(%rax)
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = (~zmm1 & zmm0 & zmm16) | (zmm1 & ~zmm0 & ~zmm16) | (zmm1 & ~zmm0 & zmm16) | (zmm1 & zmm0 & zmm16)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm0 & (zmm1 ^ zmm16))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 320(%rax)
 ; AVX512-FCP-NEXT:    popq %rax
 ; AVX512-FCP-NEXT:    vzeroupper
@@ -4568,19 +4568,19 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm20, %zmm19, %zmm10
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm10 = (~zmm10 & zmm11 & zmm16) | (zmm10 & ~zmm11 & ~zmm16) | (zmm10 & ~zmm11 & zmm16) | (zmm10 & zmm11 & zmm16)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm10 = zmm10 ^ (zmm11 & (zmm10 ^ zmm16))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm23, %zmm21, %zmm12
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm12 = (~zmm12 & zmm11 & zmm17) | (zmm12 & ~zmm11 & ~zmm17) | (zmm12 & ~zmm11 & zmm17) | (zmm12 & zmm11 & zmm17)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm12 = zmm12 ^ (zmm11 & (zmm12 ^ zmm17))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm0 = (~zmm0 & zmm4 & zmm22) | (zmm0 & ~zmm4 & ~zmm22) | (zmm0 & ~zmm4 & zmm22) | (zmm0 & zmm4 & zmm22)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm4 & (zmm0 ^ zmm22))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm7, %zmm7
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm7 = (~zmm7 & zmm4 & zmm18) | (zmm7 & ~zmm4 & ~zmm18) | (zmm7 & ~zmm4 & zmm18) | (zmm7 & zmm4 & zmm18)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (zmm4 & (zmm7 ^ zmm18))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm4
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm4 = (~zmm4 & zmm5 & zmm2) | (zmm4 & ~zmm5 & ~zmm2) | (zmm4 & ~zmm5 & zmm2) | (zmm4 & zmm5 & zmm2)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm5 & (zmm4 ^ zmm2))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm9, %zmm1
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm1 = (~zmm1 & zmm5 & zmm3) | (zmm1 & ~zmm5 & ~zmm3) | (zmm1 & ~zmm5 & zmm3) | (zmm1 & zmm5 & zmm3)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm5 & (zmm1 ^ zmm3))
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm1, 256(%rax)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 64(%rax)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm12, (%rax)
@@ -4787,22 +4787,22 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,0,2,1,4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm5, %zmm10, %zmm6
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = (~zmm6 & zmm5 & zmm4) | (zmm6 & ~zmm5 & ~zmm4) | (zmm6 & ~zmm5 & zmm4) | (zmm6 & zmm5 & zmm4)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm5 & (zmm6 ^ zmm4))
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, (%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = (~zmm2 & zmm5 & zmm3) | (zmm2 & ~zmm5 & ~zmm3) | (zmm2 & ~zmm5 & zmm3) | (zmm2 & zmm5 & zmm3)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (zmm5 & (zmm2 ^ zmm3))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, 192(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = (~zmm1 & zmm2 & zmm0) | (zmm1 & ~zmm2 & ~zmm0) | (zmm1 & ~zmm2 & zmm0) | (zmm1 & zmm2 & zmm0)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm2 & (zmm1 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 128(%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm14 = (~zmm14 & zmm2 & zmm18) | (zmm14 & ~zmm2 & ~zmm18) | (zmm14 & ~zmm2 & zmm18) | (zmm14 & zmm2 & zmm18)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm14 = zmm14 ^ (zmm2 & (zmm14 ^ zmm18))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm14, 320(%rax)
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm20, %zmm17, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm13 = (~zmm13 & zmm1 & zmm0) | (zmm13 & ~zmm1 & ~zmm0) | (zmm13 & ~zmm1 & zmm0) | (zmm13 & zmm1 & zmm0)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm13 = zmm13 ^ (zmm1 & (zmm13 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm13, 256(%rax)
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm23, %zmm0
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm15 = (~zmm15 & zmm1 & zmm0) | (zmm15 & ~zmm1 & ~zmm0) | (zmm15 & ~zmm1 & zmm0) | (zmm15 & zmm1 & zmm0)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm15 = zmm15 ^ (zmm1 & (zmm15 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm15, 64(%rax)
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
@@ -8710,7 +8710,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm25, %zmm24, %zmm24
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm25 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
 ; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm24 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm24 = (~zmm24 & zmm25 & mem) | (zmm24 & ~zmm25 & ~mem) | (zmm24 & ~zmm25 & mem) | (zmm24 & zmm25 & mem)
+; AVX512-NEXT:    # zmm24 = zmm24 ^ (zmm25 & (zmm24 ^ mem))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm27, %zmm26, %zmm26
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[2,3,2,3]
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm13[0,2,2,1,4,5,6,7]
@@ -8723,7 +8723,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm10[0,2,2,1,4,5,6,7]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,1,0,1]
 ; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm26 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm26 = (~zmm26 & zmm25 & mem) | (zmm26 & ~zmm25 & ~mem) | (zmm26 & ~zmm25 & mem) | (zmm26 & zmm25 & mem)
+; AVX512-NEXT:    # zmm26 = zmm26 ^ (zmm25 & (zmm26 ^ mem))
 ; AVX512-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm5[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,2,2]
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm0[0,0,2,1,4,5,6,7]
@@ -8738,30 +8738,30 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,4,4]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]
 ; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm17 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm17 = (~zmm17 & zmm25 & mem) | (zmm17 & ~zmm25 & ~mem) | (zmm17 & ~zmm25 & mem) | (zmm17 & zmm25 & mem)
+; AVX512-NEXT:    # zmm17 = zmm17 ^ (zmm25 & (zmm17 ^ mem))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm16, %zmm18, %zmm16
 ; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm16 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm16 = (~zmm16 & zmm25 & mem) | (zmm16 & ~zmm25 & ~mem) | (zmm16 & ~zmm25 & mem) | (zmm16 & zmm25 & mem)
+; AVX512-NEXT:    # zmm16 = zmm16 ^ (zmm25 & (zmm16 ^ mem))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm29, %zmm23, %zmm18
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
 ; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm23, %zmm18 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm18 = (~zmm18 & zmm23 & mem) | (zmm18 & ~zmm23 & ~mem) | (zmm18 & ~zmm23 & mem) | (zmm18 & zmm23 & mem)
+; AVX512-NEXT:    # zmm18 = zmm18 ^ (zmm23 & (zmm18 ^ mem))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm11, %zmm4
 ; AVX512-NEXT:    vpternlogd $184, (%rsp), %zmm23, %zmm4 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm4 = (~zmm4 & zmm23 & mem) | (zmm4 & ~zmm23 & ~mem) | (zmm4 & ~zmm23 & mem) | (zmm4 & zmm23 & mem)
+; AVX512-NEXT:    # zmm4 = zmm4 ^ (zmm23 & (zmm4 ^ mem))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm13, %zmm7
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm7 = (~zmm7 & zmm23 & zmm20) | (zmm7 & ~zmm23 & ~zmm20) | (zmm7 & ~zmm23 & zmm20) | (zmm7 & zmm23 & zmm20)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (zmm23 & (zmm7 ^ zmm20))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm10, %zmm5
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm5 = (~zmm5 & zmm23 & zmm19) | (zmm5 & ~zmm23 & ~zmm19) | (zmm5 & ~zmm23 & zmm19) | (zmm5 & zmm23 & zmm19)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm5 ^ (zmm23 & (zmm5 ^ zmm19))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm12, %zmm8
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm8 = (~zmm8 & zmm10 & zmm22) | (zmm8 & ~zmm10 & ~zmm22) | (zmm8 & ~zmm10 & zmm22) | (zmm8 & zmm10 & zmm22)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm8 ^ (zmm10 & (zmm8 ^ zmm22))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm14, %zmm6
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm6 = (~zmm6 & zmm10 & zmm28) | (zmm6 & ~zmm10 & ~zmm28) | (zmm6 & ~zmm10 & zmm28) | (zmm6 & zmm10 & zmm28)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm10 & (zmm6 ^ zmm28))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm9, %zmm0
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = (~zmm0 & zmm10 & zmm15) | (zmm0 & ~zmm10 & ~zmm15) | (zmm0 & ~zmm10 & zmm15) | (zmm0 & zmm10 & zmm15)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm10 & (zmm0 ^ zmm15))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm2 = (~zmm2 & zmm10 & zmm1) | (zmm2 & ~zmm10 & ~zmm1) | (zmm2 & ~zmm10 & zmm1) | (zmm2 & zmm10 & zmm1)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (zmm10 & (zmm2 ^ zmm1))
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-NEXT:    vmovdqa64 %zmm2, (%rax)
 ; AVX512-NEXT:    vmovdqa64 %zmm0, 192(%rax)
@@ -9172,45 +9172,45 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpermt2q %zmm8, %zmm28, %zmm6
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm27, %zmm8
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = (~zmm6 & zmm9 & zmm8) | (zmm6 & ~zmm9 & ~zmm8) | (zmm6 & ~zmm9 & zmm8) | (zmm6 & zmm9 & zmm8)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm9 & (zmm6 ^ zmm8))
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, 256(%rax)
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm19, %zmm0
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm12 = (~zmm12 & zmm9 & zmm0) | (zmm12 & ~zmm9 & ~zmm0) | (zmm12 & ~zmm9 & zmm0) | (zmm12 & zmm9 & zmm0)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm12 = zmm12 ^ (zmm9 & (zmm12 ^ zmm0))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm12, 448(%rax)
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm29, %zmm0
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm11 = (~zmm11 & zmm9 & zmm0) | (zmm11 & ~zmm9 & ~zmm0) | (zmm11 & ~zmm9 & zmm0) | (zmm11 & zmm9 & zmm0)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm11 = zmm11 ^ (zmm9 & (zmm11 ^ zmm0))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm11, 640(%rax)
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm0
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = (~zmm3 & zmm9 & zmm0) | (zmm3 & ~zmm9 & ~zmm0) | (zmm3 & ~zmm9 & zmm0) | (zmm3 & zmm9 & zmm0)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (zmm9 & (zmm3 ^ zmm0))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, 64(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = (~zmm4 & zmm0 & zmm17) | (zmm4 & ~zmm0 & ~zmm17) | (zmm4 & ~zmm0 & zmm17) | (zmm4 & zmm0 & zmm17)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm0 & (zmm4 ^ zmm17))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, (%rax)
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm5 = (~zmm5 & zmm0 & zmm16) | (zmm5 & ~zmm0 & ~zmm16) | (zmm5 & ~zmm0 & zmm16) | (zmm5 & zmm0 & zmm16)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm5 ^ (zmm0 & (zmm5 ^ zmm16))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, 192(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm13 = (~zmm13 & zmm1 & zmm25) | (zmm13 & ~zmm1 & ~zmm25) | (zmm13 & ~zmm1 & zmm25) | (zmm13 & zmm1 & zmm25)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm13 = zmm13 ^ (zmm1 & (zmm13 ^ zmm25))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm13, 128(%rax)
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm2 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm2 = (~zmm2 & zmm1 & mem) | (zmm2 & ~zmm1 & ~mem) | (zmm2 & ~zmm1 & mem) | (zmm2 & zmm1 & mem)
+; AVX512-FCP-NEXT:    # zmm2 = zmm2 ^ (zmm1 & (zmm2 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, 320(%rax)
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm2 = (~zmm2 & zmm0 & mem) | (zmm2 & ~zmm0 & ~mem) | (zmm2 & ~zmm0 & mem) | (zmm2 & zmm0 & mem)
+; AVX512-FCP-NEXT:    # zmm2 = zmm2 ^ (zmm0 & (zmm2 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, 384(%rax)
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm2 = (~zmm2 & zmm0 & mem) | (zmm2 & ~zmm0 & ~mem) | (zmm2 & ~zmm0 & mem) | (zmm2 & zmm0 & mem)
+; AVX512-FCP-NEXT:    # zmm2 = zmm2 ^ (zmm0 & (zmm2 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, 576(%rax)
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm0 = (~zmm0 & zmm1 & mem) | (zmm0 & ~zmm1 & ~mem) | (zmm0 & ~zmm1 & mem) | (zmm0 & zmm1 & mem)
+; AVX512-FCP-NEXT:    # zmm0 = zmm0 ^ (zmm1 & (zmm0 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 512(%rax)
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm0 = (~zmm0 & zmm1 & mem) | (zmm0 & ~zmm1 & ~mem) | (zmm0 & ~zmm1 & mem) | (zmm0 & zmm1 & mem)
+; AVX512-FCP-NEXT:    # zmm0 = zmm0 ^ (zmm1 & (zmm0 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 704(%rax)
 ; AVX512-FCP-NEXT:    addq $1240, %rsp # imm = 0x4D8
 ; AVX512-FCP-NEXT:    vzeroupper
@@ -9556,7 +9556,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm16
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm28 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
 ; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm16 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm16 = (~zmm16 & zmm28 & mem) | (zmm16 & ~zmm28 & ~mem) | (zmm16 & ~zmm28 & mem) | (zmm16 & zmm28 & mem)
+; AVX512DQ-NEXT:    # zmm16 = zmm16 ^ (zmm28 & (zmm16 ^ mem))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm4[2,2,2,3]
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm1, %zmm2, %zmm3 {%k1}
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm3, %ymm1
@@ -9633,7 +9633,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm3[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,1,2,3]
 ; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm1 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm1 = (~zmm1 & zmm28 & mem) | (zmm1 & ~zmm28 & ~mem) | (zmm1 & ~zmm28 & mem) | (zmm1 & zmm28 & mem)
+; AVX512DQ-NEXT:    # zmm1 = zmm1 ^ (zmm28 & (zmm1 ^ mem))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm0, %zmm12
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm15, %zmm2, %zmm2
 ; AVX512DQ-NEXT:    vmovdqa (%r9), %ymm15
@@ -9664,31 +9664,31 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm10 = ymm10[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,2,2]
 ; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm12 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm12 = (~zmm12 & zmm28 & mem) | (zmm12 & ~zmm28 & ~mem) | (zmm12 & ~zmm28 & mem) | (zmm12 & zmm28 & mem)
+; AVX512DQ-NEXT:    # zmm12 = zmm12 ^ (zmm28 & (zmm12 ^ mem))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm30, %zmm29, %zmm17
 ; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm17 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm17 = (~zmm17 & zmm28 & mem) | (zmm17 & ~zmm28 & ~mem) | (zmm17 & ~zmm28 & mem) | (zmm17 & zmm28 & mem)
+; AVX512DQ-NEXT:    # zmm17 = zmm17 ^ (zmm28 & (zmm17 ^ mem))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm31, %zmm24, %zmm22
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
 ; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm22 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm22 = (~zmm22 & zmm24 & mem) | (zmm22 & ~zmm24 & ~mem) | (zmm22 & ~zmm24 & mem) | (zmm22 & zmm24 & mem)
+; AVX512DQ-NEXT:    # zmm22 = zmm22 ^ (zmm24 & (zmm22 ^ mem))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm18, %zmm7
 ; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm7 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm7 = (~zmm7 & zmm24 & mem) | (zmm7 & ~zmm24 & ~mem) | (zmm7 & ~zmm24 & mem) | (zmm7 & zmm24 & mem)
+; AVX512DQ-NEXT:    # zmm7 = zmm7 ^ (zmm24 & (zmm7 ^ mem))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm5, %zmm3
 ; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm3 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm3 = (~zmm3 & zmm24 & mem) | (zmm3 & ~zmm24 & ~mem) | (zmm3 & ~zmm24 & mem) | (zmm3 & zmm24 & mem)
+; AVX512DQ-NEXT:    # zmm3 = zmm3 ^ (zmm24 & (zmm3 ^ mem))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm13, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm0 = (~zmm0 & zmm24 & zmm23) | (zmm0 & ~zmm24 & ~zmm23) | (zmm0 & ~zmm24 & zmm23) | (zmm0 & zmm24 & zmm23)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm24 & (zmm0 ^ zmm23))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm15, %zmm14, %zmm5
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm13 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm5 = (~zmm5 & zmm13 & zmm25) | (zmm5 & ~zmm13 & ~zmm25) | (zmm5 & ~zmm13 & zmm25) | (zmm5 & zmm13 & zmm25)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm5 ^ (zmm13 & (zmm5 ^ zmm25))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm6, %zmm6
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm6 = (~zmm6 & zmm13 & zmm26) | (zmm6 & ~zmm13 & ~zmm26) | (zmm6 & ~zmm13 & zmm26) | (zmm6 & zmm13 & zmm26)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm13 & (zmm6 ^ zmm26))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm9, %zmm4
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm4 = (~zmm4 & zmm13 & zmm27) | (zmm4 & ~zmm13 & ~zmm27) | (zmm4 & ~zmm13 & zmm27) | (zmm4 & zmm13 & zmm27)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm13 & (zmm4 ^ zmm27))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm8, %zmm8
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm8 = (~zmm8 & zmm13 & zmm2) | (zmm8 & ~zmm13 & ~zmm2) | (zmm8 & ~zmm13 & zmm2) | (zmm8 & zmm13 & zmm2)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm8 ^ (zmm13 & (zmm8 ^ zmm2))
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm8, 256(%rax)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 448(%rax)
@@ -10112,45 +10112,45 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm9[0,0,2,1,4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm8, %zmm16, %zmm11
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm11 = (~zmm11 & zmm8 & zmm0) | (zmm11 & ~zmm8 & ~zmm0) | (zmm11 & ~zmm8 & zmm0) | (zmm11 & zmm8 & zmm0)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm11 = zmm11 ^ (zmm8 & (zmm11 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm11, (%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm12 = (~zmm12 & zmm8 & zmm10) | (zmm12 & ~zmm8 & ~zmm10) | (zmm12 & ~zmm8 & zmm10) | (zmm12 & zmm8 & zmm10)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm12 = zmm12 ^ (zmm8 & (zmm12 ^ zmm10))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm12, 192(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = (~zmm7 & zmm0 & zmm2) | (zmm7 & ~zmm0 & ~zmm2) | (zmm7 & ~zmm0 & zmm2) | (zmm7 & zmm0 & zmm2)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (zmm0 & (zmm7 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm7, 128(%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = (~zmm6 & zmm0 & zmm27) | (zmm6 & ~zmm0 & ~zmm27) | (zmm6 & ~zmm0 & zmm27) | (zmm6 & zmm0 & zmm27)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm0 & (zmm6 ^ zmm27))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, 320(%rax)
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm24, %zmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = (~zmm7 & zmm6 & zmm2) | (zmm7 & ~zmm6 & ~zmm2) | (zmm7 & ~zmm6 & zmm2) | (zmm7 & zmm6 & zmm2)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (zmm6 & (zmm7 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm7, 256(%rax)
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm28, %zmm20, %zmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = (~zmm7 & zmm6 & zmm2) | (zmm7 & ~zmm6 & ~zmm2) | (zmm7 & ~zmm6 & zmm2) | (zmm7 & zmm6 & zmm2)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (zmm6 & (zmm7 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm7, 448(%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm5 = (~zmm5 & zmm8 & zmm23) | (zmm5 & ~zmm8 & ~zmm23) | (zmm5 & ~zmm8 & zmm23) | (zmm5 & zmm8 & zmm23)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm5 ^ (zmm8 & (zmm5 ^ zmm23))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, 384(%rax)
 ; AVX512DQ-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm4 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm4 = (~zmm4 & zmm8 & mem) | (zmm4 & ~zmm8 & ~mem) | (zmm4 & ~zmm8 & mem) | (zmm4 & zmm8 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm4 = zmm4 ^ (zmm8 & (zmm4 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, 576(%rax)
 ; AVX512DQ-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm3 = (~zmm3 & zmm0 & mem) | (zmm3 & ~zmm0 & ~mem) | (zmm3 & ~zmm0 & mem) | (zmm3 & zmm0 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm3 = zmm3 ^ (zmm0 & (zmm3 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, 512(%rax)
 ; AVX512DQ-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm1 = (~zmm1 & zmm0 & mem) | (zmm1 & ~zmm0 & ~mem) | (zmm1 & ~zmm0 & mem) | (zmm1 & zmm0 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm1 = zmm1 ^ (zmm0 & (zmm1 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 704(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm25, %zmm0, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = (~zmm1 & zmm6 & zmm0) | (zmm1 & ~zmm6 & ~zmm0) | (zmm1 & ~zmm6 & zmm0) | (zmm1 & zmm6 & zmm0)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm6 & (zmm1 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 640(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm21, %zmm0, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = (~zmm1 & zmm6 & zmm0) | (zmm1 & ~zmm6 & ~zmm0) | (zmm1 & ~zmm6 & zmm0) | (zmm1 & zmm6 & zmm0)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm6 & (zmm1 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 64(%rax)
 ; AVX512DQ-FCP-NEXT:    addq $1176, %rsp # imm = 0x498
 ; AVX512DQ-FCP-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
index 51f173bc1a9bc..6e251f8675bea 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
@@ -647,7 +647,7 @@ define void @store_i16_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[4,5,12,13,u,u,u,u,u,u],zero,zero,zero,zero,ymm3[6,7],zero,zero,ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
 ; AVX512-NEXT:    vporq %zmm1, %zmm2, %zmm1
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm1 = (~zmm1 & zmm0 & ~mem) | (zmm1 & ~zmm0 & mem) | (zmm1 & zmm0 & ~mem) | (zmm1 & zmm0 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm0 ^ (mem & (zmm1 ^ zmm0))
 ; AVX512-NEXT:    vextracti32x4 $2, %zmm1, 32(%rax)
 ; AVX512-NEXT:    vextracti32x4 $3, %zmm1, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, 48(%rax)
@@ -684,7 +684,7 @@ define void @store_i16_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm2[0,1,8,9],zero,zero,zero,zero,ymm2[u,u,u,u,u,u,2,3],zero,zero,ymm2[18,19,26,27,u,u,u,u,u,u],zero,zero,zero,zero
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,ymm3[0,1,8,9,u,u,u,u,u,u],zero,zero,ymm3[26,27],zero,zero,zero,zero,ymm3[u,u,u,u,u,u,20,21,28,29]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm3 = (~ymm3 & ymm1 & mem) | (ymm3 & ~ymm1 & mem) | (ymm3 & ymm1 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm3 = mem & (ymm3 | ymm1)
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [5,7,1,3,7,0,0,0]
 ; AVX512-FCP-NEXT:    vpermd %ymm2, %ymm1, %ymm1
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,1,4,5],zero,zero,zero,zero,zero,zero,ymm1[10,11,14,15,2,3,18,19],zero,zero,zero,zero,zero,zero,ymm1[u,u,u,u,u,u,u,u]
@@ -730,7 +730,7 @@ define void @store_i16_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[4,5,12,13,u,u,u,u,u,u],zero,zero,zero,zero,ymm3[6,7],zero,zero,ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
 ; AVX512DQ-NEXT:    vporq %zmm1, %zmm2, %zmm1
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm1 = (~zmm1 & zmm0 & ~mem) | (zmm1 & ~zmm0 & mem) | (zmm1 & zmm0 & ~mem) | (zmm1 & zmm0 & mem)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm0 ^ (mem & (zmm1 ^ zmm0))
 ; AVX512DQ-NEXT:    vextracti32x4 $2, %zmm1, 32(%rax)
 ; AVX512DQ-NEXT:    vextracti32x4 $3, %zmm1, %xmm0
 ; AVX512DQ-NEXT:    vmovq %xmm0, 48(%rax)
@@ -767,7 +767,7 @@ define void @store_i16_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm2[0,1,8,9],zero,zero,zero,zero,ymm2[u,u,u,u,u,u,2,3],zero,zero,ymm2[18,19,26,27,u,u,u,u,u,u],zero,zero,zero,zero
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,ymm3[0,1,8,9,u,u,u,u,u,u],zero,zero,ymm3[26,27],zero,zero,zero,zero,ymm3[u,u,u,u,u,u,20,21,28,29]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm3 = (~ymm3 & ymm1 & mem) | (ymm3 & ~ymm1 & mem) | (ymm3 & ymm1 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm3 = mem & (ymm3 | ymm1)
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [5,7,1,3,7,0,0,0]
 ; AVX512DQ-FCP-NEXT:    vpermd %ymm2, %ymm1, %ymm1
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,1,4,5],zero,zero,zero,zero,zero,zero,ymm1[10,11,14,15,2,3,18,19],zero,zero,zero,zero,zero,zero,ymm1[u,u,u,u,u,u,u,u]
@@ -1381,8 +1381,8 @@ define void @store_i16_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpbroadcastd (%r10), %ymm11
 ; AVX512-NEXT:    vpbroadcastd 4(%r10), %ymm12
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm11, %zmm11
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm11 = (~zmm11 & zmm10 & ~mem) | (~zmm11 & zmm10 & mem) | (zmm11 & ~zmm10 & mem) | (zmm11 & zmm10 & ~mem) | (zmm11 & zmm10 & mem)
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm11 = (~zmm11 & zmm9 & mem) | (zmm11 & ~zmm9 & ~mem) | (zmm11 & zmm9 & ~mem) | (zmm11 & zmm9 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm11 = (zmm11 & mem) | zmm10
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm11 = zmm11 ^ (mem & (zmm11 ^ zmm9))
 ; AVX512-NEXT:    vpsrlq $48, %xmm4, %xmm4
 ; AVX512-NEXT:    vpunpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm4[1]
 ; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
@@ -1401,8 +1401,8 @@ define void @store_i16_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm8[1,3,3,1]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[0,1,8,9],zero,zero,ymm2[u,u,u,u,u,u,u,u,2,3,18,19],zero,zero,ymm2[u,u,u,u,u,u,u,u,28,29,20,21]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm3 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogd {{.*#+}} ymm3 = (~ymm3 & ~ymm2 & mem) | (~ymm3 & ymm2 & ~mem) | (~ymm3 & ymm2 & mem) | (ymm3 & ymm2 & ~mem) | (ymm3 & ymm2 & mem)
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm3 = (~ymm3 & ymm1 & ~mem) | (ymm3 & ~ymm1 & mem) | (ymm3 & ymm1 & ~mem) | (ymm3 & ymm1 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} ymm3 = (mem & ~ymm3) | ymm2
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm3 = ymm1 ^ (mem & (ymm3 ^ ymm1))
 ; AVX512-NEXT:    vinserti32x4 $2, %xmm0, %zmm3, %zmm1
 ; AVX512-NEXT:    vmovdqa %xmm0, 96(%rax)
 ; AVX512-NEXT:    vmovdqa %ymm1, 64(%rax)
@@ -1441,8 +1441,8 @@ define void @store_i16_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm8[1,3,3,1]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[0,1,8,9],zero,zero,ymm2[u,u,u,u,u,u,u,u,2,3,18,19],zero,zero,ymm2[u,u,u,u,u,u,u,u,28,29,20,21]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} ymm3 = (~ymm3 & ~ymm2 & mem) | (~ymm3 & ymm2 & ~mem) | (~ymm3 & ymm2 & mem) | (ymm3 & ymm2 & ~mem) | (ymm3 & ymm2 & mem)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm3 = (~ymm3 & ymm1 & ~mem) | (ymm3 & ~ymm1 & mem) | (ymm3 & ymm1 & ~mem) | (ymm3 & ymm1 & mem)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} ymm3 = (mem & ~ymm3) | ymm2
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm3 = ymm1 ^ (mem & (ymm3 ^ ymm1))
 ; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm3, %zmm1
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm7[0,2,0,2]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,ymm2[0,1,8,9,u,u,u,u,u,u],zero,zero,zero,zero,ymm2[18,19,26,27,u,u,u,u,u,u],zero,zero,zero,zero
@@ -1463,8 +1463,8 @@ define void @store_i16_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpbroadcastd (%r10), %ymm4
 ; AVX512-FCP-NEXT:    vpbroadcastd 4(%r10), %ymm5
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm3 & ~mem) | (~zmm4 & zmm3 & mem) | (zmm4 & ~zmm3 & mem) | (zmm4 & zmm3 & ~mem) | (zmm4 & zmm3 & mem)
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = (~zmm4 & zmm2 & mem) | (zmm4 & ~zmm2 & ~mem) | (zmm4 & zmm2 & ~mem) | (zmm4 & zmm2 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (zmm4 & mem) | zmm3
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm2))
 ; AVX512-FCP-NEXT:    vmovdqa %xmm0, 96(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, (%rax)
 ; AVX512-FCP-NEXT:    vmovdqa %ymm1, 64(%rax)
@@ -1506,8 +1506,8 @@ define void @store_i16_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpbroadcastd (%r10), %ymm11
 ; AVX512DQ-NEXT:    vpbroadcastd 4(%r10), %ymm12
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm11, %zmm11
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm11 = (~zmm11 & zmm10 & ~mem) | (~zmm11 & zmm10 & mem) | (zmm11 & ~zmm10 & mem) | (zmm11 & zmm10 & ~mem) | (zmm11 & zmm10 & mem)
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm11 = (~zmm11 & zmm9 & mem) | (zmm11 & ~zmm9 & ~mem) | (zmm11 & zmm9 & ~mem) | (zmm11 & zmm9 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm11 = (zmm11 & mem) | zmm10
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm11 = zmm11 ^ (mem & (zmm11 ^ zmm9))
 ; AVX512DQ-NEXT:    vpsrlq $48, %xmm4, %xmm4
 ; AVX512DQ-NEXT:    vpunpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm4[1]
 ; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
@@ -1526,8 +1526,8 @@ define void @store_i16_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm8[1,3,3,1]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[0,1,8,9],zero,zero,ymm2[u,u,u,u,u,u,u,u,2,3,18,19],zero,zero,ymm2[u,u,u,u,u,u,u,u,28,29,20,21]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm3 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} ymm3 = (~ymm3 & ~ymm2 & mem) | (~ymm3 & ymm2 & ~mem) | (~ymm3 & ymm2 & mem) | (ymm3 & ymm2 & ~mem) | (ymm3 & ymm2 & mem)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm3 = (~ymm3 & ymm1 & ~mem) | (ymm3 & ~ymm1 & mem) | (ymm3 & ymm1 & ~mem) | (ymm3 & ymm1 & mem)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} ymm3 = (mem & ~ymm3) | ymm2
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm3 = ymm1 ^ (mem & (ymm3 ^ ymm1))
 ; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm0, %zmm3, %zmm1
 ; AVX512DQ-NEXT:    vmovdqa %xmm0, 96(%rax)
 ; AVX512DQ-NEXT:    vmovdqa %ymm1, 64(%rax)
@@ -1566,8 +1566,8 @@ define void @store_i16_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm8[1,3,3,1]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[0,1,8,9],zero,zero,ymm2[u,u,u,u,u,u,u,u,2,3,18,19],zero,zero,ymm2[u,u,u,u,u,u,u,u,28,29,20,21]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} ymm3 = (~ymm3 & ~ymm2 & mem) | (~ymm3 & ymm2 & ~mem) | (~ymm3 & ymm2 & mem) | (ymm3 & ymm2 & ~mem) | (ymm3 & ymm2 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm3 = (~ymm3 & ymm1 & ~mem) | (ymm3 & ~ymm1 & mem) | (ymm3 & ymm1 & ~mem) | (ymm3 & ymm1 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} ymm3 = (mem & ~ymm3) | ymm2
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm3 = ymm1 ^ (mem & (ymm3 ^ ymm1))
 ; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm3, %zmm1
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm7[0,2,0,2]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,ymm2[0,1,8,9,u,u,u,u,u,u],zero,zero,zero,zero,ymm2[18,19,26,27,u,u,u,u,u,u],zero,zero,zero,zero
@@ -1588,8 +1588,8 @@ define void @store_i16_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd (%r10), %ymm4
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 4(%r10), %ymm5
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm3 & ~mem) | (~zmm4 & zmm3 & mem) | (zmm4 & ~zmm3 & mem) | (zmm4 & zmm3 & ~mem) | (zmm4 & zmm3 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = (~zmm4 & zmm2 & mem) | (zmm4 & ~zmm2 & ~mem) | (zmm4 & zmm2 & ~mem) | (zmm4 & zmm2 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (zmm4 & mem) | zmm3
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, 96(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, (%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm1, 64(%rax)
@@ -2937,27 +2937,27 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm16, %zmm7, %zmm7
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm17, %zmm9, %zmm9
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm9 = (~zmm9 & zmm7 & ~mem) | (zmm9 & ~zmm7 & mem) | (zmm9 & zmm7 & ~mem) | (zmm9 & zmm7 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm9 = zmm7 ^ (mem & (zmm9 ^ zmm7))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm19, %zmm4, %zmm4
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm4 & mem) | (zmm5 & ~zmm4 & ~mem) | (zmm5 & ~zmm4 & mem) | (zmm5 & zmm4 & ~mem) | (zmm5 & zmm4 & mem)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm9 & mem) | (zmm5 & ~zmm9 & ~mem) | (zmm5 & zmm9 & ~mem) | (zmm5 & zmm9 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm4 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm9))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm4
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm7 = zmm20[2,2,2,3,6,6,6,7]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm7 = (~zmm7 & zmm4 & ~mem) | (zmm7 & ~zmm4 & mem) | (zmm7 & zmm4 & ~mem) | (zmm7 & zmm4 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm4 ^ (mem & (zmm7 ^ zmm4))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm12, %zmm0
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm3 = (~zmm3 & zmm0 & mem) | (zmm3 & ~zmm0 & ~mem) | (zmm3 & zmm0 & ~mem) | (zmm3 & zmm0 & mem)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & zmm7 & mem) | (zmm3 & ~zmm7 & ~mem) | (zmm3 & zmm7 & ~mem) | (zmm3 & zmm7 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm0))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm7))
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm0 = zmm1[0,0,1,1,4,4,5,5]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = (~zmm0 & zmm18 & ~mem) | (zmm0 & ~zmm18 & mem) | (zmm0 & zmm18 & ~mem) | (zmm0 & zmm18 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm18 ^ (mem & (zmm0 ^ zmm18))
 ; AVX512-NEXT:    vpbroadcastd (%rax), %ymm1
 ; AVX512-NEXT:    vpbroadcastd 4(%rax), %ymm4
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm1, %zmm1
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm1 = (~zmm1 & zmm15 & mem) | (zmm1 & ~zmm15 & ~mem) | (zmm1 & zmm15 & ~mem) | (zmm1 & zmm15 & mem)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & zmm0 & mem) | (zmm1 & ~zmm0 & ~mem) | (zmm1 & zmm0 & ~mem) | (zmm1 & zmm0 & mem)
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ymm8 & mem) | (ymm6 & ~ymm8 & ~mem) | (ymm6 & ymm8 & ~mem) | (ymm6 & ymm8 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm15))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm0))
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm6 ^ (mem & (ymm6 ^ ymm8))
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm13[2,1,3,2]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ymm2 & mem) | (ymm0 & ~ymm2 & ~mem) | (ymm0 & ymm2 & ~mem) | (ymm0 & ymm2 & mem)
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ymm6 & mem) | (ymm0 & ~ymm6 & ~mem) | (ymm0 & ymm6 & ~mem) | (ymm0 & ymm6 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm2))
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm6))
 ; AVX512-NEXT:    vmovdqa %ymm0, 192(%rcx)
 ; AVX512-NEXT:    vmovdqa64 %zmm1, (%rcx)
 ; AVX512-NEXT:    vmovdqa64 %zmm3, 128(%rcx)
@@ -3065,24 +3065,24 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm16, %zmm3, %zmm2
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm17, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = (~zmm0 & zmm2 & mem) | (zmm0 & ~zmm2 & ~mem) | (zmm0 & zmm2 & ~mem) | (zmm0 & zmm2 & mem)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm18, %zmm8, %zmm2
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & zmm2 & mem) | (zmm10 & ~zmm2 & ~mem) | (zmm10 & ~zmm2 & mem) | (zmm10 & zmm2 & ~mem) | (zmm10 & zmm2 & mem)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & zmm0 & mem) | (zmm10 & ~zmm0 & ~mem) | (zmm10 & zmm0 & ~mem) | (zmm10 & zmm0 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 | (zmm2 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm0))
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm19[2,2,2,3,6,6,6,7]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = (~zmm0 & zmm9 & ~mem) | (zmm0 & ~zmm9 & mem) | (zmm0 & zmm9 & ~mem) | (zmm0 & zmm9 & mem)
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = (~zmm6 & zmm13 & mem) | (zmm6 & ~zmm13 & ~mem) | (zmm6 & zmm13 & ~mem) | (zmm6 & zmm13 & mem)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = (~zmm6 & zmm0 & mem) | (zmm6 & ~zmm0 & ~mem) | (zmm6 & zmm0 & ~mem) | (zmm6 & zmm0 & mem)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm9 ^ (mem & (zmm0 ^ zmm9))
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm13))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm0))
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm11[0,0,1,1,4,4,5,5]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = (~zmm0 & zmm15 & ~mem) | (zmm0 & ~zmm15 & mem) | (zmm0 & zmm15 & ~mem) | (zmm0 & zmm15 & mem)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm15 ^ (mem & (zmm0 ^ zmm15))
 ; AVX512-FCP-NEXT:    vpbroadcastd (%rax), %ymm2
 ; AVX512-FCP-NEXT:    vpbroadcastd 4(%rax), %ymm3
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = (~zmm2 & zmm20 & mem) | (zmm2 & ~zmm20 & ~mem) | (zmm2 & zmm20 & ~mem) | (zmm2 & zmm20 & mem)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm0 & mem) | (zmm2 & ~zmm0 & ~mem) | (zmm2 & zmm0 & ~mem) | (zmm2 & zmm0 & mem)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = (~ymm4 & ymm5 & mem) | (ymm4 & ~ymm5 & ~mem) | (ymm4 & ymm5 & ~mem) | (ymm4 & ymm5 & mem)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ymm7 & ~mem) | (ymm1 & ~ymm7 & mem) | (ymm1 & ymm7 & ~mem) | (ymm1 & ymm7 & mem)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ymm4 & mem) | (ymm1 & ~ymm4 & ~mem) | (ymm1 & ymm4 & ~mem) | (ymm1 & ymm4 & mem)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm20))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm0))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = ymm4 ^ (mem & (ymm4 ^ ymm5))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm7 ^ (mem & (ymm1 ^ ymm7))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm4))
 ; AVX512-FCP-NEXT:    vmovdqa %ymm1, 192(%rcx)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, (%rcx)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, 128(%rcx)
@@ -3199,27 +3199,27 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm16, %zmm7, %zmm7
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm17, %zmm9, %zmm9
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm9 = (~zmm9 & zmm7 & ~mem) | (zmm9 & ~zmm7 & mem) | (zmm9 & zmm7 & ~mem) | (zmm9 & zmm7 & mem)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm9 = zmm7 ^ (mem & (zmm9 ^ zmm7))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm19, %zmm4, %zmm4
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm4 & mem) | (zmm5 & ~zmm4 & ~mem) | (zmm5 & ~zmm4 & mem) | (zmm5 & zmm4 & ~mem) | (zmm5 & zmm4 & mem)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm9 & mem) | (zmm5 & ~zmm9 & ~mem) | (zmm5 & zmm9 & ~mem) | (zmm5 & zmm9 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm4 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm9))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm4
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm7 = zmm20[2,2,2,3,6,6,6,7]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm7 = (~zmm7 & zmm4 & ~mem) | (zmm7 & ~zmm4 & mem) | (zmm7 & zmm4 & ~mem) | (zmm7 & zmm4 & mem)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm4 ^ (mem & (zmm7 ^ zmm4))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm12, %zmm0
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm3 = (~zmm3 & zmm0 & mem) | (zmm3 & ~zmm0 & ~mem) | (zmm3 & zmm0 & ~mem) | (zmm3 & zmm0 & mem)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & zmm7 & mem) | (zmm3 & ~zmm7 & ~mem) | (zmm3 & zmm7 & ~mem) | (zmm3 & zmm7 & mem)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm0))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm7))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm0 = zmm1[0,0,1,1,4,4,5,5]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm0 = (~zmm0 & zmm18 & ~mem) | (zmm0 & ~zmm18 & mem) | (zmm0 & zmm18 & ~mem) | (zmm0 & zmm18 & mem)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm18 ^ (mem & (zmm0 ^ zmm18))
 ; AVX512DQ-NEXT:    vpbroadcastd (%rax), %ymm1
 ; AVX512DQ-NEXT:    vpbroadcastd 4(%rax), %ymm4
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm1, %zmm1
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm1 = (~zmm1 & zmm15 & mem) | (zmm1 & ~zmm15 & ~mem) | (zmm1 & zmm15 & ~mem) | (zmm1 & zmm15 & mem)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & zmm0 & mem) | (zmm1 & ~zmm0 & ~mem) | (zmm1 & zmm0 & ~mem) | (zmm1 & zmm0 & mem)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ymm8 & mem) | (ymm6 & ~ymm8 & ~mem) | (ymm6 & ymm8 & ~mem) | (ymm6 & ymm8 & mem)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm15))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm0))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm6 ^ (mem & (ymm6 ^ ymm8))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm13[2,1,3,2]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ymm2 & mem) | (ymm0 & ~ymm2 & ~mem) | (ymm0 & ymm2 & ~mem) | (ymm0 & ymm2 & mem)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm0 = (~ymm0 & ymm6 & mem) | (ymm0 & ~ymm6 & ~mem) | (ymm0 & ymm6 & ~mem) | (ymm0 & ymm6 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm2))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm6))
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, 192(%rcx)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm1, (%rcx)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm3, 128(%rcx)
@@ -3327,24 +3327,24 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm16, %zmm3, %zmm2
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm17, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = (~zmm0 & zmm2 & mem) | (zmm0 & ~zmm2 & ~mem) | (zmm0 & zmm2 & ~mem) | (zmm0 & zmm2 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm18, %zmm8, %zmm2
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & zmm2 & mem) | (zmm10 & ~zmm2 & ~mem) | (zmm10 & ~zmm2 & mem) | (zmm10 & zmm2 & ~mem) | (zmm10 & zmm2 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & zmm0 & mem) | (zmm10 & ~zmm0 & ~mem) | (zmm10 & zmm0 & ~mem) | (zmm10 & zmm0 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 | (zmm2 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm19[2,2,2,3,6,6,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = (~zmm0 & zmm9 & ~mem) | (zmm0 & ~zmm9 & mem) | (zmm0 & zmm9 & ~mem) | (zmm0 & zmm9 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = (~zmm6 & zmm13 & mem) | (zmm6 & ~zmm13 & ~mem) | (zmm6 & zmm13 & ~mem) | (zmm6 & zmm13 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = (~zmm6 & zmm0 & mem) | (zmm6 & ~zmm0 & ~mem) | (zmm6 & zmm0 & ~mem) | (zmm6 & zmm0 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm9 ^ (mem & (zmm0 ^ zmm9))
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm13))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm11[0,0,1,1,4,4,5,5]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = (~zmm0 & zmm15 & ~mem) | (zmm0 & ~zmm15 & mem) | (zmm0 & zmm15 & ~mem) | (zmm0 & zmm15 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm15 ^ (mem & (zmm0 ^ zmm15))
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd (%rax), %ymm2
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 4(%rax), %ymm3
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = (~zmm2 & zmm20 & mem) | (zmm2 & ~zmm20 & ~mem) | (zmm2 & zmm20 & ~mem) | (zmm2 & zmm20 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm0 & mem) | (zmm2 & ~zmm0 & ~mem) | (zmm2 & zmm0 & ~mem) | (zmm2 & zmm0 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = (~ymm4 & ymm5 & mem) | (ymm4 & ~ymm5 & ~mem) | (ymm4 & ymm5 & ~mem) | (ymm4 & ymm5 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ymm7 & ~mem) | (ymm1 & ~ymm7 & mem) | (ymm1 & ymm7 & ~mem) | (ymm1 & ymm7 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = (~ymm1 & ymm4 & mem) | (ymm1 & ~ymm4 & ~mem) | (ymm1 & ymm4 & ~mem) | (ymm1 & ymm4 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm20))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm0))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = ymm4 ^ (mem & (ymm4 ^ ymm5))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm7 ^ (mem & (ymm1 ^ ymm7))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 ^ (mem & (ymm1 ^ ymm4))
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm1, 192(%rcx)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, (%rcx)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, 128(%rcx)
@@ -6136,19 +6136,19 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm29, %zmm28, %zmm28
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm31, %zmm30, %zmm29
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm30 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm29 = (~zmm29 & zmm30 & zmm28) | (zmm29 & ~zmm30 & ~zmm28) | (zmm29 & ~zmm30 & zmm28) | (zmm29 & zmm30 & zmm28)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm29 = zmm29 ^ (zmm30 & (zmm29 ^ zmm28))
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,1,1,3]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm1, %zmm1
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm0, %zmm9
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm9 = (~zmm9 & ~zmm30 & zmm1) | (zmm9 & ~zmm30 & zmm1) | (zmm9 & zmm30 & ~zmm1) | (zmm9 & zmm30 & zmm1)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm1 ^ (zmm30 & (zmm9 ^ zmm1))
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm0 # 32-byte Folded Reload
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm1 # 32-byte Folded Reload
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm2 & zmm0) | (zmm1 & ~zmm2 & zmm0) | (zmm1 & zmm2 & ~zmm0) | (zmm1 & zmm2 & zmm0)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm0 ^ (zmm2 & (zmm1 ^ zmm0))
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm0 # 32-byte Folded Reload
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm3 & ~mem) | (~zmm0 & zmm3 & mem) | (zmm0 & ~zmm3 & mem) | (zmm0 & zmm3 & ~mem) | (zmm0 & zmm3 & mem)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm1 & mem) | (zmm0 & ~zmm1 & ~mem) | (zmm0 & zmm1 & ~mem) | (zmm0 & zmm1 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = (zmm0 & mem) | zmm3
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
 ; AVX512-NEXT:    vpshufd $254, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
 ; AVX512-NEXT:    # ymm1 = mem[2,3,3,3,6,7,7,7]
 ; AVX512-NEXT:    vpermq $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
@@ -6177,48 +6177,48 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,2,3,3]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,1,3,2]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm6, %zmm5
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm29 & mem) | (zmm5 & ~zmm29 & ~mem) | (zmm5 & zmm29 & ~mem) | (zmm5 & zmm29 & mem)
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm27 = (~zmm27 & zmm5 & mem) | (zmm27 & ~zmm5 & ~mem) | (zmm27 & zmm5 & ~mem) | (zmm27 & zmm5 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm29))
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm27 = zmm27 ^ (mem & (zmm27 ^ zmm5))
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm15[0,0,0,1]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,1,3,2]
 ; AVX512-NEXT:    vpbroadcastd 32(%rax), %ymm5
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm1, %zmm1
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm1 = (~zmm1 & zmm4 & mem) | (zmm1 & ~zmm4 & ~mem) | (zmm1 & zmm4 & ~mem) | (zmm1 & zmm4 & mem)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & zmm9 & mem) | (zmm1 & ~zmm9 & ~mem) | (zmm1 & zmm9 & ~mem) | (zmm1 & zmm9 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm4))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm9))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm3, %zmm3
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm14, %zmm12, %zmm4
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & ~zmm2 & zmm3) | (zmm4 & ~zmm2 & zmm3) | (zmm4 & zmm2 & ~zmm3) | (zmm4 & zmm2 & zmm3)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm3 ^ (zmm2 & (zmm4 ^ zmm3))
 ; AVX512-NEXT:    vpbroadcastd 36(%rax), %ymm2
 ; AVX512-NEXT:    vpbroadcastd 40(%rax), %ymm3
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm2 = (~zmm2 & zmm3 & mem) | (zmm2 & ~zmm3 & ~mem) | (zmm2 & zmm3 & ~mem) | (zmm2 & zmm3 & mem)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm4 & mem) | (zmm2 & ~zmm4 & ~mem) | (zmm2 & zmm4 & ~mem) | (zmm2 & zmm4 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm3))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm4))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm17, %zmm3, %zmm3
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm18, %zmm4, %zmm4
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm3 & mem) | (zmm4 & ~zmm3 & ~mem) | (zmm4 & zmm3 & ~mem) | (zmm4 & zmm3 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm3))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm19[0,1,2,3]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm20 = (~zmm20 & zmm3 & mem) | (zmm20 & ~zmm3 & ~mem) | (zmm20 & ~zmm3 & mem) | (zmm20 & zmm3 & ~mem) | (zmm20 & zmm3 & mem)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm20 = (~zmm20 & zmm4 & mem) | (zmm20 & ~zmm4 & ~mem) | (zmm20 & zmm4 & ~mem) | (zmm20 & zmm4 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm20 | (zmm3 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm20 ^ (mem & (zmm20 ^ zmm4))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm28, %zmm21, %zmm3
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & zmm26 & ~mem) | (zmm3 & ~zmm26 & mem) | (zmm3 & zmm26 & ~mem) | (zmm3 & zmm26 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm26 ^ (mem & (zmm3 ^ zmm26))
 ; AVX512-NEXT:    vpbroadcastd (%rax), %ymm4
 ; AVX512-NEXT:    vpbroadcastd 4(%rax), %ymm5
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm4 = (~zmm4 & zmm25 & mem) | (zmm4 & ~zmm25 & ~mem) | (zmm4 & zmm25 & ~mem) | (zmm4 & zmm25 & mem)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm3 & mem) | (zmm4 & ~zmm3 & ~mem) | (zmm4 & zmm3 & ~mem) | (zmm4 & zmm3 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm25))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm3))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm31, %zmm30, %zmm3
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm5
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm3 & mem) | (zmm5 & ~zmm3 & ~mem) | (zmm5 & zmm3 & ~mem) | (zmm5 & zmm3 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm3))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm8, %zmm3
 ; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [0,0,4,0,0,0,5,0,0,5,0,0,0,6,0,0]
 ; AVX512-NEXT:    vpermd (%rax), %zmm6, %zmm6
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm6 = (~zmm6 & zmm3 & mem) | (zmm6 & ~zmm3 & ~mem) | (zmm6 & zmm3 & ~mem) | (zmm6 & zmm3 & mem)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm6 = (~zmm6 & zmm5 & mem) | (zmm6 & ~zmm5 & ~mem) | (zmm6 & zmm5 & ~mem) | (zmm6 & zmm5 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm3))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-NEXT:    vmovdqa64 %zmm6, 128(%rax)
 ; AVX512-NEXT:    vmovdqa64 %zmm4, (%rax)
@@ -6447,20 +6447,20 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm7[0,1],ymm6[2],ymm7[3,4],ymm6[5],ymm7[6,7,8,9],ymm6[10],ymm7[11,12],ymm6[13],ymm7[14,15]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm4, %zmm5, %zmm6
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = (~zmm12 & ~zmm16 & zmm24) | (zmm12 & ~zmm16 & zmm24) | (zmm12 & zmm16 & ~zmm24) | (zmm12 & zmm16 & zmm24)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = (~zmm15 & zmm12 & mem) | (zmm15 & ~zmm12 & ~mem) | (zmm15 & zmm12 & ~mem) | (zmm15 & zmm12 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm24 ^ (zmm16 & (zmm12 ^ zmm24))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = zmm15 ^ (mem & (zmm15 ^ zmm12))
 ; AVX512-FCP-NEXT:    vmovdqa64 (%rax), %zmm5
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm7 = [4,5,2,2,6,6,6,6,30,31,27,27,31,31,30,31]
 ; AVX512-FCP-NEXT:    vpermi2d %zmm5, %zmm2, %zmm7
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = (~zmm7 & zmm15 & mem) | (zmm7 & ~zmm15 & ~mem) | (zmm7 & zmm15 & ~mem) | (zmm7 & zmm15 & mem)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm15))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & ~zmm2 & zmm26) | (zmm8 & ~zmm2 & zmm26) | (zmm8 & zmm2 & ~zmm26) | (zmm8 & zmm2 & zmm26)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm26 ^ (zmm2 & (zmm8 ^ zmm26))
 ; AVX512-FCP-NEXT:    vpbroadcastd 36(%rax), %ymm12
 ; AVX512-FCP-NEXT:    vpbroadcastd 40(%rax), %ymm13
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm12 = (~zmm12 & zmm29 & mem) | (zmm12 & ~zmm29 & ~mem) | (zmm12 & zmm29 & ~mem) | (zmm12 & zmm29 & mem)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = (~zmm12 & zmm8 & mem) | (zmm12 & ~zmm8 & ~mem) | (zmm12 & zmm8 & ~mem) | (zmm12 & zmm8 & mem)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm27 = (~zmm27 & zmm3 & mem) | (zmm27 & ~zmm3 & ~mem) | (zmm27 & zmm3 & ~mem) | (zmm27 & zmm3 & mem)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm12 = zmm12 ^ (mem & (zmm12 ^ zmm29))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm12 ^ (mem & (zmm12 ^ zmm8))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm27 = zmm27 ^ (mem & (zmm27 ^ zmm3))
 ; AVX512-FCP-NEXT:    vmovdqu (%rsp), %ymm4 # 32-byte Reload
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm4[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm24 # 32-byte Reload
@@ -6477,36 +6477,36 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm24[1,1,1,1,5,5,5,5]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm15[0,1],ymm4[2],ymm15[3,4],ymm4[5],ymm15[6,7,8,9],ymm4[10],ymm15[11,12],ymm4[13],ymm15[14,15]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm20, %zmm4
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm6 & mem) | (zmm4 & ~zmm6 & ~mem) | (zmm4 & zmm6 & ~mem) | (zmm4 & zmm6 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm6))
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [6,0,0,0,7,0,0,7]
 ; AVX512-FCP-NEXT:    vpermd %ymm10, %ymm3, %ymm3
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm5, %zmm5
 ; AVX512-FCP-NEXT:    vpermd %zmm5, %zmm22, %zmm5
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm5 = (~zmm5 & zmm31 & mem) | (zmm5 & ~zmm31 & ~mem) | (zmm5 & zmm31 & ~mem) | (zmm5 & zmm31 & mem)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm4 & mem) | (zmm5 & ~zmm4 & ~mem) | (zmm5 & zmm4 & ~mem) | (zmm5 & zmm4 & mem)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm31))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm4))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm4 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm13, %zmm6 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = (~zmm6 & ~zmm2 & zmm4) | (zmm6 & ~zmm2 & zmm4) | (zmm6 & zmm2 & ~zmm4) | (zmm6 & zmm2 & zmm4)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm21 = (~zmm21 & ~zmm16 & zmm25) | (zmm21 & ~zmm16 & zmm25) | (zmm21 & zmm16 & ~zmm25) | (zmm21 & zmm16 & zmm25)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm4 ^ (zmm2 & (zmm6 ^ zmm4))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm21 = zmm25 ^ (zmm16 & (zmm21 ^ zmm25))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm30 & ~mem) | (~zmm0 & zmm30 & mem) | (zmm0 & ~zmm30 & mem) | (zmm0 & zmm30 & ~mem) | (zmm0 & zmm30 & mem)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm6 & mem) | (zmm0 & ~zmm6 & ~mem) | (zmm0 & zmm6 & ~mem) | (zmm0 & zmm6 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = (zmm0 & mem) | zmm30
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm6))
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm2, %zmm2
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm4, %zmm4
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm2 & mem) | (zmm4 & ~zmm2 & ~mem) | (zmm4 & zmm2 & ~mem) | (zmm4 & zmm2 & mem)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & zmm9 & mem) | (zmm1 & ~zmm9 & ~mem) | (zmm1 & ~zmm9 & mem) | (zmm1 & zmm9 & ~mem) | (zmm1 & zmm9 & mem)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & zmm4 & mem) | (zmm1 & ~zmm4 & ~mem) | (zmm1 & zmm4 & ~mem) | (zmm1 & zmm4 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm2))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm9 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm4))
 ; AVX512-FCP-NEXT:    vpbroadcastd 32(%rax), %ymm2
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = (~zmm2 & zmm23 & mem) | (zmm2 & ~zmm23 & ~mem) | (zmm2 & zmm23 & ~mem) | (zmm2 & zmm23 & mem)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm21 & mem) | (zmm2 & ~zmm21 & ~mem) | (zmm2 & zmm21 & ~mem) | (zmm2 & zmm21 & mem)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm23))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm21))
 ; AVX512-FCP-NEXT:    vpbroadcastd (%rax), %ymm3
 ; AVX512-FCP-NEXT:    vpbroadcastd 4(%rax), %ymm4
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm3
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = (~zmm3 & zmm28 & mem) | (zmm3 & ~zmm28 & ~mem) | (zmm3 & zmm28 & ~mem) | (zmm3 & zmm28 & mem)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & zmm27 & mem) | (zmm3 & ~zmm27 & ~mem) | (zmm3 & zmm27 & ~mem) | (zmm3 & zmm27 & mem)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm28))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm27))
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, 128(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, (%rax)
@@ -6764,19 +6764,19 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm29, %zmm28, %zmm28
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm31, %zmm30, %zmm29
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm30 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm29 = (~zmm29 & zmm30 & zmm28) | (zmm29 & ~zmm30 & ~zmm28) | (zmm29 & ~zmm30 & zmm28) | (zmm29 & zmm30 & zmm28)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm29 = zmm29 ^ (zmm30 & (zmm29 ^ zmm28))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,1,1,3]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm1, %zmm1
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm0, %zmm9
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm9 = (~zmm9 & ~zmm30 & zmm1) | (zmm9 & ~zmm30 & zmm1) | (zmm9 & zmm30 & ~zmm1) | (zmm9 & zmm30 & zmm1)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm1 ^ (zmm30 & (zmm9 ^ zmm1))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm0 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm1 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm2 & zmm0) | (zmm1 & ~zmm2 & zmm0) | (zmm1 & zmm2 & ~zmm0) | (zmm1 & zmm2 & zmm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm0 ^ (zmm2 & (zmm1 ^ zmm0))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm0 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm3 & ~mem) | (~zmm0 & zmm3 & mem) | (zmm0 & ~zmm3 & mem) | (zmm0 & zmm3 & ~mem) | (zmm0 & zmm3 & mem)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm1 & mem) | (zmm0 & ~zmm1 & ~mem) | (zmm0 & zmm1 & ~mem) | (zmm0 & zmm1 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = (zmm0 & mem) | zmm3
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
 ; AVX512DQ-NEXT:    vpshufd $254, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    # ymm1 = mem[2,3,3,3,6,7,7,7]
 ; AVX512DQ-NEXT:    vpermq $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
@@ -6805,48 +6805,48 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,2,3,3]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,1,3,2]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm6, %zmm5
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm29 & mem) | (zmm5 & ~zmm29 & ~mem) | (zmm5 & zmm29 & ~mem) | (zmm5 & zmm29 & mem)
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm27 = (~zmm27 & zmm5 & mem) | (zmm27 & ~zmm5 & ~mem) | (zmm27 & zmm5 & ~mem) | (zmm27 & zmm5 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm29))
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm27 = zmm27 ^ (mem & (zmm27 ^ zmm5))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm15[0,0,0,1]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,1,3,2]
 ; AVX512DQ-NEXT:    vpbroadcastd 32(%rax), %ymm5
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm1, %zmm1
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm1 = (~zmm1 & zmm4 & mem) | (zmm1 & ~zmm4 & ~mem) | (zmm1 & zmm4 & ~mem) | (zmm1 & zmm4 & mem)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & zmm9 & mem) | (zmm1 & ~zmm9 & ~mem) | (zmm1 & zmm9 & ~mem) | (zmm1 & zmm9 & mem)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm4))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm9))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm3, %zmm3
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm14, %zmm12, %zmm4
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & ~zmm2 & zmm3) | (zmm4 & ~zmm2 & zmm3) | (zmm4 & zmm2 & ~zmm3) | (zmm4 & zmm2 & zmm3)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm3 ^ (zmm2 & (zmm4 ^ zmm3))
 ; AVX512DQ-NEXT:    vpbroadcastd 36(%rax), %ymm2
 ; AVX512DQ-NEXT:    vpbroadcastd 40(%rax), %ymm3
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm2 = (~zmm2 & zmm3 & mem) | (zmm2 & ~zmm3 & ~mem) | (zmm2 & zmm3 & ~mem) | (zmm2 & zmm3 & mem)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm4 & mem) | (zmm2 & ~zmm4 & ~mem) | (zmm2 & zmm4 & ~mem) | (zmm2 & zmm4 & mem)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm3))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm4))
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm17, %zmm3, %zmm3
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm18, %zmm4, %zmm4
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm3 & mem) | (zmm4 & ~zmm3 & ~mem) | (zmm4 & zmm3 & ~mem) | (zmm4 & zmm3 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm3))
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm19[0,1,2,3]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm20 = (~zmm20 & zmm3 & mem) | (zmm20 & ~zmm3 & ~mem) | (zmm20 & ~zmm3 & mem) | (zmm20 & zmm3 & ~mem) | (zmm20 & zmm3 & mem)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm20 = (~zmm20 & zmm4 & mem) | (zmm20 & ~zmm4 & ~mem) | (zmm20 & zmm4 & ~mem) | (zmm20 & zmm4 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm20 | (zmm3 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm20 ^ (mem & (zmm20 ^ zmm4))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm28, %zmm21, %zmm3
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & zmm26 & ~mem) | (zmm3 & ~zmm26 & mem) | (zmm3 & zmm26 & ~mem) | (zmm3 & zmm26 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm26 ^ (mem & (zmm3 ^ zmm26))
 ; AVX512DQ-NEXT:    vpbroadcastd (%rax), %ymm4
 ; AVX512DQ-NEXT:    vpbroadcastd 4(%rax), %ymm5
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm4 = (~zmm4 & zmm25 & mem) | (zmm4 & ~zmm25 & ~mem) | (zmm4 & zmm25 & ~mem) | (zmm4 & zmm25 & mem)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm3 & mem) | (zmm4 & ~zmm3 & ~mem) | (zmm4 & zmm3 & ~mem) | (zmm4 & zmm3 & mem)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm25))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm3))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm31, %zmm30, %zmm3
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm5
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm3 & mem) | (zmm5 & ~zmm3 & ~mem) | (zmm5 & zmm3 & ~mem) | (zmm5 & zmm3 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm3))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm8, %zmm3
 ; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [0,0,4,0,0,0,5,0,0,5,0,0,0,6,0,0]
 ; AVX512DQ-NEXT:    vpermd (%rax), %zmm6, %zmm6
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm6 = (~zmm6 & zmm3 & mem) | (zmm6 & ~zmm3 & ~mem) | (zmm6 & zmm3 & ~mem) | (zmm6 & zmm3 & mem)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm6 = (~zmm6 & zmm5 & mem) | (zmm6 & ~zmm5 & ~mem) | (zmm6 & zmm5 & ~mem) | (zmm6 & zmm5 & mem)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm3))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm6, 128(%rax)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm4, (%rax)
@@ -7075,20 +7075,20 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm7[0,1],ymm6[2],ymm7[3,4],ymm6[5],ymm7[6,7,8,9],ymm6[10],ymm7[11,12],ymm6[13],ymm7[14,15]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm5, %zmm6
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = (~zmm12 & ~zmm16 & zmm24) | (zmm12 & ~zmm16 & zmm24) | (zmm12 & zmm16 & ~zmm24) | (zmm12 & zmm16 & zmm24)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = (~zmm15 & zmm12 & mem) | (zmm15 & ~zmm12 & ~mem) | (zmm15 & zmm12 & ~mem) | (zmm15 & zmm12 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm24 ^ (zmm16 & (zmm12 ^ zmm24))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = zmm15 ^ (mem & (zmm15 ^ zmm12))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 (%rax), %zmm5
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm7 = [4,5,2,2,6,6,6,6,30,31,27,27,31,31,30,31]
 ; AVX512DQ-FCP-NEXT:    vpermi2d %zmm5, %zmm2, %zmm7
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = (~zmm7 & zmm15 & mem) | (zmm7 & ~zmm15 & ~mem) | (zmm7 & zmm15 & ~mem) | (zmm7 & zmm15 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm15))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & ~zmm2 & zmm26) | (zmm8 & ~zmm2 & zmm26) | (zmm8 & zmm2 & ~zmm26) | (zmm8 & zmm2 & zmm26)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm26 ^ (zmm2 & (zmm8 ^ zmm26))
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 36(%rax), %ymm12
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 40(%rax), %ymm13
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm12 = (~zmm12 & zmm29 & mem) | (zmm12 & ~zmm29 & ~mem) | (zmm12 & zmm29 & ~mem) | (zmm12 & zmm29 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = (~zmm12 & zmm8 & mem) | (zmm12 & ~zmm8 & ~mem) | (zmm12 & zmm8 & ~mem) | (zmm12 & zmm8 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm27 = (~zmm27 & zmm3 & mem) | (zmm27 & ~zmm3 & ~mem) | (zmm27 & zmm3 & ~mem) | (zmm27 & zmm3 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm12 = zmm12 ^ (mem & (zmm12 ^ zmm29))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm12 ^ (mem & (zmm12 ^ zmm8))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm27 = zmm27 ^ (mem & (zmm27 ^ zmm3))
 ; AVX512DQ-FCP-NEXT:    vmovdqu (%rsp), %ymm4 # 32-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm4[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm24 # 32-byte Reload
@@ -7105,36 +7105,36 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm24[1,1,1,1,5,5,5,5]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm15[0,1],ymm4[2],ymm15[3,4],ymm4[5],ymm15[6,7,8,9],ymm4[10],ymm15[11,12],ymm4[13],ymm15[14,15]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm20, %zmm4
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm6 & mem) | (zmm4 & ~zmm6 & ~mem) | (zmm4 & zmm6 & ~mem) | (zmm4 & zmm6 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm6))
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [6,0,0,0,7,0,0,7]
 ; AVX512DQ-FCP-NEXT:    vpermd %ymm10, %ymm3, %ymm3
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm5, %zmm5
 ; AVX512DQ-FCP-NEXT:    vpermd %zmm5, %zmm22, %zmm5
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm5 = (~zmm5 & zmm31 & mem) | (zmm5 & ~zmm31 & ~mem) | (zmm5 & zmm31 & ~mem) | (zmm5 & zmm31 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm4 & mem) | (zmm5 & ~zmm4 & ~mem) | (zmm5 & zmm4 & ~mem) | (zmm5 & zmm4 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm31))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm4))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm4 # 32-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm13, %zmm6 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = (~zmm6 & ~zmm2 & zmm4) | (zmm6 & ~zmm2 & zmm4) | (zmm6 & zmm2 & ~zmm4) | (zmm6 & zmm2 & zmm4)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm21 = (~zmm21 & ~zmm16 & zmm25) | (zmm21 & ~zmm16 & zmm25) | (zmm21 & zmm16 & ~zmm25) | (zmm21 & zmm16 & zmm25)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm4 ^ (zmm2 & (zmm6 ^ zmm4))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm21 = zmm25 ^ (zmm16 & (zmm21 ^ zmm25))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm30 & ~mem) | (~zmm0 & zmm30 & mem) | (zmm0 & ~zmm30 & mem) | (zmm0 & zmm30 & ~mem) | (zmm0 & zmm30 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm6 & mem) | (zmm0 & ~zmm6 & ~mem) | (zmm0 & zmm6 & ~mem) | (zmm0 & zmm6 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = (zmm0 & mem) | zmm30
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm6))
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm2, %zmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm4, %zmm4
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm2 & mem) | (zmm4 & ~zmm2 & ~mem) | (zmm4 & zmm2 & ~mem) | (zmm4 & zmm2 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & zmm9 & mem) | (zmm1 & ~zmm9 & ~mem) | (zmm1 & ~zmm9 & mem) | (zmm1 & zmm9 & ~mem) | (zmm1 & zmm9 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & zmm4 & mem) | (zmm1 & ~zmm4 & ~mem) | (zmm1 & zmm4 & ~mem) | (zmm1 & zmm4 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm2))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm9 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm4))
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 32(%rax), %ymm2
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = (~zmm2 & zmm23 & mem) | (zmm2 & ~zmm23 & ~mem) | (zmm2 & zmm23 & ~mem) | (zmm2 & zmm23 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm21 & mem) | (zmm2 & ~zmm21 & ~mem) | (zmm2 & zmm21 & ~mem) | (zmm2 & zmm21 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm23))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm21))
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd (%rax), %ymm3
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 4(%rax), %ymm4
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm3
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = (~zmm3 & zmm28 & mem) | (zmm3 & ~zmm28 & ~mem) | (zmm3 & zmm28 & ~mem) | (zmm3 & zmm28 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & zmm27 & mem) | (zmm3 & ~zmm27 & ~mem) | (zmm3 & zmm27 & ~mem) | (zmm3 & zmm27 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm28))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm27))
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, 128(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, (%rax)
@@ -12769,24 +12769,24 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpshufb %ymm12, %ymm9, %ymm9
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm17, %zmm6
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm18, %zmm7
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & zmm6 & ~mem) | (zmm7 & ~zmm6 & mem) | (zmm7 & zmm6 & ~mem) | (zmm7 & zmm6 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm6 ^ (mem & (zmm7 ^ zmm6))
 ; AVX512-NEXT:    vmovdqa 96(%r8), %ymm6
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm10 = ymm6[1,2,2,3,5,6,6,7]
 ; AVX512-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm10 = ymm6[0,0,2,1,4,4,6,5]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm6[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm6[16,17,u,u,u,u],zero,zero
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ymm7 & ymm11) | (ymm6 & ~ymm7 & ~ymm11) | (ymm6 & ~ymm7 & ymm11) | (ymm6 & ymm7 & ~ymm11) | (ymm6 & ymm7 & ymm11)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm7 & ymm11)
 ; AVX512-NEXT:    vmovdqa64 %zmm11, %zmm12
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm9 = (~ymm9 & ymm6 & ymm11) | (ymm9 & ~ymm6 & ~ymm11) | (ymm9 & ~ymm6 & ymm11) | (ymm9 & ymm6 & ~ymm11) | (ymm9 & ymm6 & ymm11)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm6 & ymm11)
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm7, %ymm6
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm10[2,1,3,3]
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm7 = (~ymm7 & ymm10 & ymm6) | (ymm7 & ~ymm10 & ~ymm6) | (ymm7 & ~ymm10 & ymm6) | (ymm7 & ymm10 & ymm6)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm7 = ymm7 ^ (ymm10 & (ymm7 ^ ymm6))
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm8[2,2,2,2]
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ymm8 & ymm7) | (ymm6 & ~ymm8 & ~ymm7) | (ymm6 & ~ymm8 & ymm7) | (ymm6 & ymm8 & ymm7)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm6 ^ (ymm8 & (ymm6 ^ ymm7))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm6
 ; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm9[0,1,2,3],zmm6[4,5,6,7]
 ; AVX512-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -13024,7 +13024,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpbroadcastd 100(%rax), %ymm1
 ; AVX512-NEXT:    vpbroadcastd 104(%rax), %ymm2
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm31
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm31 = (~zmm31 & zmm0 & mem) | (zmm31 & ~zmm0 & ~mem) | (zmm31 & zmm0 & ~mem) | (zmm31 & zmm0 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm31 = zmm31 ^ (mem & (zmm31 ^ zmm0))
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm7[0],xmm9[0],xmm7[1],xmm9[1],xmm7[2],xmm9[2],xmm7[3],xmm9[3]
 ; AVX512-NEXT:    vpshufb %xmm4, %xmm9, %xmm1
 ; AVX512-NEXT:    vmovdqa %xmm4, %xmm6
@@ -13042,7 +13042,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm14
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm14 = (~zmm14 & ~zmm0 & zmm1) | (zmm14 & ~zmm0 & zmm1) | (zmm14 & zmm0 & ~zmm1) | (zmm14 & zmm0 & zmm1)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm1 ^ (zmm0 & (zmm14 ^ zmm1))
 ; AVX512-NEXT:    vmovdqa64 %xmm17, %xmm1
 ; AVX512-NEXT:    vmovdqa64 %xmm20, %xmm3
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
@@ -13054,7 +13054,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpbroadcastd 68(%rax), %ymm5
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm25
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm25 = (~zmm25 & zmm19 & zmm1) | (zmm25 & ~zmm19 & ~zmm1) | (zmm25 & ~zmm19 & zmm1) | (zmm25 & zmm19 & zmm1)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm25 = zmm25 ^ (zmm19 & (zmm25 ^ zmm1))
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
 ; AVX512-NEXT:    vpshufb %xmm6, %xmm8, %xmm3
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm13[1,1,2,2]
@@ -13071,7 +13071,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm2 = xmm5[0,1],xmm2[2],xmm5[3,4],xmm2[5],xmm5[6,7]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm5
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & ~zmm0 & zmm3) | (zmm5 & ~zmm0 & zmm3) | (zmm5 & zmm0 & ~zmm3) | (zmm5 & zmm0 & zmm3)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm3 ^ (zmm0 & (zmm5 ^ zmm3))
 ; AVX512-NEXT:    vmovdqa64 %xmm29, %xmm0
 ; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm1
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -13081,7 +13081,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpbroadcastd (%rax), %ymm1
 ; AVX512-NEXT:    vpbroadcastd 4(%rax), %ymm2
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm20
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm20 = (~zmm20 & zmm19 & zmm0) | (zmm20 & ~zmm19 & ~zmm0) | (zmm20 & ~zmm19 & zmm0) | (zmm20 & zmm19 & zmm0)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm20 = zmm20 ^ (zmm19 & (zmm20 ^ zmm0))
 ; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
 ; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm6[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,2,1,4,4,6,5]
@@ -13156,7 +13156,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpbroadcastd 36(%rax), %ymm2
 ; AVX512-NEXT:    vpbroadcastd 40(%rax), %ymm4
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm13
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm13 = (~zmm13 & zmm1 & mem) | (zmm13 & ~zmm1 & ~mem) | (zmm13 & zmm1 & ~mem) | (zmm13 & zmm1 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm13 = zmm13 ^ (mem & (zmm13 ^ zmm1))
 ; AVX512-NEXT:    vmovdqa 32(%rcx), %xmm7
 ; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm6
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
@@ -13169,103 +13169,103 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 # 32-byte Folded Reload
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & zmm4 & zmm3) | (zmm1 & ~zmm4 & ~zmm3) | (zmm1 & ~zmm4 & zmm3) | (zmm1 & zmm4 & zmm3)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm4 & (zmm1 ^ zmm3))
 ; AVX512-NEXT:    vpermq $182, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
 ; AVX512-NEXT:    # ymm3 = mem[2,1,3,2]
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm3 = (~ymm3 & ymm29 & ymm1) | (ymm3 & ~ymm29 & ~ymm1) | (ymm3 & ~ymm29 & ymm1) | (ymm3 & ymm29 & ymm1)
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm30 = (~ymm30 & ymm28 & ymm3) | (ymm30 & ~ymm28 & ~ymm3) | (ymm30 & ~ymm28 & ymm3) | (ymm30 & ymm28 & ymm3)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm3 = ymm3 ^ (ymm29 & (ymm3 ^ ymm1))
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm30 = ymm30 ^ (ymm28 & (ymm30 ^ ymm3))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 32-byte Folded Reload
 ; AVX512-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm3 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm3 = (~zmm3 & ~zmm28 & mem) | (~zmm3 & zmm28 & mem) | (zmm3 & ~zmm28 & mem) | (zmm3 & zmm28 & ~mem) | (zmm3 & zmm28 & mem)
+; AVX512-NEXT:    # zmm3 = (zmm3 & zmm28) | mem
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
 ; AVX512-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm0 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm0 = (~zmm0 & ~zmm28 & mem) | (~zmm0 & zmm28 & mem) | (zmm0 & ~zmm28 & mem) | (zmm0 & zmm28 & ~mem) | (zmm0 & zmm28 & mem)
+; AVX512-NEXT:    # zmm0 = (zmm0 & zmm28) | mem
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm23 # 32-byte Folded Reload
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm18 # 32-byte Folded Reload
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm28 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm18 = (~zmm18 & ~zmm28 & zmm23) | (zmm18 & ~zmm28 & zmm23) | (zmm18 & zmm28 & ~zmm23) | (zmm18 & zmm28 & zmm23)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm18 = zmm23 ^ (zmm28 & (zmm18 ^ zmm23))
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm18 = (~zmm18 & ~zmm2 & zmm3) | (zmm18 & ~zmm2 & zmm3) | (zmm18 & zmm2 & ~zmm3) | (zmm18 & zmm2 & zmm3)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm18 = zmm3 ^ (zmm2 & (zmm18 ^ zmm3))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 32-byte Folded Reload
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm16, %zmm23 # 32-byte Folded Reload
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm23 = (~zmm23 & ~zmm28 & zmm3) | (zmm23 & ~zmm28 & zmm3) | (zmm23 & zmm28 & ~zmm3) | (zmm23 & zmm28 & zmm3)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm23 = (~zmm23 & ~zmm2 & zmm0) | (zmm23 & ~zmm2 & zmm0) | (zmm23 & zmm2 & ~zmm0) | (zmm23 & zmm2 & zmm0)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm23 = zmm3 ^ (zmm28 & (zmm23 ^ zmm3))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm23 = zmm0 ^ (zmm2 & (zmm23 ^ zmm0))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm17 # 32-byte Folded Reload
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm17 = (~zmm17 & zmm0 & ~mem) | (zmm17 & ~zmm0 & mem) | (zmm17 & zmm0 & ~mem) | (zmm17 & zmm0 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm17 = zmm0 ^ (mem & (zmm17 ^ zmm0))
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm1, %ymm0
-; AVX512-NEXT:    vpternlogd {{.*#+}} ymm0 = (~ymm0 & ~ymm19 & mem) | (ymm0 & ~ymm19 & mem) | (ymm0 & ymm19 & ~mem) | (ymm0 & ymm19 & mem)
+; AVX512-NEXT:    vpternlogd {{.*#+}} ymm0 = mem ^ (ymm19 & (ymm0 ^ mem))
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
 ; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ymm1 & ymm0) | (ymm2 & ~ymm1 & ~ymm0) | (ymm2 & ~ymm1 & ymm0) | (ymm2 & ymm1 & ymm0)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm2 ^ (ymm1 & (ymm2 ^ ymm0))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
 ; AVX512-NEXT:    vpternlogq $248, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm19 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm19 = (~zmm19 & zmm1 & mem) | (zmm19 & ~zmm1 & ~mem) | (zmm19 & ~zmm1 & mem) | (zmm19 & zmm1 & ~mem) | (zmm19 & zmm1 & mem)
+; AVX512-NEXT:    # zmm19 = zmm19 | (zmm1 & mem)
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512-NEXT:    vshufi64x2 $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
 ; AVX512-NEXT:    # zmm0 = zmm0[0,1,2,3],mem[0,1,2,3]
 ; AVX512-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
-; AVX512-NEXT:    # zmm0 = (~zmm0 & ~zmm1 & mem) | (~zmm0 & zmm1 & mem) | (zmm0 & ~zmm1 & mem) | (zmm0 & zmm1 & ~mem) | (zmm0 & zmm1 & mem)
+; AVX512-NEXT:    # zmm0 = (zmm0 & zmm1) | mem
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm1
 ; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm30[0,1,2,3],zmm1[4,5,6,7]
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm30 = (~zmm30 & zmm2 & zmm1) | (zmm30 & ~zmm2 & ~zmm1) | (zmm30 & ~zmm2 & zmm1) | (zmm30 & zmm2 & zmm1)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm30 = zmm30 ^ (zmm2 & (zmm30 ^ zmm1))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm11 = (~zmm11 & zmm2 & zmm1) | (zmm11 & ~zmm2 & ~zmm1) | (zmm11 & ~zmm2 & zmm1) | (zmm11 & zmm2 & zmm1)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm11 = zmm11 ^ (zmm2 & (zmm11 ^ zmm1))
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm25 = (~zmm25 & zmm1 & zmm14) | (zmm25 & ~zmm1 & ~zmm14) | (zmm25 & ~zmm1 & zmm14) | (zmm25 & zmm1 & zmm14)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm20 = (~zmm20 & zmm1 & zmm5) | (zmm20 & ~zmm1 & ~zmm5) | (zmm20 & ~zmm1 & zmm5) | (zmm20 & zmm1 & zmm5)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm25 = zmm25 ^ (zmm1 & (zmm25 ^ zmm14))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm20 ^ (zmm1 & (zmm20 ^ zmm5))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 32-byte Folded Reload
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & ~zmm5 & zmm1) | (zmm2 & ~zmm5 & zmm1) | (zmm2 & zmm5 & ~zmm1) | (zmm2 & zmm5 & zmm1)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm1 ^ (zmm5 & (zmm2 ^ zmm1))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm22, %zmm3, %zmm22
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm22 = (~zmm22 & ~zmm5 & zmm1) | (zmm22 & ~zmm5 & zmm1) | (zmm22 & zmm5 & ~zmm1) | (zmm22 & zmm5 & zmm1)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm22 = zmm1 ^ (zmm5 & (zmm22 ^ zmm1))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
 ; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm14 = [0,0,4,0,0,0,5,0,0,5,0,0,0,6,0,0]
 ; AVX512-NEXT:    vpermd 64(%rax), %zmm14, %zmm5
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm5 = (~zmm5 & zmm3 & zmm1) | (zmm5 & ~zmm3 & ~zmm1) | (zmm5 & ~zmm3 & zmm1) | (zmm5 & zmm3 & zmm1)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm5 ^ (zmm3 & (zmm5 ^ zmm1))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm24, %zmm26, %zmm1
 ; AVX512-NEXT:    vpermd (%rax), %zmm14, %zmm14
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm14 = (~zmm14 & zmm3 & zmm1) | (zmm14 & ~zmm3 & ~zmm1) | (zmm14 & ~zmm3 & zmm1) | (zmm14 & zmm3 & zmm1)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm14 = zmm14 ^ (zmm3 & (zmm14 ^ zmm1))
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm1 & zmm2) | (zmm5 & ~zmm1 & ~zmm2) | (zmm5 & ~zmm1 & zmm2) | (zmm5 & zmm1 & zmm2)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm14 = (~zmm14 & zmm1 & zmm22) | (zmm14 & ~zmm1 & ~zmm22) | (zmm14 & ~zmm1 & zmm22) | (zmm14 & zmm1 & zmm22)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm1 & (zmm5 ^ zmm2))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 ^ (zmm1 & (zmm14 ^ zmm22))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 32-byte Folded Reload
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm4 & zmm1) | (zmm2 & ~zmm4 & ~zmm1) | (zmm2 & ~zmm4 & zmm1) | (zmm2 & zmm4 & zmm1)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm4 & (zmm2 ^ zmm1))
 ; AVX512-NEXT:    vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
 ; AVX512-NEXT:    # ymm1 = mem[0,1,1,3]
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm22 # 32-byte Folded Reload
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm22 = (~zmm22 & ~zmm4 & zmm1) | (zmm22 & ~zmm4 & zmm1) | (zmm22 & zmm4 & ~zmm1) | (zmm22 & zmm4 & zmm1)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm22 = zmm1 ^ (zmm4 & (zmm22 ^ zmm1))
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm12[0,1,1,3]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm21, %zmm1
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm15, %zmm8
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & ~zmm4 & zmm1) | (zmm8 & ~zmm4 & zmm1) | (zmm8 & zmm4 & ~zmm1) | (zmm8 & zmm4 & zmm1)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm1 ^ (zmm4 & (zmm8 ^ zmm1))
 ; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm4 = xmm7[u,u,4,5,u,u,u,u,6,7,u,u,u,u,8,9]
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm9 = xmm6[1,1,2,2]
@@ -13309,27 +13309,27 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm10[2,1,3,2]
 ; AVX512-NEXT:    vpbroadcastd 96(%rax), %ymm10
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm9 = (~zmm9 & zmm29 & zmm7) | (zmm9 & ~zmm29 & ~zmm7) | (zmm9 & ~zmm29 & zmm7) | (zmm9 & zmm29 & zmm7)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm9 = zmm9 ^ (zmm29 & (zmm9 ^ zmm7))
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,0,1]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm24, %zmm3
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm26[2,1,3,2]
 ; AVX512-NEXT:    vpbroadcastd 32(%rax), %ymm10
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm7, %zmm7
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm7 = (~zmm7 & zmm29 & zmm3) | (zmm7 & ~zmm29 & ~zmm3) | (zmm7 & ~zmm29 & zmm3) | (zmm7 & zmm29 & zmm3)
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (zmm29 & (zmm7 ^ zmm3))
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm9 = (~zmm9 & zmm3 & zmm22) | (zmm9 & ~zmm3 & ~zmm22) | (zmm9 & ~zmm3 & zmm22) | (zmm9 & zmm3 & zmm22)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & zmm3 & zmm8) | (zmm7 & ~zmm3 & ~zmm8) | (zmm7 & ~zmm3 & zmm8) | (zmm7 & zmm3 & zmm8)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 ^ (zmm3 & (zmm9 ^ zmm22))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm3 & (zmm7 ^ zmm8))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm16, %zmm3
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm15, %zmm21, %zmm8
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & ~zmm28 & zmm3) | (zmm8 & ~zmm28 & zmm3) | (zmm8 & zmm28 & ~zmm3) | (zmm8 & zmm28 & zmm3)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm3 ^ (zmm28 & (zmm8 ^ zmm3))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm27, %zmm1
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm3
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & ~zmm28 & zmm1) | (zmm3 & ~zmm28 & zmm1) | (zmm3 & zmm28 & ~zmm1) | (zmm3 & zmm28 & zmm1)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm1 ^ (zmm28 & (zmm3 ^ zmm1))
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm31 = (~zmm31 & zmm1 & zmm8) | (zmm31 & ~zmm1 & ~zmm8) | (zmm31 & ~zmm1 & zmm8) | (zmm31 & zmm1 & zmm8)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm13 = (~zmm13 & zmm1 & zmm3) | (zmm13 & ~zmm1 & ~zmm3) | (zmm13 & ~zmm1 & zmm3) | (zmm13 & zmm1 & zmm3)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm17 & mem) | (zmm0 & ~zmm17 & ~mem) | (zmm0 & zmm17 & ~mem) | (zmm0 & zmm17 & mem)
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm11 = (~zmm11 & zmm2 & mem) | (zmm11 & ~zmm2 & ~mem) | (zmm11 & zmm2 & ~mem) | (zmm11 & zmm2 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm31 = zmm31 ^ (zmm1 & (zmm31 ^ zmm8))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm13 ^ (zmm1 & (zmm13 ^ zmm3))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm17))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 ^ (mem & (zmm11 ^ zmm2))
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-NEXT:    vmovdqa64 %zmm0, 320(%rax)
 ; AVX512-NEXT:    vmovdqa64 %zmm13, 256(%rax)
@@ -13446,26 +13446,26 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,2,2,3]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm16, %zmm11
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm7, %zmm12
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = (~zmm12 & zmm11 & ~mem) | (zmm12 & ~zmm11 & mem) | (zmm12 & zmm11 & ~mem) | (zmm12 & zmm11 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm11 ^ (mem & (zmm12 ^ zmm11))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm13 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
 ; AVX512-FCP-NEXT:    vmovdqa 96(%r8), %ymm7
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm7[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm7[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm7[16,17,u,u,u,u],zero,zero
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ymm12 & ymm13) | (ymm11 & ~ymm12 & ~ymm13) | (ymm11 & ~ymm12 & ymm13) | (ymm11 & ymm12 & ~ymm13) | (ymm11 & ymm12 & ymm13)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm12 & ymm13)
 ; AVX512-FCP-NEXT:    vmovdqa 96(%r9), %ymm10
 ; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm10, %ymm6
 ; AVX512-FCP-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ymm11 & ymm16) | (ymm6 & ~ymm11 & ~ymm16) | (ymm6 & ~ymm11 & ymm16) | (ymm6 & ymm11 & ~ymm16) | (ymm6 & ymm11 & ymm16)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm11 & ymm16)
 ; AVX512-FCP-NEXT:    vextracti64x4 $1, %zmm12, %ymm11
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm12 = [0,4,0,0,0,5,0,0]
 ; AVX512-FCP-NEXT:    vpermd %ymm7, %ymm12, %ymm12
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm12 = (~ymm12 & ymm16 & ymm11) | (ymm12 & ~ymm16 & ~ymm11) | (ymm12 & ~ymm16 & ymm11) | (ymm12 & ymm16 & ymm11)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm12 = ymm12 ^ (ymm16 & (ymm12 ^ ymm11))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm16, %zmm19
 ; AVX512-FCP-NEXT:    vprold $16, %ymm10, %ymm11
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,2,2,2]
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ymm10 & ymm12) | (ymm11 & ~ymm10 & ~ymm12) | (ymm11 & ~ymm10 & ymm12) | (ymm11 & ymm10 & ymm12)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm11 ^ (ymm10 & (ymm11 ^ ymm12))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm11
 ; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm6[0,1,2,3],zmm11[4,5,6,7]
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -13503,7 +13503,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm20 = [2,2,2,3,8,10,10,11]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm11, %zmm20, %zmm1
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm27 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm27 & zmm4) | (zmm1 & ~zmm27 & zmm4) | (zmm1 & zmm27 & ~zmm4) | (zmm1 & zmm27 & zmm4)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm4 ^ (zmm27 & (zmm1 ^ zmm4))
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [5,0,0,0,6,0,0,6]
 ; AVX512-FCP-NEXT:    vpermd %ymm7, %ymm1, %ymm1
@@ -13555,7 +13555,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm15[2,2,2,2,6,6,6,6]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0,1],ymm6[2],ymm5[3,4],ymm6[5],ymm5[6,7,8,9],ymm6[10],ymm5[11,12],ymm6[13],ymm5[14,15]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm20, %zmm0
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & ~zmm27 & zmm4) | (zmm0 & ~zmm27 & zmm4) | (zmm0 & zmm27 & ~zmm4) | (zmm0 & zmm27 & zmm4)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm4 ^ (zmm27 & (zmm0 ^ zmm4))
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm8[3,3,3,3,7,7,7,7]
@@ -13570,7 +13570,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rax), %ymm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vpermi2d %zmm0, %zmm16, %zmm3
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = (~zmm3 & zmm19 & zmm4) | (zmm3 & ~zmm19 & ~zmm4) | (zmm3 & ~zmm19 & zmm4) | (zmm3 & zmm19 & zmm4)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (zmm19 & (zmm3 ^ zmm4))
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm29, %ymm10
 ; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm10, %ymm3
@@ -13591,7 +13591,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm4[2],ymm6[3,4],ymm4[5],ymm6[6,7,8,9],ymm4[10],ymm6[11,12],ymm4[13],ymm6[14,15]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm11, %zmm6
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm27 = [0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = (~zmm6 & ~zmm27 & zmm5) | (zmm6 & ~zmm27 & zmm5) | (zmm6 & zmm27 & ~zmm5) | (zmm6 & zmm27 & zmm5)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm5 ^ (zmm27 & (zmm6 ^ zmm5))
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm26, %ymm12
 ; AVX512-FCP-NEXT:    vprold $16, %ymm26, %ymm3
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm28, %ymm22
@@ -13607,8 +13607,8 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm25, %zmm1, %zmm1
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm0 = [0,0,4,0,0,0,5,0,0,13,0,0,0,14,0,0]
 ; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = (~zmm0 & zmm18 & zmm5) | (zmm0 & ~zmm18 & ~zmm5) | (zmm0 & ~zmm18 & zmm5) | (zmm0 & zmm18 & zmm5)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm6 & mem) | (zmm0 & ~zmm6 & ~mem) | (zmm0 & zmm6 & ~mem) | (zmm0 & zmm6 & mem)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm18 & (zmm0 ^ zmm5))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm6))
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm8, %ymm0
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm1 = ymm21[3,3,3,3,7,7,7,7]
@@ -13641,7 +13641,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm28 = [0,0,1,1,8,8,10,9]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm28, %zmm5
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & ~zmm3 & zmm2) | (zmm5 & ~zmm3 & zmm2) | (zmm5 & zmm3 & ~zmm2) | (zmm5 & zmm3 & zmm2)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm2 ^ (zmm3 & (zmm5 ^ zmm2))
 ; AVX512-FCP-NEXT:    vmovdqa 64(%r9), %xmm2
 ; AVX512-FCP-NEXT:    vmovdqa 64(%r8), %xmm6
 ; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
@@ -13657,9 +13657,9 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpbroadcastd 68(%rax), %ymm8
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm6, %zmm31
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm31 = (~zmm31 & zmm21 & zmm2) | (zmm31 & ~zmm21 & ~zmm2) | (zmm31 & ~zmm21 & zmm2) | (zmm31 & zmm21 & zmm2)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm31 = zmm31 ^ (zmm21 & (zmm31 ^ zmm2))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm31 = (~zmm31 & zmm2 & zmm5) | (zmm31 & ~zmm2 & ~zmm5) | (zmm31 & ~zmm2 & zmm5) | (zmm31 & zmm2 & zmm5)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm31 = zmm31 ^ (zmm2 & (zmm31 ^ zmm5))
 ; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm0
 ; AVX512-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm8
@@ -13681,7 +13681,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
 ; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm8, %xmm8
 ; AVX512-FCP-NEXT:    vpermt2q %zmm5, %zmm28, %zmm8
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & ~zmm3 & zmm6) | (zmm8 & ~zmm3 & zmm6) | (zmm8 & zmm3 & ~zmm6) | (zmm8 & zmm3 & zmm6)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm6 ^ (zmm3 & (zmm8 ^ zmm6))
 ; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm3
 ; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm5
 ; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
@@ -13694,8 +13694,8 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpbroadcastd (%rax), %ymm5
 ; AVX512-FCP-NEXT:    vpbroadcastd 4(%rax), %ymm6
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm26
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm26 = (~zmm26 & zmm21 & zmm3) | (zmm26 & ~zmm21 & ~zmm3) | (zmm26 & ~zmm21 & zmm3) | (zmm26 & zmm21 & zmm3)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm26 = (~zmm26 & zmm2 & zmm8) | (zmm26 & ~zmm2 & ~zmm8) | (zmm26 & ~zmm2 & zmm8) | (zmm26 & zmm2 & zmm8)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm26 = zmm26 ^ (zmm21 & (zmm26 ^ zmm3))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm26 = zmm26 ^ (zmm2 & (zmm26 ^ zmm8))
 ; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm14[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
 ; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
@@ -13726,7 +13726,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm10, %ymm2
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm8 = ymm19[3,3,3,3,7,7,7,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm8[0,1,2],ymm2[3],ymm8[4,5],ymm2[6],ymm8[7,8,9,10],ymm2[11],ymm8[12,13],ymm2[14],ymm8[15]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & ~zmm27 & zmm3) | (zmm5 & ~zmm27 & zmm3) | (zmm5 & zmm27 & ~zmm3) | (zmm5 & zmm27 & zmm3)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm3 ^ (zmm27 & (zmm5 ^ zmm3))
 ; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
 ; AVX512-FCP-NEXT:    vprold $16, %ymm4, %ymm2
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm19 # 32-byte Reload
@@ -13744,7 +13744,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm20 = [2,1,3,3,8,8,9,9]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm20, %zmm10
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & ~zmm21 & zmm1) | (zmm10 & ~zmm21 & zmm1) | (zmm10 & zmm21 & ~zmm1) | (zmm10 & zmm21 & zmm1)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm1 ^ (zmm21 & (zmm10 ^ zmm1))
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm30, %ymm1
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm22[3,3,3,3,7,7,7,7]
@@ -13753,8 +13753,8 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm23, %zmm16, %zmm1
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,4,0,0,0,5,0,0,13,0,0,0,14,0,0]
 ; AVX512-FCP-NEXT:    vpermd %zmm1, %zmm16, %zmm22
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm22 = (~zmm22 & zmm8 & mem) | (zmm22 & ~zmm8 & ~mem) | (zmm22 & zmm8 & ~mem) | (zmm22 & zmm8 & mem)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm22 = (~zmm22 & zmm5 & mem) | (zmm22 & ~zmm5 & ~mem) | (zmm22 & zmm5 & ~mem) | (zmm22 & zmm5 & mem)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm22 = zmm22 ^ (mem & (zmm22 ^ zmm8))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm22 = zmm22 ^ (mem & (zmm22 ^ zmm5))
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm5
 ; AVX512-FCP-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm1
@@ -13785,14 +13785,14 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpbroadcastd 96(%rax), %ymm20
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm20, %zmm16, %zmm25
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm25 = (~zmm25 & zmm18 & zmm2) | (zmm25 & ~zmm18 & ~zmm2) | (zmm25 & ~zmm18 & zmm2) | (zmm25 & zmm18 & zmm2)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm25 = zmm25 ^ (zmm18 & (zmm25 ^ zmm2))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm27 = [0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm25 = (~zmm25 & zmm27 & zmm10) | (zmm25 & ~zmm27 & ~zmm10) | (zmm25 & ~zmm27 & zmm10) | (zmm25 & zmm27 & zmm10)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm25 = zmm25 ^ (zmm27 & (zmm25 ^ zmm10))
 ; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
 ; AVX512-FCP-NEXT:    vprold $16, %xmm3, %xmm3
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[1,1,2,3]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm3[2],xmm4[3,4],xmm3[5],xmm4[6,7]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm21 & zmm0) | (zmm1 & ~zmm21 & zmm0) | (zmm1 & zmm21 & ~zmm0) | (zmm1 & zmm21 & zmm0)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm0 ^ (zmm21 & (zmm1 ^ zmm0))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm2[4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm21 = [0,0,2,1,8,8,9,11]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm21, %zmm3
@@ -13804,11 +13804,11 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,3,3,4,5,6,7]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm28, %zmm7
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm28 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & ~zmm28 & zmm3) | (zmm7 & ~zmm28 & zmm3) | (zmm7 & zmm28 & ~zmm3) | (zmm7 & zmm28 & zmm3)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm3 ^ (zmm28 & (zmm7 ^ zmm3))
 ; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm16 # 32-byte Reload
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm16 = (~ymm16 & ymm18 & ymm0) | (ymm16 & ~ymm18 & ~ymm0) | (ymm16 & ~ymm18 & ymm0) | (ymm16 & ymm18 & ymm0)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm16 = ymm16 ^ (ymm18 & (ymm16 ^ ymm0))
 ; AVX512-FCP-NEXT:    vextracti64x4 $1, %zmm0, %ymm13
 ; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm2, %xmm2
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm8[0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
@@ -13827,7 +13827,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpbroadcastd 104(%rax), %ymm19
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm5, %zmm5
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm5 = (~zmm5 & zmm24 & zmm6) | (zmm5 & ~zmm24 & ~zmm6) | (zmm5 & ~zmm24 & zmm6) | (zmm5 & zmm24 & zmm6)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm5 ^ (zmm24 & (zmm5 ^ zmm6))
 ; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
 ; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} ymm6 = ymm4[0,1,2,3,5,5,7,6,8,9,10,11,13,13,15,14]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[3,3,3,3]
@@ -13843,10 +13843,10 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpermd %ymm23, %ymm17, %ymm17
 ; AVX512-FCP-NEXT:    vpbroadcastd 32(%rax), %ymm19
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm17, %zmm20
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm20 = (~zmm20 & zmm18 & zmm8) | (zmm20 & ~zmm18 & ~zmm8) | (zmm20 & ~zmm18 & zmm8) | (zmm20 & zmm18 & zmm8)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm20 = (~zmm20 & zmm27 & zmm1) | (zmm20 & ~zmm27 & ~zmm1) | (zmm20 & ~zmm27 & zmm1) | (zmm20 & zmm27 & zmm1)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm20 = zmm20 ^ (zmm18 & (zmm20 ^ zmm8))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm20 ^ (zmm27 & (zmm20 ^ zmm1))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm1 & zmm7) | (zmm5 & ~zmm1 & ~zmm7) | (zmm5 & ~zmm1 & zmm7) | (zmm5 & zmm1 & zmm7)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm1 & (zmm5 ^ zmm7))
 ; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm9[4],xmm15[4],xmm9[5],xmm15[5],xmm9[6],xmm15[6],xmm9[7],xmm15[7]
 ; AVX512-FCP-NEXT:    vprold $16, %xmm9, %xmm8
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm9 = xmm15[1,1,2,3]
@@ -13864,7 +13864,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm9[0,2,3,3,4,5,6,7]
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm0 = [0,0,1,1,8,8,10,9]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm9, %zmm0, %zmm7
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & ~zmm28 & zmm8) | (zmm7 & ~zmm28 & zmm8) | (zmm7 & zmm28 & ~zmm8) | (zmm7 & zmm28 & zmm8)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm8 ^ (zmm28 & (zmm7 ^ zmm8))
 ; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm11[4],xmm3[4],xmm11[5],xmm3[5],xmm11[6],xmm3[6],xmm11[7],xmm3[7]
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm30, %xmm0
 ; AVX512-FCP-NEXT:    vpshufb %xmm0, %xmm3, %xmm3
@@ -13874,25 +13874,25 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpbroadcastd 36(%rax), %ymm3
 ; AVX512-FCP-NEXT:    vpbroadcastd 40(%rax), %ymm8
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm3, %zmm3
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = (~zmm3 & zmm24 & zmm2) | (zmm3 & ~zmm24 & ~zmm2) | (zmm3 & ~zmm24 & zmm2) | (zmm3 & zmm24 & zmm2)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & zmm1 & zmm7) | (zmm3 & ~zmm1 & ~zmm7) | (zmm3 & ~zmm1 & zmm7) | (zmm3 & zmm1 & zmm7)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (zmm24 & (zmm3 ^ zmm2))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm1 & (zmm3 ^ zmm7))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} ymm13 = (~ymm13 & ~ymm0 & mem) | (ymm13 & ~ymm0 & mem) | (ymm13 & ymm0 & ~mem) | (ymm13 & ymm0 & mem)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} ymm13 = mem ^ (ymm0 & (ymm13 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ymm1 & ymm13) | (ymm6 & ~ymm1 & ~ymm13) | (ymm6 & ~ymm1 & ymm13) | (ymm6 & ymm1 & ymm13)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm6 ^ (ymm1 & (ymm6 ^ ymm13))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = (~ymm4 & ymm2 & ymm16) | (ymm4 & ~ymm2 & ~ymm16) | (ymm4 & ~ymm2 & ymm16) | (ymm4 & ymm2 & ymm16)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = ymm4 ^ (ymm2 & (ymm4 ^ ymm16))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm0
 ; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm4[0,1,2,3],zmm0[4,5,6,7]
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm13 = (~zmm13 & zmm0 & mem) | (zmm13 & ~zmm0 & ~mem) | (zmm13 & zmm0 & ~mem) | (zmm13 & zmm0 & mem)
+; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm13 = zmm13 ^ (mem & (zmm13 ^ zmm0))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm14, %zmm0 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm0 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm0 = (~zmm0 & ~zmm2 & mem) | (~zmm0 & zmm2 & mem) | (zmm0 & ~zmm2 & mem) | (zmm0 & zmm2 & ~mem) | (zmm0 & zmm2 & mem)
+; AVX512-FCP-NEXT:    # zmm0 = (zmm0 & zmm2) | mem
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, %zmm4
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm2 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm2 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm2 = (~zmm2 & ~zmm4 & mem) | (~zmm2 & zmm4 & mem) | (zmm2 & ~zmm4 & mem) | (zmm2 & zmm4 & ~mem) | (zmm2 & zmm4 & mem)
+; AVX512-FCP-NEXT:    # zmm2 = (zmm2 & zmm4) | mem
 ; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
 ; AVX512-FCP-NEXT:    vpunpckhwd (%rsp), %xmm4, %xmm4 # 16-byte Folded Reload
 ; AVX512-FCP-NEXT:    # xmm4 = xmm4[4],mem[4],xmm4[5],mem[5],xmm4[6],mem[6],xmm4[7],mem[7]
@@ -13932,27 +13932,27 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = (~zmm14 & zmm12 & mem) | (zmm14 & ~zmm12 & ~mem) | (zmm14 & zmm12 & ~mem) | (zmm14 & zmm12 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 ^ (mem & (zmm14 ^ zmm12))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm11 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm4 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & ~zmm28 & zmm11) | (zmm4 & ~zmm28 & zmm11) | (zmm4 & zmm28 & ~zmm11) | (zmm4 & zmm28 & zmm11)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm11 ^ (zmm28 & (zmm4 ^ zmm11))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm7 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & ~zmm28 & zmm6) | (zmm7 & ~zmm28 & zmm6) | (zmm7 & zmm28 & ~zmm6) | (zmm7 & zmm28 & zmm6)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm6 ^ (zmm28 & (zmm7 ^ zmm6))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & ~zmm6 & zmm0) | (zmm4 & ~zmm6 & zmm0) | (zmm4 & zmm6 & ~zmm0) | (zmm4 & zmm6 & zmm0)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & ~zmm6 & zmm2) | (zmm7 & ~zmm6 & zmm2) | (zmm7 & zmm6 & ~zmm2) | (zmm7 & zmm6 & zmm2)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm0 ^ (zmm6 & (zmm4 ^ zmm0))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm2 ^ (zmm6 & (zmm7 ^ zmm2))
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm2, %zmm2
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm0 & ~mem) | (zmm2 & ~zmm0 & mem) | (zmm2 & zmm0 & ~mem) | (zmm2 & zmm0 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm0 ^ (mem & (zmm2 ^ zmm0))
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogq $248, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm0 = (~zmm0 & zmm1 & mem) | (zmm0 & ~zmm1 & ~mem) | (zmm0 & ~zmm1 & mem) | (zmm0 & zmm1 & ~mem) | (zmm0 & zmm1 & mem)
+; AVX512-FCP-NEXT:    # zmm0 = zmm0 | (zmm1 & mem)
 ; AVX512-FCP-NEXT:    vpternlogq $248, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm10 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm10 = (~zmm10 & zmm1 & mem) | (zmm10 & ~zmm1 & ~mem) | (zmm10 & ~zmm1 & mem) | (zmm10 & zmm1 & ~mem) | (zmm10 & zmm1 & mem)
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & zmm2 & mem) | (zmm10 & ~zmm2 & ~mem) | (zmm10 & zmm2 & ~mem) | (zmm10 & zmm2 & mem)
+; AVX512-FCP-NEXT:    # zmm10 = zmm10 | (zmm1 & mem)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm2))
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm10, 320(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, 256(%rax)
@@ -14105,24 +14105,24 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm9, %ymm9
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm17, %zmm6
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm18, %zmm7
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & zmm6 & ~mem) | (zmm7 & ~zmm6 & mem) | (zmm7 & zmm6 & ~mem) | (zmm7 & zmm6 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm6 ^ (mem & (zmm7 ^ zmm6))
 ; AVX512DQ-NEXT:    vmovdqa 96(%r8), %ymm6
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm10 = ymm6[1,2,2,3,5,6,6,7]
 ; AVX512DQ-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm10 = ymm6[0,0,2,1,4,4,6,5]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm6[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm6[16,17,u,u,u,u],zero,zero
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ymm7 & ymm11) | (ymm6 & ~ymm7 & ~ymm11) | (ymm6 & ~ymm7 & ymm11) | (ymm6 & ymm7 & ~ymm11) | (ymm6 & ymm7 & ymm11)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm7 & ymm11)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm11, %zmm12
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm9 = (~ymm9 & ymm6 & ymm11) | (ymm9 & ~ymm6 & ~ymm11) | (ymm9 & ~ymm6 & ymm11) | (ymm9 & ymm6 & ~ymm11) | (ymm9 & ymm6 & ymm11)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm9 = ymm9 | (ymm6 & ymm11)
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm7, %ymm6
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm10[2,1,3,3]
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm7 = (~ymm7 & ymm10 & ymm6) | (ymm7 & ~ymm10 & ~ymm6) | (ymm7 & ~ymm10 & ymm6) | (ymm7 & ymm10 & ymm6)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm7 = ymm7 ^ (ymm10 & (ymm7 ^ ymm6))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm8[2,2,2,2]
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ymm8 & ymm7) | (ymm6 & ~ymm8 & ~ymm7) | (ymm6 & ~ymm8 & ymm7) | (ymm6 & ymm8 & ymm7)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm6 ^ (ymm8 & (ymm6 ^ ymm7))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm6
 ; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm9[0,1,2,3],zmm6[4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -14360,7 +14360,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpbroadcastd 100(%rax), %ymm1
 ; AVX512DQ-NEXT:    vpbroadcastd 104(%rax), %ymm2
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm31
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm31 = (~zmm31 & zmm0 & mem) | (zmm31 & ~zmm0 & ~mem) | (zmm31 & zmm0 & ~mem) | (zmm31 & zmm0 & mem)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm31 = zmm31 ^ (mem & (zmm31 ^ zmm0))
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm7[0],xmm9[0],xmm7[1],xmm9[1],xmm7[2],xmm9[2],xmm7[3],xmm9[3]
 ; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm9, %xmm1
 ; AVX512DQ-NEXT:    vmovdqa %xmm4, %xmm6
@@ -14378,7 +14378,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm14
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm14 = (~zmm14 & ~zmm0 & zmm1) | (zmm14 & ~zmm0 & zmm1) | (zmm14 & zmm0 & ~zmm1) | (zmm14 & zmm0 & zmm1)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm1 ^ (zmm0 & (zmm14 ^ zmm1))
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm17, %xmm1
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm20, %xmm3
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
@@ -14390,7 +14390,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpbroadcastd 68(%rax), %ymm5
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm25
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm25 = (~zmm25 & zmm19 & zmm1) | (zmm25 & ~zmm19 & ~zmm1) | (zmm25 & ~zmm19 & zmm1) | (zmm25 & zmm19 & zmm1)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm25 = zmm25 ^ (zmm19 & (zmm25 ^ zmm1))
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
 ; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm8, %xmm3
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm5 = xmm13[1,1,2,2]
@@ -14407,7 +14407,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm2 = xmm5[0,1],xmm2[2],xmm5[3,4],xmm2[5],xmm5[6,7]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm5
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & ~zmm0 & zmm3) | (zmm5 & ~zmm0 & zmm3) | (zmm5 & zmm0 & ~zmm3) | (zmm5 & zmm0 & zmm3)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm3 ^ (zmm0 & (zmm5 ^ zmm3))
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm29, %xmm0
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm1
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -14417,7 +14417,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpbroadcastd (%rax), %ymm1
 ; AVX512DQ-NEXT:    vpbroadcastd 4(%rax), %ymm2
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm20
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm20 = (~zmm20 & zmm19 & zmm0) | (zmm20 & ~zmm19 & ~zmm0) | (zmm20 & ~zmm19 & zmm0) | (zmm20 & zmm19 & zmm0)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm20 = zmm20 ^ (zmm19 & (zmm20 ^ zmm0))
 ; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm6[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,2,1,4,4,6,5]
@@ -14492,7 +14492,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpbroadcastd 36(%rax), %ymm2
 ; AVX512DQ-NEXT:    vpbroadcastd 40(%rax), %ymm4
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm13
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm13 = (~zmm13 & zmm1 & mem) | (zmm13 & ~zmm1 & ~mem) | (zmm13 & zmm1 & ~mem) | (zmm13 & zmm1 & mem)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm13 = zmm13 ^ (mem & (zmm13 ^ zmm1))
 ; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %xmm7
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %xmm6
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
@@ -14505,103 +14505,103 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & zmm4 & zmm3) | (zmm1 & ~zmm4 & ~zmm3) | (zmm1 & ~zmm4 & zmm3) | (zmm1 & zmm4 & zmm3)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm4 & (zmm1 ^ zmm3))
 ; AVX512DQ-NEXT:    vpermq $182, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    # ymm3 = mem[2,1,3,2]
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm3 = (~ymm3 & ymm29 & ymm1) | (ymm3 & ~ymm29 & ~ymm1) | (ymm3 & ~ymm29 & ymm1) | (ymm3 & ymm29 & ymm1)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm30 = (~ymm30 & ymm28 & ymm3) | (ymm30 & ~ymm28 & ~ymm3) | (ymm30 & ~ymm28 & ymm3) | (ymm30 & ymm28 & ymm3)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm3 = ymm3 ^ (ymm29 & (ymm3 ^ ymm1))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm30 = ymm30 ^ (ymm28 & (ymm30 ^ ymm3))
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm3 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm3 = (~zmm3 & ~zmm28 & mem) | (~zmm3 & zmm28 & mem) | (zmm3 & ~zmm28 & mem) | (zmm3 & zmm28 & ~mem) | (zmm3 & zmm28 & mem)
+; AVX512DQ-NEXT:    # zmm3 = (zmm3 & zmm28) | mem
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm0 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm0 = (~zmm0 & ~zmm28 & mem) | (~zmm0 & zmm28 & mem) | (zmm0 & ~zmm28 & mem) | (zmm0 & zmm28 & ~mem) | (zmm0 & zmm28 & mem)
+; AVX512DQ-NEXT:    # zmm0 = (zmm0 & zmm28) | mem
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm23 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm18 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm28 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm18 = (~zmm18 & ~zmm28 & zmm23) | (zmm18 & ~zmm28 & zmm23) | (zmm18 & zmm28 & ~zmm23) | (zmm18 & zmm28 & zmm23)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm18 = zmm23 ^ (zmm28 & (zmm18 ^ zmm23))
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm18 = (~zmm18 & ~zmm2 & zmm3) | (zmm18 & ~zmm2 & zmm3) | (zmm18 & zmm2 & ~zmm3) | (zmm18 & zmm2 & zmm3)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm18 = zmm3 ^ (zmm2 & (zmm18 ^ zmm3))
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm16, %zmm23 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm23 = (~zmm23 & ~zmm28 & zmm3) | (zmm23 & ~zmm28 & zmm3) | (zmm23 & zmm28 & ~zmm3) | (zmm23 & zmm28 & zmm3)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm23 = (~zmm23 & ~zmm2 & zmm0) | (zmm23 & ~zmm2 & zmm0) | (zmm23 & zmm2 & ~zmm0) | (zmm23 & zmm2 & zmm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm23 = zmm3 ^ (zmm28 & (zmm23 ^ zmm3))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm23 = zmm0 ^ (zmm2 & (zmm23 ^ zmm0))
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm17 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm17 = (~zmm17 & zmm0 & ~mem) | (zmm17 & ~zmm0 & mem) | (zmm17 & zmm0 & ~mem) | (zmm17 & zmm0 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm17 = zmm0 ^ (mem & (zmm17 ^ zmm0))
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm0
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} ymm0 = (~ymm0 & ~ymm19 & mem) | (ymm0 & ~ymm19 & mem) | (ymm0 & ymm19 & ~mem) | (ymm0 & ymm19 & mem)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} ymm0 = mem ^ (ymm19 & (ymm0 ^ mem))
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
 ; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm2 = (~ymm2 & ymm1 & ymm0) | (ymm2 & ~ymm1 & ~ymm0) | (ymm2 & ~ymm1 & ymm0) | (ymm2 & ymm1 & ymm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm2 = ymm2 ^ (ymm1 & (ymm2 ^ ymm0))
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
 ; AVX512DQ-NEXT:    vpternlogq $248, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm19 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm19 = (~zmm19 & zmm1 & mem) | (zmm19 & ~zmm1 & ~mem) | (zmm19 & ~zmm1 & mem) | (zmm19 & zmm1 & ~mem) | (zmm19 & zmm1 & mem)
+; AVX512DQ-NEXT:    # zmm19 = zmm19 | (zmm1 & mem)
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-NEXT:    vshufi64x2 $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
 ; AVX512DQ-NEXT:    # zmm0 = zmm0[0,1,2,3],mem[0,1,2,3]
 ; AVX512DQ-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
-; AVX512DQ-NEXT:    # zmm0 = (~zmm0 & ~zmm1 & mem) | (~zmm0 & zmm1 & mem) | (zmm0 & ~zmm1 & mem) | (zmm0 & zmm1 & ~mem) | (zmm0 & zmm1 & mem)
+; AVX512DQ-NEXT:    # zmm0 = (zmm0 & zmm1) | mem
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm1
 ; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm30[0,1,2,3],zmm1[4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm30 = (~zmm30 & zmm2 & zmm1) | (zmm30 & ~zmm2 & ~zmm1) | (zmm30 & ~zmm2 & zmm1) | (zmm30 & zmm2 & zmm1)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm30 = zmm30 ^ (zmm2 & (zmm30 ^ zmm1))
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm11 = (~zmm11 & zmm2 & zmm1) | (zmm11 & ~zmm2 & ~zmm1) | (zmm11 & ~zmm2 & zmm1) | (zmm11 & zmm2 & zmm1)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm11 = zmm11 ^ (zmm2 & (zmm11 ^ zmm1))
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm25 = (~zmm25 & zmm1 & zmm14) | (zmm25 & ~zmm1 & ~zmm14) | (zmm25 & ~zmm1 & zmm14) | (zmm25 & zmm1 & zmm14)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm20 = (~zmm20 & zmm1 & zmm5) | (zmm20 & ~zmm1 & ~zmm5) | (zmm20 & ~zmm1 & zmm5) | (zmm20 & zmm1 & zmm5)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm25 = zmm25 ^ (zmm1 & (zmm25 ^ zmm14))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm20 ^ (zmm1 & (zmm20 ^ zmm5))
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & ~zmm5 & zmm1) | (zmm2 & ~zmm5 & zmm1) | (zmm2 & zmm5 & ~zmm1) | (zmm2 & zmm5 & zmm1)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm1 ^ (zmm5 & (zmm2 ^ zmm1))
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm22, %zmm3, %zmm22
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm22 = (~zmm22 & ~zmm5 & zmm1) | (zmm22 & ~zmm5 & zmm1) | (zmm22 & zmm5 & ~zmm1) | (zmm22 & zmm5 & zmm1)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm22 = zmm1 ^ (zmm5 & (zmm22 ^ zmm1))
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm14 = [0,0,4,0,0,0,5,0,0,5,0,0,0,6,0,0]
 ; AVX512DQ-NEXT:    vpermd 64(%rax), %zmm14, %zmm5
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm5 = (~zmm5 & zmm3 & zmm1) | (zmm5 & ~zmm3 & ~zmm1) | (zmm5 & ~zmm3 & zmm1) | (zmm5 & zmm3 & zmm1)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm5 ^ (zmm3 & (zmm5 ^ zmm1))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm24, %zmm26, %zmm1
 ; AVX512DQ-NEXT:    vpermd (%rax), %zmm14, %zmm14
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm14 = (~zmm14 & zmm3 & zmm1) | (zmm14 & ~zmm3 & ~zmm1) | (zmm14 & ~zmm3 & zmm1) | (zmm14 & zmm3 & zmm1)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm14 = zmm14 ^ (zmm3 & (zmm14 ^ zmm1))
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm1 & zmm2) | (zmm5 & ~zmm1 & ~zmm2) | (zmm5 & ~zmm1 & zmm2) | (zmm5 & zmm1 & zmm2)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm14 = (~zmm14 & zmm1 & zmm22) | (zmm14 & ~zmm1 & ~zmm22) | (zmm14 & ~zmm1 & zmm22) | (zmm14 & zmm1 & zmm22)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm1 & (zmm5 ^ zmm2))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 ^ (zmm1 & (zmm14 ^ zmm22))
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm4 & zmm1) | (zmm2 & ~zmm4 & ~zmm1) | (zmm2 & ~zmm4 & zmm1) | (zmm2 & zmm4 & zmm1)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm4 & (zmm2 ^ zmm1))
 ; AVX512DQ-NEXT:    vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    # ymm1 = mem[0,1,1,3]
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm22 # 32-byte Folded Reload
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm22 = (~zmm22 & ~zmm4 & zmm1) | (zmm22 & ~zmm4 & zmm1) | (zmm22 & zmm4 & ~zmm1) | (zmm22 & zmm4 & zmm1)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm22 = zmm1 ^ (zmm4 & (zmm22 ^ zmm1))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm12[0,1,1,3]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm21, %zmm1
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm15, %zmm8
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & ~zmm4 & zmm1) | (zmm8 & ~zmm4 & zmm1) | (zmm8 & zmm4 & ~zmm1) | (zmm8 & zmm4 & zmm1)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm1 ^ (zmm4 & (zmm8 ^ zmm1))
 ; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm4 = xmm7[u,u,4,5,u,u,u,u,6,7,u,u,u,u,8,9]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm9 = xmm6[1,1,2,2]
@@ -14645,27 +14645,27 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm10[2,1,3,2]
 ; AVX512DQ-NEXT:    vpbroadcastd 96(%rax), %ymm10
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm9 = (~zmm9 & zmm29 & zmm7) | (zmm9 & ~zmm29 & ~zmm7) | (zmm9 & ~zmm29 & zmm7) | (zmm9 & zmm29 & zmm7)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm9 = zmm9 ^ (zmm29 & (zmm9 ^ zmm7))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,0,1]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm24, %zmm3
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm26[2,1,3,2]
 ; AVX512DQ-NEXT:    vpbroadcastd 32(%rax), %ymm10
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm7, %zmm7
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm7 = (~zmm7 & zmm29 & zmm3) | (zmm7 & ~zmm29 & ~zmm3) | (zmm7 & ~zmm29 & zmm3) | (zmm7 & zmm29 & zmm3)
+; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (zmm29 & (zmm7 ^ zmm3))
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm9 = (~zmm9 & zmm3 & zmm22) | (zmm9 & ~zmm3 & ~zmm22) | (zmm9 & ~zmm3 & zmm22) | (zmm9 & zmm3 & zmm22)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & zmm3 & zmm8) | (zmm7 & ~zmm3 & ~zmm8) | (zmm7 & ~zmm3 & zmm8) | (zmm7 & zmm3 & zmm8)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 ^ (zmm3 & (zmm9 ^ zmm22))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm3 & (zmm7 ^ zmm8))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm16, %zmm3
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm15, %zmm21, %zmm8
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & ~zmm28 & zmm3) | (zmm8 & ~zmm28 & zmm3) | (zmm8 & zmm28 & ~zmm3) | (zmm8 & zmm28 & zmm3)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm3 ^ (zmm28 & (zmm8 ^ zmm3))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm27, %zmm1
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm3
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & ~zmm28 & zmm1) | (zmm3 & ~zmm28 & zmm1) | (zmm3 & zmm28 & ~zmm1) | (zmm3 & zmm28 & zmm1)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm1 ^ (zmm28 & (zmm3 ^ zmm1))
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm31 = (~zmm31 & zmm1 & zmm8) | (zmm31 & ~zmm1 & ~zmm8) | (zmm31 & ~zmm1 & zmm8) | (zmm31 & zmm1 & zmm8)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm13 = (~zmm13 & zmm1 & zmm3) | (zmm13 & ~zmm1 & ~zmm3) | (zmm13 & ~zmm1 & zmm3) | (zmm13 & zmm1 & zmm3)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm17 & mem) | (zmm0 & ~zmm17 & ~mem) | (zmm0 & zmm17 & ~mem) | (zmm0 & zmm17 & mem)
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm11 = (~zmm11 & zmm2 & mem) | (zmm11 & ~zmm2 & ~mem) | (zmm11 & zmm2 & ~mem) | (zmm11 & zmm2 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm31 = zmm31 ^ (zmm1 & (zmm31 ^ zmm8))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm13 ^ (zmm1 & (zmm13 ^ zmm3))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm17))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 ^ (mem & (zmm11 ^ zmm2))
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 320(%rax)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm13, 256(%rax)
@@ -14782,26 +14782,26 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,2,2,3]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm16, %zmm11
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm7, %zmm12
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = (~zmm12 & zmm11 & ~mem) | (zmm12 & ~zmm11 & mem) | (zmm12 & zmm11 & ~mem) | (zmm12 & zmm11 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm11 ^ (mem & (zmm12 ^ zmm11))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm13 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r8), %ymm7
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm7[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm7[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm7[16,17,u,u,u,u],zero,zero
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ymm12 & ymm13) | (ymm11 & ~ymm12 & ~ymm13) | (ymm11 & ~ymm12 & ymm13) | (ymm11 & ymm12 & ~ymm13) | (ymm11 & ymm12 & ymm13)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm12 & ymm13)
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r9), %ymm10
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm10, %ymm6
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ymm11 & ymm16) | (ymm6 & ~ymm11 & ~ymm16) | (ymm6 & ~ymm11 & ymm16) | (ymm6 & ymm11 & ~ymm16) | (ymm6 & ymm11 & ymm16)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm6 | (ymm11 & ymm16)
 ; AVX512DQ-FCP-NEXT:    vextracti64x4 $1, %zmm12, %ymm11
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm12 = [0,4,0,0,0,5,0,0]
 ; AVX512DQ-FCP-NEXT:    vpermd %ymm7, %ymm12, %ymm12
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm12 = (~ymm12 & ymm16 & ymm11) | (ymm12 & ~ymm16 & ~ymm11) | (ymm12 & ~ymm16 & ymm11) | (ymm12 & ymm16 & ymm11)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm12 = ymm12 ^ (ymm16 & (ymm12 ^ ymm11))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm16, %zmm19
 ; AVX512DQ-FCP-NEXT:    vprold $16, %ymm10, %ymm11
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,2,2,2]
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = (~ymm11 & ymm10 & ymm12) | (ymm11 & ~ymm10 & ~ymm12) | (ymm11 & ~ymm10 & ymm12) | (ymm11 & ymm10 & ymm12)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm11 = ymm11 ^ (ymm10 & (ymm11 ^ ymm12))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm11
 ; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm6[0,1,2,3],zmm11[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -14839,7 +14839,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm20 = [2,2,2,3,8,10,10,11]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm11, %zmm20, %zmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm27 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm27 & zmm4) | (zmm1 & ~zmm27 & zmm4) | (zmm1 & zmm27 & ~zmm4) | (zmm1 & zmm27 & zmm4)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm4 ^ (zmm27 & (zmm1 ^ zmm4))
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [5,0,0,0,6,0,0,6]
 ; AVX512DQ-FCP-NEXT:    vpermd %ymm7, %ymm1, %ymm1
@@ -14891,7 +14891,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm15[2,2,2,2,6,6,6,6]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0,1],ymm6[2],ymm5[3,4],ymm6[5],ymm5[6,7,8,9],ymm6[10],ymm5[11,12],ymm6[13],ymm5[14,15]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm20, %zmm0
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & ~zmm27 & zmm4) | (zmm0 & ~zmm27 & zmm4) | (zmm0 & zmm27 & ~zmm4) | (zmm0 & zmm27 & zmm4)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm4 ^ (zmm27 & (zmm0 ^ zmm4))
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm8[3,3,3,3,7,7,7,7]
@@ -14906,7 +14906,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rax), %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpermi2d %zmm0, %zmm16, %zmm3
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = (~zmm3 & zmm19 & zmm4) | (zmm3 & ~zmm19 & ~zmm4) | (zmm3 & ~zmm19 & zmm4) | (zmm3 & zmm19 & zmm4)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (zmm19 & (zmm3 ^ zmm4))
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm29, %ymm10
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm10, %ymm3
@@ -14927,7 +14927,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm4[2],ymm6[3,4],ymm4[5],ymm6[6,7,8,9],ymm4[10],ymm6[11,12],ymm4[13],ymm6[14,15]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm11, %zmm6
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm27 = [0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = (~zmm6 & ~zmm27 & zmm5) | (zmm6 & ~zmm27 & zmm5) | (zmm6 & zmm27 & ~zmm5) | (zmm6 & zmm27 & zmm5)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm5 ^ (zmm27 & (zmm6 ^ zmm5))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm26, %ymm12
 ; AVX512DQ-FCP-NEXT:    vprold $16, %ymm26, %ymm3
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm28, %ymm22
@@ -14943,8 +14943,8 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm25, %zmm1, %zmm1
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm0 = [0,0,4,0,0,0,5,0,0,13,0,0,0,14,0,0]
 ; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = (~zmm0 & zmm18 & zmm5) | (zmm0 & ~zmm18 & ~zmm5) | (zmm0 & ~zmm18 & zmm5) | (zmm0 & zmm18 & zmm5)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & zmm6 & mem) | (zmm0 & ~zmm6 & ~mem) | (zmm0 & zmm6 & ~mem) | (zmm0 & zmm6 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm18 & (zmm0 ^ zmm5))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm6))
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm8, %ymm0
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm1 = ymm21[3,3,3,3,7,7,7,7]
@@ -14977,7 +14977,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm28 = [0,0,1,1,8,8,10,9]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm28, %zmm5
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & ~zmm3 & zmm2) | (zmm5 & ~zmm3 & zmm2) | (zmm5 & zmm3 & ~zmm2) | (zmm5 & zmm3 & zmm2)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm2 ^ (zmm3 & (zmm5 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r9), %xmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r8), %xmm6
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
@@ -14993,9 +14993,9 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 68(%rax), %ymm8
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm6, %zmm31
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm31 = (~zmm31 & zmm21 & zmm2) | (zmm31 & ~zmm21 & ~zmm2) | (zmm31 & ~zmm21 & zmm2) | (zmm31 & zmm21 & zmm2)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm31 = zmm31 ^ (zmm21 & (zmm31 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm31 = (~zmm31 & zmm2 & zmm5) | (zmm31 & ~zmm2 & ~zmm5) | (zmm31 & ~zmm2 & zmm5) | (zmm31 & zmm2 & zmm5)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm31 = zmm31 ^ (zmm2 & (zmm31 ^ zmm5))
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm8
@@ -15017,7 +15017,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm8, %xmm8
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm5, %zmm28, %zmm8
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = (~zmm8 & ~zmm3 & zmm6) | (zmm8 & ~zmm3 & zmm6) | (zmm8 & zmm3 & ~zmm6) | (zmm8 & zmm3 & zmm6)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm6 ^ (zmm3 & (zmm8 ^ zmm6))
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm3
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm5
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
@@ -15030,8 +15030,8 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd (%rax), %ymm5
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 4(%rax), %ymm6
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm26
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm26 = (~zmm26 & zmm21 & zmm3) | (zmm26 & ~zmm21 & ~zmm3) | (zmm26 & ~zmm21 & zmm3) | (zmm26 & zmm21 & zmm3)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm26 = (~zmm26 & zmm2 & zmm8) | (zmm26 & ~zmm2 & ~zmm8) | (zmm26 & ~zmm2 & zmm8) | (zmm26 & zmm2 & zmm8)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm26 = zmm26 ^ (zmm21 & (zmm26 ^ zmm3))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm26 = zmm26 ^ (zmm2 & (zmm26 ^ zmm8))
 ; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm14[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
@@ -15062,7 +15062,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm10, %ymm2
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm8 = ymm19[3,3,3,3,7,7,7,7]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm8[0,1,2],ymm2[3],ymm8[4,5],ymm2[6],ymm8[7,8,9,10],ymm2[11],ymm8[12,13],ymm2[14],ymm8[15]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & ~zmm27 & zmm3) | (zmm5 & ~zmm27 & zmm3) | (zmm5 & zmm27 & ~zmm3) | (zmm5 & zmm27 & zmm3)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm3 ^ (zmm27 & (zmm5 ^ zmm3))
 ; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
 ; AVX512DQ-FCP-NEXT:    vprold $16, %ymm4, %ymm2
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm19 # 32-byte Reload
@@ -15080,7 +15080,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm20 = [2,1,3,3,8,8,9,9]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm20, %zmm10
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & ~zmm21 & zmm1) | (zmm10 & ~zmm21 & zmm1) | (zmm10 & zmm21 & ~zmm1) | (zmm10 & zmm21 & zmm1)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm1 ^ (zmm21 & (zmm10 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm30, %ymm1
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm22[3,3,3,3,7,7,7,7]
@@ -15089,8 +15089,8 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm23, %zmm16, %zmm1
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,4,0,0,0,5,0,0,13,0,0,0,14,0,0]
 ; AVX512DQ-FCP-NEXT:    vpermd %zmm1, %zmm16, %zmm22
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm22 = (~zmm22 & zmm8 & mem) | (zmm22 & ~zmm8 & ~mem) | (zmm22 & zmm8 & ~mem) | (zmm22 & zmm8 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm22 = (~zmm22 & zmm5 & mem) | (zmm22 & ~zmm5 & ~mem) | (zmm22 & zmm5 & ~mem) | (zmm22 & zmm5 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm22 = zmm22 ^ (mem & (zmm22 ^ zmm8))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm22 = zmm22 ^ (mem & (zmm22 ^ zmm5))
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm5
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm1
@@ -15121,14 +15121,14 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 96(%rax), %ymm20
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm20, %zmm16, %zmm25
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm25 = (~zmm25 & zmm18 & zmm2) | (zmm25 & ~zmm18 & ~zmm2) | (zmm25 & ~zmm18 & zmm2) | (zmm25 & zmm18 & zmm2)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm25 = zmm25 ^ (zmm18 & (zmm25 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm27 = [0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm25 = (~zmm25 & zmm27 & zmm10) | (zmm25 & ~zmm27 & ~zmm10) | (zmm25 & ~zmm27 & zmm10) | (zmm25 & zmm27 & zmm10)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm25 = zmm25 ^ (zmm27 & (zmm25 ^ zmm10))
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
 ; AVX512DQ-FCP-NEXT:    vprold $16, %xmm3, %xmm3
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[1,1,2,3]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm3[2],xmm4[3,4],xmm3[5],xmm4[6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm21 & zmm0) | (zmm1 & ~zmm21 & zmm0) | (zmm1 & zmm21 & ~zmm0) | (zmm1 & zmm21 & zmm0)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm0 ^ (zmm21 & (zmm1 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm2[4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm21 = [0,0,2,1,8,8,9,11]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm21, %zmm3
@@ -15140,11 +15140,11 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,3,3,4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm28, %zmm7
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm28 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & ~zmm28 & zmm3) | (zmm7 & ~zmm28 & zmm3) | (zmm7 & zmm28 & ~zmm3) | (zmm7 & zmm28 & zmm3)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm3 ^ (zmm28 & (zmm7 ^ zmm3))
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm16 # 32-byte Reload
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm16 = (~ymm16 & ymm18 & ymm0) | (ymm16 & ~ymm18 & ~ymm0) | (ymm16 & ~ymm18 & ymm0) | (ymm16 & ymm18 & ymm0)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm16 = ymm16 ^ (ymm18 & (ymm16 ^ ymm0))
 ; AVX512DQ-FCP-NEXT:    vextracti64x4 $1, %zmm0, %ymm13
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm2, %xmm2
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm6 = xmm8[0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
@@ -15163,7 +15163,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 104(%rax), %ymm19
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm5, %zmm5
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm5 = (~zmm5 & zmm24 & zmm6) | (zmm5 & ~zmm24 & ~zmm6) | (zmm5 & ~zmm24 & zmm6) | (zmm5 & zmm24 & zmm6)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm5 ^ (zmm24 & (zmm5 ^ zmm6))
 ; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} ymm6 = ymm4[0,1,2,3,5,5,7,6,8,9,10,11,13,13,15,14]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[3,3,3,3]
@@ -15179,10 +15179,10 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpermd %ymm23, %ymm17, %ymm17
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 32(%rax), %ymm19
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm17, %zmm20
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm20 = (~zmm20 & zmm18 & zmm8) | (zmm20 & ~zmm18 & ~zmm8) | (zmm20 & ~zmm18 & zmm8) | (zmm20 & zmm18 & zmm8)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm20 = (~zmm20 & zmm27 & zmm1) | (zmm20 & ~zmm27 & ~zmm1) | (zmm20 & ~zmm27 & zmm1) | (zmm20 & zmm27 & zmm1)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm20 = zmm20 ^ (zmm18 & (zmm20 ^ zmm8))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm20 ^ (zmm27 & (zmm20 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm1 & zmm7) | (zmm5 & ~zmm1 & ~zmm7) | (zmm5 & ~zmm1 & zmm7) | (zmm5 & zmm1 & zmm7)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm1 & (zmm5 ^ zmm7))
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm9[4],xmm15[4],xmm9[5],xmm15[5],xmm9[6],xmm15[6],xmm9[7],xmm15[7]
 ; AVX512DQ-FCP-NEXT:    vprold $16, %xmm9, %xmm8
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm9 = xmm15[1,1,2,3]
@@ -15200,7 +15200,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm9[0,2,3,3,4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm0 = [0,0,1,1,8,8,10,9]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm9, %zmm0, %zmm7
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & ~zmm28 & zmm8) | (zmm7 & ~zmm28 & zmm8) | (zmm7 & zmm28 & ~zmm8) | (zmm7 & zmm28 & zmm8)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm8 ^ (zmm28 & (zmm7 ^ zmm8))
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm11[4],xmm3[4],xmm11[5],xmm3[5],xmm11[6],xmm3[6],xmm11[7],xmm3[7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm30, %xmm0
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm0, %xmm3, %xmm3
@@ -15210,25 +15210,25 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 36(%rax), %ymm3
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 40(%rax), %ymm8
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm3, %zmm3
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = (~zmm3 & zmm24 & zmm2) | (zmm3 & ~zmm24 & ~zmm2) | (zmm3 & ~zmm24 & zmm2) | (zmm3 & zmm24 & zmm2)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = (~zmm3 & zmm1 & zmm7) | (zmm3 & ~zmm1 & ~zmm7) | (zmm3 & ~zmm1 & zmm7) | (zmm3 & zmm1 & zmm7)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (zmm24 & (zmm3 ^ zmm2))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm1 & (zmm3 ^ zmm7))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} ymm13 = (~ymm13 & ~ymm0 & mem) | (ymm13 & ~ymm0 & mem) | (ymm13 & ymm0 & ~mem) | (ymm13 & ymm0 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} ymm13 = mem ^ (ymm0 & (ymm13 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = (~ymm6 & ymm1 & ymm13) | (ymm6 & ~ymm1 & ~ymm13) | (ymm6 & ~ymm1 & ymm13) | (ymm6 & ymm1 & ymm13)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm6 ^ (ymm1 & (ymm6 ^ ymm13))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = (~ymm4 & ymm2 & ymm16) | (ymm4 & ~ymm2 & ~ymm16) | (ymm4 & ~ymm2 & ymm16) | (ymm4 & ymm2 & ymm16)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = ymm4 ^ (ymm2 & (ymm4 ^ ymm16))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm0
 ; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm4[0,1,2,3],zmm0[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm13 = (~zmm13 & zmm0 & mem) | (zmm13 & ~zmm0 & ~mem) | (zmm13 & zmm0 & ~mem) | (zmm13 & zmm0 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm13 = zmm13 ^ (mem & (zmm13 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm14, %zmm0 # 32-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm0 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm0 = (~zmm0 & ~zmm2 & mem) | (~zmm0 & zmm2 & mem) | (zmm0 & ~zmm2 & mem) | (zmm0 & zmm2 & ~mem) | (zmm0 & zmm2 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm0 = (zmm0 & zmm2) | mem
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, %zmm4
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm2 # 32-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm2 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm2 = (~zmm2 & ~zmm4 & mem) | (~zmm2 & zmm4 & mem) | (zmm2 & ~zmm4 & mem) | (zmm2 & zmm4 & ~mem) | (zmm2 & zmm4 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm2 = (zmm2 & zmm4) | mem
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd (%rsp), %xmm4, %xmm4 # 16-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # xmm4 = xmm4[4],mem[4],xmm4[5],mem[5],xmm4[6],mem[6],xmm4[7],mem[7]
@@ -15268,27 +15268,27 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = (~zmm14 & zmm12 & mem) | (zmm14 & ~zmm12 & ~mem) | (zmm14 & zmm12 & ~mem) | (zmm14 & zmm12 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 ^ (mem & (zmm14 ^ zmm12))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm11 # 32-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm4 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & ~zmm28 & zmm11) | (zmm4 & ~zmm28 & zmm11) | (zmm4 & zmm28 & ~zmm11) | (zmm4 & zmm28 & zmm11)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm11 ^ (zmm28 & (zmm4 ^ zmm11))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 32-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm7 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & ~zmm28 & zmm6) | (zmm7 & ~zmm28 & zmm6) | (zmm7 & zmm28 & ~zmm6) | (zmm7 & zmm28 & zmm6)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm6 ^ (zmm28 & (zmm7 ^ zmm6))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & ~zmm6 & zmm0) | (zmm4 & ~zmm6 & zmm0) | (zmm4 & zmm6 & ~zmm0) | (zmm4 & zmm6 & zmm0)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = (~zmm7 & ~zmm6 & zmm2) | (zmm7 & ~zmm6 & zmm2) | (zmm7 & zmm6 & ~zmm2) | (zmm7 & zmm6 & zmm2)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm0 ^ (zmm6 & (zmm4 ^ zmm0))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm2 ^ (zmm6 & (zmm7 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm2, %zmm2
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm0 & ~mem) | (zmm2 & ~zmm0 & mem) | (zmm2 & zmm0 & ~mem) | (zmm2 & zmm0 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm0 ^ (mem & (zmm2 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm0 = (~zmm0 & zmm1 & mem) | (zmm0 & ~zmm1 & ~mem) | (zmm0 & ~zmm1 & mem) | (zmm0 & zmm1 & ~mem) | (zmm0 & zmm1 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm0 = zmm0 | (zmm1 & mem)
 ; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm10 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm10 = (~zmm10 & zmm1 & mem) | (zmm10 & ~zmm1 & ~mem) | (zmm10 & ~zmm1 & mem) | (zmm10 & zmm1 & ~mem) | (zmm10 & zmm1 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & zmm2 & mem) | (zmm10 & ~zmm2 & ~mem) | (zmm10 & zmm2 & ~mem) | (zmm10 & zmm2 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm10 = zmm10 | (zmm1 & mem)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm10, 320(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, 256(%rax)
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-8.ll
index 47690f3c60edf..20c245aa08663 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-8.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-8.ll
@@ -7226,7 +7226,7 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    # ymm5 = mem[2,1,3,3,6,5,7,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm10
 ; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm13 = [65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & zmm13 & zmm3) | (zmm10 & ~zmm13 & ~zmm3) | (zmm10 & ~zmm13 & zmm3) | (zmm10 & zmm13 & zmm3)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm13 & (zmm10 ^ zmm3))
 ; AVX512-NEXT:    vpshufd $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
 ; AVX512-NEXT:    # ymm3 = mem[0,0,2,1,4,4,6,5]
 ; AVX512-NEXT:    vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
@@ -7253,7 +7253,7 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    # ymm2 = mem[0,1,1,3,4,5,5,7]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm9[2,1,3,3,6,5,7,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm9
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm9 = (~zmm9 & zmm13 & zmm0) | (zmm9 & ~zmm13 & ~zmm0) | (zmm9 & ~zmm13 & zmm0) | (zmm9 & zmm13 & zmm0)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 ^ (zmm13 & (zmm9 ^ zmm0))
 ; AVX512-NEXT:    vpshufd $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
 ; AVX512-NEXT:    # ymm0 = mem[0,0,2,1,4,4,6,5]
 ; AVX512-NEXT:    vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
@@ -7280,7 +7280,7 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm11[0,1,1,3,4,5,5,7]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm31[2,1,3,3,6,5,7,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm11
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm11 = (~zmm11 & zmm13 & zmm0) | (zmm11 & ~zmm13 & ~zmm0) | (zmm11 & ~zmm13 & zmm0) | (zmm11 & zmm13 & zmm0)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 ^ (zmm13 & (zmm11 ^ zmm0))
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm28[0,0,2,1,4,4,6,5]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm23[0,2,2,3,4,6,6,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
@@ -7314,7 +7314,7 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm30[0,1,1,3,4,5,5,7]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm6 = ymm29[2,1,3,3,6,5,7,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm4
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm13 & zmm0) | (zmm4 & ~zmm13 & ~zmm0) | (zmm4 & ~zmm13 & zmm0) | (zmm4 & zmm13 & zmm0)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm13 & (zmm4 ^ zmm0))
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm24[0,0,2,1,4,4,6,5]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm6 = ymm22[0,2,2,3,4,6,6,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm0
@@ -7391,7 +7391,7 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpshufd $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm23 # 32-byte Folded Reload
 ; AVX512-NEXT:    # ymm23 = mem[2,1,3,3,6,5,7,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm23, %zmm5, %zmm5
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm13 & zmm12) | (zmm5 & ~zmm13 & ~zmm12) | (zmm5 & ~zmm13 & zmm12) | (zmm5 & zmm13 & zmm12)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm13 & (zmm5 ^ zmm12))
 ; AVX512-NEXT:    vpshufd $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Folded Reload
 ; AVX512-NEXT:    # ymm12 = mem[0,0,2,1,4,4,6,5]
 ; AVX512-NEXT:    vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %ymm23 # 32-byte Folded Reload
@@ -7427,7 +7427,7 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpshufd $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm30 # 32-byte Folded Reload
 ; AVX512-NEXT:    # ymm30 = mem[2,1,3,3,6,5,7,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm30, %zmm12, %zmm12
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm12 = (~zmm12 & zmm13 & zmm0) | (zmm12 & ~zmm13 & ~zmm0) | (zmm12 & ~zmm13 & zmm0) | (zmm12 & zmm13 & zmm0)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm12 ^ (zmm13 & (zmm12 ^ zmm0))
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm30 = xmm2[3,3,3,3]
 ; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm31 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
@@ -7454,7 +7454,7 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm16 = ymm28[0,1,1,3,4,5,5,7]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm18 = ymm18[2,1,3,3,6,5,7,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm18, %zmm16, %zmm16
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm16 = (~zmm16 & zmm13 & zmm7) | (zmm16 & ~zmm13 & ~zmm7) | (zmm16 & ~zmm13 & zmm7) | (zmm16 & zmm13 & zmm7)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm16 = zmm16 ^ (zmm13 & (zmm16 ^ zmm7))
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm7 = ymm21[0,0,2,1,4,4,6,5]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm18 = ymm22[0,2,2,3,4,6,6,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm18, %zmm7, %zmm7
@@ -7473,7 +7473,7 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm14[0,1,1,3,4,5,5,7]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm6 = ymm15[2,1,3,3,6,5,7,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm2, %zmm2
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm13 & zmm0) | (zmm2 & ~zmm13 & ~zmm0) | (zmm2 & ~zmm13 & zmm0) | (zmm2 & zmm13 & zmm0)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm13 & (zmm2 ^ zmm0))
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm17[0,0,2,1,4,4,6,5]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm6 = ymm20[0,2,2,3,4,6,6,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm0
@@ -7664,7 +7664,7 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm3
 ; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} zmm7 = [65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535]
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm3 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm3 = (~zmm3 & ~zmm7 & mem) | (zmm3 & ~zmm7 & mem) | (zmm3 & zmm7 & ~mem) | (zmm3 & zmm7 & mem)
+; AVX512-FCP-NEXT:    # zmm3 = mem ^ (zmm7 & (zmm3 ^ mem))
 ; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} zmm2 = [65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0]
 ; AVX512-FCP-NEXT:    vpandnq %zmm19, %zmm2, %zmm19
 ; AVX512-FCP-NEXT:    vpandq %zmm2, %zmm21, %zmm21
@@ -7748,32 +7748,32 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm8, %ymm5
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm5, %zmm4
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm6 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm6 = (~zmm6 & ~zmm7 & mem) | (zmm6 & ~zmm7 & mem) | (zmm6 & zmm7 & ~mem) | (zmm6 & zmm7 & mem)
+; AVX512-FCP-NEXT:    # zmm6 = mem ^ (zmm7 & (zmm6 ^ mem))
 ; AVX512-FCP-NEXT:    vpandnq {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm5 # 64-byte Folded Reload
 ; AVX512-FCP-NEXT:    vpandq {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm8 # 64-byte Folded Reload
 ; AVX512-FCP-NEXT:    vpord %zmm5, %zmm8, %zmm6 {%k1}
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm15 # 64-byte Folded Reload
-; AVX512-FCP-NEXT:    # zmm15 = (~zmm15 & ~zmm7 & mem) | (zmm15 & ~zmm7 & mem) | (zmm15 & zmm7 & ~mem) | (zmm15 & zmm7 & mem)
+; AVX512-FCP-NEXT:    # zmm15 = mem ^ (zmm7 & (zmm15 ^ mem))
 ; AVX512-FCP-NEXT:    vpandnq (%rsp), %zmm2, %zmm5 # 64-byte Folded Reload
 ; AVX512-FCP-NEXT:    vpandq {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm8 # 64-byte Folded Reload
 ; AVX512-FCP-NEXT:    vpord %zmm5, %zmm8, %zmm15 {%k1}
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm11 = (~zmm11 & ~zmm7 & zmm20) | (zmm11 & ~zmm7 & zmm20) | (zmm11 & zmm7 & ~zmm20) | (zmm11 & zmm7 & zmm20)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm20 ^ (zmm7 & (zmm11 ^ zmm20))
 ; AVX512-FCP-NEXT:    vpandnq %zmm22, %zmm2, %zmm5
 ; AVX512-FCP-NEXT:    vpandq %zmm2, %zmm23, %zmm8
 ; AVX512-FCP-NEXT:    vpord %zmm5, %zmm8, %zmm11 {%k1}
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = (~zmm14 & ~zmm7 & zmm24) | (zmm14 & ~zmm7 & zmm24) | (zmm14 & zmm7 & ~zmm24) | (zmm14 & zmm7 & zmm24)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm24 ^ (zmm7 & (zmm14 ^ zmm24))
 ; AVX512-FCP-NEXT:    vpandnq %zmm25, %zmm2, %zmm5
 ; AVX512-FCP-NEXT:    vpandq %zmm2, %zmm26, %zmm8
 ; AVX512-FCP-NEXT:    vpord %zmm5, %zmm8, %zmm14 {%k1}
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & ~zmm7 & zmm27) | (zmm0 & ~zmm7 & zmm27) | (zmm0 & zmm7 & ~zmm27) | (zmm0 & zmm7 & zmm27)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm27 ^ (zmm7 & (zmm0 ^ zmm27))
 ; AVX512-FCP-NEXT:    vpandnq %zmm28, %zmm2, %zmm5
 ; AVX512-FCP-NEXT:    vpandq %zmm2, %zmm29, %zmm8
 ; AVX512-FCP-NEXT:    vpord %zmm5, %zmm8, %zmm0 {%k1}
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm7 & zmm30) | (zmm1 & ~zmm7 & zmm30) | (zmm1 & zmm7 & ~zmm30) | (zmm1 & zmm7 & zmm30)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm30 ^ (zmm7 & (zmm1 ^ zmm30))
 ; AVX512-FCP-NEXT:    vpandnq %zmm31, %zmm2, %zmm5
 ; AVX512-FCP-NEXT:    vpandq %zmm2, %zmm16, %zmm8
 ; AVX512-FCP-NEXT:    vpord %zmm5, %zmm8, %zmm1 {%k1}
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & ~zmm7 & zmm19) | (zmm4 & ~zmm7 & zmm19) | (zmm4 & zmm7 & ~zmm19) | (zmm4 & zmm7 & zmm19)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm19 ^ (zmm7 & (zmm4 ^ zmm19))
 ; AVX512-FCP-NEXT:    vpandnq %zmm13, %zmm2, %zmm5
 ; AVX512-FCP-NEXT:    vpandq %zmm2, %zmm21, %zmm2
 ; AVX512-FCP-NEXT:    vpord %zmm5, %zmm2, %zmm4 {%k1}
@@ -8002,7 +8002,7 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    # ymm5 = mem[2,1,3,3,6,5,7,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm10
 ; AVX512DQ-NEXT:    vpbroadcastq {{.*#+}} zmm13 = [65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm10 = (~zmm10 & zmm13 & zmm3) | (zmm10 & ~zmm13 & ~zmm3) | (zmm10 & ~zmm13 & zmm3) | (zmm10 & zmm13 & zmm3)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm13 & (zmm10 ^ zmm3))
 ; AVX512DQ-NEXT:    vpshufd $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    # ymm3 = mem[0,0,2,1,4,4,6,5]
 ; AVX512DQ-NEXT:    vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
@@ -8029,7 +8029,7 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    # ymm2 = mem[0,1,1,3,4,5,5,7]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm9[2,1,3,3,6,5,7,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm9
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm9 = (~zmm9 & zmm13 & zmm0) | (zmm9 & ~zmm13 & ~zmm0) | (zmm9 & ~zmm13 & zmm0) | (zmm9 & zmm13 & zmm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 ^ (zmm13 & (zmm9 ^ zmm0))
 ; AVX512DQ-NEXT:    vpshufd $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    # ymm0 = mem[0,0,2,1,4,4,6,5]
 ; AVX512DQ-NEXT:    vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
@@ -8056,7 +8056,7 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm11[0,1,1,3,4,5,5,7]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm31[2,1,3,3,6,5,7,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm11
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm11 = (~zmm11 & zmm13 & zmm0) | (zmm11 & ~zmm13 & ~zmm0) | (zmm11 & ~zmm13 & zmm0) | (zmm11 & zmm13 & zmm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 ^ (zmm13 & (zmm11 ^ zmm0))
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm28[0,0,2,1,4,4,6,5]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm23[0,2,2,3,4,6,6,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
@@ -8090,7 +8090,7 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm30[0,1,1,3,4,5,5,7]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm6 = ymm29[2,1,3,3,6,5,7,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm4
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & zmm13 & zmm0) | (zmm4 & ~zmm13 & ~zmm0) | (zmm4 & ~zmm13 & zmm0) | (zmm4 & zmm13 & zmm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm13 & (zmm4 ^ zmm0))
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm24[0,0,2,1,4,4,6,5]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm6 = ymm22[0,2,2,3,4,6,6,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm0
@@ -8167,7 +8167,7 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpshufd $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm23 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    # ymm23 = mem[2,1,3,3,6,5,7,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm23, %zmm5, %zmm5
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = (~zmm5 & zmm13 & zmm12) | (zmm5 & ~zmm13 & ~zmm12) | (zmm5 & ~zmm13 & zmm12) | (zmm5 & zmm13 & zmm12)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm13 & (zmm5 ^ zmm12))
 ; AVX512DQ-NEXT:    vpshufd $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    # ymm12 = mem[0,0,2,1,4,4,6,5]
 ; AVX512DQ-NEXT:    vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %ymm23 # 32-byte Folded Reload
@@ -8203,7 +8203,7 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpshufd $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm30 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    # ymm30 = mem[2,1,3,3,6,5,7,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm30, %zmm12, %zmm12
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm12 = (~zmm12 & zmm13 & zmm0) | (zmm12 & ~zmm13 & ~zmm0) | (zmm12 & ~zmm13 & zmm0) | (zmm12 & zmm13 & zmm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm12 ^ (zmm13 & (zmm12 ^ zmm0))
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm30 = xmm2[3,3,3,3]
 ; AVX512DQ-NEXT:    vpmovzxwq {{.*#+}} xmm31 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
@@ -8230,7 +8230,7 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm16 = ymm28[0,1,1,3,4,5,5,7]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm18 = ymm18[2,1,3,3,6,5,7,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm18, %zmm16, %zmm16
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm16 = (~zmm16 & zmm13 & zmm7) | (zmm16 & ~zmm13 & ~zmm7) | (zmm16 & ~zmm13 & zmm7) | (zmm16 & zmm13 & zmm7)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm16 = zmm16 ^ (zmm13 & (zmm16 ^ zmm7))
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm7 = ymm21[0,0,2,1,4,4,6,5]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm18 = ymm22[0,2,2,3,4,6,6,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm18, %zmm7, %zmm7
@@ -8249,7 +8249,7 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm14[0,1,1,3,4,5,5,7]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm6 = ymm15[2,1,3,3,6,5,7,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm2, %zmm2
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = (~zmm2 & zmm13 & zmm0) | (zmm2 & ~zmm13 & ~zmm0) | (zmm2 & ~zmm13 & zmm0) | (zmm2 & zmm13 & zmm0)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm13 & (zmm2 ^ zmm0))
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm17[0,0,2,1,4,4,6,5]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm6 = ymm20[0,2,2,3,4,6,6,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm0
@@ -8440,7 +8440,7 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm3
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} zmm7 = [65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535]
 ; AVX512DQ-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm3 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm3 = (~zmm3 & ~zmm7 & mem) | (zmm3 & ~zmm7 & mem) | (zmm3 & zmm7 & ~mem) | (zmm3 & zmm7 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm3 = mem ^ (zmm7 & (zmm3 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} zmm2 = [65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0,65535,65535,65535,0]
 ; AVX512DQ-FCP-NEXT:    vpandnq %zmm19, %zmm2, %zmm19
 ; AVX512DQ-FCP-NEXT:    vpandq %zmm2, %zmm21, %zmm21
@@ -8524,32 +8524,32 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm8, %ymm5
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm5, %zmm4
 ; AVX512DQ-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm6 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm6 = (~zmm6 & ~zmm7 & mem) | (zmm6 & ~zmm7 & mem) | (zmm6 & zmm7 & ~mem) | (zmm6 & zmm7 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm6 = mem ^ (zmm7 & (zmm6 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vpandnq {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm5 # 64-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vpandq {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm8 # 64-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vpord %zmm5, %zmm8, %zmm6 {%k1}
 ; AVX512DQ-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm15 # 64-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    # zmm15 = (~zmm15 & ~zmm7 & mem) | (zmm15 & ~zmm7 & mem) | (zmm15 & zmm7 & ~mem) | (zmm15 & zmm7 & mem)
+; AVX512DQ-FCP-NEXT:    # zmm15 = mem ^ (zmm7 & (zmm15 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vpandnq (%rsp), %zmm2, %zmm5 # 64-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vpandq {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm8 # 64-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vpord %zmm5, %zmm8, %zmm15 {%k1}
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm11 = (~zmm11 & ~zmm7 & zmm20) | (zmm11 & ~zmm7 & zmm20) | (zmm11 & zmm7 & ~zmm20) | (zmm11 & zmm7 & zmm20)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm20 ^ (zmm7 & (zmm11 ^ zmm20))
 ; AVX512DQ-FCP-NEXT:    vpandnq %zmm22, %zmm2, %zmm5
 ; AVX512DQ-FCP-NEXT:    vpandq %zmm2, %zmm23, %zmm8
 ; AVX512DQ-FCP-NEXT:    vpord %zmm5, %zmm8, %zmm11 {%k1}
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = (~zmm14 & ~zmm7 & zmm24) | (zmm14 & ~zmm7 & zmm24) | (zmm14 & zmm7 & ~zmm24) | (zmm14 & zmm7 & zmm24)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm24 ^ (zmm7 & (zmm14 ^ zmm24))
 ; AVX512DQ-FCP-NEXT:    vpandnq %zmm25, %zmm2, %zmm5
 ; AVX512DQ-FCP-NEXT:    vpandq %zmm2, %zmm26, %zmm8
 ; AVX512DQ-FCP-NEXT:    vpord %zmm5, %zmm8, %zmm14 {%k1}
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = (~zmm0 & ~zmm7 & zmm27) | (zmm0 & ~zmm7 & zmm27) | (zmm0 & zmm7 & ~zmm27) | (zmm0 & zmm7 & zmm27)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm27 ^ (zmm7 & (zmm0 ^ zmm27))
 ; AVX512DQ-FCP-NEXT:    vpandnq %zmm28, %zmm2, %zmm5
 ; AVX512DQ-FCP-NEXT:    vpandq %zmm2, %zmm29, %zmm8
 ; AVX512DQ-FCP-NEXT:    vpord %zmm5, %zmm8, %zmm0 {%k1}
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = (~zmm1 & ~zmm7 & zmm30) | (zmm1 & ~zmm7 & zmm30) | (zmm1 & zmm7 & ~zmm30) | (zmm1 & zmm7 & zmm30)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm30 ^ (zmm7 & (zmm1 ^ zmm30))
 ; AVX512DQ-FCP-NEXT:    vpandnq %zmm31, %zmm2, %zmm5
 ; AVX512DQ-FCP-NEXT:    vpandq %zmm2, %zmm16, %zmm8
 ; AVX512DQ-FCP-NEXT:    vpord %zmm5, %zmm8, %zmm1 {%k1}
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (~zmm4 & ~zmm7 & zmm19) | (zmm4 & ~zmm7 & zmm19) | (zmm4 & zmm7 & ~zmm19) | (zmm4 & zmm7 & zmm19)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm19 ^ (zmm7 & (zmm4 ^ zmm19))
 ; AVX512DQ-FCP-NEXT:    vpandnq %zmm13, %zmm2, %zmm5
 ; AVX512DQ-FCP-NEXT:    vpandq %zmm2, %zmm21, %zmm2
 ; AVX512DQ-FCP-NEXT:    vpord %zmm5, %zmm2, %zmm4 {%k1}
diff --git a/llvm/test/CodeGen/X86/vector-pack-128.ll b/llvm/test/CodeGen/X86/vector-pack-128.ll
index d11e1a70b684c..f58f19ecd2481 100644
--- a/llvm/test/CodeGen/X86/vector-pack-128.ll
+++ b/llvm/test/CodeGen/X86/vector-pack-128.ll
@@ -327,7 +327,7 @@ define <16 x i8> @concat_trunc_packuswb_128(<8 x i16> %a0, <8 x i16> %a1) nounwi
 define void @autogen_SD10339(<1 x i32> %I49) {
 ; CHECK-LABEL: autogen_SD10339:
 ; CHECK:       # %bb.0: # %BB
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB8_1: # %CF
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movw $1, 0
diff --git a/llvm/test/CodeGen/X86/vector-replicaton-i1-mask.ll b/llvm/test/CodeGen/X86/vector-replicaton-i1-mask.ll
index 671ef61fd3c9c..834ea7385d312 100644
--- a/llvm/test/CodeGen/X86/vector-replicaton-i1-mask.ll
+++ b/llvm/test/CodeGen/X86/vector-replicaton-i1-mask.ll
@@ -97,7 +97,7 @@ define void @mask_replication_factor2_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512F-ONLY-LABEL: mask_replication_factor2_vf8:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm0
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm0, %zmm0, %k1
@@ -121,7 +121,7 @@ define void @mask_replication_factor2_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512BW-LABEL: mask_replication_factor2_vf8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    kmovw (%rdi), %k1
-; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512BW-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; AVX512BW-NEXT:    vpermd %zmm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vptestmd %zmm0, %zmm0, %k1
@@ -141,7 +141,7 @@ define void @mask_replication_factor2_vf16(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-LABEL: mask_replication_factor2_vf16:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k1
@@ -199,14 +199,14 @@ define void @mask_replication_factor2_vf32(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
 ; AVX512F-ONLY-NEXT:    kmovw 2(%rdi), %k2
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k2} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k2} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm2
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm2, %zmm2, %k2
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm2, %zmm0
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm0, %zmm0, %k3
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k1
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm2, %zmm0
@@ -305,7 +305,7 @@ define void @mask_replication_factor2_vf64(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-NEXT:    kmovw 2(%rdi), %k5
 ; AVX512F-ONLY-NEXT:    kmovw 4(%rdi), %k4
 ; AVX512F-ONLY-NEXT:    kmovw 6(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm2
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm2, %zmm2, %k1
@@ -313,17 +313,17 @@ define void @mask_replication_factor2_vf64(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm2, %zmm0
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm0, %zmm0, %k2
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k4} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k4} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm3
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm3, %zmm3, %k4
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm2, %zmm0
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm0, %zmm0, %k6
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k5} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k5} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm3
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm3, %zmm3, %k5
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm2, %zmm0
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm0, %zmm0, %k7
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k3} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k3} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k3
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm2, %zmm0
@@ -541,7 +541,7 @@ define void @mask_replication_factor3_vf4(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512F-ONLY-LABEL: mask_replication_factor3_vf4:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,1,1,1,2,2,2,3,3,3,0,0,0,0]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm0
 ; AVX512F-ONLY-NEXT:    vpslld $31, %zmm0, %zmm0
@@ -573,7 +573,7 @@ define void @mask_replication_factor3_vf4(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512BW-LABEL: mask_replication_factor3_vf4:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    kmovw (%rdi), %k1
-; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512BW-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,1,1,1,2,2,2,3,3,3,0,0,0,0]
 ; AVX512BW-NEXT:    vpermd %zmm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vpslld $31, %zmm0, %zmm0
@@ -598,11 +598,11 @@ define void @mask_replication_factor3_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512F-ONLY-LABEL: mask_replication_factor3_vf8:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,1,1,1,2,2,2,3,3,3,4,4,4,5]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k2
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm1 {%k2} {z} = -1
 ; AVX512F-ONLY-NEXT:    movw $1, %ax
 ; AVX512F-ONLY-NEXT:    kmovw %eax, %k2
 ; AVX512F-ONLY-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k2}
@@ -672,11 +672,11 @@ define void @mask_replication_factor3_vf16(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-LABEL: mask_replication_factor3_vf16:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,1,1,1,2,2,2,3,3,3,4,4,4,5]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    movw $1, %ax
 ; AVX512F-ONLY-NEXT:    kmovw %eax, %k1
 ; AVX512F-ONLY-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
@@ -726,7 +726,7 @@ define void @mask_replication_factor3_vf16(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512BW-LABEL: mask_replication_factor3_vf16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    kmovw (%rdi), %k1
-; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512BW-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,1,1,1,2,2,2,3,3,3,4,4,4,5]
 ; AVX512BW-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k1
@@ -757,11 +757,11 @@ define void @mask_replication_factor3_vf32(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k2
 ; AVX512F-ONLY-NEXT:    kmovw 2(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k2} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k2} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,1,1,1,2,2,2,3,3,3,4,4,4,5]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm2
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm2, %zmm2, %k2
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k2} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm2 {%k2} {z} = -1
 ; AVX512F-ONLY-NEXT:    movw $1, %ax
 ; AVX512F-ONLY-NEXT:    kmovw %eax, %k2
 ; AVX512F-ONLY-NEXT:    vmovdqa32 %zmm0, %zmm2 {%k2}
@@ -772,7 +772,7 @@ define void @mask_replication_factor3_vf32(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm3 = [10,11,11,11,12,12,12,13,13,13,14,14,14,15,15,15]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm3, %zmm0
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm0, %zmm0, %k4
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k1
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm2, %zmm1
@@ -1307,20 +1307,20 @@ define void @mask_replication_factor3_vf64(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-LABEL: mask_replication_factor3_vf64:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,1,1,1,2,2,2,3,3,3,4,4,4,5]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm2
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm2, %zmm2, %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    movw $1, %ax
 ; AVX512F-ONLY-NEXT:    kmovw %eax, %k1
 ; AVX512F-ONLY-NEXT:    vmovdqa32 %zmm0, %zmm2 {%k1}
 ; AVX512F-ONLY-NEXT:    kmovw 2(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm3 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    kmovw 4(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm4, %zmm4, %zmm4 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm4 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    kmovw 6(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm5, %zmm5, %zmm5 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm5 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm2, %zmm2, %k1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [5,5,6,6,6,7,7,7,8,8,8,9,9,9,10,10]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm2, %zmm6
@@ -2446,7 +2446,7 @@ define void @mask_replication_factor4_vf4(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512F-ONLY-LABEL: mask_replication_factor4_vf4:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm0
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm0, %zmm0, %k1
@@ -2470,7 +2470,7 @@ define void @mask_replication_factor4_vf4(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512BW-LABEL: mask_replication_factor4_vf4:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    kmovw (%rdi), %k1
-; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512BW-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3]
 ; AVX512BW-NEXT:    vpermd %zmm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vptestmd %zmm0, %zmm0, %k1
@@ -2490,7 +2490,7 @@ define void @mask_replication_factor4_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512F-ONLY-LABEL: mask_replication_factor4_vf8:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [4,4,4,4,5,5,5,5,6,6,6,6,7,7,7,7]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k1
@@ -2547,7 +2547,7 @@ define void @mask_replication_factor4_vf16(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-LABEL: mask_replication_factor4_vf16:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [8,8,8,8,9,9,9,9,10,10,10,10,11,11,11,11]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k1
@@ -2652,7 +2652,7 @@ define void @mask_replication_factor4_vf32(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k4
 ; AVX512F-ONLY-NEXT:    kmovw 2(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [8,8,8,8,9,9,9,9,10,10,10,10,11,11,11,11]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm2
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm2, %zmm2, %k1
@@ -2666,7 +2666,7 @@ define void @mask_replication_factor4_vf32(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [4,4,4,4,5,5,5,5,6,6,6,6,7,7,7,7]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm4, %zmm0
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm0, %zmm0, %k5
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k4} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k4} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k4
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm2, %zmm1
@@ -2824,11 +2824,11 @@ define void @mask_replication_factor4_vf64(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-LABEL: mask_replication_factor4_vf64:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw 6(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    kmovw 4(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    kmovw 2(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm3 = [12,12,12,12,13,13,13,13,14,14,14,14,15,15,15,15]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm3, %zmm4
@@ -2846,7 +2846,7 @@ define void @mask_replication_factor4_vf64(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-NEXT:    vpermd %zmm2, %zmm5, %zmm14
 ; AVX512F-ONLY-NEXT:    vpermd %zmm2, %zmm7, %zmm15
 ; AVX512F-ONLY-NEXT:    vpermd %zmm2, %zmm9, %zmm2
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm16, %zmm16, %zmm16 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm16 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpermd %zmm16, %zmm3, %zmm3
 ; AVX512F-ONLY-NEXT:    vpermd %zmm16, %zmm5, %zmm5
 ; AVX512F-ONLY-NEXT:    vpermd %zmm16, %zmm7, %zmm7
@@ -3120,7 +3120,7 @@ define void @mask_replication_factor5_vf2(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512F-ONLY-LABEL: mask_replication_factor5_vf2:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm0
 ; AVX512F-ONLY-NEXT:    vpslld $31, %zmm0, %zmm0
@@ -3154,7 +3154,7 @@ define void @mask_replication_factor5_vf2(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512BW-LABEL: mask_replication_factor5_vf2:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    kmovw (%rdi), %k1
-; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512BW-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0]
 ; AVX512BW-NEXT:    vpermd %zmm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vpslld $31, %zmm0, %zmm0
@@ -3180,7 +3180,7 @@ define void @mask_replication_factor5_vf4(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512F-ONLY-LABEL: mask_replication_factor5_vf4:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
 ; AVX512F-ONLY-NEXT:    vpslld $31, %zmm1, %zmm1
 ; AVX512F-ONLY-NEXT:    movw $15, %ax
@@ -3245,11 +3245,11 @@ define void @mask_replication_factor5_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512F-ONLY-LABEL: mask_replication_factor5_vf8:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,1,1,1,1,1,2,2,2,2,2,3]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k2
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm1 {%k2} {z} = -1
 ; AVX512F-ONLY-NEXT:    movw $1, %ax
 ; AVX512F-ONLY-NEXT:    kmovw %eax, %k2
 ; AVX512F-ONLY-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k2}
@@ -3353,11 +3353,11 @@ define void @mask_replication_factor5_vf16(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-LABEL: mask_replication_factor5_vf16:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,1,1,1,1,1,2,2,2,2,2,3]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    movw $1, %ax
 ; AVX512F-ONLY-NEXT:    kmovw %eax, %k1
 ; AVX512F-ONLY-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
@@ -3427,7 +3427,7 @@ define void @mask_replication_factor5_vf16(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512BW-LABEL: mask_replication_factor5_vf16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    kmovw (%rdi), %k1
-; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512BW-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,1,1,1,1,1,2,2,2,2,2,3]
 ; AVX512BW-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k1
@@ -3467,16 +3467,16 @@ define void @mask_replication_factor5_vf32(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-LABEL: mask_replication_factor5_vf32:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,1,1,1,1,1,2,2,2,2,2,3]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm2
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm2, %zmm2, %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    movw $1, %ax
 ; AVX512F-ONLY-NEXT:    kmovw %eax, %k1
 ; AVX512F-ONLY-NEXT:    vmovdqa32 %zmm0, %zmm2 {%k1}
 ; AVX512F-ONLY-NEXT:    kmovw 2(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm3 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm2, %zmm2, %k1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [3,3,3,3,4,4,4,4,4,5,5,5,5,5,6,6]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm2, %zmm4
@@ -4306,20 +4306,20 @@ define void @mask_replication_factor5_vf64(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-LABEL: mask_replication_factor5_vf64:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm3 = [0,0,0,0,0,1,1,1,1,1,2,2,2,2,2,3]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm3, %zmm1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    movw $1, %ax
 ; AVX512F-ONLY-NEXT:    kmovw %eax, %k1
 ; AVX512F-ONLY-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
 ; AVX512F-ONLY-NEXT:    kmovw 6(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm4, %zmm4, %zmm4 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm4 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    kmovw 4(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm5, %zmm5, %zmm5 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm5 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    kmovw 2(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm6, %zmm6, %zmm6 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm6 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm7 = [12,13,13,13,13,13,14,14,14,14,14,15,15,15,15,15]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm4, %zmm7, %zmm1
@@ -5856,7 +5856,7 @@ define void @mask_replication_factor6_vf2(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512F-ONLY-LABEL: mask_replication_factor6_vf2:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm0
 ; AVX512F-ONLY-NEXT:    vpslld $31, %zmm0, %zmm0
@@ -5888,7 +5888,7 @@ define void @mask_replication_factor6_vf2(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512BW-LABEL: mask_replication_factor6_vf2:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    kmovw (%rdi), %k1
-; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512BW-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0]
 ; AVX512BW-NEXT:    vpermd %zmm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vpslld $31, %zmm0, %zmm0
@@ -5913,7 +5913,7 @@ define void @mask_replication_factor6_vf4(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512F-SLOW-LABEL: mask_replication_factor6_vf4:
 ; AVX512F-SLOW:       # %bb.0:
 ; AVX512F-SLOW-NEXT:    kmovw (%rdi), %k1
-; AVX512F-SLOW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-SLOW-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
 ; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,1]
 ; AVX512F-SLOW-NEXT:    vpslld $31, %zmm1, %zmm1
@@ -5933,7 +5933,7 @@ define void @mask_replication_factor6_vf4(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512F-FAST-LABEL: mask_replication_factor6_vf4:
 ; AVX512F-FAST:       # %bb.0:
 ; AVX512F-FAST-NEXT:    kmovw (%rdi), %k1
-; AVX512F-FAST-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-FAST-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-FAST-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [2,2,3,3,3,3,3,3]
 ; AVX512F-FAST-NEXT:    vpermd %ymm0, %ymm1, %ymm1
 ; AVX512F-FAST-NEXT:    vpslld $31, %zmm1, %zmm1
@@ -6020,11 +6020,11 @@ define void @mask_replication_factor6_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512F-ONLY-LABEL: mask_replication_factor6_vf8:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    movw $1, %ax
 ; AVX512F-ONLY-NEXT:    kmovw %eax, %k1
 ; AVX512F-ONLY-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
@@ -6074,7 +6074,7 @@ define void @mask_replication_factor6_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512BW-LABEL: mask_replication_factor6_vf8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    kmovw (%rdi), %k1
-; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512BW-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2]
 ; AVX512BW-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k1
@@ -6104,11 +6104,11 @@ define void @mask_replication_factor6_vf16(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-LABEL: mask_replication_factor6_vf16:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    movw $1, %ax
 ; AVX512F-ONLY-NEXT:    kmovw %eax, %k1
 ; AVX512F-ONLY-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
@@ -6188,7 +6188,7 @@ define void @mask_replication_factor6_vf16(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512BW-LABEL: mask_replication_factor6_vf16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    kmovw (%rdi), %k1
-; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512BW-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2]
 ; AVX512BW-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k1
@@ -6233,16 +6233,16 @@ define void @mask_replication_factor6_vf32(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-LABEL: mask_replication_factor6_vf32:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm2
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm2, %zmm2, %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    movw $1, %ax
 ; AVX512F-ONLY-NEXT:    kmovw %eax, %k1
 ; AVX512F-ONLY-NEXT:    vmovdqa32 %zmm0, %zmm2 {%k1}
 ; AVX512F-ONLY-NEXT:    kmovw 2(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm3 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm2, %zmm2, %k1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [2,2,3,3,3,3,3,3,4,4,4,4,4,4,5,5]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm2, %zmm4
@@ -7224,20 +7224,20 @@ define void @mask_replication_factor6_vf64(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-LABEL: mask_replication_factor6_vf64:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm4, %zmm1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    movw $1, %ax
 ; AVX512F-ONLY-NEXT:    kmovw %eax, %k1
 ; AVX512F-ONLY-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
 ; AVX512F-ONLY-NEXT:    kmovw 6(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm7, %zmm7, %zmm7 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm7 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    kmovw 4(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm8, %zmm8, %zmm8 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm8 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    kmovw 2(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm9, %zmm9, %zmm9 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm9 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm10 = [13,13,13,13,14,14,14,14,14,14,15,15,15,15,15,15]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm7, %zmm10, %zmm1
@@ -9070,7 +9070,7 @@ define void @mask_replication_factor7_vf2(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512F-ONLY-LABEL: mask_replication_factor7_vf2:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,0,0,1,1,1,1,1,1,1,0,0]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm0
 ; AVX512F-ONLY-NEXT:    vpslld $31, %zmm0, %zmm0
@@ -9106,7 +9106,7 @@ define void @mask_replication_factor7_vf2(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512BW-LABEL: mask_replication_factor7_vf2:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    kmovw (%rdi), %k1
-; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512BW-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,0,0,1,1,1,1,1,1,1,0,0]
 ; AVX512BW-NEXT:    vpermd %zmm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vpslld $31, %zmm0, %zmm0
@@ -9133,7 +9133,7 @@ define void @mask_replication_factor7_vf4(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512F-ONLY-LABEL: mask_replication_factor7_vf4:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [2,2,2,2,2,3,3,3,3,3,3,3,0,0,0,0]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512F-ONLY-NEXT:    vpslld $31, %zmm1, %zmm1
@@ -9203,11 +9203,11 @@ define void @mask_replication_factor7_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512F-SLOW-LABEL: mask_replication_factor7_vf8:
 ; AVX512F-SLOW:       # %bb.0:
 ; AVX512F-SLOW-NEXT:    kmovw (%rdi), %k1
-; AVX512F-SLOW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-SLOW-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-SLOW-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,0,0,1,1,1,1,1,1,1,2,2]
 ; AVX512F-SLOW-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512F-SLOW-NEXT:    vptestmd %zmm1, %zmm1, %k2
-; AVX512F-SLOW-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
+; AVX512F-SLOW-NEXT:    vpternlogd {{.*#+}} zmm1 {%k2} {z} = -1
 ; AVX512F-SLOW-NEXT:    movw $1, %ax
 ; AVX512F-SLOW-NEXT:    kmovw %eax, %k2
 ; AVX512F-SLOW-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k2}
@@ -9237,11 +9237,11 @@ define void @mask_replication_factor7_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512F-FAST-LABEL: mask_replication_factor7_vf8:
 ; AVX512F-FAST:       # %bb.0:
 ; AVX512F-FAST-NEXT:    kmovw (%rdi), %k1
-; AVX512F-FAST-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-FAST-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-FAST-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,0,0,1,1,1,1,1,1,1,2,2]
 ; AVX512F-FAST-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512F-FAST-NEXT:    vptestmd %zmm1, %zmm1, %k2
-; AVX512F-FAST-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
+; AVX512F-FAST-NEXT:    vpternlogd {{.*#+}} zmm1 {%k2} {z} = -1
 ; AVX512F-FAST-NEXT:    movw $1, %ax
 ; AVX512F-FAST-NEXT:    kmovw %eax, %k2
 ; AVX512F-FAST-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k2}
@@ -9394,11 +9394,11 @@ define void @mask_replication_factor7_vf16(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-LABEL: mask_replication_factor7_vf16:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,0,0,1,1,1,1,1,1,1,2,2]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    movw $1, %ax
 ; AVX512F-ONLY-NEXT:    kmovw %eax, %k1
 ; AVX512F-ONLY-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
@@ -9488,7 +9488,7 @@ define void @mask_replication_factor7_vf16(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512BW-LABEL: mask_replication_factor7_vf16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    kmovw (%rdi), %k1
-; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512BW-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,0,0,1,1,1,1,1,1,1,2,2]
 ; AVX512BW-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k1
@@ -9538,16 +9538,16 @@ define void @mask_replication_factor7_vf32(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-LABEL: mask_replication_factor7_vf32:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,0,0,1,1,1,1,1,1,1,2,2]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm2
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm2, %zmm2, %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    movw $1, %ax
 ; AVX512F-ONLY-NEXT:    kmovw %eax, %k1
 ; AVX512F-ONLY-NEXT:    vmovdqa32 %zmm0, %zmm2 {%k1}
 ; AVX512F-ONLY-NEXT:    kmovw 2(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm3 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm2, %zmm2, %k1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [2,2,2,2,2,3,3,3,3,3,3,3,4,4,4,4]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm2, %zmm4
@@ -10676,20 +10676,20 @@ define void @mask_replication_factor7_vf64(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-LABEL: mask_replication_factor7_vf64:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm5 = [0,0,0,0,0,0,0,1,1,1,1,1,1,1,2,2]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm1, %zmm5, %zmm0
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm0, %zmm0, %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    movw $1, %ax
 ; AVX512F-ONLY-NEXT:    kmovw %eax, %k1
 ; AVX512F-ONLY-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
 ; AVX512F-ONLY-NEXT:    kmovw 6(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm8, %zmm8, %zmm8 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm8 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    kmovw 4(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm9, %zmm9, %zmm9 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm9 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    kmovw 2(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm11, %zmm11, %zmm11 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm11 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm0, %zmm0, %k1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm13 = [13,13,14,14,14,14,14,14,14,15,15,15,15,15,15,15]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm8, %zmm13, %zmm0
@@ -12788,7 +12788,7 @@ define void @mask_replication_factor8_vf2(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512F-ONLY-LABEL: mask_replication_factor8_vf2:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm0
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm0, %zmm0, %k1
@@ -12812,7 +12812,7 @@ define void @mask_replication_factor8_vf2(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512BW-LABEL: mask_replication_factor8_vf2:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    kmovw (%rdi), %k1
-; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512BW-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
 ; AVX512BW-NEXT:    vpermd %zmm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vptestmd %zmm0, %zmm0, %k1
@@ -12832,7 +12832,7 @@ define void @mask_replication_factor8_vf4(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512F-ONLY-LABEL: mask_replication_factor8_vf4:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k1
@@ -12889,7 +12889,7 @@ define void @mask_replication_factor8_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out
 ; AVX512F-ONLY-LABEL: mask_replication_factor8_vf8:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k1
@@ -12993,7 +12993,7 @@ define void @mask_replication_factor8_vf16(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-LABEL: mask_replication_factor8_vf16:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [12,12,12,12,12,12,12,12,13,13,13,13,13,13,13,13]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm0, %zmm1, %zmm1
 ; AVX512F-ONLY-NEXT:    vptestmd %zmm1, %zmm1, %k1
@@ -13133,7 +13133,7 @@ define void @mask_replication_factor8_vf32(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-LABEL: mask_replication_factor8_vf32:
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    kmovw 2(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [14,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm1, %zmm2, %zmm0
@@ -13151,7 +13151,7 @@ define void @mask_replication_factor8_vf32(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-NEXT:    vpermd %zmm1, %zmm13, %zmm14
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm15 = [0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm1, %zmm15, %zmm1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm16, %zmm16, %zmm16 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm16 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpermd %zmm16, %zmm2, %zmm2
 ; AVX512F-ONLY-NEXT:    vpermd %zmm16, %zmm3, %zmm3
 ; AVX512F-ONLY-NEXT:    vpermd %zmm16, %zmm5, %zmm5
@@ -13367,11 +13367,11 @@ define void @mask_replication_factor8_vf64(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY:       # %bb.0:
 ; AVX512F-ONLY-NEXT:    subq $136, %rsp
 ; AVX512F-ONLY-NEXT:    kmovw 6(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm6, %zmm6, %zmm6 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm6 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    kmovw 4(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm8, %zmm8, %zmm8 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm8 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    kmovw 2(%rdi), %k1
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm10, %zmm10, %zmm10 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm10 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    kmovw (%rdi), %k1
 ; AVX512F-ONLY-NEXT:    vpmovsxbd {{.*#+}} zmm12 = [14,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15]
 ; AVX512F-ONLY-NEXT:    vpermd %zmm6, %zmm12, %zmm0
@@ -13409,7 +13409,7 @@ define void @mask_replication_factor8_vf64(ptr %in.maskvec, ptr %in.vec, ptr %ou
 ; AVX512F-ONLY-NEXT:    vpermd %zmm10, %zmm22, %zmm3
 ; AVX512F-ONLY-NEXT:    vpermd %zmm10, %zmm24, %zmm6
 ; AVX512F-ONLY-NEXT:    vpermd %zmm10, %zmm26, %zmm2
-; AVX512F-ONLY-NEXT:    vpternlogd $255, %zmm8, %zmm8, %zmm8 {%k1} {z}
+; AVX512F-ONLY-NEXT:    vpternlogd {{.*#+}} zmm8 {%k1} {z} = -1
 ; AVX512F-ONLY-NEXT:    vpermd %zmm8, %zmm12, %zmm1
 ; AVX512F-ONLY-NEXT:    vpermd %zmm8, %zmm14, %zmm0
 ; AVX512F-ONLY-NEXT:    vpermd %zmm8, %zmm16, %zmm16
diff --git a/llvm/test/CodeGen/X86/vector-sext.ll b/llvm/test/CodeGen/X86/vector-sext.ll
index d44b11f4ca1da..7673f09b2ba29 100644
--- a/llvm/test/CodeGen/X86/vector-sext.ll
+++ b/llvm/test/CodeGen/X86/vector-sext.ll
@@ -1386,7 +1386,7 @@ define <2 x i64> @load_sext_2i1_to_2i64(ptr%ptr) {
 ; AVX512F:       # %bb.0: # %entry
 ; AVX512F-NEXT:    movzbl (%rdi), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1395,7 +1395,7 @@ define <2 x i64> @load_sext_2i1_to_2i64(ptr%ptr) {
 ; AVX512BW:       # %bb.0: # %entry
 ; AVX512BW-NEXT:    movzbl (%rdi), %eax
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1633,7 +1633,7 @@ define <4 x i32> @load_sext_4i1_to_4i32(ptr%ptr) {
 ; AVX512F:       # %bb.0: # %entry
 ; AVX512F-NEXT:    movzbl (%rdi), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1642,7 +1642,7 @@ define <4 x i32> @load_sext_4i1_to_4i32(ptr%ptr) {
 ; AVX512BW:       # %bb.0: # %entry
 ; AVX512BW-NEXT:    movzbl (%rdi), %eax
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1899,7 +1899,7 @@ define <4 x i64> @load_sext_4i1_to_4i64(ptr%ptr) {
 ; AVX512F:       # %bb.0: # %entry
 ; AVX512F-NEXT:    movzbl (%rdi), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512F-NEXT:    retq
 ;
@@ -1907,7 +1907,7 @@ define <4 x i64> @load_sext_4i1_to_4i64(ptr%ptr) {
 ; AVX512BW:       # %bb.0: # %entry
 ; AVX512BW-NEXT:    movzbl (%rdi), %eax
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512BW-NEXT:    retq
 ;
@@ -2177,7 +2177,7 @@ define <8 x i16> @load_sext_8i1_to_8i16(ptr%ptr) {
 ; AVX512F:       # %bb.0: # %entry
 ; AVX512F-NEXT:    movzbl (%rdi), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512F-NEXT:    vzeroupper
@@ -2436,7 +2436,7 @@ define <8 x i32> @load_sext_8i1_to_8i32(ptr%ptr) {
 ; AVX512F:       # %bb.0: # %entry
 ; AVX512F-NEXT:    movzbl (%rdi), %eax
 ; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512F-NEXT:    retq
 ;
@@ -2444,7 +2444,7 @@ define <8 x i32> @load_sext_8i1_to_8i32(ptr%ptr) {
 ; AVX512BW:       # %bb.0: # %entry
 ; AVX512BW-NEXT:    movzbl (%rdi), %eax
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512BW-NEXT:    retq
 ;
@@ -2607,7 +2607,7 @@ define <16 x i8> @load_sext_16i1_to_16i8(ptr%ptr) nounwind readnone {
 ; AVX512F-LABEL: load_sext_16i1_to_16i8:
 ; AVX512F:       # %bb.0: # %entry
 ; AVX512F-NEXT:    kmovw (%rdi), %k1
-; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -2721,7 +2721,7 @@ define <16 x i16> @load_sext_16i1_to_16i16(ptr%ptr) {
 ; AVX512F-LABEL: load_sext_16i1_to_16i16:
 ; AVX512F:       # %bb.0: # %entry
 ; AVX512F-NEXT:    kmovw (%rdi), %k1
-; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512F-NEXT:    retq
 ;
@@ -2816,9 +2816,9 @@ define <32 x i8> @load_sext_32i1_to_32i8(ptr%ptr) nounwind readnone {
 ; AVX512F:       # %bb.0: # %entry
 ; AVX512F-NEXT:    kmovw (%rdi), %k1
 ; AVX512F-NEXT:    kmovw 2(%rdi), %k2
-; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
+; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm1 {%k2} {z} = -1
 ; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
 ; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll b/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
index e33c99be0ed09..177d8f78b5a8d 100644
--- a/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
@@ -33,7 +33,7 @@ define void @vector_variable_shift_left_loop(ptr nocapture %arr, ptr nocapture r
 ; SSE-NEXT:    pxor %xmm8, %xmm8
 ; SSE-NEXT:    pmovzxdq {{.*#+}} xmm9 = xmm0[0],zero,xmm0[1],zero
 ; SSE-NEXT:    pmovzxdq {{.*#+}} xmm10 = xmm1[0],zero,xmm1[1],zero
-; SSE-NEXT:    .p2align 4, 0x90
+; SSE-NEXT:    .p2align 4
 ; SSE-NEXT:  .LBB0_4: # %vector.body
 ; SSE-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
@@ -119,7 +119,7 @@ define void @vector_variable_shift_left_loop(ptr nocapture %arr, ptr nocapture r
 ; SSE-NEXT:    jne .LBB0_6
 ; SSE-NEXT:  .LBB0_9: # %for.cond.cleanup
 ; SSE-NEXT:    retq
-; SSE-NEXT:    .p2align 4, 0x90
+; SSE-NEXT:    .p2align 4
 ; SSE-NEXT:  .LBB0_8: # %for.body
 ; SSE-NEXT:    # in Loop: Header=BB0_6 Depth=1
 ; SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
@@ -167,7 +167,7 @@ define void @vector_variable_shift_left_loop(ptr nocapture %arr, ptr nocapture r
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm6 = xmm8[0],zero,xmm8[1],zero
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm7 = xmm7[0],zero,xmm7[1],zero
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm8 = xmm8[0],zero,xmm8[1],zero
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB0_4: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm9 = mem[0],zero
@@ -244,7 +244,7 @@ define void @vector_variable_shift_left_loop(ptr nocapture %arr, ptr nocapture r
 ; AVX1-NEXT:  .LBB0_9: # %for.cond.cleanup
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB0_8: # %for.body
 ; AVX1-NEXT:    # in Loop: Header=BB0_6 Depth=1
 ; AVX1-NEXT:    # kill: def $cl killed $cl killed $ecx
@@ -283,7 +283,7 @@ define void @vector_variable_shift_left_loop(ptr nocapture %arr, ptr nocapture r
 ; AVX2-NEXT:    vpbroadcastd %xmm1, %ymm1
 ; AVX2-NEXT:    xorl %ecx, %ecx
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB0_4: # %vector.body
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
@@ -319,7 +319,7 @@ define void @vector_variable_shift_left_loop(ptr nocapture %arr, ptr nocapture r
 ; AVX2-NEXT:  .LBB0_9: # %for.cond.cleanup
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB0_8: # %for.body
 ; AVX2-NEXT:    # in Loop: Header=BB0_6 Depth=1
 ; AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
@@ -362,7 +362,7 @@ define void @vector_variable_shift_left_loop(ptr nocapture %arr, ptr nocapture r
 ; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; XOP-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; XOP-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; XOP-NEXT:    .p2align 4, 0x90
+; XOP-NEXT:    .p2align 4
 ; XOP-NEXT:  .LBB0_4: # %vector.body
 ; XOP-NEXT:    # =>This Inner Loop Header: Depth=1
 ; XOP-NEXT:    vmovq {{.*#+}} xmm5 = mem[0],zero
@@ -418,7 +418,7 @@ define void @vector_variable_shift_left_loop(ptr nocapture %arr, ptr nocapture r
 ; XOP-NEXT:  .LBB0_9: # %for.cond.cleanup
 ; XOP-NEXT:    vzeroupper
 ; XOP-NEXT:    retq
-; XOP-NEXT:    .p2align 4, 0x90
+; XOP-NEXT:    .p2align 4
 ; XOP-NEXT:  .LBB0_8: # %for.body
 ; XOP-NEXT:    # in Loop: Header=BB0_6 Depth=1
 ; XOP-NEXT:    # kill: def $cl killed $cl killed $ecx
@@ -547,7 +547,7 @@ define void @vector_variable_shift_left_loop_simpler(ptr nocapture %arr, ptr noc
 ; SSE-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm3[0],zero,xmm3[1],zero
 ; SSE-NEXT:    pslld %xmm0, %xmm1
 ; SSE-NEXT:    pxor %xmm3, %xmm3
-; SSE-NEXT:    .p2align 4, 0x90
+; SSE-NEXT:    .p2align 4
 ; SSE-NEXT:  .LBB1_2: # %vector.body
 ; SSE-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
@@ -578,7 +578,7 @@ define void @vector_variable_shift_left_loop_simpler(ptr nocapture %arr, ptr noc
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
 ; AVX1-NEXT:    vpslld %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    .p2align 4, 0x90
+; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB1_2: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
@@ -606,7 +606,7 @@ define void @vector_variable_shift_left_loop_simpler(ptr nocapture %arr, ptr noc
 ; AVX2-NEXT:    vpbroadcastd %xmm2, %xmm2
 ; AVX2-NEXT:    xorl %ecx, %ecx
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT:    .p2align 4, 0x90
+; AVX2-NEXT:    .p2align 4
 ; AVX2-NEXT:  .LBB1_2: # %vector.body
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} xmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
@@ -635,7 +635,7 @@ define void @vector_variable_shift_left_loop_simpler(ptr nocapture %arr, ptr noc
 ; XOP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; XOP-NEXT:    xorl %ecx, %ecx
 ; XOP-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; XOP-NEXT:    .p2align 4, 0x90
+; XOP-NEXT:    .p2align 4
 ; XOP-NEXT:  .LBB1_2: # %vector.body
 ; XOP-NEXT:    # =>This Inner Loop Header: Depth=1
 ; XOP-NEXT:    vpmovzxbd {{.*#+}} xmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining.ll
index 36c4be5f1939e..07c770abc65d6 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining.ll
@@ -3656,7 +3656,7 @@ entry:
 define void @autogen_SD25931() {
 ; CHECK-LABEL: autogen_SD25931:
 ; CHECK:       # %bb.0: # %BB
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB142_1: # %CF242
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    jmp .LBB142_1
diff --git a/llvm/test/CodeGen/X86/vselect-avx.ll b/llvm/test/CodeGen/X86/vselect-avx.ll
index 364390a4a60e5..a2c3613ecca12 100644
--- a/llvm/test/CodeGen/X86/vselect-avx.ll
+++ b/llvm/test/CodeGen/X86/vselect-avx.ll
@@ -187,7 +187,7 @@ define <32 x i8> @PR22706(<32 x i1> %x) {
 define void @PR59003(<2 x float> %0, <2 x float> %1, <8 x i1> %shuffle108) {
 ; AVX-LABEL: PR59003:
 ; AVX:       ## %bb.0: ## %entry
-; AVX-NEXT:    .p2align 4, 0x90
+; AVX-NEXT:    .p2align 4
 ; AVX-NEXT:  LBB4_1: ## %for.body.i
 ; AVX-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    jmp LBB4_1
diff --git a/llvm/test/CodeGen/X86/widen_arith-1.ll b/llvm/test/CodeGen/X86/widen_arith-1.ll
index f6be700645fb3..e08df8f76d1a6 100644
--- a/llvm/test/CodeGen/X86/widen_arith-1.ll
+++ b/llvm/test/CodeGen/X86/widen_arith-1.ll
@@ -7,7 +7,7 @@ define void @update(ptr %dst, ptr %src, i32 %n) nounwind {
 ; CHECK-NEXT:    pushl %eax
 ; CHECK-NEXT:    movl $0, (%esp)
 ; CHECK-NEXT:    pcmpeqd %xmm0, %xmm0
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %forcond
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl (%esp), %eax
diff --git a/llvm/test/CodeGen/X86/widen_arith-2.ll b/llvm/test/CodeGen/X86/widen_arith-2.ll
index a0f78db8746f4..260b87cc461a9 100644
--- a/llvm/test/CodeGen/X86/widen_arith-2.ll
+++ b/llvm/test/CodeGen/X86/widen_arith-2.ll
@@ -10,7 +10,7 @@ define void @update(ptr %dst_i, ptr %src_i, i32 %n) nounwind {
 ; CHECK-NEXT:    movl $0, (%esp)
 ; CHECK-NEXT:    pcmpeqd %xmm0, %xmm0
 ; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = [4,4,4,4,4,4,4,4,u,u,u,u,u,u,u,u]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %forcond
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl (%esp), %eax
diff --git a/llvm/test/CodeGen/X86/widen_arith-3.ll b/llvm/test/CodeGen/X86/widen_arith-3.ll
index 85640a0af597a..9031588f2690d 100644
--- a/llvm/test/CodeGen/X86/widen_arith-3.ll
+++ b/llvm/test/CodeGen/X86/widen_arith-3.ll
@@ -17,7 +17,7 @@ define void @update(ptr %dst, ptr %src, i32 %n) nounwind {
 ; CHECK-NEXT:    movw $1, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl $65537, {{[0-9]+}}(%esp) # imm = 0x10001
 ; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %forcond
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
diff --git a/llvm/test/CodeGen/X86/widen_arith-4.ll b/llvm/test/CodeGen/X86/widen_arith-4.ll
index 8bb716374b482..c49882ffe0b38 100644
--- a/llvm/test/CodeGen/X86/widen_arith-4.ll
+++ b/llvm/test/CodeGen/X86/widen_arith-4.ll
@@ -16,7 +16,7 @@ define void @update(ptr %dst, ptr %src, i32 %n) nounwind {
 ; SSE2-NEXT:    movl $0, -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [271,271,271,271,271,u,u,u]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2,4,2,2,2,u,u,u]
-; SSE2-NEXT:    .p2align 4, 0x90
+; SSE2-NEXT:    .p2align 4
 ; SSE2-NEXT:  .LBB0_1: # %forcond
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
@@ -49,7 +49,7 @@ define void @update(ptr %dst, ptr %src, i32 %n) nounwind {
 ; SSE41-NEXT:    movw $0, -{{[0-9]+}}(%rsp)
 ; SSE41-NEXT:    movl $0, -{{[0-9]+}}(%rsp)
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [271,271,271,271,271,u,u,u]
-; SSE41-NEXT:    .p2align 4, 0x90
+; SSE41-NEXT:    .p2align 4
 ; SSE41-NEXT:  .LBB0_1: # %forcond
 ; SSE41-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE41-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
diff --git a/llvm/test/CodeGen/X86/widen_arith-5.ll b/llvm/test/CodeGen/X86/widen_arith-5.ll
index dd75a4ea2cf22..466249d1bf1d4 100644
--- a/llvm/test/CodeGen/X86/widen_arith-5.ll
+++ b/llvm/test/CodeGen/X86/widen_arith-5.ll
@@ -14,7 +14,7 @@ define void @update(ptr %dst, ptr %src, i32 %n) nounwind {
 ; CHECK-NEXT:    movl $1, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movl $0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    pmovsxbd {{.*#+}} xmm0 = [3,3,3,3]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %forcond
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
diff --git a/llvm/test/CodeGen/X86/widen_arith-6.ll b/llvm/test/CodeGen/X86/widen_arith-6.ll
index 239fe25b0c4a9..6fa232f4d3227 100644
--- a/llvm/test/CodeGen/X86/widen_arith-6.ll
+++ b/llvm/test/CodeGen/X86/widen_arith-6.ll
@@ -15,7 +15,7 @@ define void @update(ptr %dst, ptr %src, i32 %n) nounwind {
 ; CHECK-NEXT:    movl $1065353216, {{[0-9]+}}(%esp) # imm = 0x3F800000
 ; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movaps {{.*#+}} xmm0 = [1.97604004E+3,1.97604004E+3,1.97604004E+3,u]
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %forcond
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
diff --git a/llvm/test/CodeGen/X86/widen_cast-1.ll b/llvm/test/CodeGen/X86/widen_cast-1.ll
index 58f0d2c2da574..566dde0ca13d3 100644
--- a/llvm/test/CodeGen/X86/widen_cast-1.ll
+++ b/llvm/test/CodeGen/X86/widen_cast-1.ll
@@ -14,7 +14,7 @@ define void @convert(ptr %dst, ptr %src) nounwind {
 ; CHECK-NEXT:    pcmpeqd %xmm0, %xmm0
 ; CHECK-NEXT:    cmpl $3, (%esp)
 ; CHECK-NEXT:    jg .LBB0_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # %forbody
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl (%esp), %eax
@@ -37,7 +37,7 @@ define void @convert(ptr %dst, ptr %src) nounwind {
 ; ATOM-NEXT:    movl $0, (%esp)
 ; ATOM-NEXT:    cmpl $3, (%esp)
 ; ATOM-NEXT:    jg .LBB0_3
-; ATOM-NEXT:    .p2align 4, 0x90
+; ATOM-NEXT:    .p2align 4
 ; ATOM-NEXT:  .LBB0_2: # %forbody
 ; ATOM-NEXT:    # =>This Inner Loop Header: Depth=1
 ; ATOM-NEXT:    movl (%esp), %eax
diff --git a/llvm/test/CodeGen/X86/widen_cast-2.ll b/llvm/test/CodeGen/X86/widen_cast-2.ll
index 6e174180d04a7..cd06f27dcc55c 100644
--- a/llvm/test/CodeGen/X86/widen_cast-2.ll
+++ b/llvm/test/CodeGen/X86/widen_cast-2.ll
@@ -10,7 +10,7 @@ define void @convert(ptr %dst, ptr %src) nounwind {
 ; CHECK-NEXT:    pcmpeqd %xmm0, %xmm0
 ; CHECK-NEXT:    cmpl $3, (%esp)
 ; CHECK-NEXT:    jg .LBB0_3
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_2: # %forbody
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl (%esp), %eax
diff --git a/llvm/test/CodeGen/X86/widen_cast-4.ll b/llvm/test/CodeGen/X86/widen_cast-4.ll
index 7dd92ce874aef..7468e229e3a84 100644
--- a/llvm/test/CodeGen/X86/widen_cast-4.ll
+++ b/llvm/test/CodeGen/X86/widen_cast-4.ll
@@ -11,7 +11,7 @@ define void @update(ptr %dst_i, ptr %src_i, i32 %n) nounwind {
 ; WIDE-NEXT:    pcmpeqd %xmm0, %xmm0
 ; WIDE-NEXT:    movdqa {{.*#+}} xmm1 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
 ; WIDE-NEXT:    movdqa {{.*#+}} xmm2 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; WIDE-NEXT:    .p2align 4, 0x90
+; WIDE-NEXT:    .p2align 4
 ; WIDE-NEXT:  .LBB0_1: # %forcond
 ; WIDE-NEXT:    # =>This Inner Loop Header: Depth=1
 ; WIDE-NEXT:    movl (%esp), %eax
diff --git a/llvm/test/CodeGen/X86/x86-shrink-wrapping.ll b/llvm/test/CodeGen/X86/x86-shrink-wrapping.ll
index fab3847b3a2c5..fe7459ea45e14 100644
--- a/llvm/test/CodeGen/X86/x86-shrink-wrapping.ll
+++ b/llvm/test/CodeGen/X86/x86-shrink-wrapping.ll
@@ -80,7 +80,7 @@ define i32 @freqSaveAndRestoreOutsideLoop(i32 %cond, i32 %N) {
 ; ENABLE-NEXT:    ## InlineAsm End
 ; ENABLE-NEXT:    xorl %eax, %eax
 ; ENABLE-NEXT:    movl $10, %ecx
-; ENABLE-NEXT:    .p2align 4, 0x90
+; ENABLE-NEXT:    .p2align 4
 ; ENABLE-NEXT:  LBB1_2: ## %for.body
 ; ENABLE-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; ENABLE-NEXT:    ## InlineAsm Start
@@ -111,7 +111,7 @@ define i32 @freqSaveAndRestoreOutsideLoop(i32 %cond, i32 %N) {
 ; DISABLE-NEXT:    ## InlineAsm End
 ; DISABLE-NEXT:    xorl %eax, %eax
 ; DISABLE-NEXT:    movl $10, %ecx
-; DISABLE-NEXT:    .p2align 4, 0x90
+; DISABLE-NEXT:    .p2align 4
 ; DISABLE-NEXT:  LBB1_2: ## %for.body
 ; DISABLE-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; DISABLE-NEXT:    ## InlineAsm Start
@@ -174,7 +174,7 @@ define i32 @freqSaveAndRestoreOutsideLoop2(i32 %cond) {
 ; ENABLE-NEXT:    ## InlineAsm End
 ; ENABLE-NEXT:    xorl %eax, %eax
 ; ENABLE-NEXT:    movl $10, %ecx
-; ENABLE-NEXT:    .p2align 4, 0x90
+; ENABLE-NEXT:    .p2align 4
 ; ENABLE-NEXT:  LBB2_1: ## %for.body
 ; ENABLE-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; ENABLE-NEXT:    ## InlineAsm Start
@@ -200,7 +200,7 @@ define i32 @freqSaveAndRestoreOutsideLoop2(i32 %cond) {
 ; DISABLE-NEXT:    ## InlineAsm End
 ; DISABLE-NEXT:    xorl %eax, %eax
 ; DISABLE-NEXT:    movl $10, %ecx
-; DISABLE-NEXT:    .p2align 4, 0x90
+; DISABLE-NEXT:    .p2align 4
 ; DISABLE-NEXT:  LBB2_1: ## %for.body
 ; DISABLE-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; DISABLE-NEXT:    ## InlineAsm Start
@@ -255,7 +255,7 @@ define i32 @loopInfoSaveOutsideLoop(i32 %cond, i32 %N) {
 ; ENABLE-NEXT:    ## InlineAsm End
 ; ENABLE-NEXT:    xorl %eax, %eax
 ; ENABLE-NEXT:    movl $10, %ecx
-; ENABLE-NEXT:    .p2align 4, 0x90
+; ENABLE-NEXT:    .p2align 4
 ; ENABLE-NEXT:  LBB3_2: ## %for.body
 ; ENABLE-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; ENABLE-NEXT:    ## InlineAsm Start
@@ -289,7 +289,7 @@ define i32 @loopInfoSaveOutsideLoop(i32 %cond, i32 %N) {
 ; DISABLE-NEXT:    ## InlineAsm End
 ; DISABLE-NEXT:    xorl %eax, %eax
 ; DISABLE-NEXT:    movl $10, %ecx
-; DISABLE-NEXT:    .p2align 4, 0x90
+; DISABLE-NEXT:    .p2align 4
 ; DISABLE-NEXT:  LBB3_2: ## %for.body
 ; DISABLE-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; DISABLE-NEXT:    ## InlineAsm Start
@@ -355,7 +355,7 @@ define i32 @loopInfoRestoreOutsideLoop(i32 %cond, i32 %N) nounwind {
 ; ENABLE-NEXT:    ## InlineAsm End
 ; ENABLE-NEXT:    xorl %eax, %eax
 ; ENABLE-NEXT:    movl $10, %ecx
-; ENABLE-NEXT:    .p2align 4, 0x90
+; ENABLE-NEXT:    .p2align 4
 ; ENABLE-NEXT:  LBB4_2: ## %for.body
 ; ENABLE-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; ENABLE-NEXT:    ## InlineAsm Start
@@ -384,7 +384,7 @@ define i32 @loopInfoRestoreOutsideLoop(i32 %cond, i32 %N) nounwind {
 ; DISABLE-NEXT:    ## InlineAsm End
 ; DISABLE-NEXT:    xorl %eax, %eax
 ; DISABLE-NEXT:    movl $10, %ecx
-; DISABLE-NEXT:    .p2align 4, 0x90
+; DISABLE-NEXT:    .p2align 4
 ; DISABLE-NEXT:  LBB4_2: ## %for.body
 ; DISABLE-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; DISABLE-NEXT:    ## InlineAsm Start
@@ -461,7 +461,7 @@ define i32 @inlineAsm(i32 %cond, i32 %N) {
 ; ENABLE-NEXT:    nop
 ; ENABLE-NEXT:    ## InlineAsm End
 ; ENABLE-NEXT:    movl $10, %eax
-; ENABLE-NEXT:    .p2align 4, 0x90
+; ENABLE-NEXT:    .p2align 4
 ; ENABLE-NEXT:  LBB6_2: ## %for.body
 ; ENABLE-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; ENABLE-NEXT:    ## InlineAsm Start
@@ -493,7 +493,7 @@ define i32 @inlineAsm(i32 %cond, i32 %N) {
 ; DISABLE-NEXT:    nop
 ; DISABLE-NEXT:    ## InlineAsm End
 ; DISABLE-NEXT:    movl $10, %eax
-; DISABLE-NEXT:    .p2align 4, 0x90
+; DISABLE-NEXT:    .p2align 4
 ; DISABLE-NEXT:  LBB6_2: ## %for.body
 ; DISABLE-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; DISABLE-NEXT:    ## InlineAsm Start
@@ -824,7 +824,7 @@ define void @infiniteloop() {
 ; ENABLE-NEXT:    ## InlineAsm Start
 ; ENABLE-NEXT:    movl $1, %edx
 ; ENABLE-NEXT:    ## InlineAsm End
-; ENABLE-NEXT:    .p2align 4, 0x90
+; ENABLE-NEXT:    .p2align 4
 ; ENABLE-NEXT:  LBB10_2: ## %for.body
 ; ENABLE-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; ENABLE-NEXT:    addl %edx, %eax
@@ -856,7 +856,7 @@ define void @infiniteloop() {
 ; DISABLE-NEXT:    ## InlineAsm Start
 ; DISABLE-NEXT:    movl $1, %edx
 ; DISABLE-NEXT:    ## InlineAsm End
-; DISABLE-NEXT:    .p2align 4, 0x90
+; DISABLE-NEXT:    .p2align 4
 ; DISABLE-NEXT:  LBB10_2: ## %for.body
 ; DISABLE-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; DISABLE-NEXT:    addl %edx, %eax
@@ -906,7 +906,7 @@ define void @infiniteloop2() {
 ; ENABLE-NEXT:    movq %rcx, %rsp
 ; ENABLE-NEXT:    xorl %edx, %edx
 ; ENABLE-NEXT:    jmp LBB11_2
-; ENABLE-NEXT:    .p2align 4, 0x90
+; ENABLE-NEXT:    .p2align 4
 ; ENABLE-NEXT:  LBB11_4: ## %body2
 ; ENABLE-NEXT:    ## in Loop: Header=BB11_2 Depth=1
 ; ENABLE-NEXT:    ## InlineAsm Start
@@ -954,7 +954,7 @@ define void @infiniteloop2() {
 ; DISABLE-NEXT:    movq %rcx, %rsp
 ; DISABLE-NEXT:    xorl %edx, %edx
 ; DISABLE-NEXT:    jmp LBB11_2
-; DISABLE-NEXT:    .p2align 4, 0x90
+; DISABLE-NEXT:    .p2align 4
 ; DISABLE-NEXT:  LBB11_4: ## %body2
 ; DISABLE-NEXT:    ## in Loop: Header=BB11_2 Depth=1
 ; DISABLE-NEXT:    ## InlineAsm Start
@@ -1024,7 +1024,7 @@ define void @infiniteloop3() {
 ; ENABLE-NEXT:    xorl %ecx, %ecx
 ; ENABLE-NEXT:    movq %rax, %rsi
 ; ENABLE-NEXT:    jmp LBB12_4
-; ENABLE-NEXT:    .p2align 4, 0x90
+; ENABLE-NEXT:    .p2align 4
 ; ENABLE-NEXT:  LBB12_3: ## %loop2b
 ; ENABLE-NEXT:    ## in Loop: Header=BB12_4 Depth=1
 ; ENABLE-NEXT:    movq %rdx, (%rsi)
@@ -1056,7 +1056,7 @@ define void @infiniteloop3() {
 ; DISABLE-NEXT:    xorl %ecx, %ecx
 ; DISABLE-NEXT:    movq %rax, %rsi
 ; DISABLE-NEXT:    jmp LBB12_4
-; DISABLE-NEXT:    .p2align 4, 0x90
+; DISABLE-NEXT:    .p2align 4
 ; DISABLE-NEXT:  LBB12_3: ## %loop2b
 ; DISABLE-NEXT:    ## in Loop: Header=BB12_4 Depth=1
 ; DISABLE-NEXT:    movq %rdx, (%rsi)
@@ -1188,7 +1188,7 @@ define i32 @useLEAForPrologue(i32 %d, i32 %a, i8 %c) #3 {
 ; ENABLE-NEXT:    ## InlineAsm Start
 ; ENABLE-NEXT:    nop
 ; ENABLE-NEXT:    ## InlineAsm End
-; ENABLE-NEXT:    .p2align 4, 0x90
+; ENABLE-NEXT:    .p2align 4
 ; ENABLE-NEXT:  LBB14_2: ## %for.body
 ; ENABLE-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; ENABLE-NEXT:    cmpl %esi, %edi
@@ -1223,7 +1223,7 @@ define i32 @useLEAForPrologue(i32 %d, i32 %a, i8 %c) #3 {
 ; DISABLE-NEXT:    ## InlineAsm Start
 ; DISABLE-NEXT:    nop
 ; DISABLE-NEXT:    ## InlineAsm End
-; DISABLE-NEXT:    .p2align 4, 0x90
+; DISABLE-NEXT:    .p2align 4
 ; DISABLE-NEXT:  LBB14_2: ## %for.body
 ; DISABLE-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; DISABLE-NEXT:    cmpl %esi, %edi
@@ -1375,7 +1375,7 @@ define i32 @irreducibleCFG() #4 {
 ; ENABLE-NEXT:    movq _irreducibleCFGf@GOTPCREL(%rip), %rax
 ; ENABLE-NEXT:    cmpb $0, (%rax)
 ; ENABLE-NEXT:    je LBB16_2
-; ENABLE-NEXT:    .p2align 4, 0x90
+; ENABLE-NEXT:    .p2align 4
 ; ENABLE-NEXT:  LBB16_1: ## %preheader
 ; ENABLE-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; ENABLE-NEXT:    jmp LBB16_1
@@ -1392,7 +1392,7 @@ define i32 @irreducibleCFG() #4 {
 ; ENABLE-NEXT:    jmp LBB16_5
 ; ENABLE-NEXT:  LBB16_3:
 ; ENABLE-NEXT:    xorl %ebx, %ebx
-; ENABLE-NEXT:    .p2align 4, 0x90
+; ENABLE-NEXT:    .p2align 4
 ; ENABLE-NEXT:  LBB16_5: ## %for.inc
 ; ENABLE-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; ENABLE-NEXT:    incl %ebx
@@ -1418,7 +1418,7 @@ define i32 @irreducibleCFG() #4 {
 ; DISABLE-NEXT:    movq _irreducibleCFGf@GOTPCREL(%rip), %rax
 ; DISABLE-NEXT:    cmpb $0, (%rax)
 ; DISABLE-NEXT:    je LBB16_2
-; DISABLE-NEXT:    .p2align 4, 0x90
+; DISABLE-NEXT:    .p2align 4
 ; DISABLE-NEXT:  LBB16_1: ## %preheader
 ; DISABLE-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; DISABLE-NEXT:    jmp LBB16_1
@@ -1435,7 +1435,7 @@ define i32 @irreducibleCFG() #4 {
 ; DISABLE-NEXT:    jmp LBB16_5
 ; DISABLE-NEXT:  LBB16_3:
 ; DISABLE-NEXT:    xorl %ebx, %ebx
-; DISABLE-NEXT:    .p2align 4, 0x90
+; DISABLE-NEXT:    .p2align 4
 ; DISABLE-NEXT:  LBB16_5: ## %for.inc
 ; DISABLE-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; DISABLE-NEXT:    incl %ebx
@@ -1515,7 +1515,7 @@ define void @infiniteLoopNoSuccessor() #5 {
 ; ENABLE-NEXT:  LBB17_3:
 ; ENABLE-NEXT:    xorl %eax, %eax
 ; ENABLE-NEXT:    callq _something
-; ENABLE-NEXT:    .p2align 4, 0x90
+; ENABLE-NEXT:    .p2align 4
 ; ENABLE-NEXT:  LBB17_4: ## =>This Inner Loop Header: Depth=1
 ; ENABLE-NEXT:    xorl %eax, %eax
 ; ENABLE-NEXT:    callq _somethingElse
@@ -1542,7 +1542,7 @@ define void @infiniteLoopNoSuccessor() #5 {
 ; DISABLE-NEXT:  LBB17_3:
 ; DISABLE-NEXT:    xorl %eax, %eax
 ; DISABLE-NEXT:    callq _something
-; DISABLE-NEXT:    .p2align 4, 0x90
+; DISABLE-NEXT:    .p2align 4
 ; DISABLE-NEXT:  LBB17_4: ## =>This Inner Loop Header: Depth=1
 ; DISABLE-NEXT:    xorl %eax, %eax
 ; DISABLE-NEXT:    callq _somethingElse
diff --git a/llvm/test/CodeGen/X86/x86-win64-shrink-wrapping.ll b/llvm/test/CodeGen/X86/x86-win64-shrink-wrapping.ll
index 6ed23b0d770cd..d52990e753d3e 100644
--- a/llvm/test/CodeGen/X86/x86-win64-shrink-wrapping.ll
+++ b/llvm/test/CodeGen/X86/x86-win64-shrink-wrapping.ll
@@ -27,7 +27,7 @@ define i32 @loopInfoSaveOutsideLoop(i32 %cond, i32 %N) #0 {
 ; ENABLE-NEXT:    #APP
 ; ENABLE-NEXT:    movl $1, %edx
 ; ENABLE-NEXT:    #NO_APP
-; ENABLE-NEXT:    .p2align 4, 0x90
+; ENABLE-NEXT:    .p2align 4
 ; ENABLE-NEXT:  .LBB0_2: # %for.body
 ; ENABLE-NEXT:    # =>This Inner Loop Header: Depth=1
 ; ENABLE-NEXT:    addl %edx, %eax
@@ -63,7 +63,7 @@ define i32 @loopInfoSaveOutsideLoop(i32 %cond, i32 %N) #0 {
 ; DISABLE-NEXT:    #APP
 ; DISABLE-NEXT:    movl $1, %edx
 ; DISABLE-NEXT:    #NO_APP
-; DISABLE-NEXT:    .p2align 4, 0x90
+; DISABLE-NEXT:    .p2align 4
 ; DISABLE-NEXT:  .LBB0_2: # %for.body
 ; DISABLE-NEXT:    # =>This Inner Loop Header: Depth=1
 ; DISABLE-NEXT:    addl %edx, %eax
@@ -129,7 +129,7 @@ define i32 @loopInfoSaveOutsideLoop2(i32 %cond, i32 %N) #0 {
 ; ENABLE-NEXT:    #NO_APP
 ; ENABLE-NEXT:    xorl %eax, %eax
 ; ENABLE-NEXT:    movl $10, %ecx
-; ENABLE-NEXT:    .p2align 4, 0x90
+; ENABLE-NEXT:    .p2align 4
 ; ENABLE-NEXT:  .LBB1_2: # %for.body
 ; ENABLE-NEXT:    # =>This Inner Loop Header: Depth=1
 ; ENABLE-NEXT:    #APP
@@ -164,7 +164,7 @@ define i32 @loopInfoSaveOutsideLoop2(i32 %cond, i32 %N) #0 {
 ; DISABLE-NEXT:    #NO_APP
 ; DISABLE-NEXT:    xorl %eax, %eax
 ; DISABLE-NEXT:    movl $10, %ecx
-; DISABLE-NEXT:    .p2align 4, 0x90
+; DISABLE-NEXT:    .p2align 4
 ; DISABLE-NEXT:  .LBB1_2: # %for.body
 ; DISABLE-NEXT:    # =>This Inner Loop Header: Depth=1
 ; DISABLE-NEXT:    #APP
diff --git a/llvm/test/CodeGen/X86/xor.ll b/llvm/test/CodeGen/X86/xor.ll
index 7f4006792ef1a..2bef66825d8c0 100644
--- a/llvm/test/CodeGen/X86/xor.ll
+++ b/llvm/test/CodeGen/X86/xor.ll
@@ -59,7 +59,7 @@ define i32 @test4(i32 %a, i32 %b) nounwind  {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB3_1: # %bb
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    xorl %ecx, %eax
@@ -75,7 +75,7 @@ define i32 @test4(i32 %a, i32 %b) nounwind  {
 ; X64-LIN-LABEL: test4:
 ; X64-LIN:       # %bb.0: # %entry
 ; X64-LIN-NEXT:    movl %edi, %eax
-; X64-LIN-NEXT:    .p2align 4, 0x90
+; X64-LIN-NEXT:    .p2align 4
 ; X64-LIN-NEXT:  .LBB3_1: # %bb
 ; X64-LIN-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-LIN-NEXT:    xorl %esi, %eax
@@ -91,7 +91,7 @@ define i32 @test4(i32 %a, i32 %b) nounwind  {
 ; X64-WIN-LABEL: test4:
 ; X64-WIN:       # %bb.0: # %entry
 ; X64-WIN-NEXT:    movl %ecx, %eax
-; X64-WIN-NEXT:    .p2align 4, 0x90
+; X64-WIN-NEXT:    .p2align 4
 ; X64-WIN-NEXT:  .LBB3_1: # %bb
 ; X64-WIN-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-WIN-NEXT:    xorl %edx, %eax
@@ -123,7 +123,7 @@ define i16 @test5(i16 %a, i16 %b) nounwind  {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB4_1: # %bb
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    xorl %ecx, %eax
@@ -141,7 +141,7 @@ define i16 @test5(i16 %a, i16 %b) nounwind  {
 ; X64-LIN-LABEL: test5:
 ; X64-LIN:       # %bb.0: # %entry
 ; X64-LIN-NEXT:    movl %edi, %eax
-; X64-LIN-NEXT:    .p2align 4, 0x90
+; X64-LIN-NEXT:    .p2align 4
 ; X64-LIN-NEXT:  .LBB4_1: # %bb
 ; X64-LIN-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-LIN-NEXT:    xorl %esi, %eax
@@ -160,7 +160,7 @@ define i16 @test5(i16 %a, i16 %b) nounwind  {
 ; X64-WIN:       # %bb.0: # %entry
 ; X64-WIN-NEXT:    # kill: def $dx killed $dx def $edx
 ; X64-WIN-NEXT:    movl %ecx, %eax
-; X64-WIN-NEXT:    .p2align 4, 0x90
+; X64-WIN-NEXT:    .p2align 4
 ; X64-WIN-NEXT:  .LBB4_1: # %bb
 ; X64-WIN-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-WIN-NEXT:    xorl %edx, %eax
@@ -194,7 +194,7 @@ define i8 @test6(i8 %a, i8 %b) nounwind  {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB5_1: # %bb
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    xorb %cl, %al
@@ -210,7 +210,7 @@ define i8 @test6(i8 %a, i8 %b) nounwind  {
 ; X64-LIN-LABEL: test6:
 ; X64-LIN:       # %bb.0: # %entry
 ; X64-LIN-NEXT:    movl %edi, %eax
-; X64-LIN-NEXT:    .p2align 4, 0x90
+; X64-LIN-NEXT:    .p2align 4
 ; X64-LIN-NEXT:  .LBB5_1: # %bb
 ; X64-LIN-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-LIN-NEXT:    xorb %sil, %al
@@ -227,7 +227,7 @@ define i8 @test6(i8 %a, i8 %b) nounwind  {
 ; X64-WIN-LABEL: test6:
 ; X64-WIN:       # %bb.0: # %entry
 ; X64-WIN-NEXT:    movl %ecx, %eax
-; X64-WIN-NEXT:    .p2align 4, 0x90
+; X64-WIN-NEXT:    .p2align 4
 ; X64-WIN-NEXT:  .LBB5_1: # %bb
 ; X64-WIN-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-WIN-NEXT:    xorb %dl, %al
@@ -259,7 +259,7 @@ define i32 @test7(i32 %a, i32 %b) nounwind  {
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:    .p2align 4
 ; X86-NEXT:  .LBB6_1: # %bb
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    xorl %ecx, %eax
@@ -275,7 +275,7 @@ define i32 @test7(i32 %a, i32 %b) nounwind  {
 ; X64-LIN-LABEL: test7:
 ; X64-LIN:       # %bb.0: # %entry
 ; X64-LIN-NEXT:    movl %edi, %eax
-; X64-LIN-NEXT:    .p2align 4, 0x90
+; X64-LIN-NEXT:    .p2align 4
 ; X64-LIN-NEXT:  .LBB6_1: # %bb
 ; X64-LIN-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-LIN-NEXT:    xorl %esi, %eax
@@ -291,7 +291,7 @@ define i32 @test7(i32 %a, i32 %b) nounwind  {
 ; X64-WIN-LABEL: test7:
 ; X64-WIN:       # %bb.0: # %entry
 ; X64-WIN-NEXT:    movl %ecx, %eax
-; X64-WIN-NEXT:    .p2align 4, 0x90
+; X64-WIN-NEXT:    .p2align 4
 ; X64-WIN-NEXT:  .LBB6_1: # %bb
 ; X64-WIN-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-WIN-NEXT:    xorl %edx, %eax
diff --git a/llvm/test/CodeGen/X86/xray-attribute-instrumentation.ll b/llvm/test/CodeGen/X86/xray-attribute-instrumentation.ll
index 9cebffc3abe9a..fc1f3352ce818 100644
--- a/llvm/test/CodeGen/X86/xray-attribute-instrumentation.ll
+++ b/llvm/test/CodeGen/X86/xray-attribute-instrumentation.ll
@@ -3,12 +3,12 @@
 ; RUN: llc -mtriple=x86_64-darwin-unknown                          < %s | FileCheck %s --check-prefixes=CHECK,CHECK-MACOS
 
 define i32 @foo() nounwind noinline uwtable "function-instrument"="xray-always" {
-; CHECK:       .p2align 1, 0x90
+; CHECK:       .p2align 1
 ; CHECK-LABEL: Lxray_sled_0:
 ; CHECK:       .ascii "\353\t"
 ; CHECK-NEXT:  nopw 512(%rax,%rax)
   ret i32 0
-; CHECK:       .p2align 1, 0x90
+; CHECK:       .p2align 1
 ; CHECK-LABEL: Lxray_sled_1:
 ; CHECK:       retq
 ; CHECK-NEXT:  nopw %cs:512(%rax,%rax)
@@ -38,7 +38,7 @@ define i32 @foo() nounwind noinline uwtable "function-instrument"="xray-always"
 ; We test multiple returns in a single function to make sure we're getting all
 ; of them with XRay instrumentation.
 define i32 @bar(i32 %i) nounwind noinline uwtable "function-instrument"="xray-always" {
-; CHECK:       .p2align 1, 0x90
+; CHECK:       .p2align 1
 ; CHECK-LABEL: Lxray_sled_2:
 ; CHECK:       .ascii "\353\t"
 ; CHECK-NEXT:  nopw 512(%rax,%rax)
@@ -47,13 +47,13 @@ Test:
   br i1 %cond, label %IsEqual, label %NotEqual
 IsEqual:
   ret i32 0
-; CHECK:       .p2align 1, 0x90
+; CHECK:       .p2align 1
 ; CHECK-LABEL: Lxray_sled_3:
 ; CHECK:       retq
 ; CHECK-NEXT:  nopw %cs:512(%rax,%rax)
 NotEqual:
   ret i32 1
-; CHECK:       .p2align 1, 0x90
+; CHECK:       .p2align 1
 ; CHECK-LABEL: Lxray_sled_4:
 ; CHECK:       retq
 ; CHECK-NEXT:  nopw %cs:512(%rax,%rax)
diff --git a/llvm/test/CodeGen/X86/xray-custom-log.ll b/llvm/test/CodeGen/X86/xray-custom-log.ll
index 9ccda8ecb0480..8f23055aca97c 100644
--- a/llvm/test/CodeGen/X86/xray-custom-log.ll
+++ b/llvm/test/CodeGen/X86/xray-custom-log.ll
@@ -84,7 +84,7 @@ define void @leaf_func() "function-instrument"="xray-always" "frame-pointer"="no
   ; CHECK:         pushq %rax
   ; CHECK:         movl $leaf_func.event_id, %eax
   ; CHECK-NEXT:    movl $4, %ecx
-  ; CHECK-NEXT:    .p2align 1, 0x90
+  ; CHECK-NEXT:    .p2align 1
   ; CHECK-NEXT:  .Lxray_event_sled_1:
   call void @llvm.xray.customevent(ptr @leaf_func.event_id, i64 4)
   ret void
diff --git a/llvm/test/CodeGen/X86/xray-partial-instrumentation-skip-entry.ll b/llvm/test/CodeGen/X86/xray-partial-instrumentation-skip-entry.ll
index f58d743d465dd..51b047c5a0e63 100644
--- a/llvm/test/CodeGen/X86/xray-partial-instrumentation-skip-entry.ll
+++ b/llvm/test/CodeGen/X86/xray-partial-instrumentation-skip-entry.ll
@@ -5,7 +5,7 @@
 define i32 @foo() nounwind noinline uwtable "function-instrument"="xray-always" "xray-skip-entry" {
 ; CHECK-NOT: Lxray_sled_0:
   ret i32 0
-; CHECK:       .p2align 1, 0x90
+; CHECK:       .p2align 1
 ; CHECK-LABEL: Lxray_sled_0:
 ; CHECK:       retq
 ; CHECK-NEXT:  nopw %cs:512(%rax,%rax)
@@ -39,13 +39,13 @@ Test:
   br i1 %cond, label %IsEqual, label %NotEqual
 IsEqual:
   ret i32 0
-; CHECK:       .p2align 1, 0x90
+; CHECK:       .p2align 1
 ; CHECK-LABEL: Lxray_sled_1:
 ; CHECK:       retq
 ; CHECK-NEXT:  nopw %cs:512(%rax,%rax)
 NotEqual:
   ret i32 1
-; CHECK:       .p2align 1, 0x90
+; CHECK:       .p2align 1
 ; CHECK-LABEL: Lxray_sled_2:
 ; CHECK:       retq
 ; CHECK-NEXT:  nopw %cs:512(%rax,%rax)
diff --git a/llvm/test/CodeGen/X86/xray-partial-instrumentation-skip-exit.ll b/llvm/test/CodeGen/X86/xray-partial-instrumentation-skip-exit.ll
index 3d631d73b68c5..ab841d734a23e 100644
--- a/llvm/test/CodeGen/X86/xray-partial-instrumentation-skip-exit.ll
+++ b/llvm/test/CodeGen/X86/xray-partial-instrumentation-skip-exit.ll
@@ -4,7 +4,7 @@
 ; RUN: llc -mtriple=x86_64-darwin-unknown    < %s | FileCheck %s
 
 define i32 @foo() nounwind noinline uwtable "function-instrument"="xray-always" "xray-skip-exit" {
-; CHECK:       .p2align 1, 0x90
+; CHECK:       .p2align 1
 ; CHECK-LABEL: Lxray_sled_0:
 ; CHECK:       .ascii "\353\t"
 ; CHECK-NEXT:  nopw 512(%rax,%rax)
@@ -35,7 +35,7 @@ define i32 @foo() nounwind noinline uwtable "function-instrument"="xray-always"
 ; We test multiple returns in a single function to make sure we're skipping all
 ; of them with XRay instrumentation.
 define i32 @bar(i32 %i) nounwind noinline uwtable "function-instrument"="xray-always" "xray-skip-exit" {
-; CHECK:       .p2align 1, 0x90
+; CHECK:       .p2align 1
 ; CHECK-LABEL: Lxray_sled_1:
 ; CHECK:       .ascii "\353\t"
 ; CHECK-NEXT:  nopw 512(%rax,%rax)
diff --git a/llvm/test/CodeGen/X86/xray-selective-instrumentation.ll b/llvm/test/CodeGen/X86/xray-selective-instrumentation.ll
index 7bf47ea2894c0..868c35363f6e9 100644
--- a/llvm/test/CodeGen/X86/xray-selective-instrumentation.ll
+++ b/llvm/test/CodeGen/X86/xray-selective-instrumentation.ll
@@ -7,12 +7,12 @@ target triple = "x86_64-apple-darwin8"
 define i32 @foo() nounwind uwtable "xray-instruction-threshold"="1" {
 ; CHECK-LABEL: foo:
 ; CHECK:       ## %bb.0: ## %entry
-; CHECK-NEXT:    .p2align 1, 0x90
+; CHECK-NEXT:    .p2align 1
 ; CHECK-NEXT:  Lxray_sled_0:
 ; CHECK-NEXT:    .ascii "\353\t"
 ; CHECK-NEXT:    nopw 512(%rax,%rax)
 ; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    .p2align 1, 0x90
+; CHECK-NEXT:    .p2align 1
 ; CHECK-NEXT:  Lxray_sled_1:
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
diff --git a/llvm/test/CodeGen/X86/xray-tail-call-sled.ll b/llvm/test/CodeGen/X86/xray-tail-call-sled.ll
index 126e5db52a5b7..91b91f5360f92 100644
--- a/llvm/test/CodeGen/X86/xray-tail-call-sled.ll
+++ b/llvm/test/CodeGen/X86/xray-tail-call-sled.ll
@@ -2,12 +2,12 @@
 ; RUN: llc -mtriple=x86_64-darwin-unknown    < %s | FileCheck %s --check-prefixes=CHECK,CHECK-MACOS
 
 define dso_local i32 @callee() nounwind noinline uwtable "function-instrument"="xray-always" {
-; CHECK:       .p2align 1, 0x90
+; CHECK:       .p2align 1
 ; CHECK-LABEL: Lxray_sled_0:
 ; CHECK:       .ascii "\353\t"
 ; CHECK-NEXT:  nopw 512(%rax,%rax)
   ret i32 0
-; CHECK:       .p2align 1, 0x90
+; CHECK:       .p2align 1
 ; CHECK-LABEL: Lxray_sled_1:
 ; CHECK:       retq
 ; CHECK-NEXT:  nopw %cs:512(%rax,%rax)
@@ -34,11 +34,11 @@ define dso_local i32 @callee() nounwind noinline uwtable "function-instrument"="
 ; CHECK-MACOS-NEXT:    .quad 2
 
 define dso_local i32 @caller() nounwind noinline uwtable "function-instrument"="xray-always" {
-; CHECK:       .p2align 1, 0x90
+; CHECK:       .p2align 1
 ; CHECK-LABEL: Lxray_sled_2:
 ; CHECK:       .ascii "\353\t"
 ; CHECK-NEXT:  nopw 512(%rax,%rax)
-; CHECK:       .p2align 1, 0x90
+; CHECK:       .p2align 1
 ; CHECK-LABEL: Lxray_sled_3:
 ; CHECK-NEXT:  .ascii "\353\t"
 ; CHECK-NEXT:  nopw 512(%rax,%rax)
@@ -69,13 +69,13 @@ define dso_local i32 @caller() nounwind noinline uwtable "function-instrument"="
 
 define dso_local i32 @conditional_tail_call(i32 %cond) nounwind noinline uwtable "function-instrument"="xray-always" {
 ; CHECK-LABEL: conditional_tail_call:
-; CHECK:         .p2align 1, 0x90
+; CHECK:         .p2align 1
 ; CHECK-LABEL: Lxray_sled_4:
 ; CHECK:         .ascii "\353\t"
 ; CHECK-NEXT:    nopw 512(%rax,%rax)
 ; CHECK-NEXT:    testl %edi, %edi
 ; CHECK-NEXT:    je {{\.?Ltmp5}}
-; CHECK:         .p2align 1, 0x90
+; CHECK:         .p2align 1
 ; CHECK-LABEL: Lxray_sled_5:
 ; CHECK-NEXT:    .ascii "\353\t"
 ; CHECK-NEXT:    nopw 512(%rax,%rax)
@@ -83,7 +83,7 @@ define dso_local i32 @conditional_tail_call(i32 %cond) nounwind noinline uwtable
 ; CHECK-NEXT:    jmp {{.*}}callee {{.*}}# TAILCALL
 ; CHECK-LABEL: Ltmp5:
 ; CHECK:         xorl %eax, %eax
-; CHECK-NEXT:   .p2align  1, 0x90
+; CHECK-NEXT:   .p2align  1
 ; CHECK-LABEL: Lxray_sled_6:
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    nopw %cs:512(%rax,%rax)
diff --git a/llvm/test/CodeGen/Xtensa/blockaddress.ll b/llvm/test/CodeGen/Xtensa/blockaddress.ll
index bbeb1790a1b78..e3c23cf8cedf0 100644
--- a/llvm/test/CodeGen/Xtensa/blockaddress.ll
+++ b/llvm/test/CodeGen/Xtensa/blockaddress.ll
@@ -1,22 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc --mtriple=xtensa < %s | FileCheck %s
 
 @addr = global ptr null
 
 define void @test_blockaddress() {
-
-  store volatile ptr blockaddress(@test_blockaddress, %block), ptr @addr
-; CHECK:      .literal_position
-; CHECK-NEXT: .literal .LCPI0_0, addr
-; CHECK-NEXT: .literal .LCPI0_1, .Ltmp0
 ; CHECK-LABEL: test_blockaddress:
-; CHECK:      # %bb.0:
-; CHECK-NEXT: l32r a8, .LCPI0_0
-; CHECK-NEXT: l32r a9, .LCPI0_1
-; CHECK-NEXT: s32i a9, a8, 0
-; CHECK-NEXT: l32i a8, a8, 0
-; CHECK-NEXT: jx a8
-; CHECK-NEXT: .Ltmp0:
-; CHECK-NEXT: .LBB0_1:
+; CHECK:         l32r a8, .LCPI0_0
+; CHECK-NEXT:    l32r a9, .LCPI0_1
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    s32i a9, a8, 0
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    l32i a8, a8, 0
+; CHECK-NEXT:    jx a8
+; CHECK-NEXT:  .Ltmp0: # Block address taken
+; CHECK-NEXT:  .LBB0_1: # %block
+; CHECK-NEXT:    ret
+  store volatile ptr blockaddress(@test_blockaddress, %block), ptr @addr
 
   %val = load volatile ptr, ptr @addr
   indirectbr ptr %val, [label %block]
diff --git a/llvm/test/CodeGen/Xtensa/volatile.ll b/llvm/test/CodeGen/Xtensa/volatile.ll
new file mode 100644
index 0000000000000..6b27c28b86899
--- /dev/null
+++ b/llvm/test/CodeGen/Xtensa/volatile.ll
@@ -0,0 +1,176 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=xtensa -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s
+
+@x_i8 = common dso_local global i8 0, align 8
+@y_i8 = common dso_local global i8 0, align 8
+@x_i16 = common dso_local global i16 0, align 8
+@y_i16 = common dso_local global i16 0, align 8
+@x_i32 = common dso_local global i32 0, align 8
+@y_i32 = common dso_local global i32 0, align 8
+@x_i64 = common dso_local global i64 0, align 8
+@y_i64 = common dso_local global i64 0, align 8
+@x_float = common dso_local global float 0.0, align 8
+@y_float = common dso_local global float 0.0, align 8
+@x_double = common dso_local global double 0.0, align 8
+@y_double = common dso_local global double 0.0, align 8
+@x_vec = common dso_local global <4 x i32> <i32 0, i32 0, i32 0, i32 0>, align 8
+@y_vec = common dso_local global <4 x i32> <i32 0, i32 0, i32 0, i32 0>, align 8
+
+define void @test() {
+; CHECK-LABEL: test:
+; CHECK:         l32r a8, .LCPI0_0
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    l8ui a8, a8, 0
+; CHECK-NEXT:    l32r a9, .LCPI0_1
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    s8i a8, a9, 0
+; CHECK-NEXT:    l32r a8, .LCPI0_2
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    l16ui a8, a8, 0
+; CHECK-NEXT:    l32r a9, .LCPI0_3
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    s16i a8, a9, 0
+; CHECK-NEXT:    l32r a8, .LCPI0_4
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    l32i a8, a8, 0
+; CHECK-NEXT:    l32r a9, .LCPI0_5
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    s32i a8, a9, 0
+; CHECK-NEXT:    ret
+
+entry:
+  %a = load volatile i8, ptr @x_i8, align 4
+  store volatile i8 %a, ptr @y_i8, align 4
+  %b = load volatile i16, ptr @x_i16, align 4
+  store volatile i16 %b, ptr @y_i16, align 4
+  %c = load volatile i32, ptr @x_i32, align 4
+  store volatile i32 %c, ptr @y_i32, align 4
+  ret void
+}
+
+
+define void @test_i8() {
+; CHECK-LABEL: test_i8:
+; CHECK:         l32r a8, .LCPI1_0
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    l8ui a8, a8, 0
+; CHECK-NEXT:    l32r a9, .LCPI1_1
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    s8i a8, a9, 0
+; CHECK-NEXT:    ret
+entry:
+  %a = load volatile i8, ptr @x_i8, align 4
+  store volatile i8 %a, ptr @y_i8, align 4
+  ret void
+}
+
+define void @test_i16() {
+; CHECK-LABEL: test_i16:
+; CHECK:         l32r a8, .LCPI2_0
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    l16ui a8, a8, 0
+; CHECK-NEXT:    l32r a9, .LCPI2_1
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    s16i a8, a9, 0
+; CHECK-NEXT:    ret
+entry:
+  %a = load volatile i16, ptr @x_i16, align 4
+  store volatile i16 %a, ptr @y_i16, align 4
+  ret void
+}
+
+define void @test_i32() {
+; CHECK-LABEL: test_i32:
+; CHECK:         l32r a8, .LCPI3_0
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    l32i a8, a8, 0
+; CHECK-NEXT:    l32r a9, .LCPI3_1
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    s32i a8, a9, 0
+; CHECK-NEXT:    ret
+entry:
+  %a = load volatile i32, ptr @x_i32, align 4
+  store volatile i32 %a, ptr @y_i32, align 4
+  ret void
+}
+
+define void @test_i64() {
+; CHECK-LABEL: test_i64:
+; CHECK:         l32r a8, .LCPI4_0
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    l32i a9, a8, 0
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    l32i a8, a8, 4
+; CHECK-NEXT:    l32r a10, .LCPI4_1
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    s32i a8, a10, 4
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    s32i a9, a10, 0
+; CHECK-NEXT:    ret
+entry:
+  %a = load volatile i64, ptr @x_i64, align 4
+  store volatile i64 %a, ptr @y_i64, align 4
+  ret void
+}
+
+define void @test_float() {
+; CHECK-LABEL: test_float:
+; CHECK:         l32r a8, .LCPI5_0
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    l32i a8, a8, 0
+; CHECK-NEXT:    l32r a9, .LCPI5_1
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    s32i a8, a9, 0
+; CHECK-NEXT:    ret
+entry:
+  %a = load volatile float, ptr @x_float, align 4
+  store volatile float %a, ptr @y_float, align 4
+  ret void
+}
+
+define void @test_double() {
+; CHECK-LABEL: test_double:
+; CHECK:         l32r a8, .LCPI6_0
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    l32i a9, a8, 0
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    l32i a8, a8, 4
+; CHECK-NEXT:    l32r a10, .LCPI6_1
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    s32i a8, a10, 4
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    s32i a9, a10, 0
+; CHECK-NEXT:    ret
+entry:
+  %a = load volatile double, ptr @x_double, align 4
+  store volatile double %a, ptr @y_double, align 4
+  ret void
+}
+
+define void @test_vec() {
+; CHECK-LABEL: test_vec:
+; CHECK:         l32r a8, .LCPI7_0
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    l32i a9, a8, 0
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    l32i a10, a8, 4
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    l32i a11, a8, 8
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    l32i a8, a8, 12
+; CHECK-NEXT:    l32r a7, .LCPI7_1
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    s32i a8, a7, 12
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    s32i a11, a7, 8
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    s32i a10, a7, 4
+; CHECK-NEXT:    memw
+; CHECK-NEXT:    s32i a9, a7, 0
+; CHECK-NEXT:    ret
+entry:
+  %a = load volatile <4 x i32>, ptr @x_vec, align 4
+  store volatile <4 x i32> %a, ptr @y_vec, align 4
+  ret void
+}
diff --git a/llvm/test/DebugInfo/COFF/pieces.ll b/llvm/test/DebugInfo/COFF/pieces.ll
index 8f5b6b2d7c1af..8e62ad0093aa3 100644
--- a/llvm/test/DebugInfo/COFF/pieces.ll
+++ b/llvm/test/DebugInfo/COFF/pieces.ll
@@ -41,7 +41,7 @@
 ; ASM:         xorl    %edi, %edi
 ; ASM:         xorl    %esi, %esi
 ; ASM: [[oy_ox_start:\.Ltmp[0-9]+]]:
-; ASM:         .p2align        4, 0x90
+; ASM:         .p2align        4
 ; ASM: .LBB0_3:                                # %for.body
 ; ASM:        #DEBUG_VALUE: loop_csr:o <- [DW_OP_LLVM_fragment 0 32] $edi
 ; ASM:        #DEBUG_VALUE: loop_csr:o <- [DW_OP_LLVM_fragment 32 32] $esi
diff --git a/llvm/test/DebugInfo/NVPTX/cu-range-hole.ll b/llvm/test/DebugInfo/NVPTX/cu-range-hole.ll
index 2d927b18d976d..1f4c44eaf455c 100644
--- a/llvm/test/DebugInfo/NVPTX/cu-range-hole.ll
+++ b/llvm/test/DebugInfo/NVPTX/cu-range-hole.ll
@@ -73,89 +73,85 @@ entry:
   ret i32 %add, !dbg !16
 }
 
-; CHECK: .section .debug_abbrev
-; CHECK-NEXT: {
-; CHECK-NEXT: .b8 1                                // Abbreviation Code
-; CHECK-NEXT: .b8 17                               // DW_TAG_compile_unit
-; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
-; CHECK-NEXT: .b8 37                               // DW_AT_producer
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 19                               // DW_AT_language
-; CHECK-NEXT: .b8 5                                // DW_FORM_data2
-; CHECK-NEXT: .b8 3                                // DW_AT_name
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 16                               // DW_AT_stmt_list
-; CHECK-NEXT: .b8 6                                // DW_FORM_data4
-; CHECK-NEXT: .b8 27                               // DW_AT_comp_dir
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 17                               // DW_AT_low_pc
-; CHECK-NEXT: .b8 1                                // DW_FORM_addr
-; CHECK-NEXT: .b8 18                               // DW_AT_high_pc
-; CHECK-NEXT: .b8 1                                // DW_FORM_addr
-; CHECK-NEXT: .b8 0                                // EOM(1)
-; CHECK-NEXT: .b8 0                                // EOM(2)
-; CHECK-NEXT: .b8 2                                // Abbreviation Code
-; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
-; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
-; CHECK-NEXT: .b8 17                               // DW_AT_low_pc
-; CHECK-NEXT: .b8 1                                // DW_FORM_addr
-; CHECK-NEXT: .b8 18                               // DW_AT_high_pc
-; CHECK-NEXT: .b8 1                                // DW_FORM_addr
-; CHECK-NEXT: .b8 64                               // DW_AT_frame_base
-; CHECK-NEXT: .b8 10                               // DW_FORM_block1
-; CHECK-NEXT: .b8 3                                // DW_AT_name
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 39                               // DW_AT_prototyped
-; CHECK-NEXT: .b8 12                               // DW_FORM_flag
-; CHECK-NEXT: .b8 73                               // DW_AT_type
-; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
-; CHECK-NEXT: .b8 63                               // DW_AT_external
-; CHECK-NEXT: .b8 12                               // DW_FORM_flag
-; CHECK-NEXT: .b8 0                                // EOM(1)
-; CHECK-NEXT: .b8 0                                // EOM(2)
-; CHECK-NEXT: .b8 3                                // Abbreviation Code
-; CHECK-NEXT: .b8 5                                // DW_TAG_formal_parameter
-; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
-; CHECK-NEXT: .b8 51                               // DW_AT_address_class
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 2                                // DW_AT_location
-; CHECK-NEXT: .b8 10                               // DW_FORM_block1
-; CHECK-NEXT: .b8 3                                // DW_AT_name
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 73                               // DW_AT_type
-; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
-; CHECK-NEXT: .b8 0                                // EOM(1)
-; CHECK-NEXT: .b8 0                                // EOM(2)
-; CHECK-NEXT: .b8 4                                // Abbreviation Code
-; CHECK-NEXT: .b8 36                               // DW_TAG_base_type
-; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
-; CHECK-NEXT: .b8 3                                // DW_AT_name
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 62                               // DW_AT_encoding
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 11                               // DW_AT_byte_size
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 0                                // EOM(1)
-; CHECK-NEXT: .b8 0                                // EOM(2)
-; CHECK-NEXT: .b8 0                                // EOM(3)
-; CHECK-NEXT: }
-; CHECK-NEXT: .section .debug_info
-; CHECK-NEXT: {
-; CHECK-NEXT: .b32 197                             // Length of Unit
-; CHECK-NEXT: .b8 2                                // DWARF version number
+; CHECK: 	.section	.debug_abbrev
+; CHECK-NEXT: 	{
+; CHECK-NEXT: .b8 1                                   // Abbreviation Code
+; CHECK-NEXT: .b8 17                                  // DW_TAG_compile_unit
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 37                                  // DW_AT_producer
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 19                                  // DW_AT_language
+; CHECK-NEXT: .b8 5                                   // DW_FORM_data2
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 16                                  // DW_AT_stmt_list
+; CHECK-NEXT: .b8 6                                   // DW_FORM_data4
+; CHECK-NEXT: .b8 27                                  // DW_AT_comp_dir
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 2                                   // Abbreviation Code
+; CHECK-NEXT: .b8 46                                  // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 17                                  // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                   // DW_FORM_addr
+; CHECK-NEXT: .b8 18                                  // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_FORM_addr
+; CHECK-NEXT: .b8 64                                  // DW_AT_frame_base
+; CHECK-NEXT: .b8 10                                  // DW_FORM_block1
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 39                                  // DW_AT_prototyped
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 63                                  // DW_AT_external
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 3                                   // Abbreviation Code
+; CHECK-NEXT: .b8 5                                   // DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 51                                  // DW_AT_address_class
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 2                                   // DW_AT_location
+; CHECK-NEXT: .b8 10                                  // DW_FORM_block1
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 4                                   // Abbreviation Code
+; CHECK-NEXT: .b8 36                                  // DW_TAG_base_type
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 62                                  // DW_AT_encoding
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 11                                  // DW_AT_byte_size
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 0                                   // EOM(3)
+; CHECK-NEXT: 	}
+; CHECK-NEXT: 	.section	.debug_info
+; CHECK-NEXT: 	{
+; CHECK-NEXT: .b32 181                                // Length of Unit
+; CHECK-NEXT: .b8 2                                   // DWARF version number
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b32 .debug_abbrev                   // Offset Into Abbrev. Section
-; CHECK-NEXT: .b8 8                                // Address Size (in bytes)
-; CHECK-NEXT: .b8 1                                // Abbrev [1] 0xb:0xbe DW_TAG_compile_unit
-; CHECK-NEXT: .b8 99                               // DW_AT_producer
+; CHECK-NEXT: .b32 .debug_abbrev                      // Offset Into Abbrev. Section
+; CHECK-NEXT: .b8 8                                   // Address Size (in bytes)
+; CHECK-NEXT: .b8 1                                   // Abbrev [1] 0xb:0xae DW_TAG_compile_unit
+; CHECK-NEXT: .b8 99                                  // DW_AT_producer
 ; CHECK-NEXT: .b8 108
 ; CHECK-NEXT: .b8 97
 ; CHECK-NEXT: .b8 110
@@ -210,14 +206,14 @@ entry:
 ; CHECK-NEXT: .b8 51
 ; CHECK-NEXT: .b8 41
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 12                               // DW_AT_language
+; CHECK-NEXT: .b8 12                                  // DW_AT_language
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 98                               // DW_AT_name
+; CHECK-NEXT: .b8 98                                  // DW_AT_name
 ; CHECK-NEXT: .b8 46
 ; CHECK-NEXT: .b8 99
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b32 .debug_line                     // DW_AT_stmt_list
-; CHECK-NEXT: .b8 47                               // DW_AT_comp_dir
+; CHECK-NEXT: .b32 .debug_line                        // DW_AT_stmt_list
+; CHECK-NEXT: .b8 47                                  // DW_AT_comp_dir
 ; CHECK-NEXT: .b8 115
 ; CHECK-NEXT: .b8 111
 ; CHECK-NEXT: .b8 117
@@ -225,70 +221,68 @@ entry:
 ; CHECK-NEXT: .b8 99
 ; CHECK-NEXT: .b8 101
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b64 $L__func_begin0                 // DW_AT_low_pc
-; CHECK-NEXT: .b64 $L__func_end2                   // DW_AT_high_pc
-; CHECK-NEXT: .b8 2                                // Abbrev [2] 0x65:0x2e DW_TAG_subprogram
-; CHECK-NEXT: .b64 $L__func_begin0                 // DW_AT_low_pc
-; CHECK-NEXT: .b64 $L__func_end0                   // DW_AT_high_pc
-; CHECK-NEXT: .b8 1                                // DW_AT_frame_base
+; CHECK-NEXT: .b8 2                                   // Abbrev [2] 0x55:0x2e DW_TAG_subprogram
+; CHECK-NEXT: .b64 $L__func_begin0                    // DW_AT_low_pc
+; CHECK-NEXT: .b64 $L__func_end0                      // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_AT_frame_base
 ; CHECK-NEXT: .b8 156
-; CHECK-NEXT: .b8 98                               // DW_AT_name
+; CHECK-NEXT: .b8 98                                  // DW_AT_name
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
-; CHECK-NEXT: .b8 1                                // DW_AT_decl_line
-; CHECK-NEXT: .b8 1                                // DW_AT_prototyped
-; CHECK-NEXT: .b32 193                             // DW_AT_type
-; CHECK-NEXT: .b8 1                                // DW_AT_external
-; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x82:0x10 DW_TAG_formal_parameter
-; CHECK-NEXT: .b8 2                                // DW_AT_address_class
-; CHECK-NEXT: .b8 5                                // DW_AT_location
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                   // DW_AT_prototyped
+; CHECK-NEXT: .b32 177                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // Abbrev [3] 0x72:0x10 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 2                                   // DW_AT_address_class
+; CHECK-NEXT: .b8 5                                   // DW_AT_location
 ; CHECK-NEXT: .b8 144
 ; CHECK-NEXT: .b8 177
 ; CHECK-NEXT: .b8 228
 ; CHECK-NEXT: .b8 149
 ; CHECK-NEXT: .b8 1
-; CHECK-NEXT: .b8 99                               // DW_AT_name
+; CHECK-NEXT: .b8 99                                  // DW_AT_name
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
-; CHECK-NEXT: .b8 1                                // DW_AT_decl_line
-; CHECK-NEXT: .b32 193                             // DW_AT_type
-; CHECK-NEXT: .b8 0                                // End Of Children Mark
-; CHECK-NEXT: .b8 2                                // Abbrev [2] 0x93:0x2e DW_TAG_subprogram
-; CHECK-NEXT: .b64 $L__func_begin2                 // DW_AT_low_pc
-; CHECK-NEXT: .b64 $L__func_end2                   // DW_AT_high_pc
-; CHECK-NEXT: .b8 1                                // DW_AT_frame_base
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_line
+; CHECK-NEXT: .b32 177                                // DW_AT_type
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 2                                   // Abbrev [2] 0x83:0x2e DW_TAG_subprogram
+; CHECK-NEXT: .b64 $L__func_begin2                    // DW_AT_low_pc
+; CHECK-NEXT: .b64 $L__func_end2                      // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_AT_frame_base
 ; CHECK-NEXT: .b8 156
-; CHECK-NEXT: .b8 100                              // DW_AT_name
+; CHECK-NEXT: .b8 100                                 // DW_AT_name
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
-; CHECK-NEXT: .b8 3                                // DW_AT_decl_line
-; CHECK-NEXT: .b8 1                                // DW_AT_prototyped
-; CHECK-NEXT: .b32 193                             // DW_AT_type
-; CHECK-NEXT: .b8 1                                // DW_AT_external
-; CHECK-NEXT: .b8 3                                // Abbrev [3] 0xb0:0x10 DW_TAG_formal_parameter
-; CHECK-NEXT: .b8 2                                // DW_AT_address_class
-; CHECK-NEXT: .b8 5                                // DW_AT_location
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                   // DW_AT_prototyped
+; CHECK-NEXT: .b32 177                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // Abbrev [3] 0xa0:0x10 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 2                                   // DW_AT_address_class
+; CHECK-NEXT: .b8 5                                   // DW_AT_location
 ; CHECK-NEXT: .b8 144
 ; CHECK-NEXT: .b8 177
 ; CHECK-NEXT: .b8 228
 ; CHECK-NEXT: .b8 149
 ; CHECK-NEXT: .b8 1
-; CHECK-NEXT: .b8 101                              // DW_AT_name
+; CHECK-NEXT: .b8 101                                 // DW_AT_name
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
-; CHECK-NEXT: .b8 3                                // DW_AT_decl_line
-; CHECK-NEXT: .b32 193                             // DW_AT_type
-; CHECK-NEXT: .b8 0                                // End Of Children Mark
-; CHECK-NEXT: .b8 4                                // Abbrev [4] 0xc1:0x7 DW_TAG_base_type
-; CHECK-NEXT: .b8 105                              // DW_AT_name
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_line
+; CHECK-NEXT: .b32 177                                // DW_AT_type
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 4                                   // Abbrev [4] 0xb1:0x7 DW_TAG_base_type
+; CHECK-NEXT: .b8 105                                 // DW_AT_name
 ; CHECK-NEXT: .b8 110
 ; CHECK-NEXT: .b8 116
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 5                                // DW_AT_encoding
-; CHECK-NEXT: .b8 4                                // DW_AT_byte_size
-; CHECK-NEXT: .b8 0                                // End Of Children Mark
-; CHECK-NEXT: }
-; CHECK-NEXT: .section .debug_loc { }
+; CHECK-NEXT: .b8 5                                   // DW_AT_encoding
+; CHECK-NEXT: .b8 4                                   // DW_AT_byte_size
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: 	}
+; CHECK-NEXT: 	.section	.debug_macinfo	{	}
 ; CHECK-NOT: debug_
 
 attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll b/llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll
index de367dfa4acb4..116ab7e3978cf 100644
--- a/llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll
+++ b/llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll
@@ -23,140 +23,134 @@
 
 ; CHECK: .file 1 "test{{(/|\\\\)}}t.c"
 
-; CHECK: .section .debug_abbrev
-; CHECK-NEXT: {
-; CHECK-NEXT: .b8 1                                // Abbreviation Code
-; CHECK-NEXT: .b8 17                               // DW_TAG_compile_unit
-; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
-; CHECK-NEXT: .b8 37                               // DW_AT_producer
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 19                               // DW_AT_language
-; CHECK-NEXT: .b8 5                                // DW_FORM_data2
-; CHECK-NEXT: .b8 3                                // DW_AT_name
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 16                               // DW_AT_stmt_list
-; CHECK-NEXT: .b8 6                                // DW_FORM_data4
-; CHECK-NEXT: .b8 27                               // DW_AT_comp_dir
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 17                               // DW_AT_low_pc
-; CHECK-NEXT: .b8 1                                // DW_FORM_addr
-; CHECK-NEXT: .b8 18                               // DW_AT_high_pc
-; CHECK-NEXT: .b8 1                                // DW_FORM_addr
-; CHECK-NEXT: .b8 0                                // EOM(1)
-; CHECK-NEXT: .b8 0                                // EOM(2)
-; CHECK-NEXT: .b8 2                                // Abbreviation Code
-; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
-; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
-; CHECK-NEXT: .b8 17                               // DW_AT_low_pc
-; CHECK-NEXT: .b8 1                                // DW_FORM_addr
-; CHECK-NEXT: .b8 18                               // DW_AT_high_pc
-; CHECK-NEXT: .b8 1                                // DW_FORM_addr
-; CHECK-NEXT: .b8 64                               // DW_AT_frame_base
-; CHECK-NEXT: .b8 10                               // DW_FORM_block1
-; CHECK-NEXT: .b8 3                                // DW_AT_name
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 39                               // DW_AT_prototyped
-; CHECK-NEXT: .b8 12                               // DW_FORM_flag
-; CHECK-NEXT: .b8 63                               // DW_AT_external
-; CHECK-NEXT: .b8 12                               // DW_FORM_flag
-; CHECK-NEXT: .b8 0                                // EOM(1)
-; CHECK-NEXT: .b8 0                                // EOM(2)
-; CHECK-NEXT: .b8 3                                // Abbreviation Code
-; CHECK-NEXT: .b8 52                               // DW_TAG_variable
-; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
-; CHECK-NEXT: .b8 51                               // DW_AT_address_class
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 2                                // DW_AT_location
-; CHECK-NEXT: .b8 10                               // DW_FORM_block1
-; CHECK-NEXT: .b8 3                                // DW_AT_name
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 73                               // DW_AT_type
-; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
-; CHECK-NEXT: .b8 0                                // EOM(1)
-; CHECK-NEXT: .b8 0                                // EOM(2)
-; CHECK-NEXT: .b8 4                                // Abbreviation Code
-; CHECK-NEXT: .b8 19                               // DW_TAG_structure_type
-; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
-; CHECK-NEXT: .b8 3                                // DW_AT_name
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 11                               // DW_AT_byte_size
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 0                                // EOM(1)
-; CHECK-NEXT: .b8 0                                // EOM(2)
-; CHECK-NEXT: .b8 5                                // Abbreviation Code
-; CHECK-NEXT: .b8 13                               // DW_TAG_member
-; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
-; CHECK-NEXT: .b8 3                                // DW_AT_name
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 73                               // DW_AT_type
-; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
-; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 56                               // DW_AT_data_member_location
-; CHECK-NEXT: .b8 10                               // DW_FORM_block1
-; CHECK-NEXT: .b8 0                                // EOM(1)
-; CHECK-NEXT: .b8 0                                // EOM(2)
-; CHECK-NEXT: .b8 6                                // Abbreviation Code
-; CHECK-NEXT: .b8 36                               // DW_TAG_base_type
-; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
-; CHECK-NEXT: .b8 3                                // DW_AT_name
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 62                               // DW_AT_encoding
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 11                               // DW_AT_byte_size
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 0                                // EOM(1)
-; CHECK-NEXT: .b8 0                                // EOM(2)
-; CHECK-NEXT: .b8 0                                // EOM(3)
-; CHECK-NEXT: }
-; CHECK-NEXT: .section .debug_info
-; CHECK-NEXT: {
-; CHECK-NEXT: .b32 134                             // Length of Unit
-; CHECK-NEXT: .b8 2                                // DWARF version number
+; CHECK: 	.section	.debug_abbrev
+; CHECK-NEXT: 	{
+; CHECK-NEXT: .b8 1                                   // Abbreviation Code
+; CHECK-NEXT: .b8 17                                  // DW_TAG_compile_unit
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 37                                  // DW_AT_producer
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 19                                  // DW_AT_language
+; CHECK-NEXT: .b8 5                                   // DW_FORM_data2
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 16                                  // DW_AT_stmt_list
+; CHECK-NEXT: .b8 6                                   // DW_FORM_data4
+; CHECK-NEXT: .b8 27                                  // DW_AT_comp_dir
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 2                                   // Abbreviation Code
+; CHECK-NEXT: .b8 46                                  // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 17                                  // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                   // DW_FORM_addr
+; CHECK-NEXT: .b8 18                                  // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_FORM_addr
+; CHECK-NEXT: .b8 64                                  // DW_AT_frame_base
+; CHECK-NEXT: .b8 10                                  // DW_FORM_block1
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 39                                  // DW_AT_prototyped
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 63                                  // DW_AT_external
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 3                                   // Abbreviation Code
+; CHECK-NEXT: .b8 52                                  // DW_TAG_variable
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 51                                  // DW_AT_address_class
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 2                                   // DW_AT_location
+; CHECK-NEXT: .b8 10                                  // DW_FORM_block1
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 4                                   // Abbreviation Code
+; CHECK-NEXT: .b8 19                                  // DW_TAG_structure_type
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 11                                  // DW_AT_byte_size
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 5                                   // Abbreviation Code
+; CHECK-NEXT: .b8 13                                  // DW_TAG_member
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 56                                  // DW_AT_data_member_location
+; CHECK-NEXT: .b8 10                                  // DW_FORM_block1
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 6                                   // Abbreviation Code
+; CHECK-NEXT: .b8 36                                  // DW_TAG_base_type
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 62                                  // DW_AT_encoding
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 11                                  // DW_AT_byte_size
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 0                                   // EOM(3)
+; CHECK-NEXT: 	}
+; CHECK-NEXT: 	.section	.debug_info
+; CHECK-NEXT: 	{
+; CHECK-NEXT: .b32 118                                // Length of Unit
+; CHECK-NEXT: .b8 2                                   // DWARF version number
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b32 .debug_abbrev                   // Offset Into Abbrev. Section
-; CHECK-NEXT: .b8 8                                // Address Size (in bytes)
-; CHECK-NEXT: .b8 1                                // Abbrev [1] 0xb:0x7f DW_TAG_compile_unit
-; CHECK-NEXT: .b8 99                               // DW_AT_producer
+; CHECK-NEXT: .b32 .debug_abbrev                      // Offset Into Abbrev. Section
+; CHECK-NEXT: .b8 8                                   // Address Size (in bytes)
+; CHECK-NEXT: .b8 1                                   // Abbrev [1] 0xb:0x6f DW_TAG_compile_unit
+; CHECK-NEXT: .b8 99                                  // DW_AT_producer
 ; CHECK-NEXT: .b8 108
 ; CHECK-NEXT: .b8 97
 ; CHECK-NEXT: .b8 110
 ; CHECK-NEXT: .b8 103
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 12                               // DW_AT_language
+; CHECK-NEXT: .b8 12                                  // DW_AT_language
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 116                              // DW_AT_name
+; CHECK-NEXT: .b8 116                                 // DW_AT_name
 ; CHECK-NEXT: .b8 46
 ; CHECK-NEXT: .b8 99
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b32 .debug_line                     // DW_AT_stmt_list
-; CHECK-NEXT: .b8 116                              // DW_AT_comp_dir
+; CHECK-NEXT: .b32 .debug_line                        // DW_AT_stmt_list
+; CHECK-NEXT: .b8 116                                 // DW_AT_comp_dir
 ; CHECK-NEXT: .b8 101
 ; CHECK-NEXT: .b8 115
 ; CHECK-NEXT: .b8 116
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b64 $L__func_begin0                 // DW_AT_low_pc
-; CHECK-NEXT: .b64 $L__func_end0                   // DW_AT_high_pc
-; CHECK-NEXT: .b8 2                                // Abbrev [2] 0x31:0x3c DW_TAG_subprogram
-; CHECK-NEXT: .b64 $L__func_begin0                 // DW_AT_low_pc
-; CHECK-NEXT: .b64 $L__func_end0                   // DW_AT_high_pc
-; CHECK-NEXT: .b8 1                                // DW_AT_frame_base
+; CHECK-NEXT: .b8 2                                   // Abbrev [2] 0x21:0x3c DW_TAG_subprogram
+; CHECK-NEXT: .b64 $L__func_begin0                    // DW_AT_low_pc
+; CHECK-NEXT: .b64 $L__func_end0                      // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_AT_frame_base
 ; CHECK-NEXT: .b8 156
-; CHECK-NEXT: .b8 117                              // DW_AT_name
+; CHECK-NEXT: .b8 117                                 // DW_AT_name
 ; CHECK-NEXT: .b8 115
 ; CHECK-NEXT: .b8 101
 ; CHECK-NEXT: .b8 95
@@ -172,48 +166,48 @@
 ; CHECK-NEXT: .b8 114
 ; CHECK-NEXT: .b8 101
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
-; CHECK-NEXT: .b8 3                                // DW_AT_decl_line
-; CHECK-NEXT: .b8 1                                // DW_AT_prototyped
-; CHECK-NEXT: .b8 1                                // DW_AT_external
-; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x58:0x14 DW_TAG_variable
-; CHECK-NEXT: .b8 6                                // DW_AT_address_class
-; CHECK-NEXT: .b8 9                                // DW_AT_location
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                   // DW_AT_prototyped
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 3                                   // Abbrev [3] 0x48:0x14 DW_TAG_variable
+; CHECK-NEXT: .b8 6                                   // DW_AT_address_class
+; CHECK-NEXT: .b8 9                                   // DW_AT_location
 ; CHECK-NEXT: .b8 3
 ; CHECK-NEXT: .b64 __local_depot0
-; CHECK-NEXT: .b8 111                              // DW_AT_name
+; CHECK-NEXT: .b8 111                                 // DW_AT_name
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
-; CHECK-NEXT: .b8 4                                // DW_AT_decl_line
-; CHECK-NEXT: .b32 109                             // DW_AT_type
-; CHECK-NEXT: .b8 0                                // End Of Children Mark
-; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x6d:0x15 DW_TAG_structure_type
-; CHECK-NEXT: .b8 70                               // DW_AT_name
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 4                                   // DW_AT_decl_line
+; CHECK-NEXT: .b32 93                                 // DW_AT_type
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 4                                   // Abbrev [4] 0x5d:0x15 DW_TAG_structure_type
+; CHECK-NEXT: .b8 70                                  // DW_AT_name
 ; CHECK-NEXT: .b8 111
 ; CHECK-NEXT: .b8 111
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 4                                // DW_AT_byte_size
-; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
-; CHECK-NEXT: .b8 1                                // DW_AT_decl_line
-; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x75:0xc DW_TAG_member
-; CHECK-NEXT: .b8 120                              // DW_AT_name
+; CHECK-NEXT: .b8 4                                   // DW_AT_byte_size
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_line
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0x65:0xc DW_TAG_member
+; CHECK-NEXT: .b8 120                                 // DW_AT_name
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b32 130                             // DW_AT_type
-; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
-; CHECK-NEXT: .b8 1                                // DW_AT_decl_line
-; CHECK-NEXT: .b8 2                                // DW_AT_data_member_location
+; CHECK-NEXT: .b32 114                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_line
+; CHECK-NEXT: .b8 2                                   // DW_AT_data_member_location
 ; CHECK-NEXT: .b8 35
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 0                                // End Of Children Mark
-; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x82:0x7 DW_TAG_base_type
-; CHECK-NEXT: .b8 105                              // DW_AT_name
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 6                                   // Abbrev [6] 0x72:0x7 DW_TAG_base_type
+; CHECK-NEXT: .b8 105                                 // DW_AT_name
 ; CHECK-NEXT: .b8 110
 ; CHECK-NEXT: .b8 116
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 5                                // DW_AT_encoding
-; CHECK-NEXT: .b8 4                                // DW_AT_byte_size
-; CHECK-NEXT: .b8 0                                // End Of Children Mark
-; CHECK-NEXT: }
+; CHECK-NEXT: .b8 5                                   // DW_AT_encoding
+; CHECK-NEXT: .b8 4                                   // DW_AT_byte_size
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: 	}
 
 %struct.Foo = type { i32 }
 
diff --git a/llvm/test/DebugInfo/NVPTX/debug-addr-class.ll b/llvm/test/DebugInfo/NVPTX/debug-addr-class.ll
index 3585095151181..26ad59723abf0 100644
--- a/llvm/test/DebugInfo/NVPTX/debug-addr-class.ll
+++ b/llvm/test/DebugInfo/NVPTX/debug-addr-class.ll
@@ -84,306 +84,300 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 !39 = !DILocation(line: 10, column: 1, scope: !17)
 !40 = !{!22, !24, !26, !28}
 
-; CHECK:        .section        .debug_abbrev
-; CHECK-NEXT:        {
-; CHECK-NEXT:.b8 1                                   // Abbreviation Code
-; CHECK-NEXT:.b8 17                                  // DW_TAG_compile_unit
-; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
-; CHECK-NEXT:.b8 37                                  // DW_AT_producer
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 19                                  // DW_AT_language
-; CHECK-NEXT:.b8 5                                   // DW_FORM_data2
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 16                                  // DW_AT_stmt_list
-; CHECK-NEXT:.b8 6                                   // DW_FORM_data4
-; CHECK-NEXT:.b8 27                                  // DW_AT_comp_dir
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 17                                  // DW_AT_low_pc
-; CHECK-NEXT:.b8 1                                   // DW_FORM_addr
-; CHECK-NEXT:.b8 18                                  // DW_AT_high_pc
-; CHECK-NEXT:.b8 1                                   // DW_FORM_addr
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 2                                   // Abbreviation Code
-; CHECK-NEXT:.b8 52                                  // DW_TAG_variable
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 63                                  // DW_AT_external
-; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 51                                  // DW_AT_address_class
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 2                                   // DW_AT_location
-; CHECK-NEXT:.b8 10                                  // DW_FORM_block1
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 3                                   // Abbreviation Code
-; CHECK-NEXT:.b8 36                                  // DW_TAG_base_type
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 62                                  // DW_AT_encoding
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 11                                  // DW_AT_byte_size
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 4                                   // Abbreviation Code
-; CHECK-NEXT:.b8 46                                  // DW_TAG_subprogram
-; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
-; CHECK-NEXT:.b8 17                                  // DW_AT_low_pc
-; CHECK-NEXT:.b8 1                                   // DW_FORM_addr
-; CHECK-NEXT:.b8 18                                  // DW_AT_high_pc
-; CHECK-NEXT:.b8 1                                   // DW_FORM_addr
-; CHECK-NEXT:.b8 64                                  // DW_AT_frame_base
-; CHECK-NEXT:.b8 10                                  // DW_FORM_block1
-; CHECK-NEXT:.b8 135                                 // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 64
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 63                                  // DW_AT_external
-; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 5                                   // Abbreviation Code
-; CHECK-NEXT:.b8 5                                   // DW_TAG_formal_parameter
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 51                                  // DW_AT_address_class
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 2                                   // DW_AT_location
-; CHECK-NEXT:.b8 10                                  // DW_FORM_block1
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 6                                   // Abbreviation Code
-; CHECK-NEXT:.b8 5                                   // DW_TAG_formal_parameter
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 3                                   // DW_AT_name
-; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 7                                   // Abbreviation Code
-; CHECK-NEXT:.b8 15                                  // DW_TAG_pointer_type
-; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
-; CHECK-NEXT:.b8 73                                  // DW_AT_type
-; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
-; CHECK-NEXT:.b8 0                                   // EOM(1)
-; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 0                                   // EOM(3)
-; CHECK-NEXT:	}
-; CHECK-NEXT:	.section	.debug_info
-; CHECK-NEXT:	{
-; CHECK-NEXT:.b32 254                                // Length of Unit
-; CHECK-NEXT:.b8 2                                   // DWARF version number
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 .debug_abbrev                      // Offset Into Abbrev. Section
-; CHECK-NEXT:.b8 8                                   // Address Size (in bytes)
-; CHECK-NEXT:.b8 1                                   // Abbrev [1] 0xb:0xf7 DW_TAG_compile_unit
-; CHECK-NEXT:.b8 99                                  // DW_AT_producer
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 103
-; CHECK-NEXT:.b8 32
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 115
-; CHECK-NEXT:.b8 105
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 32
-; CHECK-NEXT:.b8 57
-; CHECK-NEXT:.b8 46
-; CHECK-NEXT:.b8 48
-; CHECK-NEXT:.b8 46
-; CHECK-NEXT:.b8 48
-; CHECK-NEXT:.b8 32
-; CHECK-NEXT:.b8 40
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 107
-; CHECK-NEXT:.b8 32
-; CHECK-NEXT:.b8 51
-; CHECK-NEXT:.b8 53
-; CHECK-NEXT:.b8 49
-; CHECK-NEXT:.b8 57
-; CHECK-NEXT:.b8 54
-; CHECK-NEXT:.b8 57
-; CHECK-NEXT:.b8 41
-; CHECK-NEXT:.b8 32
-; CHECK-NEXT:.b8 40
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 118
-; CHECK-NEXT:.b8 109
-; CHECK-NEXT:.b8 47
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 114
-; CHECK-NEXT:.b8 117
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 107
-; CHECK-NEXT:.b8 32
-; CHECK-NEXT:.b8 51
-; CHECK-NEXT:.b8 53
-; CHECK-NEXT:.b8 49
-; CHECK-NEXT:.b8 57
-; CHECK-NEXT:.b8 55
-; CHECK-NEXT:.b8 51
-; CHECK-NEXT:.b8 41
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 4                                   // DW_AT_language
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 110                                 // DW_AT_name
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 119
-; CHECK-NEXT:.b8 46
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 99
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 .debug_line                        // DW_AT_stmt_list
-; CHECK-NEXT:.b8 47                                  // DW_AT_comp_dir
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 109
-; CHECK-NEXT:.b8 112
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b64 $L__func_begin0                    // DW_AT_low_pc
-; CHECK-NEXT:.b64 $L__func_end0                      // DW_AT_high_pc
-; CHECK-NEXT:.b8 2                                   // Abbrev [2] 0x65:0x1a DW_TAG_variable
-; CHECK-NEXT:.b8 71                                  // DW_AT_name
-; CHECK-NEXT:.b8 76
-; CHECK-NEXT:.b8 79
-; CHECK-NEXT:.b8 66
-; CHECK-NEXT:.b8 65
-; CHECK-NEXT:.b8 76
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 127                                // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 3                                   // DW_AT_decl_line
-; CHECK-NEXT:.b8 5                                   // DW_AT_address_class
-; CHECK-NEXT:.b8 9                                   // DW_AT_location
-; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b64 GLOBAL
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x7f:0x7 DW_TAG_base_type
-; CHECK-NEXT:.b8 105                                 // DW_AT_name
-; CHECK-NEXT:.b8 110
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 5                                   // DW_AT_encoding
-; CHECK-NEXT:.b8 4                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 2                                   // Abbrev [2] 0x86:0x1a DW_TAG_variable
-; CHECK-NEXT:.b8 83                                  // DW_AT_name
-; CHECK-NEXT:.b8 72
-; CHECK-NEXT:.b8 65
-; CHECK-NEXT:.b8 82
-; CHECK-NEXT:.b8 69
-; CHECK-NEXT:.b8 68
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 127                                // DW_AT_type
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 4                                   // DW_AT_decl_line
-; CHECK-NEXT:.b8 8                                   // DW_AT_address_class
-; CHECK-NEXT:.b8 9                                   // DW_AT_location
-; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b64 SHARED
-; CHECK-NEXT:.b8 4                                   // Abbrev [4] 0xa0:0x53 DW_TAG_subprogram
-; CHECK-NEXT:.b64 $L__func_begin0                    // DW_AT_low_pc
-; CHECK-NEXT:.b64 $L__func_end0                      // DW_AT_high_pc
-; CHECK-NEXT:.b8 1                                   // DW_AT_frame_base
-; CHECK-NEXT:.b8 156
-; CHECK-NEXT:.b8 116                                 // DW_AT_MIPS_linkage_name
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 115
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 116                                 // DW_AT_name
-; CHECK-NEXT:.b8 101
-; CHECK-NEXT:.b8 115
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 6                                   // DW_AT_decl_line
-; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0xc0:0x10 DW_TAG_formal_parameter
-; CHECK-NEXT:.b8 2                                   // DW_AT_address_class
-; CHECK-NEXT:.b8 5                                   // DW_AT_location
-; CHECK-NEXT:.b8 144
-; CHECK-NEXT:.b8 177
-; CHECK-NEXT:.b8 204
-; CHECK-NEXT:.b8 149
-; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b8 97                                  // DW_AT_name
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 6                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 243                                // DW_AT_type
-; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0xd0:0x9 DW_TAG_formal_parameter
-; CHECK-NEXT:.b8 120                                 // DW_AT_name
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 6                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 252                                // DW_AT_type
-; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0xd9:0x9 DW_TAG_formal_parameter
-; CHECK-NEXT:.b8 121                                 // DW_AT_name
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 6                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 252                                // DW_AT_type
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0xe2:0x10 DW_TAG_formal_parameter
-; CHECK-NEXT:.b8 2                                   // DW_AT_address_class
-; CHECK-NEXT:.b8 5                                   // DW_AT_location
-; CHECK-NEXT:.b8 144
-; CHECK-NEXT:.b8 177
-; CHECK-NEXT:.b8 228
-; CHECK-NEXT:.b8 149
-; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b8 105                                 // DW_AT_name
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
-; CHECK-NEXT:.b8 6                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 127                                // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0xf3:0x9 DW_TAG_base_type
-; CHECK-NEXT:.b8 102                                 // DW_AT_name
-; CHECK-NEXT:.b8 108
-; CHECK-NEXT:.b8 111
-; CHECK-NEXT:.b8 97
-; CHECK-NEXT:.b8 116
-; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 4                                   // DW_AT_encoding
-; CHECK-NEXT:.b8 4                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xfc:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 243                                // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:        }
-; CHECK-NEXT:         .section        .debug_loc      {       }
+; CHECK: 	.section	.debug_abbrev
+; CHECK-NEXT: 	{
+; CHECK-NEXT: .b8 1                                   // Abbreviation Code
+; CHECK-NEXT: .b8 17                                  // DW_TAG_compile_unit
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 37                                  // DW_AT_producer
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 19                                  // DW_AT_language
+; CHECK-NEXT: .b8 5                                   // DW_FORM_data2
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 16                                  // DW_AT_stmt_list
+; CHECK-NEXT: .b8 6                                   // DW_FORM_data4
+; CHECK-NEXT: .b8 27                                  // DW_AT_comp_dir
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 2                                   // Abbreviation Code
+; CHECK-NEXT: .b8 52                                  // DW_TAG_variable
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 63                                  // DW_AT_external
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 51                                  // DW_AT_address_class
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 2                                   // DW_AT_location
+; CHECK-NEXT: .b8 10                                  // DW_FORM_block1
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 3                                   // Abbreviation Code
+; CHECK-NEXT: .b8 36                                  // DW_TAG_base_type
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 62                                  // DW_AT_encoding
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 11                                  // DW_AT_byte_size
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 4                                   // Abbreviation Code
+; CHECK-NEXT: .b8 46                                  // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 17                                  // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                   // DW_FORM_addr
+; CHECK-NEXT: .b8 18                                  // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_FORM_addr
+; CHECK-NEXT: .b8 64                                  // DW_AT_frame_base
+; CHECK-NEXT: .b8 10                                  // DW_FORM_block1
+; CHECK-NEXT: .b8 135                                 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 64
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 63                                  // DW_AT_external
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 5                                   // Abbreviation Code
+; CHECK-NEXT: .b8 5                                   // DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 51                                  // DW_AT_address_class
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 2                                   // DW_AT_location
+; CHECK-NEXT: .b8 10                                  // DW_FORM_block1
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 6                                   // Abbreviation Code
+; CHECK-NEXT: .b8 5                                   // DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 7                                   // Abbreviation Code
+; CHECK-NEXT: .b8 15                                  // DW_TAG_pointer_type
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 0                                   // EOM(3)
+; CHECK-NEXT: 	}
+; CHECK-NEXT: 	.section	.debug_info
+; CHECK-NEXT: 	{
+; CHECK-NEXT: .b32 238                                // Length of Unit
+; CHECK-NEXT: .b8 2                                   // DWARF version number
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 .debug_abbrev                      // Offset Into Abbrev. Section
+; CHECK-NEXT: .b8 8                                   // Address Size (in bytes)
+; CHECK-NEXT: .b8 1                                   // Abbrev [1] 0xb:0xe7 DW_TAG_compile_unit
+; CHECK-NEXT: .b8 99                                  // DW_AT_producer
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 103
+; CHECK-NEXT: .b8 32
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 115
+; CHECK-NEXT: .b8 105
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 32
+; CHECK-NEXT: .b8 57
+; CHECK-NEXT: .b8 46
+; CHECK-NEXT: .b8 48
+; CHECK-NEXT: .b8 46
+; CHECK-NEXT: .b8 48
+; CHECK-NEXT: .b8 32
+; CHECK-NEXT: .b8 40
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 107
+; CHECK-NEXT: .b8 32
+; CHECK-NEXT: .b8 51
+; CHECK-NEXT: .b8 53
+; CHECK-NEXT: .b8 49
+; CHECK-NEXT: .b8 57
+; CHECK-NEXT: .b8 54
+; CHECK-NEXT: .b8 57
+; CHECK-NEXT: .b8 41
+; CHECK-NEXT: .b8 32
+; CHECK-NEXT: .b8 40
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 109
+; CHECK-NEXT: .b8 47
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 114
+; CHECK-NEXT: .b8 117
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 107
+; CHECK-NEXT: .b8 32
+; CHECK-NEXT: .b8 51
+; CHECK-NEXT: .b8 53
+; CHECK-NEXT: .b8 49
+; CHECK-NEXT: .b8 57
+; CHECK-NEXT: .b8 55
+; CHECK-NEXT: .b8 51
+; CHECK-NEXT: .b8 41
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                   // DW_AT_language
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 110                                 // DW_AT_name
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 119
+; CHECK-NEXT: .b8 46
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 99
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 .debug_line                        // DW_AT_stmt_list
+; CHECK-NEXT: .b8 47                                  // DW_AT_comp_dir
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 109
+; CHECK-NEXT: .b8 112
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                   // Abbrev [2] 0x55:0x1a DW_TAG_variable
+; CHECK-NEXT: .b8 71                                  // DW_AT_name
+; CHECK-NEXT: .b8 76
+; CHECK-NEXT: .b8 79
+; CHECK-NEXT: .b8 66
+; CHECK-NEXT: .b8 65
+; CHECK-NEXT: .b8 76
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 111                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_line
+; CHECK-NEXT: .b8 5                                   // DW_AT_address_class
+; CHECK-NEXT: .b8 9                                   // DW_AT_location
+; CHECK-NEXT: .b8 3
+; CHECK-NEXT: .b64 GLOBAL
+; CHECK-NEXT: .b8 3                                   // Abbrev [3] 0x6f:0x7 DW_TAG_base_type
+; CHECK-NEXT: .b8 105                                 // DW_AT_name
+; CHECK-NEXT: .b8 110
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 5                                   // DW_AT_encoding
+; CHECK-NEXT: .b8 4                                   // DW_AT_byte_size
+; CHECK-NEXT: .b8 2                                   // Abbrev [2] 0x76:0x1a DW_TAG_variable
+; CHECK-NEXT: .b8 83                                  // DW_AT_name
+; CHECK-NEXT: .b8 72
+; CHECK-NEXT: .b8 65
+; CHECK-NEXT: .b8 82
+; CHECK-NEXT: .b8 69
+; CHECK-NEXT: .b8 68
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 111                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 4                                   // DW_AT_decl_line
+; CHECK-NEXT: .b8 8                                   // DW_AT_address_class
+; CHECK-NEXT: .b8 9                                   // DW_AT_location
+; CHECK-NEXT: .b8 3
+; CHECK-NEXT: .b64 SHARED
+; CHECK-NEXT: .b8 4                                   // Abbrev [4] 0x90:0x53 DW_TAG_subprogram
+; CHECK-NEXT: .b64 $L__func_begin0                    // DW_AT_low_pc
+; CHECK-NEXT: .b64 $L__func_end0                      // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_AT_frame_base
+; CHECK-NEXT: .b8 156
+; CHECK-NEXT: .b8 116                                 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 115
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 116                                 // DW_AT_name
+; CHECK-NEXT: .b8 101
+; CHECK-NEXT: .b8 115
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 6                                   // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0xb0:0x10 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 2                                   // DW_AT_address_class
+; CHECK-NEXT: .b8 5                                   // DW_AT_location
+; CHECK-NEXT: .b8 144
+; CHECK-NEXT: .b8 177
+; CHECK-NEXT: .b8 204
+; CHECK-NEXT: .b8 149
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b8 97                                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 6                                   // DW_AT_decl_line
+; CHECK-NEXT: .b32 227                                // DW_AT_type
+; CHECK-NEXT: .b8 6                                   // Abbrev [6] 0xc0:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 120                                 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 6                                   // DW_AT_decl_line
+; CHECK-NEXT: .b32 236                                // DW_AT_type
+; CHECK-NEXT: .b8 6                                   // Abbrev [6] 0xc9:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 121                                 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 6                                   // DW_AT_decl_line
+; CHECK-NEXT: .b32 236                                // DW_AT_type
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0xd2:0x10 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 2                                   // DW_AT_address_class
+; CHECK-NEXT: .b8 5                                   // DW_AT_location
+; CHECK-NEXT: .b8 144
+; CHECK-NEXT: .b8 177
+; CHECK-NEXT: .b8 228
+; CHECK-NEXT: .b8 149
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b8 105                                 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 6                                   // DW_AT_decl_line
+; CHECK-NEXT: .b32 111                                // DW_AT_type
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 3                                   // Abbrev [3] 0xe3:0x9 DW_TAG_base_type
+; CHECK-NEXT: .b8 102                                 // DW_AT_name
+; CHECK-NEXT: .b8 108
+; CHECK-NEXT: .b8 111
+; CHECK-NEXT: .b8 97
+; CHECK-NEXT: .b8 116
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                   // DW_AT_encoding
+; CHECK-NEXT: .b8 4                                   // DW_AT_byte_size
+; CHECK-NEXT: .b8 7                                   // Abbrev [7] 0xec:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 227                                // DW_AT_type
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: 	}
+; CHECK-NEXT: 	.section	.debug_macinfo	{	}
 ; CHECK-NOT: debug_
diff --git a/llvm/test/DebugInfo/NVPTX/debug-empty.ll b/llvm/test/DebugInfo/NVPTX/debug-empty.ll
index e34c61059c0d6..2e764940df7cc 100644
--- a/llvm/test/DebugInfo/NVPTX/debug-empty.ll
+++ b/llvm/test/DebugInfo/NVPTX/debug-empty.ll
@@ -2,7 +2,7 @@
 ; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64-nvidia-cuda | %ptxas-verify %}
 
 ; CHECK: .target sm_{{[0-9]+$}}
-; CHECK: .section .debug_loc { }
+; CHECK: .section .debug_macinfo { }
 ; CHECK-NOT: }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/NVPTX/debug-file-loc.ll b/llvm/test/DebugInfo/NVPTX/debug-file-loc.ll
index 7386056d9da21..9d1d3506a3f19 100644
--- a/llvm/test/DebugInfo/NVPTX/debug-file-loc.ll
+++ b/llvm/test/DebugInfo/NVPTX/debug-file-loc.ll
@@ -30,49 +30,45 @@ bb:
 
 ; CHECK-DAG: .file [[FOO]] "/source/dir{{/|\\\\}}foo.h"
 ; CHECK-DAG: .file [[BAR]] "/source/dir{{/|\\\\}}bar.cu"
-; CHECK: .section .debug_abbrev
-; CHECK-NEXT: {
-; CHECK-NEXT: .b8 1                                // Abbreviation Code
-; CHECK-NEXT: .b8 17                               // DW_TAG_compile_unit
-; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
-; CHECK-NEXT: .b8 37                               // DW_AT_producer
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 19                               // DW_AT_language
-; CHECK-NEXT: .b8 5                                // DW_FORM_data2
-; CHECK-NEXT: .b8 3                                // DW_AT_name
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 16                               // DW_AT_stmt_list
-; CHECK-NEXT: .b8 6                                // DW_FORM_data4
-; CHECK-NEXT: .b8 27                               // DW_AT_comp_dir
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 17                               // DW_AT_low_pc
-; CHECK-NEXT: .b8 1                                // DW_FORM_addr
-; CHECK-NEXT: .b8 18                               // DW_AT_high_pc
-; CHECK-NEXT: .b8 1                                // DW_FORM_addr
-; CHECK-NEXT: .b8 0                                // EOM(1)
-; CHECK-NEXT: .b8 0                                // EOM(2)
-; CHECK-NEXT: .b8 0                                // EOM(3)
-; CHECK-NEXT: }
-; CHECK-NEXT: .section .debug_info
-; CHECK-NEXT: {
-; CHECK-NEXT: .b32 50                              // Length of Unit
-; CHECK-NEXT: .b8 2                                // DWARF version number
+; CHECK: 	.section	.debug_abbrev
+; CHECK-NEXT: 	{
+; CHECK-NEXT: .b8 1                                   // Abbreviation Code
+; CHECK-NEXT: .b8 17                                  // DW_TAG_compile_unit
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 37                                  // DW_AT_producer
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 19                                  // DW_AT_language
+; CHECK-NEXT: .b8 5                                   // DW_FORM_data2
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 16                                  // DW_AT_stmt_list
+; CHECK-NEXT: .b8 6                                   // DW_FORM_data4
+; CHECK-NEXT: .b8 27                                  // DW_AT_comp_dir
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 0                                   // EOM(3)
+; CHECK-NEXT: 	}
+; CHECK-NEXT: 	.section	.debug_info
+; CHECK-NEXT: 	{
+; CHECK-NEXT: .b32 34                                 // Length of Unit
+; CHECK-NEXT: .b8 2                                   // DWARF version number
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b32 .debug_abbrev                   // Offset Into Abbrev. Section
-; CHECK-NEXT: .b8 8                                // Address Size (in bytes)
-; CHECK-NEXT: .b8 1                                // Abbrev [1] 0xb:0x2b DW_TAG_compile_unit
-; CHECK-NEXT: .b8 0                                // DW_AT_producer
-; CHECK-NEXT: .b8 4                                // DW_AT_language
+; CHECK-NEXT: .b32 .debug_abbrev                      // Offset Into Abbrev. Section
+; CHECK-NEXT: .b8 8                                   // Address Size (in bytes)
+; CHECK-NEXT: .b8 1                                   // Abbrev [1] 0xb:0x1b DW_TAG_compile_unit
+; CHECK-NEXT: .b8 0                                   // DW_AT_producer
+; CHECK-NEXT: .b8 4                                   // DW_AT_language
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 98                               // DW_AT_name
+; CHECK-NEXT: .b8 98                                  // DW_AT_name
 ; CHECK-NEXT: .b8 97
 ; CHECK-NEXT: .b8 114
 ; CHECK-NEXT: .b8 46
 ; CHECK-NEXT: .b8 99
 ; CHECK-NEXT: .b8 117
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b32 .debug_line                     // DW_AT_stmt_list
-; CHECK-NEXT: .b8 47                               // DW_AT_comp_dir
+; CHECK-NEXT: .b32 .debug_line                        // DW_AT_stmt_list
+; CHECK-NEXT: .b8 47                                  // DW_AT_comp_dir
 ; CHECK-NEXT: .b8 115
 ; CHECK-NEXT: .b8 111
 ; CHECK-NEXT: .b8 117
@@ -84,10 +80,8 @@ bb:
 ; CHECK-NEXT: .b8 105
 ; CHECK-NEXT: .b8 114
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b64 $L__func_begin0                 // DW_AT_low_pc
-; CHECK-NEXT: .b64 $L__func_end1                   // DW_AT_high_pc
-; CHECK-NEXT: }
-; CHECK-NEXT: .section .debug_loc { }
+; CHECK-NEXT: 	}
+; CHECK-NEXT: 	.section	.debug_macinfo	{	}
 ; CHECK-NOT: debug_
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/DebugInfo/NVPTX/debug-info.ll b/llvm/test/DebugInfo/NVPTX/debug-info.ll
index a255717926d6b..64509b22ed365 100644
--- a/llvm/test/DebugInfo/NVPTX/debug-info.ll
+++ b/llvm/test/DebugInfo/NVPTX/debug-info.ll
@@ -106,7 +106,46 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-DAG: .file [[DEBUG_INFO_CU]] "{{.*}}debug-info.cu"
 ; CHECK-DAG: .file [[BUILTUIN_VARS_H]] "{{.*}}clang/include{{/|\\\\}}__clang_cuda_builtin_vars.h"
 
-; CHECK:	.section	.debug_abbrev
+; CHECK:	.section	.debug_loc
+; CHECK-NEXT:	{
+; CHECK-NEXT:$L__debug_loc0:
+; CHECK-NEXT:.b64 $L__tmp8
+; CHECK-NEXT:.b64 $L__tmp10
+; CHECK-NEXT:.b8 5                                   // Loc expr size
+; CHECK-NEXT:.b8 0
+; CHECK-NEXT:.b8 144                                 // DW_OP_regx
+; CHECK-NEXT:.b8 177                                 // 2450993
+; CHECK-NEXT:.b8 204                                 // 
+; CHECK-NEXT:.b8 149                                 // 
+; CHECK-NEXT:.b8 1                                   // 
+; CHECK-NEXT:.b64 0
+; CHECK-NEXT:.b64 0
+; CHECK-NEXT:$L__debug_loc1:
+; CHECK-NEXT:.b64 $L__tmp0
+; CHECK-NEXT:.b64 $L__tmp7
+; CHECK-NEXT:.b8 5                                   // Loc expr size
+; CHECK-NEXT:.b8 0
+; CHECK-NEXT:.b8 144                                 // DW_OP_regx
+; CHECK-NEXT:.b8 178                                 // 2454066
+; CHECK-NEXT:.b8 228                                 // 
+; CHECK-NEXT:.b8 149                                 // 
+; CHECK-NEXT:.b8 1                                   // 
+; CHECK-NEXT:.b64 0
+; CHECK-NEXT:.b64 0
+; CHECK-NEXT:$L__debug_loc2:
+; CHECK-NEXT:.b64 $L__tmp5
+; CHECK-NEXT:.b64 $L__tmp7
+; CHECK-NEXT:.b8 5                                   // Loc expr size
+; CHECK-NEXT:.b8 0
+; CHECK-NEXT:.b8 144                                 // DW_OP_regx
+; CHECK-NEXT:.b8 177                                 // 2454065
+; CHECK-NEXT:.b8 228                                 // 
+; CHECK-NEXT:.b8 149                                 // 
+; CHECK-NEXT:.b8 1                                   // 
+; CHECK-NEXT:.b64 0
+; CHECK-NEXT:.b64 0
+; CHECK-NEXT:	}
+; CHECK-NEXT:	.section	.debug_abbrev
 ; CHECK-NEXT:	{
 ; CHECK-NEXT:.b8 1                                   // Abbreviation Code
 ; CHECK-NEXT:.b8 17                                  // DW_TAG_compile_unit
@@ -121,10 +160,6 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 6                                   // DW_FORM_data4
 ; CHECK-NEXT:.b8 27                                  // DW_AT_comp_dir
 ; CHECK-NEXT:.b8 8                                   // DW_FORM_string
-; CHECK-NEXT:.b8 17                                  // DW_AT_low_pc
-; CHECK-NEXT:.b8 1                                   // DW_FORM_addr
-; CHECK-NEXT:.b8 18                                  // DW_AT_high_pc
-; CHECK-NEXT:.b8 1                                   // DW_FORM_addr
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
 ; CHECK-NEXT:.b8 2                                   // Abbreviation Code
@@ -441,8 +476,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
 ; CHECK-NEXT:.b8 24                                  // Abbreviation Code
-; CHECK-NEXT:.b8 52                                  // DW_TAG_variable
+; CHECK-NEXT:.b8 5                                   // DW_TAG_formal_parameter
 ; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT:.b8 2                                   // DW_AT_location
+; CHECK-NEXT:.b8 6                                   // DW_FORM_data4
 ; CHECK-NEXT:.b8 3                                   // DW_AT_name
 ; CHECK-NEXT:.b8 8                                   // DW_FORM_string
 ; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
@@ -454,6 +491,21 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
 ; CHECK-NEXT:.b8 25                                  // Abbreviation Code
+; CHECK-NEXT:.b8 52                                  // DW_TAG_variable
+; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT:.b8 2                                   // DW_AT_location
+; CHECK-NEXT:.b8 6                                   // DW_FORM_data4
+; CHECK-NEXT:.b8 3                                   // DW_AT_name
+; CHECK-NEXT:.b8 8                                   // DW_FORM_string
+; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
+; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
+; CHECK-NEXT:.b8 73                                  // DW_AT_type
+; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT:.b8 0                                   // EOM(1)
+; CHECK-NEXT:.b8 0                                   // EOM(2)
+; CHECK-NEXT:.b8 26                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 29                                  // DW_TAG_inlined_subroutine
 ; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
 ; CHECK-NEXT:.b8 49                                  // DW_AT_abstract_origin
@@ -470,7 +522,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 26                                  // Abbreviation Code
+; CHECK-NEXT:.b8 27                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 29                                  // DW_TAG_inlined_subroutine
 ; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
 ; CHECK-NEXT:.b8 49                                  // DW_AT_abstract_origin
@@ -487,7 +539,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 27                                  // Abbreviation Code
+; CHECK-NEXT:.b8 28                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 5                                   // DW_TAG_formal_parameter
 ; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
 ; CHECK-NEXT:.b8 51                                  // DW_AT_address_class
@@ -498,14 +550,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 28                                  // Abbreviation Code
+; CHECK-NEXT:.b8 29                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 57                                  // DW_TAG_namespace
 ; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
 ; CHECK-NEXT:.b8 3                                   // DW_AT_name
 ; CHECK-NEXT:.b8 8                                   // DW_FORM_string
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 29                                  // Abbreviation Code
+; CHECK-NEXT:.b8 30                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 8                                   // DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
 ; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
@@ -516,7 +568,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 30                                  // Abbreviation Code
+; CHECK-NEXT:.b8 31                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 8                                   // DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
 ; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
@@ -527,7 +579,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 31                                  // Abbreviation Code
+; CHECK-NEXT:.b8 32                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 46                                  // DW_TAG_subprogram
 ; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
 ; CHECK-NEXT:.b8 135                                 // DW_AT_MIPS_linkage_name
@@ -545,7 +597,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 32                                  // Abbreviation Code
+; CHECK-NEXT:.b8 33                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 46                                  // DW_TAG_subprogram
 ; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
 ; CHECK-NEXT:.b8 3                                   // DW_AT_name
@@ -562,7 +614,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 33                                  // Abbreviation Code
+; CHECK-NEXT:.b8 34                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 22                                  // DW_TAG_typedef
 ; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
 ; CHECK-NEXT:.b8 73                                  // DW_AT_type
@@ -575,14 +627,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 34                                  // Abbreviation Code
+; CHECK-NEXT:.b8 35                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 19                                  // DW_TAG_structure_type
 ; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
 ; CHECK-NEXT:.b8 60                                  // DW_AT_declaration
 ; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 35                                  // Abbreviation Code
+; CHECK-NEXT:.b8 36                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 19                                  // DW_TAG_structure_type
 ; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
 ; CHECK-NEXT:.b8 11                                  // DW_AT_byte_size
@@ -593,7 +645,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 36                                  // Abbreviation Code
+; CHECK-NEXT:.b8 37                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 46                                  // DW_TAG_subprogram
 ; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
 ; CHECK-NEXT:.b8 3                                   // DW_AT_name
@@ -611,7 +663,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 37                                  // Abbreviation Code
+; CHECK-NEXT:.b8 38                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 46                                  // DW_TAG_subprogram
 ; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
 ; CHECK-NEXT:.b8 3                                   // DW_AT_name
@@ -628,29 +680,29 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 38                                  // Abbreviation Code
+; CHECK-NEXT:.b8 39                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 21                                  // DW_TAG_subroutine_type
 ; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 39                                  // Abbreviation Code
+; CHECK-NEXT:.b8 40                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 15                                  // DW_TAG_pointer_type
 ; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 40                                  // Abbreviation Code
+; CHECK-NEXT:.b8 41                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 38                                  // DW_TAG_const_type
 ; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 41                                  // Abbreviation Code
+; CHECK-NEXT:.b8 42                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 21                                  // DW_TAG_subroutine_type
 ; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
 ; CHECK-NEXT:.b8 73                                  // DW_AT_type
 ; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 42                                  // Abbreviation Code
+; CHECK-NEXT:.b8 43                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 46                                  // DW_TAG_subprogram
 ; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
 ; CHECK-NEXT:.b8 3                                   // DW_AT_name
@@ -668,7 +720,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 43                                  // Abbreviation Code
+; CHECK-NEXT:.b8 44                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 46                                  // DW_TAG_subprogram
 ; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
 ; CHECK-NEXT:.b8 3                                   // DW_AT_name
@@ -685,7 +737,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_FORM_flag
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
-; CHECK-NEXT:.b8 44                                  // Abbreviation Code
+; CHECK-NEXT:.b8 45                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 46                                  // DW_TAG_subprogram
 ; CHECK-NEXT:.b8 1                                   // DW_CHILDREN_yes
 ; CHECK-NEXT:.b8 135                                 // DW_AT_MIPS_linkage_name
@@ -707,12 +759,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:	}
 ; CHECK-NEXT:	.section	.debug_info
 ; CHECK-NEXT:	{
-; CHECK-NEXT:.b32 10036                              // Length of Unit
+; CHECK-NEXT:.b32 10032                              // Length of Unit
 ; CHECK-NEXT:.b8 2                                   // DWARF version number
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b32 .debug_abbrev                      // Offset Into Abbrev. Section
 ; CHECK-NEXT:.b8 8                                   // Address Size (in bytes)
-; CHECK-NEXT:.b8 1                                   // Abbrev [1] 0xb:0x272d DW_TAG_compile_unit
+; CHECK-NEXT:.b8 1                                   // Abbrev [1] 0xb:0x2729 DW_TAG_compile_unit
 ; CHECK-NEXT:.b8 0                                   // DW_AT_producer
 ; CHECK-NEXT:.b8 4                                   // DW_AT_language
 ; CHECK-NEXT:.b8 0
@@ -747,9 +799,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 114
 ; CHECK-NEXT:.b8 121
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b64 $L__func_begin0                    // DW_AT_low_pc
-; CHECK-NEXT:.b64 $L__func_end0                      // DW_AT_high_pc
-; CHECK-NEXT:.b8 2                                   // Abbrev [2] 0x41:0x22a DW_TAG_structure_type
+; CHECK-NEXT:.b8 2                                   // Abbrev [2] 0x31:0x22a DW_TAG_structure_type
 ; CHECK-NEXT:.b8 95                                  // DW_AT_name
 ; CHECK-NEXT:.b8 95
 ; CHECK-NEXT:.b8 99
@@ -779,7 +829,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1                                   // DW_AT_byte_size
 ; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 77                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x5f:0x4f DW_TAG_subprogram
+; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x4f:0x4f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -852,10 +902,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 78                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 619                                // DW_AT_type
+; CHECK-NEXT:.b32 603                                // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0xae:0x4f DW_TAG_subprogram
+; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x9e:0x4f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -928,10 +978,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 79                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 619                                // DW_AT_type
+; CHECK-NEXT:.b32 603                                // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0xfd:0x4f DW_TAG_subprogram
+; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0xed:0x4f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -1004,10 +1054,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 80                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 619                                // DW_AT_type
+; CHECK-NEXT:.b32 603                                // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 4                                   // Abbrev [4] 0x14c:0x49 DW_TAG_subprogram
+; CHECK-NEXT:.b8 4                                   // Abbrev [4] 0x13c:0x49 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -1067,14 +1117,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 83                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 635                                // DW_AT_type
+; CHECK-NEXT:.b32 619                                // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x18e:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 682                                // DW_AT_type
+; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x17e:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 666                                // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0x195:0x27 DW_TAG_subprogram
+; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0x185:0x27 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_name
 ; CHECK-NEXT:.b8 95
 ; CHECK-NEXT:.b8 99
@@ -1107,11 +1157,11 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
 ; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
 ; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x1b5:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 692                                // DW_AT_type
+; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x1a5:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 676                                // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0x1bc:0x2c DW_TAG_subprogram
+; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0x1ac:0x2c DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_name
 ; CHECK-NEXT:.b8 95
 ; CHECK-NEXT:.b8 99
@@ -1144,13 +1194,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
 ; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
 ; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x1dc:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 692                                // DW_AT_type
+; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x1cc:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 676                                // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1e2:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 697                                // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 681                                // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 8                                   // Abbrev [8] 0x1e8:0x43 DW_TAG_subprogram
+; CHECK-NEXT:.b8 8                                   // Abbrev [8] 0x1d8:0x43 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -1206,13 +1256,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
 ; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
 ; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x21f:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 682                                // DW_AT_type
+; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x20f:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 666                                // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x225:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 697                                // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x215:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 681                                // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 9                                   // Abbrev [9] 0x22b:0x3f DW_TAG_subprogram
+; CHECK-NEXT:.b8 9                                   // Abbrev [9] 0x21b:0x3f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -1261,17 +1311,17 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 85                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 702                                // DW_AT_type
+; CHECK-NEXT:.b32 686                                // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
 ; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
 ; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x263:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 682                                // DW_AT_type
+; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x253:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 666                                // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x26b:0x10 DW_TAG_base_type
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x25b:0x10 DW_TAG_base_type
 ; CHECK-NEXT:.b8 117                                 // DW_AT_name
 ; CHECK-NEXT:.b8 110
 ; CHECK-NEXT:.b8 115
@@ -1287,7 +1337,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 4                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 2                                   // Abbrev [2] 0x27b:0x2f DW_TAG_structure_type
+; CHECK-NEXT:.b8 2                                   // Abbrev [2] 0x26b:0x2f DW_TAG_structure_type
 ; CHECK-NEXT:.b8 117                                 // DW_AT_name
 ; CHECK-NEXT:.b8 105
 ; CHECK-NEXT:.b8 110
@@ -1297,48 +1347,48 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_byte_size
 ; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 190                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x285:0xc DW_TAG_member
+; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x275:0xc DW_TAG_member
 ; CHECK-NEXT:.b8 120                                 // DW_AT_name
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 619                                // DW_AT_type
+; CHECK-NEXT:.b32 603                                // DW_AT_type
 ; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 192                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2                                   // DW_AT_data_member_location
 ; CHECK-NEXT:.b8 35
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x291:0xc DW_TAG_member
+; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x281:0xc DW_TAG_member
 ; CHECK-NEXT:.b8 121                                 // DW_AT_name
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 619                                // DW_AT_type
+; CHECK-NEXT:.b32 603                                // DW_AT_type
 ; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 192                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2                                   // DW_AT_data_member_location
 ; CHECK-NEXT:.b8 35
 ; CHECK-NEXT:.b8 4
-; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x29d:0xc DW_TAG_member
+; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x28d:0xc DW_TAG_member
 ; CHECK-NEXT:.b8 122                                 // DW_AT_name
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 619                                // DW_AT_type
+; CHECK-NEXT:.b32 603                                // DW_AT_type
 ; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 192                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2                                   // DW_AT_data_member_location
 ; CHECK-NEXT:.b8 35
 ; CHECK-NEXT:.b8 8
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x2aa:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 687                                // DW_AT_type
-; CHECK-NEXT:.b8 13                                  // Abbrev [13] 0x2af:0x5 DW_TAG_const_type
-; CHECK-NEXT:.b32 65                                 // DW_AT_type
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x2b4:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 65                                 // DW_AT_type
-; CHECK-NEXT:.b8 14                                  // Abbrev [14] 0x2b9:0x5 DW_TAG_reference_type
-; CHECK-NEXT:.b32 687                                // DW_AT_type
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x2be:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 65                                 // DW_AT_type
-; CHECK-NEXT:.b8 15                                  // Abbrev [15] 0x2c3:0x6 DW_TAG_subprogram
-; CHECK-NEXT:.b32 95                                 // DW_AT_specification
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x29a:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 671                                // DW_AT_type
+; CHECK-NEXT:.b8 13                                  // Abbrev [13] 0x29f:0x5 DW_TAG_const_type
+; CHECK-NEXT:.b32 49                                 // DW_AT_type
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x2a4:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 49                                 // DW_AT_type
+; CHECK-NEXT:.b8 14                                  // Abbrev [14] 0x2a9:0x5 DW_TAG_reference_type
+; CHECK-NEXT:.b32 671                                // DW_AT_type
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x2ae:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 49                                 // DW_AT_type
+; CHECK-NEXT:.b8 15                                  // Abbrev [15] 0x2b3:0x6 DW_TAG_subprogram
+; CHECK-NEXT:.b32 79                                 // DW_AT_specification
 ; CHECK-NEXT:.b8 1                                   // DW_AT_inline
-; CHECK-NEXT:.b8 2                                   // Abbrev [2] 0x2c9:0x228 DW_TAG_structure_type
+; CHECK-NEXT:.b8 2                                   // Abbrev [2] 0x2b9:0x228 DW_TAG_structure_type
 ; CHECK-NEXT:.b8 95                                  // DW_AT_name
 ; CHECK-NEXT:.b8 95
 ; CHECK-NEXT:.b8 99
@@ -1368,7 +1418,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1                                   // DW_AT_byte_size
 ; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 88                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x2e7:0x4f DW_TAG_subprogram
+; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x2d7:0x4f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -1441,10 +1491,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 89                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 619                                // DW_AT_type
+; CHECK-NEXT:.b32 603                                // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x336:0x4f DW_TAG_subprogram
+; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x326:0x4f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -1517,10 +1567,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 90                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 619                                // DW_AT_type
+; CHECK-NEXT:.b32 603                                // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x385:0x4f DW_TAG_subprogram
+; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x375:0x4f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -1593,10 +1643,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 91                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 619                                // DW_AT_type
+; CHECK-NEXT:.b32 603                                // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 4                                   // Abbrev [4] 0x3d4:0x47 DW_TAG_subprogram
+; CHECK-NEXT:.b8 4                                   // Abbrev [4] 0x3c4:0x47 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -1654,14 +1704,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 94                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 1265                               // DW_AT_type
+; CHECK-NEXT:.b32 1249                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x414:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1441                               // DW_AT_type
+; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x404:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 1425                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0x41b:0x27 DW_TAG_subprogram
+; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0x40b:0x27 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_name
 ; CHECK-NEXT:.b8 95
 ; CHECK-NEXT:.b8 99
@@ -1694,11 +1744,11 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
 ; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
 ; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x43b:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1451                               // DW_AT_type
+; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x42b:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 1435                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0x442:0x2c DW_TAG_subprogram
+; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0x432:0x2c DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_name
 ; CHECK-NEXT:.b8 95
 ; CHECK-NEXT:.b8 99
@@ -1731,13 +1781,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
 ; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
 ; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x462:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1451                               // DW_AT_type
+; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x452:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 1435                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x468:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1456                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x458:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 1440                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 8                                   // Abbrev [8] 0x46e:0x43 DW_TAG_subprogram
+; CHECK-NEXT:.b8 8                                   // Abbrev [8] 0x45e:0x43 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -1793,13 +1843,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
 ; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
 ; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x4a5:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1441                               // DW_AT_type
+; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x495:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 1425                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x4ab:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1456                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x49b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 1440                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 9                                   // Abbrev [9] 0x4b1:0x3f DW_TAG_subprogram
+; CHECK-NEXT:.b8 9                                   // Abbrev [9] 0x4a1:0x3f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -1848,17 +1898,17 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 96                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 1461                               // DW_AT_type
+; CHECK-NEXT:.b32 1445                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
 ; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
 ; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x4e9:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1441                               // DW_AT_type
+; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x4d9:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 1425                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 16                                  // Abbrev [16] 0x4f1:0x9d DW_TAG_structure_type
+; CHECK-NEXT:.b8 16                                  // Abbrev [16] 0x4e1:0x9d DW_TAG_structure_type
 ; CHECK-NEXT:.b8 100                                 // DW_AT_name
 ; CHECK-NEXT:.b8 105
 ; CHECK-NEXT:.b8 109
@@ -1868,37 +1918,37 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 161                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b8 17                                  // Abbrev [17] 0x4fb:0xd DW_TAG_member
+; CHECK-NEXT:.b8 17                                  // Abbrev [17] 0x4eb:0xd DW_TAG_member
 ; CHECK-NEXT:.b8 120                                 // DW_AT_name
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 619                                // DW_AT_type
+; CHECK-NEXT:.b32 603                                // DW_AT_type
 ; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 163                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
 ; CHECK-NEXT:.b8 2                                   // DW_AT_data_member_location
 ; CHECK-NEXT:.b8 35
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 17                                  // Abbrev [17] 0x508:0xd DW_TAG_member
+; CHECK-NEXT:.b8 17                                  // Abbrev [17] 0x4f8:0xd DW_TAG_member
 ; CHECK-NEXT:.b8 121                                 // DW_AT_name
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 619                                // DW_AT_type
+; CHECK-NEXT:.b32 603                                // DW_AT_type
 ; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 163                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
 ; CHECK-NEXT:.b8 2                                   // DW_AT_data_member_location
 ; CHECK-NEXT:.b8 35
 ; CHECK-NEXT:.b8 4
-; CHECK-NEXT:.b8 17                                  // Abbrev [17] 0x515:0xd DW_TAG_member
+; CHECK-NEXT:.b8 17                                  // Abbrev [17] 0x505:0xd DW_TAG_member
 ; CHECK-NEXT:.b8 122                                 // DW_AT_name
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 619                                // DW_AT_type
+; CHECK-NEXT:.b32 603                                // DW_AT_type
 ; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 163                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
 ; CHECK-NEXT:.b8 2                                   // DW_AT_data_member_location
 ; CHECK-NEXT:.b8 35
 ; CHECK-NEXT:.b8 8
-; CHECK-NEXT:.b8 18                                  // Abbrev [18] 0x522:0x21 DW_TAG_subprogram
+; CHECK-NEXT:.b8 18                                  // Abbrev [18] 0x512:0x21 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 100                                 // DW_AT_name
 ; CHECK-NEXT:.b8 105
 ; CHECK-NEXT:.b8 109
@@ -1909,17 +1959,17 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x52d:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1422                               // DW_AT_type
+; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x51d:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 1406                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x533:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 619                                // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x538:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 619                                // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x53d:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 619                                // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 18                                  // Abbrev [18] 0x543:0x17 DW_TAG_subprogram
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x523:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 603                                // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x528:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 603                                // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x52d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 603                                // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 18                                  // Abbrev [18] 0x533:0x17 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 100                                 // DW_AT_name
 ; CHECK-NEXT:.b8 105
 ; CHECK-NEXT:.b8 109
@@ -1930,13 +1980,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x54e:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1422                               // DW_AT_type
+; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x53e:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 1406                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x554:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1427                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x544:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 1411                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 19                                  // Abbrev [19] 0x55a:0x33 DW_TAG_subprogram
+; CHECK-NEXT:.b8 19                                  // Abbrev [19] 0x54a:0x33 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -1974,18 +2024,18 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 167                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 1427                               // DW_AT_type
+; CHECK-NEXT:.b32 1411                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x586:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 1422                               // DW_AT_type
+; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x576:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 1406                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x58e:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 1265                               // DW_AT_type
-; CHECK-NEXT:.b8 20                                  // Abbrev [20] 0x593:0xe DW_TAG_typedef
-; CHECK-NEXT:.b32 635                                // DW_AT_type
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x57e:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 1249                               // DW_AT_type
+; CHECK-NEXT:.b8 20                                  // Abbrev [20] 0x583:0xe DW_TAG_typedef
+; CHECK-NEXT:.b32 619                                // DW_AT_type
 ; CHECK-NEXT:.b8 117                                 // DW_AT_name
 ; CHECK-NEXT:.b8 105
 ; CHECK-NEXT:.b8 110
@@ -1995,20 +2045,20 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 3                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 127                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x5a1:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 1446                               // DW_AT_type
-; CHECK-NEXT:.b8 13                                  // Abbrev [13] 0x5a6:0x5 DW_TAG_const_type
-; CHECK-NEXT:.b32 713                                // DW_AT_type
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x5ab:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 713                                // DW_AT_type
-; CHECK-NEXT:.b8 14                                  // Abbrev [14] 0x5b0:0x5 DW_TAG_reference_type
-; CHECK-NEXT:.b32 1446                               // DW_AT_type
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x5b5:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 713                                // DW_AT_type
-; CHECK-NEXT:.b8 15                                  // Abbrev [15] 0x5ba:0x6 DW_TAG_subprogram
-; CHECK-NEXT:.b32 743                                // DW_AT_specification
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x591:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 1430                               // DW_AT_type
+; CHECK-NEXT:.b8 13                                  // Abbrev [13] 0x596:0x5 DW_TAG_const_type
+; CHECK-NEXT:.b32 697                                // DW_AT_type
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x59b:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 697                                // DW_AT_type
+; CHECK-NEXT:.b8 14                                  // Abbrev [14] 0x5a0:0x5 DW_TAG_reference_type
+; CHECK-NEXT:.b32 1430                               // DW_AT_type
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x5a5:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 697                                // DW_AT_type
+; CHECK-NEXT:.b8 15                                  // Abbrev [15] 0x5aa:0x6 DW_TAG_subprogram
+; CHECK-NEXT:.b32 727                                // DW_AT_specification
 ; CHECK-NEXT:.b8 1                                   // DW_AT_inline
-; CHECK-NEXT:.b8 2                                   // Abbrev [2] 0x5c0:0x233 DW_TAG_structure_type
+; CHECK-NEXT:.b8 2                                   // Abbrev [2] 0x5b0:0x233 DW_TAG_structure_type
 ; CHECK-NEXT:.b8 95                                  // DW_AT_name
 ; CHECK-NEXT:.b8 95
 ; CHECK-NEXT:.b8 99
@@ -2039,7 +2089,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1                                   // DW_AT_byte_size
 ; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 66                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x5df:0x50 DW_TAG_subprogram
+; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x5cf:0x50 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -2113,10 +2163,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 67                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 619                                // DW_AT_type
+; CHECK-NEXT:.b32 603                                // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x62f:0x50 DW_TAG_subprogram
+; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x61f:0x50 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -2190,10 +2240,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 68                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 619                                // DW_AT_type
+; CHECK-NEXT:.b32 603                                // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x67f:0x50 DW_TAG_subprogram
+; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0x66f:0x50 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -2267,10 +2317,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 69                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 619                                // DW_AT_type
+; CHECK-NEXT:.b32 603                                // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 4                                   // Abbrev [4] 0x6cf:0x4a DW_TAG_subprogram
+; CHECK-NEXT:.b8 4                                   // Abbrev [4] 0x6bf:0x4a DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -2331,14 +2381,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 72                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 635                                // DW_AT_type
+; CHECK-NEXT:.b32 619                                // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x712:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2035                               // DW_AT_type
+; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x702:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2019                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0x719:0x28 DW_TAG_subprogram
+; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0x709:0x28 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_name
 ; CHECK-NEXT:.b8 95
 ; CHECK-NEXT:.b8 99
@@ -2372,11 +2422,11 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
 ; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
 ; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x73a:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2045                               // DW_AT_type
+; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x72a:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2029                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0x741:0x2d DW_TAG_subprogram
+; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0x731:0x2d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_name
 ; CHECK-NEXT:.b8 95
 ; CHECK-NEXT:.b8 99
@@ -2410,13 +2460,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
 ; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
 ; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x762:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2045                               // DW_AT_type
+; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x752:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2029                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x768:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2050                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x758:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2034                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 8                                   // Abbrev [8] 0x76e:0x44 DW_TAG_subprogram
+; CHECK-NEXT:.b8 8                                   // Abbrev [8] 0x75e:0x44 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -2473,13 +2523,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
 ; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
 ; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x7a6:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2035                               // DW_AT_type
+; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x796:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2019                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x7ac:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2050                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x79c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2034                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 9                                   // Abbrev [9] 0x7b2:0x40 DW_TAG_subprogram
+; CHECK-NEXT:.b8 9                                   // Abbrev [9] 0x7a2:0x40 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -2529,30 +2579,30 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 2                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 74                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2055                               // DW_AT_type
+; CHECK-NEXT:.b32 2039                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
 ; CHECK-NEXT:.b8 3                                   // DW_AT_accessibility
 ; CHECK-NEXT:                                        // DW_ACCESS_private
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x7eb:0x6 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2035                               // DW_AT_type
+; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0x7db:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2019                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_artificial
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x7f3:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 2040                               // DW_AT_type
-; CHECK-NEXT:.b8 13                                  // Abbrev [13] 0x7f8:0x5 DW_TAG_const_type
-; CHECK-NEXT:.b32 1472                               // DW_AT_type
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x7fd:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 1472                               // DW_AT_type
-; CHECK-NEXT:.b8 14                                  // Abbrev [14] 0x802:0x5 DW_TAG_reference_type
-; CHECK-NEXT:.b32 2040                               // DW_AT_type
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x807:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 1472                               // DW_AT_type
-; CHECK-NEXT:.b8 15                                  // Abbrev [15] 0x80c:0x6 DW_TAG_subprogram
-; CHECK-NEXT:.b32 1503                               // DW_AT_specification
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x7e3:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 2024                               // DW_AT_type
+; CHECK-NEXT:.b8 13                                  // Abbrev [13] 0x7e8:0x5 DW_TAG_const_type
+; CHECK-NEXT:.b32 1456                               // DW_AT_type
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x7ed:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 1456                               // DW_AT_type
+; CHECK-NEXT:.b8 14                                  // Abbrev [14] 0x7f2:0x5 DW_TAG_reference_type
+; CHECK-NEXT:.b32 2024                               // DW_AT_type
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x7f7:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 1456                               // DW_AT_type
+; CHECK-NEXT:.b8 15                                  // Abbrev [15] 0x7fc:0x6 DW_TAG_subprogram
+; CHECK-NEXT:.b32 1487                               // DW_AT_specification
 ; CHECK-NEXT:.b8 1                                   // DW_AT_inline
-; CHECK-NEXT:.b8 21                                  // Abbrev [21] 0x812:0x32 DW_TAG_subprogram
+; CHECK-NEXT:.b8 21                                  // Abbrev [21] 0x802:0x32 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 51
@@ -2572,28 +2622,28 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 3                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
 ; CHECK-NEXT:.b8 1                                   // DW_AT_inline
-; CHECK-NEXT:.b8 22                                  // Abbrev [22] 0x826:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 22                                  // Abbrev [22] 0x816:0x9 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b8 120                                 // DW_AT_name
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 3                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 22                                  // Abbrev [22] 0x82f:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 22                                  // Abbrev [22] 0x81f:0x9 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b8 121                                 // DW_AT_name
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 3                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 22                                  // Abbrev [22] 0x838:0xb DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 22                                  // Abbrev [22] 0x828:0xb DW_TAG_formal_parameter
 ; CHECK-NEXT:.b8 114                                 // DW_AT_name
 ; CHECK-NEXT:.b8 101
 ; CHECK-NEXT:.b8 115
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 3                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 2125                               // DW_AT_type
+; CHECK-NEXT:.b32 2109                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x844:0x9 DW_TAG_base_type
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x834:0x9 DW_TAG_base_type
 ; CHECK-NEXT:.b8 102                                 // DW_AT_name
 ; CHECK-NEXT:.b8 108
 ; CHECK-NEXT:.b8 111
@@ -2602,9 +2652,9 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 4                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x84d:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 23                                  // Abbrev [23] 0x852:0xc6 DW_TAG_subprogram
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x83d:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 23                                  // Abbrev [23] 0x842:0xd2 DW_TAG_subprogram
 ; CHECK-NEXT:.b64 $L__func_begin0                    // DW_AT_low_pc
 ; CHECK-NEXT:.b64 $L__func_end0                      // DW_AT_high_pc
 ; CHECK-NEXT:.b8 1                                   // DW_AT_frame_base
@@ -2633,65 +2683,68 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 22                                  // Abbrev [22] 0x87d:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 24                                  // Abbrev [24] 0x86d:0xd DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 $L__debug_loc1                     // DW_AT_location
 ; CHECK-NEXT:.b8 110                                 // DW_AT_name
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
-; CHECK-NEXT:.b8 22                                  // Abbrev [22] 0x886:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
+; CHECK-NEXT:.b8 24                                  // Abbrev [24] 0x87a:0xd DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 $L__debug_loc0                     // DW_AT_location
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 22                                  // Abbrev [22] 0x88f:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 22                                  // Abbrev [22] 0x887:0x9 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b8 120                                 // DW_AT_name
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 2125                               // DW_AT_type
-; CHECK-NEXT:.b8 22                                  // Abbrev [22] 0x898:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2109                               // DW_AT_type
+; CHECK-NEXT:.b8 22                                  // Abbrev [22] 0x890:0x9 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b8 121                                 // DW_AT_name
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 2125                               // DW_AT_type
-; CHECK-NEXT:.b8 24                                  // Abbrev [24] 0x8a1:0x9 DW_TAG_variable
+; CHECK-NEXT:.b32 2109                               // DW_AT_type
+; CHECK-NEXT:.b8 25                                  // Abbrev [25] 0x899:0xd DW_TAG_variable
+; CHECK-NEXT:.b32 $L__debug_loc2                     // DW_AT_location
 ; CHECK-NEXT:.b8 105                                 // DW_AT_name
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
-; CHECK-NEXT:.b8 25                                  // Abbrev [25] 0x8aa:0x18 DW_TAG_inlined_subroutine
-; CHECK-NEXT:.b32 707                                // DW_AT_abstract_origin
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
+; CHECK-NEXT:.b8 26                                  // Abbrev [26] 0x8a6:0x18 DW_TAG_inlined_subroutine
+; CHECK-NEXT:.b32 691                                // DW_AT_abstract_origin
 ; CHECK-NEXT:.b64 $L__tmp1                           // DW_AT_low_pc
 ; CHECK-NEXT:.b64 $L__tmp2                           // DW_AT_high_pc
 ; CHECK-NEXT:.b8 1                                   // DW_AT_call_file
 ; CHECK-NEXT:.b8 6                                   // DW_AT_call_line
 ; CHECK-NEXT:.b8 11                                  // DW_AT_call_column
-; CHECK-NEXT:.b8 25                                  // Abbrev [25] 0x8c2:0x18 DW_TAG_inlined_subroutine
-; CHECK-NEXT:.b32 1466                               // DW_AT_abstract_origin
+; CHECK-NEXT:.b8 26                                  // Abbrev [26] 0x8be:0x18 DW_TAG_inlined_subroutine
+; CHECK-NEXT:.b32 1450                               // DW_AT_abstract_origin
 ; CHECK-NEXT:.b64 $L__tmp2                           // DW_AT_low_pc
 ; CHECK-NEXT:.b64 $L__tmp3                           // DW_AT_high_pc
 ; CHECK-NEXT:.b8 1                                   // DW_AT_call_file
 ; CHECK-NEXT:.b8 6                                   // DW_AT_call_line
 ; CHECK-NEXT:.b8 24                                  // DW_AT_call_column
-; CHECK-NEXT:.b8 25                                  // Abbrev [25] 0x8da:0x18 DW_TAG_inlined_subroutine
-; CHECK-NEXT:.b32 2060                               // DW_AT_abstract_origin
+; CHECK-NEXT:.b8 26                                  // Abbrev [26] 0x8d6:0x18 DW_TAG_inlined_subroutine
+; CHECK-NEXT:.b32 2044                               // DW_AT_abstract_origin
 ; CHECK-NEXT:.b64 $L__tmp3                           // DW_AT_low_pc
 ; CHECK-NEXT:.b64 $L__tmp4                           // DW_AT_high_pc
 ; CHECK-NEXT:.b8 1                                   // DW_AT_call_file
 ; CHECK-NEXT:.b8 6                                   // DW_AT_call_line
 ; CHECK-NEXT:.b8 37                                  // DW_AT_call_column
-; CHECK-NEXT:.b8 26                                  // Abbrev [26] 0x8f2:0x25 DW_TAG_inlined_subroutine
-; CHECK-NEXT:.b32 2066                               // DW_AT_abstract_origin
+; CHECK-NEXT:.b8 27                                  // Abbrev [27] 0x8ee:0x25 DW_TAG_inlined_subroutine
+; CHECK-NEXT:.b32 2050                               // DW_AT_abstract_origin
 ; CHECK-NEXT:.b64 $L__tmp9                           // DW_AT_low_pc
 ; CHECK-NEXT:.b64 $L__tmp10                          // DW_AT_high_pc
 ; CHECK-NEXT:.b8 1                                   // DW_AT_call_file
 ; CHECK-NEXT:.b8 8                                   // DW_AT_call_line
 ; CHECK-NEXT:.b8 5                                   // DW_AT_call_column
-; CHECK-NEXT:.b8 27                                  // Abbrev [27] 0x90a:0xc DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 28                                  // Abbrev [28] 0x906:0xc DW_TAG_formal_parameter
 ; CHECK-NEXT:.b8 2                                   // DW_AT_address_class
 ; CHECK-NEXT:.b8 5                                   // DW_AT_location
 ; CHECK-NEXT:.b8 144
@@ -2699,859 +2752,859 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 204
 ; CHECK-NEXT:.b8 149
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 2095                               // DW_AT_abstract_origin
+; CHECK-NEXT:.b32 2079                               // DW_AT_abstract_origin
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 28                                  // Abbrev [28] 0x918:0x588 DW_TAG_namespace
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x914:0x588 DW_TAG_namespace
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 100
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x91d:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x919:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 202                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3744                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x924:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 3740                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x920:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 203                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3788                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x92b:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 3784                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x927:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 204                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3817                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x932:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 3813                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x92e:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 205                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3848                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x939:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 3844                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x935:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 206                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3877                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x940:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 3873                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x93c:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 207                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3908                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x947:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 3904                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x943:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 208                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3937                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x94e:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 3933                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x94a:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 209                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3974                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x955:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 3970                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x951:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 210                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4005                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x95c:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4001                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x958:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 211                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4034                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x963:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4030                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x95f:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 212                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4063                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x96a:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4059                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x966:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 213                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4106                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x971:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4102                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x96d:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 214                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4133                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x978:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4129                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x974:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 215                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4162                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x97f:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4158                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x97b:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 216                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4189                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x986:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4185                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x982:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 217                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4218                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x98d:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4214                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x989:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 218                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4245                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x994:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4241                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x990:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 219                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4274                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x99b:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4270                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x997:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 220                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4305                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9a2:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4301                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x99e:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 221                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4334                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9a9:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4330                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x9a5:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 222                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4369                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9b0:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4365                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x9ac:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 223                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4400                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9b7:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4396                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x9b3:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 224                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4439                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9be:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4435                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x9ba:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 225                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4474                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9c5:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4470                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x9c1:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 226                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4509                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9cc:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4505                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x9c8:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 227                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4544                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9d3:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4540                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x9cf:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 228                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4593                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9da:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4589                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x9d6:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 229                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4636                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9e1:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4632                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x9dd:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 230                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4673                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9e8:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4669                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x9e4:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 231                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4704                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9ef:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4700                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x9eb:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 232                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4749                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9f6:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4745                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x9f2:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 233                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4794                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9fd:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4790                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x9f9:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 234                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4850                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa04:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4846                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa00:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 235                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4881                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa0b:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4877                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa07:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 236                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4920                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa12:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4916                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa0e:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 237                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4970                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa19:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4966                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa15:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 238                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5024                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa20:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5020                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa1c:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 239                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5055                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa27:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5051                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa23:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 240                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5092                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa2e:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5088                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa2a:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 241                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5142                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa35:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5138                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa31:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 242                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5183                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa3c:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5179                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa38:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 243                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5220                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa43:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5216                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa3f:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 244                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5253                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa4a:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5249                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa46:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 245                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5284                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa51:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5280                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa4d:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 246                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5317                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa58:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5313                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa54:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 247                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5344                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa5f:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5340                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa5b:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 248                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5375                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa66:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5371                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa62:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 249                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5406                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa6d:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5402                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa69:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 250                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5435                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa74:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5431                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa70:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 251                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5464                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa7b:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5460                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa77:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 252                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5495                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa82:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5491                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa7e:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 253                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5528                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa89:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5524                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa85:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 254                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5563                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa90:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5559                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa8c:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 255                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5599                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa97:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5595                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xa93:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 0                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5656                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa9f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5652                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xa9b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 1                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5687                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xaa7:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5683                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xaa3:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 2                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5726                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xaaf:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5722                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xaab:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 3                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5771                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xab7:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5767                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xab3:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5804                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xabf:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5800                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xabb:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5849                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xac7:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5845                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xac3:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5895                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xacf:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5891                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xacb:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5924                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xad7:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5920                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xad3:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5955                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xadf:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5951                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xadb:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 9                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5996                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xae7:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5992                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xae3:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 10                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6035                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xaef:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6031                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xaeb:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 11                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6070                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xaf7:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6066                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xaf3:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6097                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xaff:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6093                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xafb:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6126                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb07:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6122                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xb03:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 14                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6155                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb0f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6151                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xb0b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 15                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6182                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb17:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6178                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xb13:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 16                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6211                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb1f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6207                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xb1b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 17                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6244                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb27:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6240                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb23:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 102                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6275                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb2e:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6271                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb2a:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 121                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6295                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb35:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6291                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb31:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 140                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6315                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb3c:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6311                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb38:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 159                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6335                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb43:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6331                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb3f:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 180                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6361                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb4a:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6357                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb46:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 199                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6381                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb51:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6377                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb4d:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 218                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6400                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb58:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6396                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb54:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 237                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6420                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb5f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6416                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xb5b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 0                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6439                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb67:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6435                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xb63:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 19                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6459                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb6f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6455                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xb6b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 38                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6480                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb77:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6476                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xb73:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6505                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb7f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6501                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xb7b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 78                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6531                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb87:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6527                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xb83:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 97                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6557                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb8f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6553                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xb8b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 116                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6576                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb97:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6572                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xb93:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 135                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6597                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb9f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6593                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xb9b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 147                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6627                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xba7:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6623                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xba3:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 184                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6651                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xbaf:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6647                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xbab:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 203                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6670                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xbb7:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6666                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xbb3:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 222                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6690                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xbbf:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6686                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xbbb:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 241                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6710                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xbc7:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6706                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xbc3:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2
-; CHECK-NEXT:.b32 6729                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbcf:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6725                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xbcb:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 118                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6749                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbd6:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6745                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xbd2:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 119                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6764                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbdd:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6760                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xbd9:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 121                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6812                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbe4:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6808                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xbe0:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 122                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6825                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbeb:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6821                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xbe7:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 123                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6845                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbf2:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6841                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xbee:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 129                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6874                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbf9:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6870                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xbf5:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 130                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6894                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc00:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6890                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xbfc:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 131                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6915                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc07:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6911                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc03:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 132                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6936                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc0e:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6932                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc0a:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 133                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7064                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc15:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7060                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc11:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 134                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7092                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc1c:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7088                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc18:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 135                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7117                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc23:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7113                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc1f:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 136                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7135                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc2a:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7131                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc26:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 137                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7152                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc31:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7148                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc2d:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 138                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7180                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc38:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7176                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc34:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 139                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7201                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc3f:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7197                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc3b:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 140                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7227                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc46:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7223                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc42:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 142                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7250                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc4d:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7246                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc49:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 143                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7277                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc54:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7273                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc50:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 144                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7328                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc5b:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7324                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc57:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 146                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7361                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc62:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7357                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc5e:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 152                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7394                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc69:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7390                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc65:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 153                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7409                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc70:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7405                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc6c:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 154                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7438                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc77:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7434                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc73:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 155                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7456                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc7e:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7452                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc7a:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 156                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7488                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc85:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7484                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc81:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 157                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7520                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc8c:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7516                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc88:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 158                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7553                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc93:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7549                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc8f:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 160                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7576                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc9a:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7572                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc96:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 161                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7621                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xca1:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7617                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xc9d:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 241                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7769                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xca8:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7765                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xca4:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 243                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7818                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xcaf:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7814                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xcab:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 245                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7837                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xcb6:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7833                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xcb2:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 246                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7723                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xcbd:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7719                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xcb9:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 247                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7859                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xcc4:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7855                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xcc0:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 249                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7886                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xccb:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7882                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xcc7:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 250                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 8001                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xcd2:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7997                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xcce:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 251                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7908                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xcd9:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7904                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xcd5:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 252                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7941                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xce0:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7937                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xcdc:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 253                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 8028                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xce7:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8024                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xce3:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 149                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8071                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xcef:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8067                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xceb:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 150                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8103                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xcf7:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8099                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xcf3:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 151                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8137                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xcff:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8133                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xcfb:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 152                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8169                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd07:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8165                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xd03:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 153                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8203                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd0f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8199                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xd0b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 154                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8243                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd17:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8239                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xd13:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 155                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8275                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd1f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8271                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xd1b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 156                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8309                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd27:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8305                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xd23:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 157                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8341                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd2f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8337                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xd2b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 158                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8373                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd37:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8369                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xd33:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 159                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8419                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd3f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8415                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xd3b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 160                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8449                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd47:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8445                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xd43:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 161                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8481                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd4f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8477                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xd4b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 162                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8513                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd57:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8509                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xd53:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 163                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8543                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd5f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8539                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xd5b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 164                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8575                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd67:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8571                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xd63:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 165                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8605                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd6f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8601                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xd6b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 166                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8639                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd77:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8635                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xd73:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 167                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8671                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd7f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8667                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xd7b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 168                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8709                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd87:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8705                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xd83:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 169                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8743                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd8f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8739                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xd8b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 170                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8785                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd97:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8781                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xd93:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 171                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8823                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd9f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8819                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xd9b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 172                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8861                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xda7:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8857                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xda3:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 173                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8899                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdaf:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8895                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xdab:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 174                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8940                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdb7:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8936                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xdb3:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 175                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8980                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdbf:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8976                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xdbb:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 176                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9014                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdc7:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9010                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xdc3:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 177                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9054                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdcf:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9050                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xdcb:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 178                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9090                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdd7:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9086                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xdd3:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 179                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9126                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xddf:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9122                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xddb:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 180                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9164                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xde7:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9160                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xde3:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 181                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9198                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdef:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9194                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xdeb:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 182                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9232                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdf7:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9228                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xdf3:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 183                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9264                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdff:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9260                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xdfb:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 184                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9296                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe07:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9292                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xe03:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 185                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9326                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe0f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9322                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xe0b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 186                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9360                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe17:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9356                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xe13:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 187                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9396                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe1f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9392                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xe1b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 188                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9435                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe27:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9431                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xe23:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 189                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9478                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe2f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9474                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xe2b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 190                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9527                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe37:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9523                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xe33:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 191                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9563                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe3f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9559                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xe3b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 192                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9612                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe47:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9608                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xe43:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 193                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9661                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe4f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9657                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xe4b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 194                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9693                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe57:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9689                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xe53:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 195                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9727                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe5f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9723                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xe5b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 196                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9771                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe67:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9767                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xe63:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 197                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9813                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe6f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9809                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xe6b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 198                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9843                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe77:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9839                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xe73:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 199                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9875                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe7f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9871                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xe7b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 200                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9907                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe87:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9903                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xe83:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 201                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9937                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe8f:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9933                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xe8b:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 202                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9969                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe97:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9965                               // DW_AT_import
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xe93:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 203                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 10005                              // DW_AT_import
+; CHECK-NEXT:.b32 10001                              // DW_AT_import
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xea0:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0xe9c:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3567,12 +3620,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 44                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 3771                               // DW_AT_type
+; CHECK-NEXT:.b32 3767                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xeb5:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 3771                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xeb1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 3767                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0xebb:0x11 DW_TAG_base_type
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0xeb7:0x11 DW_TAG_base_type
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 110
@@ -3589,7 +3642,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 8                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xecc:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0xec8:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3607,12 +3660,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 46                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xee3:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xedf:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xee9:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0xee5:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3632,12 +3685,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 48                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf02:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xefe:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xf08:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0xf04:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3655,12 +3708,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 50                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf1f:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf1b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xf25:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0xf21:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3680,12 +3733,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 52                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf3e:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf3a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xf44:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0xf40:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3703,12 +3756,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 56                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf5b:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf57:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xf61:0x25 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0xf5d:0x25 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3729,14 +3782,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 54                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf7b:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf80:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf77:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf7c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xf86:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0xf82:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3756,12 +3809,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 58                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf9f:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf9b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xfa5:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0xfa1:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3779,12 +3832,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 60                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xfbc:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xfb8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xfc2:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0xfbe:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3802,12 +3855,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 62                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xfd9:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xfd5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xfdf:0x2b DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0xfdb:0x2b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3834,14 +3887,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 64                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xfff:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1004:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xffb:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1000:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x100a:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1006:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3857,12 +3910,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 66                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x101f:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x101b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1025:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1021:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3880,12 +3933,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 68                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x103c:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1038:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1042:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x103e:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3901,12 +3954,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 72                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1057:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1053:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x105d:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1059:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3924,12 +3977,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 70                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1074:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1070:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x107a:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1076:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3945,12 +3998,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 76                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x108f:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x108b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1095:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1091:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3968,12 +4021,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 74                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x10ac:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x10a8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x10b2:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x10ae:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3993,12 +4046,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 78                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x10cb:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x10c7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x10d1:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x10cd:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4016,12 +4069,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 80                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x10e8:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x10e4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x10ee:0x23 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x10ea:0x23 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4040,14 +4093,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 82                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1106:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x110b:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1102:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1107:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1111:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x110d:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4067,12 +4120,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 84                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x112a:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1126:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1130:0x27 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x112c:0x27 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4090,16 +4143,16 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 86                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1147:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x114c:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1151:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1143:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1148:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x114d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1157:0x23 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1153:0x23 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4118,14 +4171,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 88                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x116f:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1174:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x116b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1170:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x117a:0x23 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1176:0x23 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4144,14 +4197,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 90                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1192:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1197:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x118e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1193:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x119d:0x23 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1199:0x23 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4170,14 +4223,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 92                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x11b5:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x11ba:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x11b1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x11b6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x11c0:0x2a DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x11bc:0x2a DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4208,19 +4261,19 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 94                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x11e4:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x11e0:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x11ea:0x7 DW_TAG_base_type
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x11e6:0x7 DW_TAG_base_type
 ; CHECK-NEXT:.b8 105                                 // DW_AT_name
 ; CHECK-NEXT:.b8 110
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 4                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x11f1:0x26 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x11ed:0x26 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4242,16 +4295,16 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 96                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x120c:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1211:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4631                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1208:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x120d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4627                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1217:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x121c:0x25 DW_TAG_subprogram
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1213:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1218:0x25 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4272,14 +4325,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 98                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1236:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x123b:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1232:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1237:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1241:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x123d:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4299,12 +4352,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 100                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x125a:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1256:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1260:0x25 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x125c:0x25 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4330,12 +4383,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 102                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4741                               // DW_AT_type
+; CHECK-NEXT:.b32 4737                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x127f:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x127b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1285:0x8 DW_TAG_base_type
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1281:0x8 DW_TAG_base_type
 ; CHECK-NEXT:.b8 98                                  // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 111
@@ -4343,7 +4396,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 2                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 1                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x128d:0x2d DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1289:0x2d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4372,14 +4425,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 106                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4741                               // DW_AT_type
+; CHECK-NEXT:.b32 4737                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x12af:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x12b4:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x12ab:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x12b0:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x12ba:0x38 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x12b6:0x38 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4419,14 +4472,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 105                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4741                               // DW_AT_type
+; CHECK-NEXT:.b32 4737                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x12e7:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x12ec:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x12e3:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x12e8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x12f2:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x12ee:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4446,12 +4499,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 108                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4741                               // DW_AT_type
+; CHECK-NEXT:.b32 4737                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x130b:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1307:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1311:0x27 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x130d:0x27 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4474,14 +4527,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 112                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4741                               // DW_AT_type
+; CHECK-NEXT:.b32 4737                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x132d:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1332:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1329:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x132e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1338:0x32 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1334:0x32 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4515,14 +4568,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 111                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4741                               // DW_AT_type
+; CHECK-NEXT:.b32 4737                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x135f:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1364:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x135b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1360:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x136a:0x36 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1366:0x36 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4560,14 +4613,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 114                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4741                               // DW_AT_type
+; CHECK-NEXT:.b32 4737                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1395:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x139a:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1391:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1396:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x13a0:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x139c:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4587,12 +4640,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 116                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4741                               // DW_AT_type
+; CHECK-NEXT:.b32 4737                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x13b9:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x13b5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x13bf:0x25 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x13bb:0x25 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4618,12 +4671,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 118                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4741                               // DW_AT_type
+; CHECK-NEXT:.b32 4737                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x13de:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x13da:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x13e4:0x32 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x13e0:0x32 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4657,14 +4710,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 120                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4741                               // DW_AT_type
+; CHECK-NEXT:.b32 4737                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x140b:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1410:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1407:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x140c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1416:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1412:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4682,12 +4735,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 121                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5171                               // DW_AT_type
+; CHECK-NEXT:.b32 5167                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x142d:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5171                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1429:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5167                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1433:0xc DW_TAG_base_type
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x142f:0xc DW_TAG_base_type
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 110
@@ -4699,7 +4752,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 8                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x143f:0x25 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x143b:0x25 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4720,14 +4773,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 123                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1459:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x145e:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1455:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x145a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1464:0x21 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1460:0x21 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4749,12 +4802,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 125                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x147f:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x147b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1485:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1481:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4774,12 +4827,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 126                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3771                               // DW_AT_type
+; CHECK-NEXT:.b32 3767                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x149e:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 3771                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x149a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 3767                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x14a4:0x21 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x14a0:0x21 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4801,12 +4854,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 128                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3771                               // DW_AT_type
+; CHECK-NEXT:.b32 3767                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x14bf:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x14bb:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x14c5:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x14c1:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4822,12 +4875,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 138                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x14da:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x14d6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x14e0:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x14dc:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4847,12 +4900,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 130                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x14f9:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x14f5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x14ff:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x14fb:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4872,12 +4925,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 132                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1518:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1514:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x151e:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x151a:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4895,12 +4948,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 134                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1535:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1531:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x153b:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1537:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4918,12 +4971,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 136                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1552:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x154e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1558:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1554:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4943,12 +4996,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 140                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5171                               // DW_AT_type
+; CHECK-NEXT:.b32 5167                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1571:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x156d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1577:0x21 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1573:0x21 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4970,12 +5023,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 142                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5171                               // DW_AT_type
+; CHECK-NEXT:.b32 5167                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1592:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x158e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1598:0x23 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1594:0x23 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4999,12 +5052,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 143                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3771                               // DW_AT_type
+; CHECK-NEXT:.b32 3767                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x15b5:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x15b1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x15bb:0x24 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x15b7:0x24 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5024,14 +5077,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 145                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x15d4:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x15d9:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2125                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x15d0:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x15d5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2109                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x15df:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x15db:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5049,12 +5102,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 146                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x15f6:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5638                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x15f2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5634                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x15fc:0xa DW_TAG_base_type
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x15f8:0xa DW_TAG_base_type
 ; CHECK-NEXT:.b8 100                                 // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 117
@@ -5064,11 +5117,11 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 8                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1606:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 5643                               // DW_AT_type
-; CHECK-NEXT:.b8 13                                  // Abbrev [13] 0x160b:0x5 DW_TAG_const_type
-; CHECK-NEXT:.b32 5648                               // DW_AT_type
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1610:0x8 DW_TAG_base_type
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1602:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 5639                               // DW_AT_type
+; CHECK-NEXT:.b8 13                                  // Abbrev [13] 0x1607:0x5 DW_TAG_const_type
+; CHECK-NEXT:.b32 5644                               // DW_AT_type
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x160c:0x8 DW_TAG_base_type
 ; CHECK-NEXT:.b8 99                                  // DW_AT_name
 ; CHECK-NEXT:.b8 104
 ; CHECK-NEXT:.b8 97
@@ -5076,7 +5129,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 8                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 1                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1618:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1614:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5096,12 +5149,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 147                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1631:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5638                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x162d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5634                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1637:0x27 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1633:0x27 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5129,12 +5182,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 149                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1658:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1654:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x165e:0x2d DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x165a:0x2d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5163,14 +5216,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 151                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1680:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1685:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x167c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1681:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x168b:0x21 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1687:0x21 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5187,14 +5240,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 155                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16a1:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16a6:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x169d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16a2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x16ac:0x2d DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x16a8:0x2d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5223,14 +5276,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 157                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16ce:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16d3:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16ca:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16cf:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x16d9:0x2e DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x16d5:0x2e DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5255,16 +5308,16 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 159                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16f7:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16fc:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1701:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4631                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16f3:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16f8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16fd:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4627                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1707:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1703:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5282,12 +5335,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 161                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x171e:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x171a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1724:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1720:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5307,12 +5360,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 163                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x173d:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1739:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1743:0x29 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x173f:0x29 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5337,14 +5390,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 165                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1761:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1766:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5171                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x175d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1762:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5167                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x176c:0x27 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1768:0x27 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5367,14 +5420,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 167                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1788:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x178d:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1784:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1789:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1793:0x23 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x178f:0x23 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5398,12 +5451,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 169                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4741                               // DW_AT_type
+; CHECK-NEXT:.b32 4737                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x17b0:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x17ac:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x17b6:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x17b2:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5419,12 +5472,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 171                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x17cb:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x17c7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x17d1:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x17cd:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5442,12 +5495,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 173                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x17e8:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x17e4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x17ee:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x17ea:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5465,12 +5518,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 175                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1805:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1801:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x180b:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1807:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5486,12 +5539,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 177                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1820:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x181c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1826:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1822:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5509,12 +5562,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 179                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x183d:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1839:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1843:0x21 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x183f:0x21 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5536,12 +5589,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 181                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x185e:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x185a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1864:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1860:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5561,12 +5614,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 183                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x187d:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1879:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1883:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x187f:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 99
 ; CHECK-NEXT:.b8 111
@@ -5574,13 +5627,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 54                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1891:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x188d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1897:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1893:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 115
 ; CHECK-NEXT:.b8 105
@@ -5588,13 +5641,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 56                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18a5:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18a1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x18ab:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x18a7:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 97
@@ -5602,13 +5655,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 58                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18b9:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18b5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x18bf:0x1a DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x18bb:0x1a DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 97
@@ -5617,15 +5670,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 60                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18ce:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18d3:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18ca:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18cf:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x18d9:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x18d5:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 99                                  // DW_AT_name
 ; CHECK-NEXT:.b8 101
 ; CHECK-NEXT:.b8 105
@@ -5633,26 +5686,26 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 178                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18e7:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18e3:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x18ed:0x13 DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x18e9:0x13 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 99                                  // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 115
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 63                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18fa:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18f6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1900:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x18fc:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 99                                  // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 115
@@ -5660,26 +5713,26 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 72                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x190e:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x190a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1914:0x13 DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1910:0x13 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 101                                 // DW_AT_name
 ; CHECK-NEXT:.b8 120
 ; CHECK-NEXT:.b8 112
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 100                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1921:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x191d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1927:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1923:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 102                                 // DW_AT_name
 ; CHECK-NEXT:.b8 97
 ; CHECK-NEXT:.b8 98
@@ -5687,13 +5740,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 181                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1935:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1931:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x193b:0x15 DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1937:0x15 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 102                                 // DW_AT_name
 ; CHECK-NEXT:.b8 108
 ; CHECK-NEXT:.b8 111
@@ -5702,13 +5755,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 184                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x194a:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1946:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1950:0x19 DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x194c:0x19 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 102                                 // DW_AT_name
 ; CHECK-NEXT:.b8 109
 ; CHECK-NEXT:.b8 111
@@ -5716,15 +5769,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 187                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x195e:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1963:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x195a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x195f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1969:0x1a DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1965:0x1a DW_TAG_subprogram
 ; CHECK-NEXT:.b8 102                                 // DW_AT_name
 ; CHECK-NEXT:.b8 114
 ; CHECK-NEXT:.b8 101
@@ -5733,15 +5786,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 103                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1978:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x197d:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4631                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1974:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1979:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4627                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1983:0x1a DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x197f:0x1a DW_TAG_subprogram
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 100
 ; CHECK-NEXT:.b8 101
@@ -5750,28 +5803,28 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 106                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1992:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1997:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x198e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1993:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x199d:0x13 DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1999:0x13 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 103
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 109                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19aa:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19a6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x19b0:0x15 DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x19ac:0x15 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 103
@@ -5780,13 +5833,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 112                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19bf:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19bb:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x19c5:0x19 DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x19c1:0x19 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 109                                 // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 100
@@ -5794,45 +5847,45 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 115                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19d3:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19d8:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6622                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x19de:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x19e3:0x18 DW_TAG_subprogram
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19cf:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19d4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6618                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x19da:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x19df:0x18 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 112                                 // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 119
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 153                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19f0:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19f5:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19ec:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19f1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x19fb:0x13 DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x19f7:0x13 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 105
 ; CHECK-NEXT:.b8 110
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 65                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1a08:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1a04:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1a0e:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1a0a:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 105
 ; CHECK-NEXT:.b8 110
@@ -5840,13 +5893,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 74                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1a1c:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1a18:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1a22:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1a1e:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 113
 ; CHECK-NEXT:.b8 114
@@ -5854,26 +5907,26 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 156                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1a30:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1a2c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1a36:0x13 DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1a32:0x13 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 116                                 // DW_AT_name
 ; CHECK-NEXT:.b8 97
 ; CHECK-NEXT:.b8 110
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 67                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1a43:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1a3f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1a49:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1a45:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 116                                 // DW_AT_name
 ; CHECK-NEXT:.b8 97
 ; CHECK-NEXT:.b8 110
@@ -5881,14 +5934,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 76                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1a57:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1a53:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1a5d:0xd DW_TAG_typedef
-; CHECK-NEXT:.b32 6762                               // DW_AT_type
+; CHECK-NEXT:.b8 34                                  // Abbrev [34] 0x1a59:0xd DW_TAG_typedef
+; CHECK-NEXT:.b32 6758                               // DW_AT_type
 ; CHECK-NEXT:.b8 100                                 // DW_AT_name
 ; CHECK-NEXT:.b8 105
 ; CHECK-NEXT:.b8 118
@@ -5897,10 +5950,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 101                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 34                                  // Abbrev [34] 0x1a6a:0x2 DW_TAG_structure_type
+; CHECK-NEXT:.b8 35                                  // Abbrev [35] 0x1a66:0x2 DW_TAG_structure_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1a6c:0xe DW_TAG_typedef
-; CHECK-NEXT:.b32 6778                               // DW_AT_type
+; CHECK-NEXT:.b8 34                                  // Abbrev [34] 0x1a68:0xe DW_TAG_typedef
+; CHECK-NEXT:.b32 6774                               // DW_AT_type
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 100
 ; CHECK-NEXT:.b8 105
@@ -5910,35 +5963,35 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 109                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 35                                  // Abbrev [35] 0x1a7a:0x22 DW_TAG_structure_type
+; CHECK-NEXT:.b8 36                                  // Abbrev [36] 0x1a76:0x22 DW_TAG_structure_type
 ; CHECK-NEXT:.b8 16                                  // DW_AT_byte_size
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 105                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x1a7e:0xf DW_TAG_member
+; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x1a7a:0xf DW_TAG_member
 ; CHECK-NEXT:.b8 113                                 // DW_AT_name
 ; CHECK-NEXT:.b8 117
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 5171                               // DW_AT_type
+; CHECK-NEXT:.b32 5167                               // DW_AT_type
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 107                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2                                   // DW_AT_data_member_location
 ; CHECK-NEXT:.b8 35
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x1a8d:0xe DW_TAG_member
+; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x1a89:0xe DW_TAG_member
 ; CHECK-NEXT:.b8 114                                 // DW_AT_name
 ; CHECK-NEXT:.b8 101
 ; CHECK-NEXT:.b8 109
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 5171                               // DW_AT_type
+; CHECK-NEXT:.b32 5167                               // DW_AT_type
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 108                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2                                   // DW_AT_data_member_location
 ; CHECK-NEXT:.b8 35
 ; CHECK-NEXT:.b8 8
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 36                                  // Abbrev [36] 0x1a9c:0xd DW_TAG_subprogram
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1a98:0xd DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 98
 ; CHECK-NEXT:.b8 111
@@ -5951,7 +6004,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
 ; CHECK-NEXT:.b8 1                                   // DW_AT_noreturn
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1aa9:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1aa5:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 98
 ; CHECK-NEXT:.b8 115
@@ -5959,13 +6012,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ab7:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ab3:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1abd:0x17 DW_TAG_subprogram
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1ab9:0x17 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 101
@@ -5976,16 +6029,16 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ace:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6868                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1aca:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6864                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1ad4:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 6873                               // DW_AT_type
-; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1ad9:0x1 DW_TAG_subroutine_type
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1ada:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1ad0:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 6869                               // DW_AT_type
+; CHECK-NEXT:.b8 39                                  // Abbrev [39] 0x1ad5:0x1 DW_TAG_subroutine_type
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1ad6:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 111
@@ -5993,13 +6046,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 9                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 26                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ae8:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5638                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ae4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5634                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1aee:0x15 DW_TAG_subprogram
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1aea:0x15 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 111
@@ -6008,13 +6061,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 22                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1afd:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5638                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1af9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5634                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1b03:0x15 DW_TAG_subprogram
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1aff:0x15 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 111
@@ -6023,13 +6076,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 27                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5171                               // DW_AT_type
+; CHECK-NEXT:.b32 5167                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b12:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5638                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b0e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5634                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1b18:0x2b DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1b14:0x2b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 98                                  // DW_AT_name
 ; CHECK-NEXT:.b8 115
 ; CHECK-NEXT:.b8 101
@@ -6040,26 +6093,26 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 10                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 20                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 6979                               // DW_AT_type
+; CHECK-NEXT:.b32 6975                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b29:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6980                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b2e:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6980                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b33:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6986                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b38:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6986                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b3d:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7021                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 39                                  // Abbrev [39] 0x1b43:0x1 DW_TAG_pointer_type
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1b44:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 6985                               // DW_AT_type
-; CHECK-NEXT:.b8 40                                  // Abbrev [40] 0x1b49:0x1 DW_TAG_const_type
-; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1b4a:0xe DW_TAG_typedef
-; CHECK-NEXT:.b32 7000                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b25:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6976                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b2a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6976                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b2f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6982                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b34:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6982                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b39:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7017                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 40                                  // Abbrev [40] 0x1b3f:0x1 DW_TAG_pointer_type
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1b40:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 6981                               // DW_AT_type
+; CHECK-NEXT:.b8 41                                  // Abbrev [41] 0x1b45:0x1 DW_TAG_const_type
+; CHECK-NEXT:.b8 34                                  // Abbrev [34] 0x1b46:0xe DW_TAG_typedef
+; CHECK-NEXT:.b32 6996                               // DW_AT_type
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 105
 ; CHECK-NEXT:.b8 122
@@ -6069,7 +6122,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 11                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 62                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1b58:0x15 DW_TAG_base_type
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1b54:0x15 DW_TAG_base_type
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 110
@@ -6090,8 +6143,8 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 8                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 20                                  // Abbrev [20] 0x1b6d:0x16 DW_TAG_typedef
-; CHECK-NEXT:.b32 7043                               // DW_AT_type
+; CHECK-NEXT:.b8 20                                  // Abbrev [20] 0x1b69:0x16 DW_TAG_typedef
+; CHECK-NEXT:.b32 7039                               // DW_AT_type
 ; CHECK-NEXT:.b8 95                                  // DW_AT_name
 ; CHECK-NEXT:.b8 95
 ; CHECK-NEXT:.b8 99
@@ -6109,16 +6162,16 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 230                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1b83:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 7048                               // DW_AT_type
-; CHECK-NEXT:.b8 41                                  // Abbrev [41] 0x1b88:0x10 DW_TAG_subroutine_type
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b8d:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6980                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b92:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6980                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1b98:0x1c DW_TAG_subprogram
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1b7f:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 7044                               // DW_AT_type
+; CHECK-NEXT:.b8 42                                  // Abbrev [42] 0x1b84:0x10 DW_TAG_subroutine_type
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b89:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6976                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b8e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6976                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1b94:0x1c DW_TAG_subprogram
 ; CHECK-NEXT:.b8 99                                  // DW_AT_name
 ; CHECK-NEXT:.b8 97
 ; CHECK-NEXT:.b8 108
@@ -6129,15 +6182,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 212                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6979                               // DW_AT_type
+; CHECK-NEXT:.b32 6975                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ba9:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6986                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1bae:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6986                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ba5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6982                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1baa:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6982                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1bb4:0x19 DW_TAG_subprogram
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1bb0:0x19 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 100                                 // DW_AT_name
 ; CHECK-NEXT:.b8 105
 ; CHECK-NEXT:.b8 118
@@ -6145,15 +6198,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 21                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 6749                               // DW_AT_type
+; CHECK-NEXT:.b32 6745                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1bc2:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1bc7:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1bbe:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1bc3:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 42                                  // Abbrev [42] 0x1bcd:0x12 DW_TAG_subprogram
+; CHECK-NEXT:.b8 43                                  // Abbrev [43] 0x1bc9:0x12 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 101                                 // DW_AT_name
 ; CHECK-NEXT:.b8 120
 ; CHECK-NEXT:.b8 105
@@ -6165,10 +6218,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
 ; CHECK-NEXT:.b8 1                                   // DW_AT_noreturn
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1bd9:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1bd5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 18                                  // Abbrev [18] 0x1bdf:0x11 DW_TAG_subprogram
+; CHECK-NEXT:.b8 18                                  // Abbrev [18] 0x1bdb:0x11 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 102                                 // DW_AT_name
 ; CHECK-NEXT:.b8 114
 ; CHECK-NEXT:.b8 101
@@ -6179,10 +6232,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1bea:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6979                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1be6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6975                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1bf0:0x17 DW_TAG_subprogram
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1bec:0x17 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 103                                 // DW_AT_name
 ; CHECK-NEXT:.b8 101
 ; CHECK-NEXT:.b8 116
@@ -6193,15 +6246,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 52                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2
-; CHECK-NEXT:.b32 7175                               // DW_AT_type
+; CHECK-NEXT:.b32 7171                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c01:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5638                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1bfd:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5634                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1c07:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 5648                               // DW_AT_type
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1c0c:0x15 DW_TAG_subprogram
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1c03:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 5644                               // DW_AT_type
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1c08:0x15 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 97
 ; CHECK-NEXT:.b8 98
@@ -6210,13 +6263,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 5171                               // DW_AT_type
+; CHECK-NEXT:.b32 5167                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c1b:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5171                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c17:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5167                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1c21:0x1a DW_TAG_subprogram
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1c1d:0x1a DW_TAG_subprogram
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 100
 ; CHECK-NEXT:.b8 105
@@ -6225,15 +6278,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 23                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 6764                               // DW_AT_type
+; CHECK-NEXT:.b32 6760                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c30:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5171                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c35:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5171                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c2c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5167                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c31:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5167                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1c3b:0x17 DW_TAG_subprogram
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1c37:0x17 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 109                                 // DW_AT_name
 ; CHECK-NEXT:.b8 97
 ; CHECK-NEXT:.b8 108
@@ -6244,13 +6297,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 210                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6979                               // DW_AT_type
+; CHECK-NEXT:.b32 6975                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c4c:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6986                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c48:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6982                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1c52:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1c4e:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 109                                 // DW_AT_name
 ; CHECK-NEXT:.b8 98
 ; CHECK-NEXT:.b8 108
@@ -6260,15 +6313,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 95                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c62:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5638                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c67:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6986                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c5e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5634                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c63:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6982                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1c6d:0x23 DW_TAG_subprogram
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1c69:0x23 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 109                                 // DW_AT_name
 ; CHECK-NEXT:.b8 98
 ; CHECK-NEXT:.b8 115
@@ -6281,19 +6334,19 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 106                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 6986                               // DW_AT_type
+; CHECK-NEXT:.b32 6982                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c80:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7312                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c85:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5638                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c8a:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6986                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1c90:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 7317                               // DW_AT_type
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1c95:0xb DW_TAG_base_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c7c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7308                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c81:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5634                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c86:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6982                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1c8c:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 7313                               // DW_AT_type
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1c91:0xb DW_TAG_base_type
 ; CHECK-NEXT:.b8 119                                 // DW_AT_name
 ; CHECK-NEXT:.b8 99
 ; CHECK-NEXT:.b8 104
@@ -6304,7 +6357,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 4                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1ca0:0x21 DW_TAG_subprogram
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1c9c:0x21 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 109                                 // DW_AT_name
 ; CHECK-NEXT:.b8 98
 ; CHECK-NEXT:.b8 116
@@ -6315,17 +6368,17 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 98                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cb1:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7312                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cb6:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5638                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cbb:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6986                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 18                                  // Abbrev [18] 0x1cc1:0x21 DW_TAG_subprogram
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cad:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7308                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cb2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5634                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cb7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6982                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 18                                  // Abbrev [18] 0x1cbd:0x21 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 113                                 // DW_AT_name
 ; CHECK-NEXT:.b8 115
 ; CHECK-NEXT:.b8 111
@@ -6337,16 +6390,16 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 2
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ccd:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6979                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cd2:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6986                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cd7:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6986                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cdc:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7021                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 43                                  // Abbrev [43] 0x1ce2:0xf DW_TAG_subprogram
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cc9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6975                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cce:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6982                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cd3:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6982                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cd8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7017                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x1cde:0xf DW_TAG_subprogram
 ; CHECK-NEXT:.b8 114                                 // DW_AT_name
 ; CHECK-NEXT:.b8 97
 ; CHECK-NEXT:.b8 110
@@ -6355,10 +6408,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 118                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1cf1:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1ced:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 114                                 // DW_AT_name
 ; CHECK-NEXT:.b8 101
 ; CHECK-NEXT:.b8 97
@@ -6370,15 +6423,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 224                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6979                               // DW_AT_type
+; CHECK-NEXT:.b32 6975                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d03:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6979                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d08:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6986                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cff:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6975                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d04:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6982                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 18                                  // Abbrev [18] 0x1d0e:0x12 DW_TAG_subprogram
+; CHECK-NEXT:.b8 18                                  // Abbrev [18] 0x1d0a:0x12 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 114
 ; CHECK-NEXT:.b8 97
@@ -6390,10 +6443,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d1a:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 619                                // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d16:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 603                                // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1d20:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1d1c:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 114
@@ -6403,17 +6456,17 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 164                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b32 5624                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d30:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5638                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d35:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7483                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1d3b:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 7175                               // DW_AT_type
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1d40:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d2c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5634                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d31:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7479                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1d37:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 7171                               // DW_AT_type
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1d3c:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 114
@@ -6423,17 +6476,17 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 183                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5171                               // DW_AT_type
+; CHECK-NEXT:.b32 5167                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d50:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5638                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d55:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7483                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d5a:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1d60:0x21 DW_TAG_subprogram
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d4c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5634                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d51:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7479                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d56:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1d5c:0x21 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 114
@@ -6444,17 +6497,17 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 187                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7000                               // DW_AT_type
+; CHECK-NEXT:.b32 6996                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d71:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5638                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d76:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7483                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d7b:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1d81:0x17 DW_TAG_subprogram
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d6d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5634                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d72:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7479                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d77:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1d7d:0x17 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 121
 ; CHECK-NEXT:.b8 115
@@ -6465,13 +6518,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 205                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d92:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5638                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d8e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5634                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1d98:0x23 DW_TAG_subprogram
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1d94:0x23 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 119                                 // DW_AT_name
 ; CHECK-NEXT:.b8 99
 ; CHECK-NEXT:.b8 115
@@ -6484,21 +6537,21 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 109                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 6986                               // DW_AT_type
+; CHECK-NEXT:.b32 6982                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1dab:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7175                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1db0:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7611                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1db5:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6986                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1dbb:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 7616                               // DW_AT_type
-; CHECK-NEXT:.b8 13                                  // Abbrev [13] 0x1dc0:0x5 DW_TAG_const_type
-; CHECK-NEXT:.b32 7317                               // DW_AT_type
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1dc5:0x1c DW_TAG_subprogram
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1da7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7171                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1dac:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7607                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1db1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6982                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1db7:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 7612                               // DW_AT_type
+; CHECK-NEXT:.b8 13                                  // Abbrev [13] 0x1dbc:0x5 DW_TAG_const_type
+; CHECK-NEXT:.b32 7313                               // DW_AT_type
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1dc1:0x1c DW_TAG_subprogram
 ; CHECK-NEXT:.b8 119                                 // DW_AT_name
 ; CHECK-NEXT:.b8 99
 ; CHECK-NEXT:.b8 116
@@ -6509,15 +6562,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 102                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1dd6:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7175                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ddb:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7317                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1dd2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7171                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1dd7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7313                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 28                                  // Abbrev [28] 0x1de1:0x78 DW_TAG_namespace
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1ddd:0x78 DW_TAG_namespace
 ; CHECK-NEXT:.b8 95                                  // DW_AT_name
 ; CHECK-NEXT:.b8 95
 ; CHECK-NEXT:.b8 103
@@ -6528,43 +6581,43 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 120
 ; CHECK-NEXT:.b8 120
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1dec:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x1de8:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 201                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7769                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1df3:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7765                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x1def:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 207                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7818                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1dfa:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7814                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x1df6:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 211                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7837                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1e01:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7833                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x1dfd:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 217                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7859                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1e08:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7855                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x1e04:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 228                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7886                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1e0f:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7882                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x1e0b:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 229                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7908                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1e16:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7904                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x1e12:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 230                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7941                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1e1d:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7937                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x1e19:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 232                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 8001                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1e24:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7997                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0x1e20:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 233                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 8028                               // DW_AT_import
-; CHECK-NEXT:.b8 4                                   // Abbrev [4] 0x1e2b:0x2d DW_TAG_subprogram
+; CHECK-NEXT:.b32 8024                               // DW_AT_import
+; CHECK-NEXT:.b8 4                                   // Abbrev [4] 0x1e27:0x2d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -6592,17 +6645,17 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 214                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7769                               // DW_AT_type
+; CHECK-NEXT:.b32 7765                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1e4d:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 3771                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1e52:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 3771                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1e49:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 3767                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1e4e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 3767                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1e59:0xf DW_TAG_typedef
-; CHECK-NEXT:.b32 7784                               // DW_AT_type
+; CHECK-NEXT:.b8 34                                  // Abbrev [34] 0x1e55:0xf DW_TAG_typedef
+; CHECK-NEXT:.b32 7780                               // DW_AT_type
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 108
 ; CHECK-NEXT:.b8 100
@@ -6613,35 +6666,35 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 121                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 35                                  // Abbrev [35] 0x1e68:0x22 DW_TAG_structure_type
+; CHECK-NEXT:.b8 36                                  // Abbrev [36] 0x1e64:0x22 DW_TAG_structure_type
 ; CHECK-NEXT:.b8 16                                  // DW_AT_byte_size
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 117                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x1e6c:0xf DW_TAG_member
+; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x1e68:0xf DW_TAG_member
 ; CHECK-NEXT:.b8 113                                 // DW_AT_name
 ; CHECK-NEXT:.b8 117
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 3771                               // DW_AT_type
+; CHECK-NEXT:.b32 3767                               // DW_AT_type
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 119                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2                                   // DW_AT_data_member_location
 ; CHECK-NEXT:.b8 35
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x1e7b:0xe DW_TAG_member
+; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x1e77:0xe DW_TAG_member
 ; CHECK-NEXT:.b8 114                                 // DW_AT_name
 ; CHECK-NEXT:.b8 101
 ; CHECK-NEXT:.b8 109
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 3771                               // DW_AT_type
+; CHECK-NEXT:.b32 3767                               // DW_AT_type
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 120                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2                                   // DW_AT_data_member_location
 ; CHECK-NEXT:.b8 35
 ; CHECK-NEXT:.b8 8
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 42                                  // Abbrev [42] 0x1e8a:0x13 DW_TAG_subprogram
+; CHECK-NEXT:.b8 43                                  // Abbrev [43] 0x1e86:0x13 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_name
 ; CHECK-NEXT:.b8 69
 ; CHECK-NEXT:.b8 120
@@ -6654,10 +6707,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
 ; CHECK-NEXT:.b8 1                                   // DW_AT_noreturn
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1e97:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1e93:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1e9d:0x16 DW_TAG_subprogram
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1e99:0x16 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 108
 ; CHECK-NEXT:.b8 97
@@ -6667,13 +6720,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 3771                               // DW_AT_type
+; CHECK-NEXT:.b32 3767                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ead:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 3771                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ea9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 3767                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1eb3:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1eaf:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 108
 ; CHECK-NEXT:.b8 100
@@ -6683,15 +6736,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 29                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 7769                               // DW_AT_type
+; CHECK-NEXT:.b32 7765                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ec3:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 3771                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ec8:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 3771                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ebf:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 3767                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ec4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 3767                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1ece:0x16 DW_TAG_subprogram
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1eca:0x16 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 111
@@ -6701,13 +6754,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 36                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 3771                               // DW_AT_type
+; CHECK-NEXT:.b32 3767                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ede:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5638                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1eda:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5634                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1ee4:0x21 DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1ee0:0x21 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 114
@@ -6718,17 +6771,17 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 209                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3771                               // DW_AT_type
+; CHECK-NEXT:.b32 3767                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ef5:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5638                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1efa:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7483                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1eff:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1f05:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ef1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5634                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ef6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7479                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1efb:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1f01:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 114
@@ -6740,17 +6793,17 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 214                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7975                               // DW_AT_type
+; CHECK-NEXT:.b32 7971                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f17:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5638                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f1c:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7483                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f21:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1f27:0x1a DW_TAG_base_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f13:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5634                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f18:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7479                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f1d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1f23:0x1a DW_TAG_base_type
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 110
@@ -6776,7 +6829,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 8                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1f41:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1f3d:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 114
@@ -6786,15 +6839,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 172                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f51:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5638                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f56:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7483                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f4d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5634                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f52:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7479                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1f5c:0x1c DW_TAG_subprogram
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1f58:0x1c DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 114
@@ -6805,15 +6858,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 175                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 8056                               // DW_AT_type
+; CHECK-NEXT:.b32 8052                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f6d:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5638                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f72:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7483                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f69:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5634                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f6e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7479                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1f78:0xf DW_TAG_base_type
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1f74:0xf DW_TAG_base_type
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 110
@@ -6828,7 +6881,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 8                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x1f87:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x1f83:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -6849,12 +6902,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 62                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1fa1:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f9d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x1fa7:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x1fa3:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -6877,12 +6930,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 90                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1fc3:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1fbf:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x1fc9:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x1fc5:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -6903,12 +6956,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 57                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1fe3:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1fdf:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x1fe9:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x1fe5:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -6931,12 +6984,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 95                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2005:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2001:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x200b:0x28 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x2007:0x28 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -6960,14 +7013,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 47                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2028:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x202d:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2024:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2029:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2033:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x202f:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -6988,12 +7041,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 52                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x204d:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2049:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2053:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x204f:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7016,12 +7069,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 100                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x206f:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x206b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2075:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x2071:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7042,12 +7095,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 150                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x208f:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x208b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2095:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x2091:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7068,12 +7121,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 14                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 155                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x20af:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x20ab:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x20b5:0x2e DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x20b1:0x2e DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7103,14 +7156,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 165                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 4
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x20d8:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x20dd:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x20d4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x20d9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x20e3:0x1e DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x20df:0x1e DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7129,12 +7182,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 219                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 4
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x20fb:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x20f7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2101:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x20fd:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7155,12 +7208,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 32                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x211b:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2117:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2121:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x211d:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7181,12 +7234,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 210                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x213b:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2137:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2141:0x1e DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x213d:0x1e DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7205,12 +7258,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 200                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2159:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2155:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x215f:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x215b:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7231,12 +7284,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 14                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 145                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2179:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2175:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x217f:0x1e DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x217b:0x1e DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7255,12 +7308,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 14                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2197:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2193:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x219d:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x2199:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7283,12 +7336,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 105                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x21b9:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x21b5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x21bf:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x21bb:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7309,12 +7362,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 14                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 95                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x21d9:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x21d5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x21df:0x26 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x21db:0x26 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7336,14 +7389,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 80                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 6
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x21fa:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x21ff:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x21f6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x21fb:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2205:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x2201:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7366,12 +7419,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 14                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 85                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2221:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x221d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2227:0x2a DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x2223:0x2a DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7392,16 +7445,16 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 32                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 6
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2241:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2246:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x224b:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x223d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2242:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2247:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2251:0x26 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x224d:0x26 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7423,14 +7476,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 14                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 110                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x226c:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2271:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2268:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x226d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2277:0x26 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x2273:0x26 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7452,14 +7505,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 14                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 105                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2292:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2297:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x228e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2293:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x229d:0x26 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x2299:0x26 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7481,14 +7534,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 17                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 6
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x22b8:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x22bd:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x22b4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x22b9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x22c3:0x29 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x22bf:0x29 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7513,14 +7566,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 6
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x22e1:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x22e6:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4631                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x22dd:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x22e2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4627                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x22ec:0x28 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x22e8:0x28 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7544,14 +7597,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 110                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2309:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x230e:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2305:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x230a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2314:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x2310:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7574,12 +7627,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 85                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 6
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2330:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x232c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2336:0x28 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x2332:0x28 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7603,14 +7656,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 240                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2353:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2358:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x234f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2354:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x235e:0x24 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x235a:0x24 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7635,12 +7688,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 235                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x237c:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2378:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2382:0x24 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x237e:0x24 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7665,12 +7718,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 125                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 4
-; CHECK-NEXT:.b32 3771                               // DW_AT_type
+; CHECK-NEXT:.b32 3767                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x23a0:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x239c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x23a6:0x26 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x23a2:0x26 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7697,12 +7750,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 66                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 6
-; CHECK-NEXT:.b32 3771                               // DW_AT_type
+; CHECK-NEXT:.b32 3767                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x23c6:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x23c2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x23cc:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x23c8:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7725,12 +7778,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 76                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x23e8:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x23e4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x23ee:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x23ea:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7753,12 +7806,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 85                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x240a:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2406:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2410:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x240c:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7779,12 +7832,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x242a:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2426:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2430:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x242c:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7805,12 +7858,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 90                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 6
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x244a:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2446:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2450:0x1e DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x244c:0x1e DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7829,12 +7882,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 67                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2468:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2464:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x246e:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x246a:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7857,12 +7910,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 116                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 4
-; CHECK-NEXT:.b32 5171                               // DW_AT_type
+; CHECK-NEXT:.b32 5167                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x248a:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2486:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2490:0x24 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x248c:0x24 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7887,12 +7940,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 71                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 6
-; CHECK-NEXT:.b32 5171                               // DW_AT_type
+; CHECK-NEXT:.b32 5167                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x24ae:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x24aa:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x24b4:0x27 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x24b0:0x27 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7915,14 +7968,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 6
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x24d0:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x24d5:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2125                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x24cc:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x24d1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2109                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x24db:0x2b DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x24d7:0x2b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7954,12 +8007,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 130                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 4
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2500:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x24fc:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2506:0x31 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x2502:0x31 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7992,14 +8045,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 194                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 4
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x252c:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2531:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2528:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x252d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2537:0x24 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x2533:0x24 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8019,14 +8072,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 47                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 6
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2550:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2555:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x254c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2551:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x255b:0x31 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x2557:0x31 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8059,14 +8112,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 22                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 6
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2581:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2586:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x257d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2582:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x258c:0x31 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x2588:0x31 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8094,16 +8147,16 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 27                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 6
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x25ad:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x25b2:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x25b7:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4631                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x25a9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x25ae:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x25b3:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4627                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x25bd:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x25b9:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8124,12 +8177,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 111                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 4
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x25d7:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x25d3:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x25dd:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x25d9:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8152,12 +8205,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 61                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 6
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x25f9:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x25f5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x25ff:0x2c DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x25fb:0x2c DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8185,14 +8238,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 250                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2620:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2625:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5171                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x261c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2621:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5167                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x262b:0x2a DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x2627:0x2a DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8218,14 +8271,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 245                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x264a:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x264f:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2646:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x264b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4582                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2655:0x1e DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x2651:0x1e DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8244,12 +8297,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 210                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 4
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x266d:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2669:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2673:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x266f:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8270,12 +8323,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 37                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x268d:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2689:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2693:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x268f:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8296,12 +8349,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 14                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 139                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x26ad:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x26a9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x26b3:0x1e DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x26af:0x1e DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8320,12 +8373,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 252                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 4
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x26cb:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x26c7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x26d1:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x26cd:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8346,12 +8399,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 42                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 5
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x26eb:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x26e7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x26f1:0x24 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x26ed:0x24 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8376,12 +8429,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 56                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 6
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x270f:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x270b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2715:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 45                                  // Abbrev [45] 0x2711:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8404,14 +8457,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 14                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 150                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2731:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 2116                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x272d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 2100                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
 ; CHECK-NEXT:	}
-; CHECK-NEXT:	.section	.debug_loc	{	}
+; CHECK-NEXT:	.section	.debug_macinfo	{	}
 ; CHECK-NOT: debug_
 
 ; Function Attrs: nounwind readnone
diff --git a/llvm/test/DebugInfo/NVPTX/debug-loc-offset.ll b/llvm/test/DebugInfo/NVPTX/debug-loc-offset.ll
index bb85e237df91f..45c387fabddac 100644
--- a/llvm/test/DebugInfo/NVPTX/debug-loc-offset.ll
+++ b/llvm/test/DebugInfo/NVPTX/debug-loc-offset.ll
@@ -132,130 +132,159 @@ attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp
 !35 = !{!21}
 !36 = !{!24, !26}
 
-; CHECK: .section .debug_abbrev
-; CHECK-NEXT: {
-; CHECK-NEXT: .b8 1                                // Abbreviation Code
-; CHECK-NEXT: .b8 17                               // DW_TAG_compile_unit
-; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
-; CHECK-NEXT: .b8 37                               // DW_AT_producer
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 19                               // DW_AT_language
-; CHECK-NEXT: .b8 5                                // DW_FORM_data2
-; CHECK-NEXT: .b8 3                                // DW_AT_name
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 16                               // DW_AT_stmt_list
-; CHECK-NEXT: .b8 6                                // DW_FORM_data4
-; CHECK-NEXT: .b8 27                               // DW_AT_comp_dir
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 17                               // DW_AT_low_pc
-; CHECK-NEXT: .b8 1                                // DW_FORM_addr
-; CHECK-NEXT: .b8 18                               // DW_AT_high_pc
-; CHECK-NEXT: .b8 1                                // DW_FORM_addr
-; CHECK-NEXT: .b8 0                                // EOM(1)
-; CHECK-NEXT: .b8 0                                // EOM(2)
-; CHECK-NEXT: .b8 2                                // Abbreviation Code
-; CHECK-NEXT: .b8 19                               // DW_TAG_structure_type
-; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
-; CHECK-NEXT: .b8 3                                // DW_AT_name
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 60                               // DW_AT_declaration
-; CHECK-NEXT: .b8 12                               // DW_FORM_flag
-; CHECK-NEXT: .b8 0                                // EOM(1)
-; CHECK-NEXT: .b8 0                                // EOM(2)
-; CHECK-NEXT: .b8 3                                // Abbreviation Code
-; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
-; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
-; CHECK-NEXT: .b8 17                               // DW_AT_low_pc
-; CHECK-NEXT: .b8 1                                // DW_FORM_addr
-; CHECK-NEXT: .b8 18                               // DW_AT_high_pc
-; CHECK-NEXT: .b8 1                                // DW_FORM_addr
-; CHECK-NEXT: .b8 64                               // DW_AT_frame_base
-; CHECK-NEXT: .b8 10                               // DW_FORM_block1
-; CHECK-NEXT: .b8 135                              // DW_AT_MIPS_linkage_name
+; CHECK: 	.section	.debug_loc
+; CHECK-NEXT: 	{
+; CHECK-NEXT: $L__debug_loc0:
+; CHECK-NEXT: .b64 $L__func_begin0
+; CHECK-NEXT: .b64 $L__tmp0
+; CHECK-NEXT: .b8 2                                   // Loc expr size
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 17                                  // DW_OP_consts
+; CHECK-NEXT: .b8 0                                   // 0
+; CHECK-NEXT: .b64 $L__tmp0
+; CHECK-NEXT: .b64 $L__func_end0
+; CHECK-NEXT: .b8 2                                   // Loc expr size
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 17                                  // DW_OP_consts
+; CHECK-NEXT: .b8 1                                   // 1
+; CHECK-NEXT: .b64 0
+; CHECK-NEXT: .b64 0
+; CHECK-NEXT: 	}
+; CHECK-NEXT: 	.section	.debug_abbrev
+; CHECK-NEXT: 	{
+; CHECK-NEXT: .b8 1                                   // Abbreviation Code
+; CHECK-NEXT: .b8 17                                  // DW_TAG_compile_unit
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 37                                  // DW_AT_producer
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 19                                  // DW_AT_language
+; CHECK-NEXT: .b8 5                                   // DW_FORM_data2
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 16                                  // DW_AT_stmt_list
+; CHECK-NEXT: .b8 6                                   // DW_FORM_data4
+; CHECK-NEXT: .b8 27                                  // DW_AT_comp_dir
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 2                                   // Abbreviation Code
+; CHECK-NEXT: .b8 19                                  // DW_TAG_structure_type
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 60                                  // DW_AT_declaration
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 3                                   // Abbreviation Code
+; CHECK-NEXT: .b8 46                                  // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 17                                  // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                   // DW_FORM_addr
+; CHECK-NEXT: .b8 18                                  // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_FORM_addr
+; CHECK-NEXT: .b8 64                                  // DW_AT_frame_base
+; CHECK-NEXT: .b8 10                                  // DW_FORM_block1
+; CHECK-NEXT: .b8 135                                 // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT: .b8 64
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 3                                // DW_AT_name
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 63                               // DW_AT_external
-; CHECK-NEXT: .b8 12                               // DW_FORM_flag
-; CHECK-NEXT: .b8 0                                // EOM(1)
-; CHECK-NEXT: .b8 0                                // EOM(2)
-; CHECK-NEXT: .b8 4                                // Abbreviation Code
-; CHECK-NEXT: .b8 5                                // DW_TAG_formal_parameter
-; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
-; CHECK-NEXT: .b8 3                                // DW_AT_name
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 73                               // DW_AT_type
-; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
-; CHECK-NEXT: .b8 0                                // EOM(1)
-; CHECK-NEXT: .b8 0                                // EOM(2)
-; CHECK-NEXT: .b8 5                                // Abbreviation Code
-; CHECK-NEXT: .b8 52                               // DW_TAG_variable
-; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
-; CHECK-NEXT: .b8 3                                // DW_AT_name
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 73                               // DW_AT_type
-; CHECK-NEXT: .b8 16                               // DW_FORM_ref_addr
-; CHECK-NEXT: .b8 0                                // EOM(1)
-; CHECK-NEXT: .b8 0                                // EOM(2)
-; CHECK-NEXT: .b8 6                                // Abbreviation Code
-; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
-; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
-; CHECK-NEXT: .b8 17                               // DW_AT_low_pc
-; CHECK-NEXT: .b8 1                                // DW_FORM_addr
-; CHECK-NEXT: .b8 18                               // DW_AT_high_pc
-; CHECK-NEXT: .b8 1                                // DW_FORM_addr
-; CHECK-NEXT: .b8 64                               // DW_AT_frame_base
-; CHECK-NEXT: .b8 10                               // DW_FORM_block1
-; CHECK-NEXT: .b8 135                              // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 63                                  // DW_AT_external
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 4                                   // Abbreviation Code
+; CHECK-NEXT: .b8 5                                   // DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 5                                   // Abbreviation Code
+; CHECK-NEXT: .b8 52                                  // DW_TAG_variable
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 16                                  // DW_FORM_ref_addr
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 6                                   // Abbreviation Code
+; CHECK-NEXT: .b8 46                                  // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 17                                  // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                   // DW_FORM_addr
+; CHECK-NEXT: .b8 18                                  // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_FORM_addr
+; CHECK-NEXT: .b8 64                                  // DW_AT_frame_base
+; CHECK-NEXT: .b8 10                                  // DW_FORM_block1
+; CHECK-NEXT: .b8 135                                 // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT: .b8 64
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 3                                // DW_AT_name
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 73                               // DW_AT_type
-; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
-; CHECK-NEXT: .b8 63                               // DW_AT_external
-; CHECK-NEXT: .b8 12                               // DW_FORM_flag
-; CHECK-NEXT: .b8 0                                // EOM(1)
-; CHECK-NEXT: .b8 0                                // EOM(2)
-; CHECK-NEXT: .b8 7                                // Abbreviation Code
-; CHECK-NEXT: .b8 36                               // DW_TAG_base_type
-; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
-; CHECK-NEXT: .b8 3                                // DW_AT_name
-; CHECK-NEXT: .b8 8                                // DW_FORM_string
-; CHECK-NEXT: .b8 62                               // DW_AT_encoding
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 11                               // DW_AT_byte_size
-; CHECK-NEXT: .b8 11                               // DW_FORM_data1
-; CHECK-NEXT: .b8 0                                // EOM(1)
-; CHECK-NEXT: .b8 0                                // EOM(2)
-; CHECK-NEXT: .b8 0                                // EOM(3)
-; CHECK-NEXT: }
-; CHECK-NEXT: .section .debug_info
-; CHECK-NEXT: {
-; CHECK-NEXT: .b32 159                             // Length of Unit
-; CHECK-NEXT: .b8 2                                // DWARF version number
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 63                                  // DW_AT_external
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 7                                   // Abbreviation Code
+; CHECK-NEXT: .b8 5                                   // DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 2                                   // DW_AT_location
+; CHECK-NEXT: .b8 6                                   // DW_FORM_data4
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 8                                   // Abbreviation Code
+; CHECK-NEXT: .b8 36                                  // DW_TAG_base_type
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 62                                  // DW_AT_encoding
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 11                                  // DW_AT_byte_size
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 0                                   // EOM(3)
+; CHECK-NEXT: 	}
+; CHECK-NEXT: 	.section	.debug_info
+; CHECK-NEXT: 	{
+; CHECK-NEXT: .b32 143                                // Length of Unit
+; CHECK-NEXT: .b8 2                                   // DWARF version number
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b32 .debug_abbrev                   // Offset Into Abbrev. Section
-; CHECK-NEXT: .b8 8                                // Address Size (in bytes)
-; CHECK-NEXT: .b8 1                                // Abbrev [1] 0xb:0x98 DW_TAG_compile_unit
-; CHECK-NEXT: .b8 99                               // DW_AT_producer
+; CHECK-NEXT: .b32 .debug_abbrev                      // Offset Into Abbrev. Section
+; CHECK-NEXT: .b8 8                                   // Address Size (in bytes)
+; CHECK-NEXT: .b8 1                                   // Abbrev [1] 0xb:0x88 DW_TAG_compile_unit
+; CHECK-NEXT: .b8 99                                  // DW_AT_producer
 ; CHECK-NEXT: .b8 108
 ; CHECK-NEXT: .b8 97
 ; CHECK-NEXT: .b8 110
@@ -284,9 +313,9 @@ attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp
 ; CHECK-NEXT: .b8 57
 ; CHECK-NEXT: .b8 41
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 4                                // DW_AT_language
+; CHECK-NEXT: .b8 4                                   // DW_AT_language
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 100                              // DW_AT_name
+; CHECK-NEXT: .b8 100                                 // DW_AT_name
 ; CHECK-NEXT: .b8 101
 ; CHECK-NEXT: .b8 98
 ; CHECK-NEXT: .b8 117
@@ -307,8 +336,8 @@ attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp
 ; CHECK-NEXT: .b8 99
 ; CHECK-NEXT: .b8 99
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b32 .debug_line                     // DW_AT_stmt_list
-; CHECK-NEXT: .b8 47                               // DW_AT_comp_dir
+; CHECK-NEXT: .b32 .debug_line                        // DW_AT_stmt_list
+; CHECK-NEXT: .b8 47                                  // DW_AT_comp_dir
 ; CHECK-NEXT: .b8 108
 ; CHECK-NEXT: .b8 108
 ; CHECK-NEXT: .b8 118
@@ -324,18 +353,16 @@ attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp
 ; CHECK-NEXT: .b8 99
 ; CHECK-NEXT: .b8 99
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b64 $L__func_begin1                 // DW_AT_low_pc
-; CHECK-NEXT: .b64 $L__func_end1                   // DW_AT_high_pc
-; CHECK-NEXT: .b8 2                                // Abbrev [2] 0x64:0x4 DW_TAG_structure_type
-; CHECK-NEXT: .b8 65                               // DW_AT_name
+; CHECK-NEXT: .b8 2                                   // Abbrev [2] 0x54:0x4 DW_TAG_structure_type
+; CHECK-NEXT: .b8 65                                  // DW_AT_name
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 1                                // DW_AT_declaration
-; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x68:0x3a DW_TAG_subprogram
-; CHECK-NEXT: .b64 $L__func_begin1                 // DW_AT_low_pc
-; CHECK-NEXT: .b64 $L__func_end1                   // DW_AT_high_pc
-; CHECK-NEXT: .b8 1                                // DW_AT_frame_base
+; CHECK-NEXT: .b8 1                                   // DW_AT_declaration
+; CHECK-NEXT: .b8 3                                   // Abbrev [3] 0x58:0x3a DW_TAG_subprogram
+; CHECK-NEXT: .b64 $L__func_begin1                    // DW_AT_low_pc
+; CHECK-NEXT: .b64 $L__func_end1                      // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_AT_frame_base
 ; CHECK-NEXT: .b8 156
-; CHECK-NEXT: .b8 95                               // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT: .b8 90
 ; CHECK-NEXT: .b8 51
 ; CHECK-NEXT: .b8 98
@@ -344,34 +371,34 @@ attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp
 ; CHECK-NEXT: .b8 49
 ; CHECK-NEXT: .b8 65
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 98                               // DW_AT_name
+; CHECK-NEXT: .b8 98                                  // DW_AT_name
 ; CHECK-NEXT: .b8 97
 ; CHECK-NEXT: .b8 122
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
-; CHECK-NEXT: .b8 6                                // DW_AT_decl_line
-; CHECK-NEXT: .b8 1                                // DW_AT_external
-; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x8b:0x9 DW_TAG_formal_parameter
-; CHECK-NEXT: .b8 97                               // DW_AT_name
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 6                                   // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 4                                   // Abbrev [4] 0x7b:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 97                                  // DW_AT_name
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
-; CHECK-NEXT: .b8 6                                // DW_AT_decl_line
-; CHECK-NEXT: .b32 100                             // DW_AT_type
-; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x94:0xd DW_TAG_variable
-; CHECK-NEXT: .b8 122                              // DW_AT_name
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 6                                   // DW_AT_decl_line
+; CHECK-NEXT: .b32 84                                 // DW_AT_type
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0x84:0xd DW_TAG_variable
+; CHECK-NEXT: .b8 122                                 // DW_AT_name
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
-; CHECK-NEXT: .b8 7                                // DW_AT_decl_line
-; CHECK-NEXT: .b64 .debug_info+311                 // DW_AT_type
-; CHECK-NEXT: .b8 0                                // End Of Children Mark
-; CHECK-NEXT: .b8 0                                // End Of Children Mark
-; CHECK-NEXT: .b32 152                             // Length of Unit
-; CHECK-NEXT: .b8 2                                // DWARF version number
+; CHECK-NEXT: .b8 2                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 7                                   // DW_AT_decl_line
+; CHECK-NEXT: .b64 .debug_info+283                    // DW_AT_type
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b32 140                                // Length of Unit
+; CHECK-NEXT: .b8 2                                   // DWARF version number
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b32 .debug_abbrev                   // Offset Into Abbrev. Section
-; CHECK-NEXT: .b8 8                                // Address Size (in bytes)
-; CHECK-NEXT: .b8 1                                // Abbrev [1] 0xb:0x91 DW_TAG_compile_unit
-; CHECK-NEXT: .b8 99                               // DW_AT_producer
+; CHECK-NEXT: .b32 .debug_abbrev                      // Offset Into Abbrev. Section
+; CHECK-NEXT: .b8 8                                   // Address Size (in bytes)
+; CHECK-NEXT: .b8 1                                   // Abbrev [1] 0xb:0x85 DW_TAG_compile_unit
+; CHECK-NEXT: .b8 99                                  // DW_AT_producer
 ; CHECK-NEXT: .b8 108
 ; CHECK-NEXT: .b8 97
 ; CHECK-NEXT: .b8 110
@@ -400,9 +427,9 @@ attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp
 ; CHECK-NEXT: .b8 57
 ; CHECK-NEXT: .b8 41
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 4                                // DW_AT_language
+; CHECK-NEXT: .b8 4                                   // DW_AT_language
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 100                              // DW_AT_name
+; CHECK-NEXT: .b8 100                                 // DW_AT_name
 ; CHECK-NEXT: .b8 101
 ; CHECK-NEXT: .b8 98
 ; CHECK-NEXT: .b8 117
@@ -423,8 +450,8 @@ attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp
 ; CHECK-NEXT: .b8 99
 ; CHECK-NEXT: .b8 99
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b32 .debug_line                     // DW_AT_stmt_list
-; CHECK-NEXT: .b8 47                               // DW_AT_comp_dir
+; CHECK-NEXT: .b32 .debug_line                        // DW_AT_stmt_list
+; CHECK-NEXT: .b8 47                                  // DW_AT_comp_dir
 ; CHECK-NEXT: .b8 108
 ; CHECK-NEXT: .b8 108
 ; CHECK-NEXT: .b8 118
@@ -440,14 +467,12 @@ attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp
 ; CHECK-NEXT: .b8 99
 ; CHECK-NEXT: .b8 99
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b64 $L__func_begin0                 // DW_AT_low_pc
-; CHECK-NEXT: .b64 $L__func_end0                   // DW_AT_high_pc
-; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x64:0x30 DW_TAG_subprogram
-; CHECK-NEXT: .b64 $L__func_begin0                 // DW_AT_low_pc
-; CHECK-NEXT: .b64 $L__func_end0                   // DW_AT_high_pc
-; CHECK-NEXT: .b8 1                                // DW_AT_frame_base
+; CHECK-NEXT: .b8 6                                   // Abbrev [6] 0x54:0x34 DW_TAG_subprogram
+; CHECK-NEXT: .b64 $L__func_begin0                    // DW_AT_low_pc
+; CHECK-NEXT: .b64 $L__func_end0                      // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_AT_frame_base
 ; CHECK-NEXT: .b8 156
-; CHECK-NEXT: .b8 95                               // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT: .b8 90
 ; CHECK-NEXT: .b8 51
 ; CHECK-NEXT: .b8 98
@@ -455,29 +480,30 @@ attributes #2 = { "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp
 ; CHECK-NEXT: .b8 114
 ; CHECK-NEXT: .b8 105
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 98                               // DW_AT_name
+; CHECK-NEXT: .b8 98                                  // DW_AT_name
 ; CHECK-NEXT: .b8 97
 ; CHECK-NEXT: .b8 114
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
-; CHECK-NEXT: .b8 1                                // DW_AT_decl_line
-; CHECK-NEXT: .b32 148                             // DW_AT_type
-; CHECK-NEXT: .b8 1                                // DW_AT_external
-; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x8a:0x9 DW_TAG_formal_parameter
-; CHECK-NEXT: .b8 98                               // DW_AT_name
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_line
+; CHECK-NEXT: .b32 136                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 7                                   // Abbrev [7] 0x7a:0xd DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 $L__debug_loc0                     // DW_AT_location
+; CHECK-NEXT: .b8 98                                  // DW_AT_name
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
-; CHECK-NEXT: .b8 1                                // DW_AT_decl_line
-; CHECK-NEXT: .b32 148                             // DW_AT_type
-; CHECK-NEXT: .b8 0                                // End Of Children Mark
-; CHECK-NEXT: .b8 7                                // Abbrev [7] 0x94:0x7 DW_TAG_base_type
-; CHECK-NEXT: .b8 105                              // DW_AT_name
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_line
+; CHECK-NEXT: .b32 136                                // DW_AT_type
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 8                                   // Abbrev [8] 0x88:0x7 DW_TAG_base_type
+; CHECK-NEXT: .b8 105                                 // DW_AT_name
 ; CHECK-NEXT: .b8 110
 ; CHECK-NEXT: .b8 116
 ; CHECK-NEXT: .b8 0
-; CHECK-NEXT: .b8 5                                // DW_AT_encoding
-; CHECK-NEXT: .b8 4                                // DW_AT_byte_size
-; CHECK-NEXT: .b8 0                                // End Of Children Mark
-; CHECK-NEXT: }
-; CHECK-NEXT: .section .debug_loc { }
+; CHECK-NEXT: .b8 5                                   // DW_AT_encoding
+; CHECK-NEXT: .b8 4                                   // DW_AT_byte_size
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: 	}
+; CHECK-NEXT: 	.section	.debug_macinfo	{	}
 ; CHECK-NOT: debug_
diff --git a/llvm/test/DebugInfo/X86/header.ll b/llvm/test/DebugInfo/X86/header.ll
index 3f3d9f95419c1..f90aa3b50f467 100644
--- a/llvm/test/DebugInfo/X86/header.ll
+++ b/llvm/test/DebugInfo/X86/header.ll
@@ -5,7 +5,7 @@
 ; CHECK:       .text
 ; CHECK-NEXT: .file	"<stdin>"
 ; CHECK-NEXT: .globl	f
-; CHECK-NEXT: .p2align	4, 0x90
+; CHECK-NEXT: .p2align	4
 ; CHECK-NEXT: .type	f,@function
 ; CHECK-NEXT: f:                                      # @f
 
diff --git a/llvm/test/DebugInfo/X86/loop-align-debug.ll b/llvm/test/DebugInfo/X86/loop-align-debug.ll
index a0302d08faa0c..d6f6395b26544 100644
--- a/llvm/test/DebugInfo/X86/loop-align-debug.ll
+++ b/llvm/test/DebugInfo/X86/loop-align-debug.ll
@@ -13,7 +13,7 @@
 
 ; ASM:      .loc    0 0 0 is_stmt 0
 ; ASM-NEXT: .L{{.*}}:
-; ASM-NEXT: .p2align        4, 0x90
+; ASM-NEXT: .p2align        4
 
 ;; $ cat test.cpp
 ;; void g();
diff --git a/llvm/test/Instrumentation/InstrProfiling/mcdc.ll b/llvm/test/Instrumentation/InstrProfiling/mcdc.ll
index fbdc7b57be428..ee294a96b04ba 100644
--- a/llvm/test/Instrumentation/InstrProfiling/mcdc.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/mcdc.ll
@@ -36,25 +36,21 @@ entry:
   ; CHECK-NEXT: %[[LAB8:[0-9]+]] = and i32 %[[TEMP]], 7
   ; CHECK-NEXT: %[[LAB9:[0-9]+]] = trunc i32 %[[LAB8]] to i8
   ; CHECK-NEXT: %[[LAB10:[0-9]+]] = shl i8 1, %[[LAB9]]
-  ; CHECK-NEXT: call void @[[RMW_OR:.+]](ptr %[[LAB7]], i8 %[[LAB10]])
+  ; CHECK-NEXT: %[[BITS:.+]] = load i8, ptr %[[LAB7]], align 1
+  ; ATOMIC-NEXT: %[[MASKED:.+]] = and i8 %[[BITS]], %[[LAB10]]
+  ; ATOMIC-NEXT: %[[SHOULDWRITE:.+]] = icmp ne i8 %[[MASKED]], %[[LAB10]]
+  ; ATOMIC-NEXT: br i1 %[[SHOULDWRITE]], label %[[WRITE:.+]], label %[[SKIP:.+]], !prof ![[MDPROF:[0-9]+]]
+  ; ATOMIC: [[WRITE]]:
+  ; BASIC-NEXT: %[[LAB11:[0-9]+]] = or i8 %[[BITS]], %[[LAB10]]
+  ; RELOC-NEXT: %[[LAB11:[0-9]+]] = or i8 %[[BITS]], %[[LAB10]]
+  ; BASIC-NEXT: store i8 %[[LAB11]], ptr %[[LAB7]], align 1
+  ; RELOC-NEXT: store i8 %[[LAB11]], ptr %[[LAB7]], align 1
+  ; ATOMIC-NEXT: %{{.+}} = atomicrmw or ptr %[[LAB7]], i8 %[[LAB10]] monotonic, align 1
+  ; ATOMIC: [[SKIP]]:
   ret void
+  ; CHECK-NEXT: ret void
 }
 
-; CHECK: define private void @[[RMW_OR]](ptr %[[ARGPTR:.+]], i8 %[[ARGVAL:.+]])
-; CHECK:      %[[BITS:.+]] = load i8, ptr %[[ARGPTR]], align 1
-; BASIC-NEXT: %[[LAB11:[0-9]+]] = or i8 %[[BITS]], %[[ARGVAL]]
-; RELOC-NEXT: %[[LAB11:[0-9]+]] = or i8 %[[BITS]], %[[ARGVAL]]
-; BASIC-NEXT: store i8 %[[LAB11]], ptr %[[ARGPTR]], align 1
-; RELOC-NEXT: store i8 %[[LAB11]], ptr %[[ARGPTR]], align 1
-; ATOMIC-NEXT: %[[MASKED:.+]] = and i8 %[[BITS]], %[[ARGVAL]]
-; ATOMIC-NEXT: %[[SHOULDWRITE:.+]] = icmp ne i8 %[[MASKED]], %[[ARGVAL]]
-; ATOMIC-NEXT: br i1 %[[SHOULDWRITE]], label %[[WRITE:.+]], label %[[SKIP:.+]], !prof ![[MDPROF:[0-9]+]]
-; ATOMIC: [[WRITE]]:
-; ATOMIC-NEXT: %{{.+}} = atomicrmw or ptr %[[ARGPTR]], i8 %[[ARGVAL]] monotonic, align 1
-; ATOMIC-NEXT: ret void
-; ATOMIC: [[SKIP]]:
-; CHECK-NEXT: ret void
-
 ; ATOMIC: ![[MDPROF]] = !{!"branch_weights", i32 1, i32 1048575}
 
 declare void @llvm.instrprof.cover(ptr, i64, i32, i32)
diff --git a/llvm/test/Instrumentation/RealtimeSanitizer/rtsan_unsafe.ll b/llvm/test/Instrumentation/RealtimeSanitizer/rtsan_unsafe.ll
new file mode 100644
index 0000000000000..5abf5de304481
--- /dev/null
+++ b/llvm/test/Instrumentation/RealtimeSanitizer/rtsan_unsafe.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --version 5
+; RUN: opt < %s -passes=rtsan -S | FileCheck %s
+
+; RealtimeSanitizer pass should create the demangled function name as a global string and,
+; at the function entrypoint, pass it as an argument to the rtsan notify method
+
+;.
+; CHECK: @[[GLOB0:[0-9]+]] = private unnamed_addr constant [20 x i8] c"blocking_function()\00", align 1
+;.
+define void @_Z17blocking_functionv() #0 {
+; CHECK-LABEL: define void @_Z17blocking_functionv(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    call void @__rtsan_notify_blocking_call(ptr @[[GLOB0]])
+; CHECK-NEXT:    ret void
+;
+  ret void
+}
+
+define noundef i32 @main() #2 {
+; CHECK-LABEL: define noundef i32 @main() {
+; CHECK-NEXT:    call void @_Z17blocking_functionv()
+; CHECK-NEXT:    ret i32 0
+;
+  call void @_Z17blocking_functionv() #4
+  ret i32 0
+}
+
+attributes #0 = { mustprogress noinline sanitize_realtime_unsafe optnone ssp uwtable(sync) }
+;.
+; CHECK: attributes #[[ATTR0]] = { mustprogress noinline optnone sanitize_realtime_unsafe ssp uwtable(sync) }
+;.
diff --git a/llvm/test/MC/AArch64/FP8/directive-arch-negative.s b/llvm/test/MC/AArch64/FP8/directive-arch-negative.s
index c4d8dbf44b03b..34fdb74fe1f37 100644
--- a/llvm/test/MC/AArch64/FP8/directive-arch-negative.s
+++ b/llvm/test/MC/AArch64/FP8/directive-arch-negative.s
@@ -51,5 +51,5 @@ luti2  z0.h, { z0.h }, z0[0]
 .arch armv9-a+sme-lutv2
 .arch armv9-a+nosme-lutv2
 luti4  { z0.b - z3.b }, zt0, { z0, z1 }
-// CHECK: error: instruction requires: sme2 sme-lutv2
+// CHECK: error: instruction requires: sme-lutv2
 // CHECK: luti4  { z0.b - z3.b }, zt0, { z0, z1 }
diff --git a/llvm/test/MC/AArch64/FP8_SME2/lut.s b/llvm/test/MC/AArch64/FP8_SME2/lut.s
index 96c88f5da9a69..ae17ae42a1a94 100644
--- a/llvm/test/MC/AArch64/FP8_SME2/lut.s
+++ b/llvm/test/MC/AArch64/FP8_SME2/lut.s
@@ -1,33 +1,33 @@
-// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2,+sme2p1,+sme-lutv2 < %s \
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p1,+sme-lutv2 < %s \
 // RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
 
 // RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
 // RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 
-// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2,+sme2p1,+sme-lutv2 < %s \
-// RUN:        | llvm-objdump -d --mattr=+sme2,+sme2p1,+sme-lutv2 --no-print-imm-hex - \
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p1,+sme-lutv2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sme2p1,+sme-lutv2 --no-print-imm-hex - \
 // RUN:        | FileCheck %s --check-prefix=CHECK-INST
 
-// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2,+sme2p1,+sme-lutv2 < %s \
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2p1,+sme-lutv2 < %s \
 // RUN:        | llvm-objdump -d --mattr=-sme-lutv2 --no-print-imm-hex - \
 // RUN:        | FileCheck %s --check-prefix=CHECK-UNKNOWN
 
 // Disassemble encoding and check the re-encoding (-show-encoding) matches.
-// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2,+sme2p1,+sme-lutv2 < %s \
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p1,+sme-lutv2 < %s \
 // RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
-// RUN:        | llvm-mc -triple=aarch64 -mattr=+sme2,+sme2p1,+sme-lutv2 -disassemble -show-encoding \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+sme2p1,+sme-lutv2 -disassemble -show-encoding \
 // RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
 
 luti4   {z0.b-z3.b}, zt0, {z0-z1}  // 11000000-10001011-00000000-00000000
 // CHECK-INST: luti4   { z0.b - z3.b }, zt0, { z0, z1 }
 // CHECK-ENCODING: [0x00,0x00,0x8b,0xc0]
-// CHECK-ERROR: instruction requires: sme2 sme-lutv2
+// CHECK-ERROR: instruction requires: sme-lutv2
 // CHECK-UNKNOWN: c08b0000 <unknown>
 
 luti4   {z28.b-z31.b}, zt0, {z30-z31}  // 11000000-10001011-00000011-11011100
 // CHECK-INST: luti4   { z28.b - z31.b }, zt0, { z30, z31 }
 // CHECK-ENCODING: [0xdc,0x03,0x8b,0xc0]
-// CHECK-ERROR: instruction requires: sme2 sme-lutv2
+// CHECK-ERROR: instruction requires: sme-lutv2
 // CHECK-UNKNOWN: c08b03dc <unknown>
 
 // Strided
diff --git a/llvm/test/MC/AArch64/FP8_SME2/movt-diagnostics.s b/llvm/test/MC/AArch64/FP8_SME2/movt-diagnostics.s
index ba45389211215..d3f334a3089b4 100644
--- a/llvm/test/MC/AArch64/FP8_SME2/movt-diagnostics.s
+++ b/llvm/test/MC/AArch64/FP8_SME2/movt-diagnostics.s
@@ -1,5 +1,5 @@
 
-// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2,+sme-lutv2  2>&1 < %s | FileCheck %s
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sme-lutv2  2>&1 < %s | FileCheck %s
 // --------------------------------------------------------------------------//
 // Invalid vector select register
 movt   z0, z31
diff --git a/llvm/test/MC/AArch64/FP8_SME2/movt.s b/llvm/test/MC/AArch64/FP8_SME2/movt.s
index fa6502e138ab6..aa601fb319277 100644
--- a/llvm/test/MC/AArch64/FP8_SME2/movt.s
+++ b/llvm/test/MC/AArch64/FP8_SME2/movt.s
@@ -1,14 +1,14 @@
-// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2,+sme-lutv2 < %s \
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme-lutv2 < %s \
 // RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
 
 // RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
 // RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 
-// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2,+sme-lutv2 < %s \
-// RUN:        | llvm-objdump -d --mattr=+sme2,+sme-lutv2 --no-print-imm-hex - \
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme-lutv2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+sme-lutv2 --no-print-imm-hex - \
 // RUN:        | FileCheck %s --check-prefix=CHECK-INST
 
-// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme2,+sme-lutv2 < %s \
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sme-lutv2 < %s \
 // RUN:        | llvm-objdump -d --mattr=-sme-lutv2 --no-print-imm-hex - \
 // RUN:        | FileCheck %s --check-prefix=CHECK-UNKNOWN
 
@@ -21,11 +21,11 @@
 movt    zt0, z0  // 11000000-01001111-00000011-11100000
 // CHECK-INST: movt    zt0, z0
 // CHECK-ENCODING: [0xe0,0x03,0x4f,0xc0]
-// CHECK-ERROR: instruction requires: sme2 sme-lutv2
+// CHECK-ERROR: instruction requires: sme-lutv2
 // CHECK-UNKNOWN: c04f03e0 <unknown>
 
 movt    zt0[3, mul vl], z31  // 11000000-01001111-00110011-11111111
 // CHECK-INST: movt    zt0[3, mul vl], z31
 // CHECK-ENCODING: [0xff,0x33,0x4f,0xc0]
-// CHECK-ERROR: instruction requires: sme2 sme-lutv2
+// CHECK-ERROR: instruction requires: sme-lutv2
 // CHECK-UNKNOWN: c04f33ff <unknown>
diff --git a/llvm/test/MC/WebAssembly/annotations-typecheck.s b/llvm/test/MC/WebAssembly/annotations-typecheck.s
new file mode 100644
index 0000000000000..1fcdb9c0e3587
--- /dev/null
+++ b/llvm/test/MC/WebAssembly/annotations-typecheck.s
@@ -0,0 +1,11 @@
+# RUN: not llvm-mc -triple=wasm32-unknown-unknown < %s | FileCheck %s
+
+# This tests annotations are handled correctly even if there is a type checking
+# error (which are unrelated to the block annotations).
+test_annotation:
+  .functype   test_annotation () -> ()
+  block (i32) -> ()
+    drop
+# CHECK-NOT: # End marker mismatch!
+  end_block
+  end_function
diff --git a/llvm/test/MC/WebAssembly/annotations.s b/llvm/test/MC/WebAssembly/annotations.s
index 3e727591afa94..59a1349f6eaf2 100644
--- a/llvm/test/MC/WebAssembly/annotations.s
+++ b/llvm/test/MC/WebAssembly/annotations.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -no-type-check -triple=wasm32-unknown-unknown -mattr=+exception-handling < %s | FileCheck %s
+# RUN: llvm-mc -triple=wasm32-unknown-unknown -mattr=+exception-handling < %s | FileCheck %s
 
 # Tests if block/loop/try/catch/end/branch/rethrow instructions are correctly
 # printed with their annotations.
@@ -12,15 +12,19 @@ test_annotation:
   try
     br        0
   catch     __cpp_exception
+    drop
     block
+      i32.const 0
       br_if     0
       loop
+        i32.const 0
         br_if     1
       end_loop
     end_block
     try
       rethrow   0
     catch     __cpp_exception
+      drop
     catch_all
       block
         try
@@ -56,15 +60,19 @@ test_annotation:
 # CHECK:        try
 # CHECK-NEXT:   br        0               # 0: down to label0
 # CHECK-NEXT:   catch     __cpp_exception # catch0:
+# CHECK-NEXT:   drop
 # CHECK-NEXT:   block
+# CHECK-NEXT:   i32.const       0
 # CHECK-NEXT:   br_if     0               # 0: down to label1
 # CHECK-NEXT:   loop                      # label2:
+# CHECK-NEXT:   i32.const       0
 # CHECK-NEXT:   br_if     1               # 1: down to label1
 # CHECK-NEXT:   end_loop
 # CHECK-NEXT:   end_block                 # label1:
 # CHECK-NEXT:   try
 # CHECK-NEXT:   rethrow   0               # down to catch3
 # CHECK-NEXT:   catch     __cpp_exception # catch3:
+# CHECK-NEXT:   drop
 # CHECK-NEXT:   catch_all{{$}}
 # CHECK-NEXT:   block
 # CHECK-NEXT:   try
diff --git a/llvm/test/MC/WebAssembly/basic-assembly.s b/llvm/test/MC/WebAssembly/basic-assembly.s
index db7ccc9759bec..2028fcea6aeae 100644
--- a/llvm/test/MC/WebAssembly/basic-assembly.s
+++ b/llvm/test/MC/WebAssembly/basic-assembly.s
@@ -82,11 +82,13 @@ test0:
     i32.const   3
     end_block            # "switch" exit.
     if                   # void
+    i32.const   0
     if          i32
+    i32.const   0
     end_if
+    drop
     else
     end_if
-    drop
     block       void
     i32.const   2
     return
@@ -222,11 +224,13 @@ empty_exnref_table:
 # CHECK-NEXT:      i32.const   3
 # CHECK-NEXT:      end_block           # label2:
 # CHECK-NEXT:      if
+# CHECK-NEXT:      i32.const   0
 # CHECK-NEXT:      if          i32
+# CHECK-NEXT:      i32.const   0
 # CHECK-NEXT:      end_if
+# CHECK-NEXT:      drop
 # CHECK-NEXT:      else
 # CHECK-NEXT:      end_if
-# CHECK-NEXT:      drop
 # CHECK-NEXT:      block
 # CHECK-NEXT:      i32.const   2
 # CHECK-NEXT:      return
diff --git a/llvm/test/MC/WebAssembly/block-assembly.s b/llvm/test/MC/WebAssembly/block-assembly.s
new file mode 100644
index 0000000000000..ef3924d26057a
--- /dev/null
+++ b/llvm/test/MC/WebAssembly/block-assembly.s
@@ -0,0 +1,182 @@
+# RUN: llvm-mc -triple=wasm32-unknown-unknown -mattr=+exception-handling < %s | FileCheck %s
+# Check that it converts to .o without errors, but don't check any output:
+# RUN: llvm-mc -triple=wasm32-unknown-unknown -mattr=+exception-handling -filetype=obj -o %t.o < %s
+
+  .tagtype  __cpp_exception i32
+
+block_branch_test:
+  .functype  block_branch_test () -> ()
+
+  # Block input paramter / return tests
+
+  i32.const 0
+  block (i32) -> (i32)
+  end_block
+  drop
+
+  i32.const 0
+  i64.const 0
+  block (i32, i64) -> (i32, f32)
+    drop
+    f32.const 0.0
+  end_block
+  drop
+  drop
+
+  i32.const 0
+  loop (i32) -> (f32)
+    drop
+    f32.const 0.0
+  end_loop
+  drop
+
+  i32.const 0
+  i32.const 0
+  if (i32) -> (i32)
+  else
+    i32.popcnt
+  end_if
+  drop
+
+  try i32
+    i32.const 0
+  catch       __cpp_exception
+    i32.clz
+  catch_all
+    i32.const 5
+  end_try
+  drop
+
+  i32.const 0
+  block (i32) -> (i32)
+    block (i32) -> (f32)
+      drop
+      f32.const 0.0
+    end_block
+    drop
+    i32.const 0
+  end_block
+  drop
+
+  # Branch tests
+
+  block f32
+    f32.const 0.0
+    i32.const 0
+    br_if 0
+    f32.const 1.0
+    br 0
+    # After 'br', we can pop any values from the polymorphic stack
+    i32.add
+    i32.sub
+    i32.mul
+    drop
+  end_block
+  drop
+
+  block () -> (f32, f64)
+    f32.const 0.0
+    f64.const 0.0
+    i32.const 0
+    br_if 0
+    block (f32, f64) -> (f32, f64)
+      i32.const 1
+      br_if 0
+    end_block
+  end_block
+  drop
+  drop
+
+  # Within a loop, branches target the start of the loop
+  i32.const 0
+  loop (i32) -> ()
+    i32.const 1
+    br 0
+  end_loop
+
+  end_function
+
+# CHECK-LABEL: block_branch_test
+
+# CHECK:         i32.const  0
+# CHECK-NEXT:    block     (i32) -> (i32)
+# CHECK-NEXT:    end_block                               # label0:
+# CHECK-NEXT:    drop
+
+# CHECK:         i32.const  0
+# CHECK-NEXT:    i64.const  0
+# CHECK-NEXT:    block     (i32, i64) -> (i32, f32)
+# CHECK-NEXT:    drop
+# CHECK-NEXT:    f32.const  0x0p0
+# CHECK-NEXT:    end_block                               # label1:
+# CHECK-NEXT:    drop
+# CHECK-NEXT:    drop
+
+# CHECK:         i32.const  0
+# CHECK-NEXT:    loop      (i32) -> (f32)                  # label2:
+# CHECK-NEXT:    drop
+# CHECK-NEXT:    f32.const  0x0p0
+# CHECK-NEXT:    end_loop
+# CHECK-NEXT:    drop
+
+# CHECK:         i32.const  0
+# CHECK-NEXT:    i32.const  0
+# CHECK-NEXT:    if      (i32) -> (i32)
+# CHECK-NEXT:    else
+# CHECK-NEXT:    i32.popcnt
+# CHECK-NEXT:    end_if
+# CHECK-NEXT:    drop
+
+# CHECK:         try       i32
+# CHECK-NEXT:    i32.const  0
+# CHECK-NEXT:    catch     __cpp_exception                 # catch3:
+# CHECK-NEXT:    i32.clz
+# CHECK-NEXT:    catch_all
+# CHECK-NEXT:    i32.const  5
+# CHECK-NEXT:    end_try                                 # label3:
+# CHECK-NEXT:    drop
+
+# CHECK:         i32.const  0
+# CHECK-NEXT:    block     (i32) -> (i32)
+# CHECK-NEXT:    block     (i32) -> (f32)
+# CHECK-NEXT:    drop
+# CHECK-NEXT:    f32.const  0x0p0
+# CHECK-NEXT:    end_block                               # label5:
+# CHECK-NEXT:    drop
+# CHECK-NEXT:    i32.const  0
+# CHECK-NEXT:    end_block                               # label4:
+# CHECK-NEXT:    drop
+
+# CHECK:         block     f32
+# CHECK-NEXT:    f32.const  0x0p0
+# CHECK-NEXT:    i32.const  0
+# CHECK-NEXT:    br_if     0                               # 0: down to label6
+# CHECK-NEXT:    f32.const  0x1p0
+# CHECK-NEXT:    br        0                               # 0: down to label6
+# CHECK-NEXT:    i32.add
+# CHECK-NEXT:    i32.sub
+# CHECK-NEXT:    i32.mul
+# CHECK-NEXT:    drop
+# CHECK-NEXT:    end_block                               # label6:
+# CHECK-NEXT:    drop
+
+# CHECK:         block     () -> (f32, f64)
+# CHECK-NEXT:    f32.const  0x0p0
+# CHECK-NEXT:    f64.const  0x0p0
+# CHECK-NEXT:    i32.const  0
+# CHECK-NEXT:    br_if     0                               # 0: down to label7
+# CHECK-NEXT:    block     (f32, f64) -> (f32, f64)
+# CHECK-NEXT:    i32.const  1
+# CHECK-NEXT:    br_if     0                               # 0: down to label8
+# CHECK-NEXT:    end_block                               # label8:
+# CHECK-NEXT:    end_block                               # label7:
+# CHECK-NEXT:    drop
+# CHECK-NEXT:    drop
+
+# CHECK:         i32.const  0
+# CHECK-NEXT:    loop      (i32) -> ()                     # label9:
+# CHECK-NEXT:    i32.const  1
+# CHECK-NEXT:    br        0                               # 0: up to label9
+# CHECK-NEXT:    end_loop
+
+# CHECK:         end_function
diff --git a/llvm/test/MC/WebAssembly/reference-types.s b/llvm/test/MC/WebAssembly/reference-types.s
index 2f8bfba68dcea..cfadede8295ef 100644
--- a/llvm/test/MC/WebAssembly/reference-types.s
+++ b/llvm/test/MC/WebAssembly/reference-types.s
@@ -76,14 +76,17 @@ ref_select_test:
 # CHECK: block externref
 # CHECK: block exnref
 ref_block_test:
-  .functype ref_block_test () -> (exnref, externref, funcref)
+  .functype ref_block_test () -> ()
   block funcref
   block externref
   block exnref
   ref.null_exn
   end_block
+  drop
   ref.null_extern
   end_block
+  drop
   ref.null_func
   end_block
+  drop
   end_function
diff --git a/llvm/test/MC/WebAssembly/type-checker-errors.s b/llvm/test/MC/WebAssembly/type-checker-errors.s
index d81c5aff0a7e9..c5187d10fdca0 100644
--- a/llvm/test/MC/WebAssembly/type-checker-errors.s
+++ b/llvm/test/MC/WebAssembly/type-checker-errors.s
@@ -297,7 +297,7 @@ end_block_insufficient_values_on_stack_2:
 end_block_type_mismatch:
   .functype end_block_type_mismatch () -> ()
   block i32
-  f32.const 1.0
+    f32.const 1.0
 # CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
   end_block
   drop
@@ -314,7 +314,7 @@ end_loop_insufficient_values_on_stack:
 end_loop_type_mismatch:
   .functype end_loop_type_mismatch () -> ()
   loop f32
-  i32.const 1
+    i32.const 1
 # CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [f32] but got [i32]
   end_loop
   drop
@@ -332,7 +332,7 @@ end_if_type_mismatch_1:
   .functype end_if_type_mismatch_1 () -> ()
   i32.const 1
   if f32
-  i32.const 1
+    i32.const 1
 # CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [f32] but got [i32]
   end_if
   drop
@@ -342,7 +342,7 @@ end_if_insufficient_values_on_stack_2:
   .functype end_if_insufficient_values_on_stack_2 () -> ()
   i32.const 1
   if i32
-  i32.const 2
+    i32.const 2
   else
 # CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   end_if
@@ -353,9 +353,9 @@ end_if_type_mismatch_2:
   .functype end_if_type_mismatch_2 () -> ()
   i32.const 1
   if i32
-  i32.const 2
+    i32.const 2
   else
-  f32.const 3.0
+    f32.const 3.0
 # CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
   end_if
   drop
@@ -367,7 +367,7 @@ else_insufficient_values_on_stack:
   if i32
 # CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   else
-  i32.const 0
+    i32.const 0
   end_if
   drop
   end_function
@@ -376,10 +376,10 @@ else_type_mismatch:
   .functype else_type_mismatch () -> ()
   i32.const 1
   if i32
-  f32.const 0.0
+    f32.const 0.0
 # CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
   else
-  i32.const 0
+    i32.const 0
   end_if
   drop
   end_function
@@ -390,7 +390,7 @@ else_type_mismatch:
 end_try_insufficient_values_on_stack:
   .functype end_try_insufficient_values_on_stack () -> ()
   try i32
-  i32.const 0
+    i32.const 0
   catch_all
 # CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   end_try
@@ -400,7 +400,7 @@ end_try_insufficient_values_on_stack:
 end_try_type_mismatch:
   .functype end_try_type_mismatch () -> ()
   try i32
-  i32.const 0
+    i32.const 0
   catch tag_f32
 # CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
   end_try
@@ -419,7 +419,7 @@ catch_insufficient_values_on_stack:
 catch_type_mismatch:
   .functype catch_type_mismatch () -> ()
   try i32
-  f32.const 1.0
+    f32.const 1.0
 # CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
   catch tag_i32
   end_try
@@ -431,7 +431,7 @@ catch_all_insufficient_values_on_stack:
   try i32
 # CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   catch_all
-  i32.const 0
+    i32.const 0
   end_try
   drop
   end_function
@@ -439,10 +439,10 @@ catch_all_insufficient_values_on_stack:
 catch_all_type_mismatch:
   .functype catch_all_type_mismatch () -> ()
   try i32
-  f32.const 1.0
+    f32.const 1.0
 # CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
   catch_all
-  i32.const 0
+    i32.const 0
   end_try
   drop
   end_function
@@ -458,7 +458,7 @@ delegate_insufficient_values_on_stack:
 delegate_type_mismatch:
   .functype delegate_type_mismatch () -> ()
   try i32
-  f32.const 1.0
+    f32.const 1.0
 # CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
   delegate 0
   drop
@@ -620,9 +620,8 @@ catch_superfluous_value_at_end:
   .functype catch_superfluous_value_at_end () -> ()
   try
   catch tag_i32
-  end_try
-# FIXME: Superfluous value should be caught at end_try?
 # CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [] but got [i32]
+  end_try
   end_function
 
 ref_is_null_empty_stack_while_popping:
@@ -680,7 +679,7 @@ other_insn_test_3:
 check_after_unreachable_within_block:
   .functype check_after_unreachable_within_block () -> ()
   block
-  unreachable
+    unreachable
   end_block
 # CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [any] but got []
   drop
@@ -690,7 +689,7 @@ check_after_unreachable_within_block:
 check_after_unreachable_within_loop:
   .functype check_after_unreachable_within_loop () -> ()
   loop
-  unreachable
+    unreachable
   end_loop
 # CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [any] but got []
   drop
@@ -701,9 +700,9 @@ check_after_unreachable_within_if_1:
   .functype check_after_unreachable_within_if_1 () -> ()
   i32.const 0
   if
-  unreachable
+    unreachable
   else
-  unreachable
+    unreachable
   end_if
 # CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [any] but got []
   drop
@@ -714,10 +713,10 @@ check_after_unreachable_within_if_2:
   .functype check_after_unreachable_within_if_2 () -> ()
   i32.const 0
   if
-  unreachable
+    unreachable
   else
-# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [any] but got []
-  drop
+# CHECK: :[[@LINE+1]]:5: error: type mismatch, expected [any] but got []
+    drop
   end_if
   end_function
 
@@ -725,9 +724,9 @@ check_after_unreachable_within_if_2:
 check_after_unreachable_within_try_1:
   .functype check_after_unreachable_within_try_1 () -> ()
   try
-  unreachable
+    unreachable
   catch_all
-  unreachable
+    unreachable
   end_try
 # CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [any] but got []
   drop
@@ -737,11 +736,11 @@ check_after_unreachable_within_try_1:
 check_after_unreachable_within_try_2:
   .functype check_after_unreachable_within_try_2 () -> ()
   try
-  unreachable
+    unreachable
   catch tag_i32
-  drop
-# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [any] but got []
-  drop
+    drop
+# CHECK: :[[@LINE+1]]:5: error: type mismatch, expected [any] but got []
+    drop
   end_try
   end_function
 
@@ -749,10 +748,10 @@ check_after_unreachable_within_try_2:
 check_after_unreachable_within_try_3:
   .functype check_after_unreachable_within_try_3 () -> ()
   try
-  unreachable
+    unreachable
   catch_all
-# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [any] but got []
-  drop
+# CHECK: :[[@LINE+1]]:5: error: type mismatch, expected [any] but got []
+    drop
   end_try
   end_function
 
@@ -760,7 +759,7 @@ check_after_unreachable_within_try_3:
 check_after_unreachable_within_try_4:
   .functype check_after_unreachable_within_try_4 () -> ()
   try
-  unreachable
+    unreachable
   delegate 0
 # CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [any] but got []
   drop
@@ -839,12 +838,12 @@ br_invalid_type_catch_all:
 br_invalid_depth_out_of_range:
   .functype br_invalid_depth_out_of_range () -> ()
   block
-  block
-  block
-# CHECK: :[[@LINE+1]]:5: error: br: invalid depth 4
-    br 4
-  end_block
-  end_block
+    block
+      block
+# CHECK: :[[@LINE+1]]:9: error: br: invalid depth 4
+        br 4
+      end_block
+    end_block
   end_block
   end_function
 
@@ -925,3 +924,23 @@ any_value_on_stack:
   drop
 
   end_function
+
+block_param_and_return:
+  .functype block_param_and_return () -> ()
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
+  block (i32) -> (f32)
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [f32] but got [i32]
+  end_block
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
+  i32.popcnt
+  drop
+
+  block f32
+    f32.const 0.0
+    br 0
+    i32.const 0
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [f32] but got [..., i32]
+  end_block
+
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [] but got [f32]
+  end_function
diff --git a/llvm/test/Other/dump-with-filter.ll b/llvm/test/Other/dump-with-filter.ll
new file mode 100644
index 0000000000000..6b6d73112fcc7
--- /dev/null
+++ b/llvm/test/Other/dump-with-filter.ll
@@ -0,0 +1,14 @@
+;; Make sure we can run -filter-print-funcs with -ir-dump-directory.
+; RUN: rm -rf %t/logs
+; RUN: opt %s -disable-output -passes='no-op-function' -print-before=no-op-function -print-after=no-op-function \
+; RUN:   -ir-dump-directory %t/logs -filter-print-funcs=test2
+; RUN: ls %t/logs | count 2
+; RUN: rm -rf %t/logs
+
+define void @test() {
+    ret void
+}
+
+define void @test2() {
+    ret void
+}
diff --git a/llvm/test/TableGen/anonymous-location.td b/llvm/test/TableGen/anonymous-location.td
new file mode 100644
index 0000000000000..c6892daf8d322
--- /dev/null
+++ b/llvm/test/TableGen/anonymous-location.td
@@ -0,0 +1,16 @@
+// RUN: llvm-tblgen --print-detailed-records %s | FileCheck %s -DFILE=%{s:basename}
+
+class A<int a> {
+  int Num = a;
+}
+
+// Verify that the location of the anonymous record instantiated
+// for A<10> and A<11> is correct. It should show the line where the
+// anonymous record was instantiated and not the line where the class
+// was defined.
+def y {
+  // CHECK: anonymous_0 |[[FILE]]:[[@LINE+1]]|
+  int x = A<10>.Num;
+  // CHECK: anonymous_1 |[[FILE]]:[[@LINE+1]]|
+  int y = A<11>.Num;
+}
diff --git a/llvm/test/TableGen/assert.td b/llvm/test/TableGen/assert.td
index d616adcb5d0d1..17c08b4313531 100644
--- a/llvm/test/TableGen/assert.td
+++ b/llvm/test/TableGen/assert.td
@@ -1,6 +1,8 @@
-// RUN: not llvm-tblgen %s 2>&1 | FileCheck %s
+// RUN: not llvm-tblgen %s 2>&1 | FileCheck %s -DFILE=%s
 
+// -----------------------------------------------------------------------------
 // Test the assert statement at top level.
+// -----------------------------------------------------------------------------
 
 // CHECK: assertion failed
 // CHECK-NOT: note: primary name is too short
@@ -48,14 +50,16 @@ foreach i = 1...3 in {
   def bar_ # i;
 }
 
+// -----------------------------------------------------------------------------
 // Test the assert statement in a record definition.
+// -----------------------------------------------------------------------------
 
-// CHECK: assertion failed
+// CHECK: [[FILE]]:[[@LINE+8]]:10: error: assertion failed
 // CHECK-NOT: primary first name is not "Grace"
-// CHECK: primary first name is not "Grack"
-// CHECK: assertion failed
-// CHECK: foo field should be
-
+// CHECK: note: primary first name is not "Grack"
+// CHECK: [[FILE]]:[[@LINE+7]]:10: error: assertion failed
+// CHECK: note: foo field should be
+// CHECK: [[FILE]]:[[@LINE+1]]:5: error: assertion failed in this record
 def Rec10 {
   assert !eq(!substr(Name, 0, 5), "Grace"), "primary first name is not \"Grace\"";
   assert !eq(!substr(Name, 0, 5), "Grack"), "primary first name is not \"Grack\"";
@@ -63,18 +67,18 @@ def Rec10 {
   assert !eq(foo, "foo"), "foo field should be \"Foo\"";
 }
 
-// CHECK: assertion failed
+// CHECK: [[FILE]]:[[@LINE+5]]:10: error: assertion failed
 // CHECK: note: magic field is incorrect: 42
-
+// CHECK: [[FILE]]:[[@LINE+1]]:5: error: assertion failed in this record
 def Rec11 {
   int magic = 13;
   assert !eq(magic, 13), "magic field is incorrect: " # magic;
   let magic = 42;       
 }
 
-// CHECK: assertion failed
+// CHECK: [[FILE]]:[[@LINE+6]]:10: error: assertion failed
 // CHECK: note: var field has wrong value
-
+// CHECK: [[FILE]]:[[@LINE+1]]:5: error: assertion failed in this record
 def Rec12 {
   defvar prefix = "foo_";
   string var = prefix # "snork";
@@ -83,11 +87,11 @@ def Rec12 {
 
 // CHECK: assertion failed
 // CHECK: note: kind field has wrong value
-
 class Kind {
   int kind = 7;
 }
 
+// CHECK: [[FILE]]:[[@LINE+1]]:5: error: assertion failed in this record
 def Rec13 : Kind {
   let kind = 8;
   assert !eq(kind, 7), "kind field has wrong value: " # kind;
@@ -95,13 +99,15 @@ def Rec13 : Kind {
 
 // CHECK: assertion failed
 // CHECK: note: double_result should be
-
+// CHECK: [[FILE]]:[[@LINE+1]]:5: error: assertion failed in this record
 def Rec14 : Cube<3> {
   int double_result = !mul(result, 2);
   assert !eq(double_result, 53), "double_result should be 54";
 }
 
+// -----------------------------------------------------------------------------
 // Test the assert statement in a class definition.
+// -----------------------------------------------------------------------------
 
 class PersonName<string name> {
   assert !le(!size(name), 32), "person name is too long: " # name;
@@ -118,32 +124,39 @@ def Rec20 : Person<"Donald Knuth", 60>;
 
 // CHECK: assertion failed
 // CHECK: note: person name is too long
-
+// CHECK: [[FILE]]:[[@LINE+1]]:5: error: assertion failed in this record
 def Rec21 : Person<"Donald Uh Oh This Name Is Too Long Knuth", 50>;
 
 // CHECK: assertion failed
 // CHECK: note: person age is invalid
-
+// CHECK: [[FILE]]:[[@LINE+1]]:5: error: assertion failed in this record
 def Rec22 : Person<"Donald Knuth", 150>;
 
 // Test the assert statement in an anonymous class invocation.
-
 def Rec30 {
   string Name = Person<"Margaret Heafield Hamilton", 25>.Name;
   int Age = Person<"Margaret Heafield Hamilton", 25>.Age;
 }
 
+// CHECK: assertion failed
+// CHECK: note: person name is too long
+// CHECK: [[FILE]]:[[@LINE+2]]:17: error: assertion failed in this record
 def Rec31 {
   string Name = Person<"Margaret Heafield And More Middle Names Hamilton", 25>.Name;
   int Age = Person<"Margaret Heafield Hamilton", 25>.Age;
 }
 
+// CHECK: assertion failed
+// CHECK: note: person age is invalid: 0
+// CHECK: [[FILE]]:[[@LINE+3]]:13: error: assertion failed in this record
 def Rec32 {
   string Name = Person<"Margaret Heafield Hamilton", 25>.Name;
   int Age = Person<"Margaret Heafield Hamilton", 0>.Age;
 }
 
+// -----------------------------------------------------------------------------
 // Test the assert statement in a multiclass.
+// -----------------------------------------------------------------------------
 
 // CHECK: assertion failed
 // CHECK: note: MC1 id string is too long
diff --git a/llvm/test/Transforms/ArgumentPromotion/actual-arguments.ll b/llvm/test/Transforms/ArgumentPromotion/actual-arguments.ll
index ca757a165fa4b..af8da57484541 100644
--- a/llvm/test/Transforms/ArgumentPromotion/actual-arguments.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/actual-arguments.ll
@@ -26,7 +26,7 @@ define internal i32 @test_cannot_promote_1(ptr %p, ptr nocapture readonly %test_
   ret i32 %sum
 }
 
-; This is called by @caller_aliased_args, from which we can see that %test_c may
+; This is called by multiple callers, from which we can see that %test_c may
 ; alias %p and so we cannot promote %test_c.
 ;
 define internal i32 @test_cannot_promote_2(ptr %p, ptr nocapture readonly %test_c) {
@@ -87,9 +87,8 @@ define internal i32 @test_can_promote_1(ptr %p, ptr nocapture readonly %test_c)
   ret i32 %sum
 }
 
-; This is called by multiple callers (@caller_safe_args_1, @caller_safe_args_2),
-; from which we can prove that %test_c does not alias %p for any Call to the
-; function, so we can promote it.
+; This is called by multiple callers, from which we can prove that %test_c does
+; not alias %p for any Call to the function, so we can promote it.
 ;
 define internal i32 @test_can_promote_2(ptr %p, ptr nocapture readonly %test_c) {
 ; CHECK-LABEL: define {{[^@]+}}@test_can_promote_2
@@ -223,4 +222,76 @@ define i32 @caller_safe_args_2(i64 %n, ptr %p) {
   ret i32 %res
 }
 
+; Invokes @test_cannot_promote_2
+define i32 @caller_invoke_aliased_args() personality ptr @__gxx_personality_v0 {
+; CHECK-LABEL: define {{[^@]+}}@caller_invoke_aliased_args() personality ptr @__gxx_personality_v0 {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALLER_C:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 5, ptr [[CALLER_C]], align 4
+; CHECK-NEXT:    [[RES:%.*]] = invoke i32 @test_cannot_promote_2(ptr [[CALLER_C]], ptr [[CALLER_C]])
+; CHECK-NEXT:            to label [[OUT:%.*]] unwind label [[CPAD:%.*]]
+; CHECK:       out:
+; CHECK-NEXT:    ret i32 [[RES]]
+; CHECK:       cpad:
+; CHECK-NEXT:    [[EXN:%.*]] = landingpad { ptr, i32 }
+; CHECK-NEXT:            catch ptr @_ZTIi
+; CHECK-NEXT:    ret i32 -1
+;
+entry:
+  %caller_c = alloca i32
+  store i32 5, ptr %caller_c
+
+  %res = invoke i32 @test_cannot_promote_2(ptr %caller_c, ptr %caller_c)
+  to label %out unwind label %cpad
+
+out:
+  ret i32 %res
+
+cpad:
+  %exn = landingpad { ptr, i32 }
+  catch ptr @_ZTIi
+  ret i32 -1
+}
+
+; Invokes @test_can_promote_2
+define i32 @caller_invoke_safe_args(i64 %n) personality ptr @__gxx_personality_v0 {
+; CHECK-LABEL: define {{[^@]+}}@caller_invoke_safe_args
+; CHECK-SAME: (i64 [[N:%.*]]) personality ptr @__gxx_personality_v0 {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P:%.*]] = alloca [5 x double], i64 [[N]], align 8
+; CHECK-NEXT:    call void @memset(ptr [[P]], i64 0, i64 [[N]])
+; CHECK-NEXT:    [[CALLER_C:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 5, ptr [[CALLER_C]], align 4
+; CHECK-NEXT:    [[CALLER_C_VAL:%.*]] = load i32, ptr [[CALLER_C]], align 4
+; CHECK-NEXT:    [[RES:%.*]] = invoke i32 @test_can_promote_2(ptr [[P]], i32 [[CALLER_C_VAL]])
+; CHECK-NEXT:            to label [[OUT:%.*]] unwind label [[CPAD:%.*]]
+; CHECK:       out:
+; CHECK-NEXT:    ret i32 [[RES]]
+; CHECK:       cpad:
+; CHECK-NEXT:    [[EXN:%.*]] = landingpad { ptr, i32 }
+; CHECK-NEXT:            catch ptr @_ZTIi
+; CHECK-NEXT:    ret i32 -1
+;
+entry:
+  %p = alloca [5 x double], i64 %n
+  call void @memset(ptr %p, i64 0, i64 %n)
+
+  %caller_c = alloca i32
+  store i32 5, ptr %caller_c
+
+  %res = invoke i32 @test_can_promote_2(ptr %p, ptr %caller_c)
+  to label %out unwind label %cpad
+
+out:
+  ret i32 %res
+
+cpad:
+  %exn = landingpad { ptr, i32 }
+  catch ptr @_ZTIi
+  ret i32 -1
+}
+
 declare void @memset(ptr, i64, i64)
+declare i32 @__gxx_personality_v0(...)
+
+@_ZTIi = external constant ptr
diff --git a/llvm/test/Transforms/Attributor/value-simplify-pointer-info.ll b/llvm/test/Transforms/Attributor/value-simplify-pointer-info.ll
index 378560cc89cd1..adcee750cae6e 100644
--- a/llvm/test/Transforms/Attributor/value-simplify-pointer-info.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify-pointer-info.ll
@@ -3389,6 +3389,197 @@ entry:
   ret ptr %R
 }
 
+define ptr @move2(ptr %p) {
+; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; CHECK-LABEL: define {{[^@]+}}@move2
+; CHECK-SAME: (ptr nofree readnone "no-capture-maybe-returned" [[P:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[G:%.*]] = getelementptr i8, ptr [[P]], i32 2
+; CHECK-NEXT:    ret ptr [[G]]
+;
+  %g = getelementptr i8, ptr %p, i32 2
+  ret ptr %g
+}
+define internal ptr @move4(ptr %p) {
+; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; CHECK-LABEL: define {{[^@]+}}@move4
+; CHECK-SAME: (ptr noalias nofree readnone "no-capture-maybe-returned" [[P:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[G:%.*]] = getelementptr i8, ptr [[P]], i32 4
+; CHECK-NEXT:    ret ptr [[G]]
+;
+  %g = getelementptr i8, ptr %p, i32 4
+  ret ptr %g
+}
+
+define ptr @move246(i32 %i, ptr %p) {
+; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; CHECK-LABEL: define {{[^@]+}}@move246
+; CHECK-SAME: (i32 [[I:%.*]], ptr nofree readnone "no-capture-maybe-returned" [[P:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[C0:%.*]] = icmp eq i32 [[I]], 0
+; CHECK-NEXT:    br i1 [[C0]], label [[BG2:%.*]], label [[BG46:%.*]]
+; CHECK:       bg2:
+; CHECK-NEXT:    [[G2:%.*]] = getelementptr i8, ptr [[P]], i32 2
+; CHECK-NEXT:    ret ptr [[G2]]
+; CHECK:       bg46:
+; CHECK-NEXT:    [[C1:%.*]] = icmp eq i32 [[I]], 1
+; CHECK-NEXT:    br i1 [[C1]], label [[BG4:%.*]], label [[BG6:%.*]]
+; CHECK:       bg4:
+; CHECK-NEXT:    [[G4:%.*]] = getelementptr i8, ptr [[P]], i32 4
+; CHECK-NEXT:    ret ptr [[G4]]
+; CHECK:       bg6:
+; CHECK-NEXT:    [[G6:%.*]] = getelementptr i8, ptr [[P]], i32 6
+; CHECK-NEXT:    ret ptr [[G6]]
+;
+  %c0 = icmp eq i32 %i, 0
+  br i1 %c0, label %bg2, label %bg46
+bg2:
+  %g2 = getelementptr i8, ptr %p, i32 2
+  ret ptr %g2
+bg46:
+  %c1 = icmp eq i32 %i, 1
+  br i1 %c1, label %bg4, label %bg6
+bg4:
+  %g4 = getelementptr i8, ptr %p, i32 4
+  ret ptr %g4
+bg6:
+  %g6 = getelementptr i8, ptr %p, i32 6
+  ret ptr %g6
+}
+
+declare void @use3i8(i8, i8, i8)
+
+define void @returnedPtrAccesses() {
+; TUNIT-LABEL: define {{[^@]+}}@returnedPtrAccesses() {
+; TUNIT-NEXT:    [[A:%.*]] = alloca i64, align 8
+; TUNIT-NEXT:    [[A2:%.*]] = call ptr @move2(ptr noalias nofree noundef nonnull readnone align 8 dereferenceable(8) "no-capture-maybe-returned" [[A]]) #[[ATTR23]]
+; TUNIT-NEXT:    [[A4:%.*]] = call ptr @move4(ptr noalias nofree noundef nonnull readnone align 8 dereferenceable(8) "no-capture-maybe-returned" [[A]]) #[[ATTR23]]
+; TUNIT-NEXT:    [[A6:%.*]] = call ptr @move4(ptr noalias nofree readnone "no-capture-maybe-returned" [[A2]]) #[[ATTR23]]
+; TUNIT-NEXT:    store i8 2, ptr [[A2]], align 1
+; TUNIT-NEXT:    store i8 4, ptr [[A4]], align 1
+; TUNIT-NEXT:    store i8 6, ptr [[A6]], align 1
+; TUNIT-NEXT:    call void @use3i8(i8 2, i8 4, i8 6)
+; TUNIT-NEXT:    ret void
+;
+; CGSCC-LABEL: define {{[^@]+}}@returnedPtrAccesses() {
+; CGSCC-NEXT:    [[A:%.*]] = alloca i64, align 8
+; CGSCC-NEXT:    [[A2:%.*]] = call nonnull dereferenceable(1) ptr @move2(ptr noalias nofree noundef nonnull readnone align 8 dereferenceable(8) [[A]]) #[[ATTR20]]
+; CGSCC-NEXT:    [[A4:%.*]] = call ptr @move4(ptr noalias nofree noundef nonnull readnone align 8 dereferenceable(8) [[A]]) #[[ATTR20]]
+; CGSCC-NEXT:    [[A6:%.*]] = call ptr @move4(ptr noalias nofree noundef nonnull readnone dereferenceable(1) [[A2]]) #[[ATTR20]]
+; CGSCC-NEXT:    [[G2:%.*]] = getelementptr i8, ptr [[A]], i32 2
+; CGSCC-NEXT:    [[G4:%.*]] = getelementptr i8, ptr [[A]], i32 4
+; CGSCC-NEXT:    [[G6:%.*]] = getelementptr i8, ptr [[A]], i32 6
+; CGSCC-NEXT:    store i8 2, ptr [[A2]], align 1
+; CGSCC-NEXT:    store i8 4, ptr [[A4]], align 1
+; CGSCC-NEXT:    store i8 6, ptr [[A6]], align 1
+; CGSCC-NEXT:    [[L2:%.*]] = load i8, ptr [[G2]], align 2
+; CGSCC-NEXT:    [[L4:%.*]] = load i8, ptr [[G4]], align 4
+; CGSCC-NEXT:    [[L6:%.*]] = load i8, ptr [[G6]], align 2
+; CGSCC-NEXT:    call void @use3i8(i8 [[L2]], i8 [[L4]], i8 [[L6]])
+; CGSCC-NEXT:    ret void
+;
+  %a = alloca i64
+  %a2 = call ptr @move2(ptr %a)
+  %a4 = call ptr @move4(ptr %a)
+  %a6 = call ptr @move4(ptr %a2)
+  %g2 = getelementptr i8, ptr %a, i32 2
+  %g4 = getelementptr i8, ptr %a, i32 4
+  %g6 = getelementptr i8, ptr %a, i32 6
+  store i8 2, ptr %a2
+  store i8 4, ptr %a4
+  store i8 6, ptr %a6
+  %l2 = load i8, ptr %g2
+  %l4 = load i8, ptr %g4
+  %l6 = load i8, ptr %g6
+  call void @use3i8(i8 %l2, i8 %l4, i8 %l6)
+  ret void
+}
+
+define void @returnedPtrAccessesMultiple(i32 %i) {
+; TUNIT-LABEL: define {{[^@]+}}@returnedPtrAccessesMultiple
+; TUNIT-SAME: (i32 [[I:%.*]]) {
+; TUNIT-NEXT:    [[A:%.*]] = alloca i64, align 8
+; TUNIT-NEXT:    [[AP:%.*]] = call ptr @move246(i32 [[I]], ptr noalias nofree noundef nonnull readnone align 8 dereferenceable(8) "no-capture-maybe-returned" [[A]]) #[[ATTR23]]
+; TUNIT-NEXT:    store i8 2, ptr [[AP]], align 1
+; TUNIT-NEXT:    call void @use3i8(i8 2, i8 2, i8 2)
+; TUNIT-NEXT:    ret void
+;
+; CGSCC-LABEL: define {{[^@]+}}@returnedPtrAccessesMultiple
+; CGSCC-SAME: (i32 [[I:%.*]]) {
+; CGSCC-NEXT:    [[A:%.*]] = alloca i64, align 8
+; CGSCC-NEXT:    [[AP:%.*]] = call ptr @move246(i32 [[I]], ptr noalias nofree noundef nonnull readnone align 8 dereferenceable(8) [[A]]) #[[ATTR20]]
+; CGSCC-NEXT:    [[G2:%.*]] = getelementptr i8, ptr [[A]], i32 2
+; CGSCC-NEXT:    [[G4:%.*]] = getelementptr i8, ptr [[A]], i32 4
+; CGSCC-NEXT:    [[G6:%.*]] = getelementptr i8, ptr [[A]], i32 6
+; CGSCC-NEXT:    store i8 2, ptr [[AP]], align 1
+; CGSCC-NEXT:    [[L2:%.*]] = load i8, ptr [[G2]], align 2
+; CGSCC-NEXT:    [[L4:%.*]] = load i8, ptr [[G4]], align 4
+; CGSCC-NEXT:    [[L6:%.*]] = load i8, ptr [[G6]], align 2
+; CGSCC-NEXT:    call void @use3i8(i8 [[L2]], i8 [[L4]], i8 [[L6]])
+; CGSCC-NEXT:    ret void
+;
+  %a = alloca i64
+  %ap = call ptr @move246(i32 %i, ptr %a)
+  %g2 = getelementptr i8, ptr %a, i32 2
+  %g4 = getelementptr i8, ptr %a, i32 4
+  %g6 = getelementptr i8, ptr %a, i32 6
+  store i8 2, ptr %ap
+  %l2 = load i8, ptr %g2
+  %l4 = load i8, ptr %g4
+  %l6 = load i8, ptr %g6
+  call void @use3i8(i8 %l2, i8 %l4, i8 %l6)
+  ret void
+}
+
+define void @returnedPtrAccessesMultiple2(i32 %i) {
+; TUNIT-LABEL: define {{[^@]+}}@returnedPtrAccessesMultiple2
+; TUNIT-SAME: (i32 [[I:%.*]]) {
+; TUNIT-NEXT:    [[A:%.*]] = alloca i64, align 8
+; TUNIT-NEXT:    [[G2:%.*]] = getelementptr i8, ptr [[A]], i32 2
+; TUNIT-NEXT:    [[G4:%.*]] = getelementptr i8, ptr [[A]], i32 4
+; TUNIT-NEXT:    [[G6:%.*]] = getelementptr i8, ptr [[A]], i32 6
+; TUNIT-NEXT:    store i8 0, ptr [[G2]], align 2
+; TUNIT-NEXT:    store i8 0, ptr [[G4]], align 4
+; TUNIT-NEXT:    store i8 0, ptr [[G6]], align 2
+; TUNIT-NEXT:    [[AP:%.*]] = call ptr @move246(i32 [[I]], ptr noalias nofree noundef nonnull readnone align 8 dereferenceable(8) "no-capture-maybe-returned" [[A]]) #[[ATTR23]]
+; TUNIT-NEXT:    store i8 2, ptr [[AP]], align 1
+; TUNIT-NEXT:    [[L2:%.*]] = load i8, ptr [[G2]], align 2
+; TUNIT-NEXT:    [[L4:%.*]] = load i8, ptr [[G4]], align 4
+; TUNIT-NEXT:    [[L6:%.*]] = load i8, ptr [[G6]], align 2
+; TUNIT-NEXT:    call void @use3i8(i8 noundef [[L2]], i8 noundef [[L4]], i8 noundef [[L6]])
+; TUNIT-NEXT:    ret void
+;
+; CGSCC-LABEL: define {{[^@]+}}@returnedPtrAccessesMultiple2
+; CGSCC-SAME: (i32 [[I:%.*]]) {
+; CGSCC-NEXT:    [[A:%.*]] = alloca i64, align 8
+; CGSCC-NEXT:    [[G2:%.*]] = getelementptr i8, ptr [[A]], i32 2
+; CGSCC-NEXT:    [[G4:%.*]] = getelementptr i8, ptr [[A]], i32 4
+; CGSCC-NEXT:    [[G6:%.*]] = getelementptr i8, ptr [[A]], i32 6
+; CGSCC-NEXT:    store i8 0, ptr [[G2]], align 2
+; CGSCC-NEXT:    store i8 0, ptr [[G4]], align 4
+; CGSCC-NEXT:    store i8 0, ptr [[G6]], align 2
+; CGSCC-NEXT:    [[AP:%.*]] = call ptr @move246(i32 [[I]], ptr noalias nofree noundef nonnull readnone align 8 dereferenceable(8) [[A]]) #[[ATTR20]]
+; CGSCC-NEXT:    store i8 2, ptr [[AP]], align 1
+; CGSCC-NEXT:    [[L2:%.*]] = load i8, ptr [[G2]], align 2
+; CGSCC-NEXT:    [[L4:%.*]] = load i8, ptr [[G4]], align 4
+; CGSCC-NEXT:    [[L6:%.*]] = load i8, ptr [[G6]], align 2
+; CGSCC-NEXT:    call void @use3i8(i8 [[L2]], i8 [[L4]], i8 [[L6]])
+; CGSCC-NEXT:    ret void
+;
+  %a = alloca i64
+  %g2 = getelementptr i8, ptr %a, i32 2
+  %g4 = getelementptr i8, ptr %a, i32 4
+  %g6 = getelementptr i8, ptr %a, i32 6
+  store i8 0, ptr %g2
+  store i8 0, ptr %g4
+  store i8 0, ptr %g6
+  %ap = call ptr @move246(i32 %i, ptr %a)
+  store i8 2, ptr %ap
+  %l2 = load i8, ptr %g2
+  %l4 = load i8, ptr %g4
+  %l6 = load i8, ptr %g6
+  call void @use3i8(i8 %l2, i8 %l4, i8 %l6)
+  ret void
+}
+
 declare void @llvm.assume(i1 noundef)
 
 
diff --git a/llvm/test/Transforms/EarlyCSE/replace-calls-def-attrs.ll b/llvm/test/Transforms/EarlyCSE/replace-calls-def-attrs.ll
new file mode 100644
index 0000000000000..2adaf1c7b67a0
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/replace-calls-def-attrs.ll
@@ -0,0 +1,256 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-attributes --check-globals all --version 5
+; RUN: opt -S -passes=early-cse  < %s | FileCheck %s
+
+declare i8 @baz(i8, i8)
+declare i8 @baz_side_effects(i8, i8)
+declare i8 @buz(i8, i8)
+
+declare ptr @baz.ptr(i8, i8)
+declare i8 @buz.ptr(ptr, ptr)
+
+declare float @baz.fp(float, float)
+declare i8 @buz.fp(float, float)
+define i8 @same_parent_combine_diff_attrs(i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR0:[0-9]+]]
+; CHECK-NEXT:    [[R:%.*]] = call i8 @buz(i8 [[C1]], i8 [[C1]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %c1 = call i8 @baz(i8 noundef %x, i8 noundef %y) readnone
+  %c0 = call i8 @baz(i8 %x, i8 noundef %y) readnone
+  %r = call i8 @buz(i8 %c0, i8 %c1)
+  ret i8 %r
+
+}
+
+define i8 @same_parent_combine_diff_attrs_needs_intersect(i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_needs_intersect(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[C1:%.*]] = call ptr @baz.ptr(i8 [[X]], i8 noundef [[Y]]) #[[ATTR0]]
+; CHECK-NEXT:    [[R:%.*]] = call i8 @buz.ptr(ptr [[C1]], ptr [[C1]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %c1 = call nonnull ptr @baz.ptr(i8 noundef %x, i8 noundef %y) readnone
+  %c0 = call ptr @baz.ptr(i8 %x, i8 noundef %y) readnone
+  %r = call i8 @buz.ptr(ptr %c0, ptr %c1)
+  ret i8 %r
+
+}
+
+define i8 @same_parent_combine_diff_attrs_fmf(float %x, float %y) {
+; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_fmf(
+; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
+; CHECK-NEXT:    [[C1:%.*]] = call nnan float @baz.fp(float [[X]], float noundef [[Y]]) #[[ATTR1:[0-9]+]]
+; CHECK-NEXT:    [[R:%.*]] = call i8 @buz.fp(float [[C1]], float [[C1]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %c1 = call nnan nsz float @baz.fp(float noundef %x, float noundef %y) readonly
+  %c0 = call nnan float @baz.fp(float %x, float noundef %y) readonly
+  %r = call i8 @buz.fp(float %c0, float %c1)
+  ret i8 %r
+
+}
+
+define i8 @same_parent_combine_diff_attrs_fmf2(float %x, float %y) {
+; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_fmf2(
+; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
+; CHECK-NEXT:    [[C1:%.*]] = call nnan float @baz.fp(float [[X]], float noundef [[Y]]) #[[ATTR0]]
+; CHECK-NEXT:    [[R:%.*]] = call i8 @buz.fp(float [[C1]], float [[C1]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %c1 = call nnan float @baz.fp(float noundef %x, float noundef %y) readnone
+  %c0 = call nnan nsz float @baz.fp(float %x, float noundef %y) readnone
+  %r = call i8 @buz.fp(float %c0, float %c1)
+  ret i8 %r
+
+}
+
+define i8 @same_parent_combine_diff_attrs_needs_intersect2(i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_needs_intersect2(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[C1:%.*]] = call ptr @baz.ptr(i8 [[X]], i8 noundef [[Y]]) #[[ATTR1]]
+; CHECK-NEXT:    [[R:%.*]] = call i8 @buz.ptr(ptr [[C1]], ptr [[C1]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %c1 = call nonnull ptr @baz.ptr(i8 noundef %x, i8 noundef %y) readonly
+  %c0 = call ptr @baz.ptr(i8 %x, i8 noundef %y) readonly
+  %r = call i8 @buz.ptr(ptr %c0, ptr %c1)
+  ret i8 %r
+
+}
+
+define i8 @same_parent_combine_diff_attrs_really_needs_intersect(i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_really_needs_intersect(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[C1:%.*]] = call ptr @baz.ptr(i8 [[X]], i8 noundef [[Y]]) #[[ATTR1]]
+; CHECK-NEXT:    [[R:%.*]] = call i8 @buz.ptr(ptr [[C1]], ptr noundef [[C1]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %c1 = call nonnull ptr @baz.ptr(i8 noundef %x, i8 noundef %y) readonly
+  %c0 = call ptr @baz.ptr(i8 %x, i8 noundef %y) readonly
+  %r = call i8 @buz.ptr(ptr %c0, ptr noundef %c1)
+  ret i8 %r
+
+}
+
+define i8 @same_parent_combine_diff_attrs_fail_side_effects(i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_fail_side_effects(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]])
+; CHECK-NEXT:    [[C0:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]])
+; CHECK-NEXT:    [[R:%.*]] = call i8 @buz(i8 [[C0]], i8 [[C1]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %c1 = call i8 @baz(i8 noundef %x, i8 noundef %y)
+  %c0 = call i8 @baz(i8 %x, i8 noundef %y)
+  %r = call i8 @buz(i8 %c0, i8 %c1)
+  ret i8 %r
+
+}
+
+define i8 @same_parent_combine_diff_attrs_quasi_side_effects2(i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_quasi_side_effects2(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR0]]
+; CHECK-NEXT:    [[C0:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]])
+; CHECK-NEXT:    [[R:%.*]] = call i8 @buz(i8 [[C0]], i8 [[C1]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %c1 = call i8 @baz(i8 noundef %x, i8 noundef %y) readnone
+  %c0 = call i8 @baz(i8 %x, i8 noundef %y)
+  %r = call i8 @buz(i8 %c0, i8 %c1)
+  ret i8 %r
+
+}
+
+define i8 @diff_parent_combine_diff_attrs(i1 %c, i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @diff_parent_combine_diff_attrs(
+; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR0]]
+; CHECK-NEXT:    br i1 [[C]], label %[[T:.*]], label %[[F:.*]]
+; CHECK:       [[T]]:
+; CHECK-NEXT:    [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR1]]
+; CHECK-NEXT:    [[R:%.*]] = call i8 @buz(i8 [[C0]], i8 [[C1]])
+; CHECK-NEXT:    ret i8 [[R]]
+; CHECK:       [[F]]:
+; CHECK-NEXT:    [[R2:%.*]] = add i8 [[C1]], 4
+; CHECK-NEXT:    ret i8 [[R2]]
+;
+  %c1 = call i8 @baz(i8 %x, i8 noundef %y) readnone
+  br i1 %c, label %T, label %F
+T:
+  %c0 = call i8 @baz(i8 noundef %x, i8 noundef %y) readonly
+  %r = call i8 @buz(i8 %c0, i8 %c1)
+  ret i8 %r
+F:
+  %r2 = add i8 %c1, 4
+  ret i8 %r2
+}
+
+define i8 @diff_parent_combine_diff_attrs_preserves_return_attrs(i1 %c, i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @diff_parent_combine_diff_attrs_preserves_return_attrs(
+; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[C1:%.*]] = call nonnull ptr @baz.ptr(i8 [[X]], i8 noundef [[Y]]) #[[ATTR1]]
+; CHECK-NEXT:    br i1 [[C]], label %[[T:.*]], label %[[F:.*]]
+; CHECK:       [[T]]:
+; CHECK-NEXT:    [[R:%.*]] = call i8 @buz.ptr(ptr [[C1]], ptr noundef [[C1]])
+; CHECK-NEXT:    ret i8 [[R]]
+; CHECK:       [[F]]:
+; CHECK-NEXT:    ret i8 9
+;
+  %c1 = call nonnull ptr @baz.ptr(i8 %x, i8 noundef %y) readonly
+  br i1 %c, label %T, label %F
+T:
+  %c0 = call nonnull ptr @baz.ptr(i8 noundef %x, i8 noundef %y) readonly
+  %r = call i8 @buz.ptr(ptr %c0, ptr noundef %c1)
+  ret i8 %r
+F:
+  ret i8 9
+}
+
+define i8 @same_parent_combine_diff_attrs_todo(i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_todo(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR0]]
+; CHECK-NEXT:    [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    [[R:%.*]] = call i8 @buz(i8 [[C0]], i8 [[C1]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %c1 = call i8 @baz(i8 %x, i8 noundef %y) readnone
+  %c0 = call i8 @baz(i8 noundef %x, i8 noundef %y) readnone alwaysinline
+  %r = call i8 @buz(i8 %c0, i8 %c1)
+  ret i8 %r
+
+}
+
+define i8 @same_parent_combine_diff_attrs_fail(i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_fail(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR0]]
+; CHECK-NEXT:    [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    [[R:%.*]] = call i8 @buz(i8 [[C0]], i8 [[C1]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %c1 = call i8 @baz(i8 %x, i8 noundef %y) readnone
+  %c0 = call i8 @baz(i8 noundef %x, i8 noundef %y) readnone strictfp
+  %r = call i8 @buz(i8 %c0, i8 %c1)
+  ret i8 %r
+
+}
+
+define i8 @diff_parent_combine_diff_attrs_todo(i1 %c, i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @diff_parent_combine_diff_attrs_todo(
+; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR0]]
+; CHECK-NEXT:    br i1 [[C]], label %[[T:.*]], label %[[F:.*]]
+; CHECK:       [[T]]:
+; CHECK-NEXT:    [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    [[R:%.*]] = call i8 @buz(i8 [[C0]], i8 [[C1]])
+; CHECK-NEXT:    ret i8 [[R]]
+; CHECK:       [[F]]:
+; CHECK-NEXT:    [[R2:%.*]] = add i8 [[C1]], 4
+; CHECK-NEXT:    ret i8 [[R2]]
+;
+  %c1 = call i8 @baz(i8 %x, i8 noundef %y) readnone
+  br i1 %c, label %T, label %F
+T:
+  %c0 = call i8 @baz(i8 noundef %x, i8 noundef %y) readnone optnone noinline
+  %r = call i8 @buz(i8 %c0, i8 %c1)
+  ret i8 %r
+F:
+  %r2 = add i8 %c1, 4
+  ret i8 %r2
+}
+
+define i8 @diff_parent_combine_diff_attrs_fail(i1 %c, i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @diff_parent_combine_diff_attrs_fail(
+; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR0]]
+; CHECK-NEXT:    br i1 [[C]], label %[[T:.*]], label %[[F:.*]]
+; CHECK:       [[T]]:
+; CHECK-NEXT:    [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR3]]
+; CHECK-NEXT:    [[R:%.*]] = call i8 @buz(i8 [[C0]], i8 [[C1]])
+; CHECK-NEXT:    ret i8 [[R]]
+; CHECK:       [[F]]:
+; CHECK-NEXT:    [[R2:%.*]] = add i8 [[C1]], 4
+; CHECK-NEXT:    ret i8 [[R2]]
+;
+  %c1 = call i8 @baz(i8 %x, i8 noundef %y) readnone
+  br i1 %c, label %T, label %F
+T:
+  %c0 = call i8 @baz(i8 noundef %x, i8 noundef %y) readnone strictfp
+  %r = call i8 @buz(i8 %c0, i8 %c1)
+  ret i8 %r
+F:
+  %r2 = add i8 %c1, 4
+  ret i8 %r2
+}
+
+;.
+; CHECK: attributes #[[ATTR0]] = { memory(none) }
+; CHECK: attributes #[[ATTR1]] = { memory(read) }
+; CHECK: attributes #[[ATTR2]] = { alwaysinline memory(none) }
+; CHECK: attributes #[[ATTR3]] = { strictfp memory(none) }
+; CHECK: attributes #[[ATTR4]] = { noinline optnone memory(none) }
+;.
diff --git a/llvm/test/Transforms/FunctionSpecialization/function-specialization2.ll b/llvm/test/Transforms/FunctionSpecialization/function-specialization2.ll
index b6cdcf18eea42..ef830a0e9a4a9 100644
--- a/llvm/test/Transforms/FunctionSpecialization/function-specialization2.ll
+++ b/llvm/test/Transforms/FunctionSpecialization/function-specialization2.ll
@@ -1,10 +1,8 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --include-generated-funcs --version 5
 ; RUN: opt -passes="ipsccp<func-spec>,deadargelim" -force-specialization -S < %s | FileCheck %s
-; RUN: opt -passes="ipsccp<func-spec>,deadargelim" -funcspec-max-iters=1 -force-specialization -S < %s | FileCheck %s
+; RUN: opt -passes="ipsccp<func-spec>,deadargelim" -funcspec-max-iters=1 -force-specialization -S < %s | FileCheck %s --check-prefix=ONE-ITER
 ; RUN: opt -passes="ipsccp<func-spec>,deadargelim" -funcspec-max-iters=0 -force-specialization -S < %s | FileCheck %s --check-prefix=DISABLED
 
-; DISABLED-NOT: @func.specialized.1(
-; DISABLED-NOT: @func.specialized.2(
 
 define internal i32 @func(ptr %0, i32 %1, ptr nocapture %2) {
   %4 = alloca i32, align 4
@@ -42,47 +40,176 @@ define internal void @decrement(ptr nocapture %0) {
 }
 
 define i32 @main(ptr %0, i32 %1) {
-; CHECK:    call void @func.specialized.2(ptr [[TMP0:%.*]], i32 [[TMP1:%.*]])
   %3 = call i32 @func(ptr %0, i32 %1, ptr nonnull @increment)
-; CHECK:    call void @func.specialized.1(ptr [[TMP0]], i32 0)
   %4 = call i32 @func(ptr %0, i32 %3, ptr nonnull @decrement)
-; CHECK:    ret i32 0
   ret i32 %4
 }
 
-; CHECK: @func.specialized.1(
-; CHECK:    [[TMP3:%.*]] = alloca i32, align 4
-; CHECK:    store i32 [[TMP1:%.*]], ptr [[TMP3]], align 4
-; CHECK:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
-; CHECK:    [[TMP5:%.*]] = icmp slt i32 [[TMP4]], 1
-; CHECK:    br i1 [[TMP5]], label [[TMP13:%.*]], label [[TMP6:%.*]]
-; CHECK:       6:
-; CHECK:    [[TMP7:%.*]] = load i32, ptr [[TMP3]], align 4
-; CHECK:    [[TMP8:%.*]] = sext i32 [[TMP7]] to i64
-; CHECK:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 [[TMP8]]
-; CHECK:    call void @decrement(ptr [[TMP9]])
-; CHECK:    [[TMP10:%.*]] = load i32, ptr [[TMP3]], align 4
-; CHECK:    [[TMP11:%.*]] = add nsw i32 [[TMP10]], -1
-; CHECK:    call void @func.specialized.1(ptr [[TMP0]], i32 [[TMP11]])
-; CHECK:    br label [[TMP12:%.*]]
-; CHECK:       12:
-; CHECK:    ret void
-;
-;
-; CHECK: @func.specialized.2(
-; CHECK:    [[TMP3:%.*]] = alloca i32, align 4
-; CHECK:    store i32 [[TMP1:%.*]], ptr [[TMP3]], align 4
-; CHECK:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
-; CHECK:    [[TMP5:%.*]] = icmp slt i32 [[TMP4]], 1
-; CHECK:    br i1 [[TMP5]], label [[TMP13:%.*]], label [[TMP6:%.*]]
-; CHECK:       6:
-; CHECK:    [[TMP7:%.*]] = load i32, ptr [[TMP3]], align 4
-; CHECK:    [[TMP8:%.*]] = sext i32 [[TMP7]] to i64
-; CHECK:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 [[TMP8]]
-; CHECK:    call void @increment(ptr [[TMP9]])
-; CHECK:    [[TMP10:%.*]] = load i32, ptr [[TMP3]], align 4
-; CHECK:    [[TMP11:%.*]] = add nsw i32 [[TMP10]], -1
-; CHECK:    call void @func.specialized.2(ptr [[TMP0]], i32 [[TMP11]])
-; CHECK:    br label [[TMP12:%.*]]
-; CHECK:       12:
-; CHECK:    ret void
+; CHECK-LABEL: define internal void @increment(
+; CHECK-SAME: ptr nocapture [[TMP0:%.*]]) {
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], 1
+; CHECK-NEXT:    store i32 [[TMP3]], ptr [[TMP0]], align 4
+; CHECK-NEXT:    ret void
+;
+;
+; CHECK-LABEL: define internal void @decrement(
+; CHECK-SAME: ptr nocapture [[TMP0:%.*]]) {
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], -1
+; CHECK-NEXT:    store i32 [[TMP3]], ptr [[TMP0]], align 4
+; CHECK-NEXT:    ret void
+;
+;
+; CHECK-LABEL: define i32 @main(
+; CHECK-SAME: ptr [[TMP0:%.*]], i32 [[TMP1:%.*]]) {
+; CHECK-NEXT:    call void @func.specialized.2(ptr [[TMP0]], i32 [[TMP1]])
+; CHECK-NEXT:    call void @func.specialized.1(ptr [[TMP0]], i32 0)
+; CHECK-NEXT:    ret i32 0
+;
+;
+; CHECK-LABEL: define internal void @func.specialized.1(
+; CHECK-SAME: ptr [[TMP0:%.*]], i32 [[TMP1:%.*]]) {
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp slt i32 [[TMP3]], 1
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[BB12:.*]], label %[[BB6:.*]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i32 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 [[TMP7]]
+; CHECK-NEXT:    call void @decrement(ptr [[TMP8]])
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP10:%.*]] = add nsw i32 [[TMP9]], -1
+; CHECK-NEXT:    call void @func.specialized.1(ptr [[TMP0]], i32 [[TMP10]])
+; CHECK-NEXT:    br label %[[BB12]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    ret void
+;
+;
+; CHECK-LABEL: define internal void @func.specialized.2(
+; CHECK-SAME: ptr [[TMP0:%.*]], i32 [[TMP1:%.*]]) {
+; CHECK-NEXT:    [[TMP3:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 [[TMP1]], ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp slt i32 [[TMP4]], 1
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[BB12:.*]], label %[[BB6:.*]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 [[TMP8]]
+; CHECK-NEXT:    call void @increment(ptr [[TMP9]])
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = add nsw i32 [[TMP10]], -1
+; CHECK-NEXT:    call void @func.specialized.2(ptr [[TMP0]], i32 [[TMP11]])
+; CHECK-NEXT:    br label %[[BB12]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    ret void
+;
+;
+; ONE-ITER-LABEL: define internal void @increment(
+; ONE-ITER-SAME: ptr nocapture [[TMP0:%.*]]) {
+; ONE-ITER-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP0]], align 4
+; ONE-ITER-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], 1
+; ONE-ITER-NEXT:    store i32 [[TMP3]], ptr [[TMP0]], align 4
+; ONE-ITER-NEXT:    ret void
+;
+;
+; ONE-ITER-LABEL: define internal void @decrement(
+; ONE-ITER-SAME: ptr nocapture [[TMP0:%.*]]) {
+; ONE-ITER-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP0]], align 4
+; ONE-ITER-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], -1
+; ONE-ITER-NEXT:    store i32 [[TMP3]], ptr [[TMP0]], align 4
+; ONE-ITER-NEXT:    ret void
+;
+;
+; ONE-ITER-LABEL: define i32 @main(
+; ONE-ITER-SAME: ptr [[TMP0:%.*]], i32 [[TMP1:%.*]]) {
+; ONE-ITER-NEXT:    call void @func.specialized.2(ptr [[TMP0]], i32 [[TMP1]])
+; ONE-ITER-NEXT:    call void @func.specialized.1(ptr [[TMP0]], i32 0)
+; ONE-ITER-NEXT:    ret i32 0
+;
+;
+; ONE-ITER-LABEL: define internal void @func.specialized.1(
+; ONE-ITER-SAME: ptr [[TMP0:%.*]], i32 [[TMP1:%.*]]) {
+; ONE-ITER-NEXT:    [[TMP2:%.*]] = alloca i32, align 4
+; ONE-ITER-NEXT:    store i32 [[TMP1]], ptr [[TMP2]], align 4
+; ONE-ITER-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
+; ONE-ITER-NEXT:    [[TMP4:%.*]] = icmp slt i32 [[TMP3]], 1
+; ONE-ITER-NEXT:    br i1 [[TMP4]], label %[[BB12:.*]], label %[[BB6:.*]]
+; ONE-ITER:       [[BB6]]:
+; ONE-ITER-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP2]], align 4
+; ONE-ITER-NEXT:    [[TMP7:%.*]] = sext i32 [[TMP6]] to i64
+; ONE-ITER-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 [[TMP7]]
+; ONE-ITER-NEXT:    call void @decrement(ptr [[TMP8]])
+; ONE-ITER-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP2]], align 4
+; ONE-ITER-NEXT:    [[TMP10:%.*]] = add nsw i32 [[TMP9]], -1
+; ONE-ITER-NEXT:    call void @func.specialized.1(ptr [[TMP0]], i32 [[TMP10]])
+; ONE-ITER-NEXT:    br label %[[BB12]]
+; ONE-ITER:       [[BB12]]:
+; ONE-ITER-NEXT:    ret void
+;
+;
+; ONE-ITER-LABEL: define internal void @func.specialized.2(
+; ONE-ITER-SAME: ptr [[TMP0:%.*]], i32 [[TMP1:%.*]]) {
+; ONE-ITER-NEXT:    [[TMP3:%.*]] = alloca i32, align 4
+; ONE-ITER-NEXT:    store i32 [[TMP1]], ptr [[TMP3]], align 4
+; ONE-ITER-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
+; ONE-ITER-NEXT:    [[TMP5:%.*]] = icmp slt i32 [[TMP4]], 1
+; ONE-ITER-NEXT:    br i1 [[TMP5]], label %[[BB12:.*]], label %[[BB6:.*]]
+; ONE-ITER:       [[BB6]]:
+; ONE-ITER-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP3]], align 4
+; ONE-ITER-NEXT:    [[TMP8:%.*]] = sext i32 [[TMP7]] to i64
+; ONE-ITER-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 [[TMP8]]
+; ONE-ITER-NEXT:    call void @increment(ptr [[TMP9]])
+; ONE-ITER-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP3]], align 4
+; ONE-ITER-NEXT:    [[TMP11:%.*]] = add nsw i32 [[TMP10]], -1
+; ONE-ITER-NEXT:    call void @func.specialized.2(ptr [[TMP0]], i32 [[TMP11]])
+; ONE-ITER-NEXT:    br label %[[BB12]]
+; ONE-ITER:       [[BB12]]:
+; ONE-ITER-NEXT:    ret void
+;
+;
+; DISABLED-LABEL: define internal void @func(
+; DISABLED-SAME: ptr [[TMP0:%.*]], i32 [[TMP1:%.*]], ptr nocapture [[TMP2:%.*]]) {
+; DISABLED-NEXT:    [[TMP4:%.*]] = alloca i32, align 4
+; DISABLED-NEXT:    store i32 [[TMP1]], ptr [[TMP4]], align 4
+; DISABLED-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
+; DISABLED-NEXT:    [[TMP6:%.*]] = icmp slt i32 [[TMP5]], 1
+; DISABLED-NEXT:    br i1 [[TMP6]], label %[[BB13:.*]], label %[[BB7:.*]]
+; DISABLED:       [[BB7]]:
+; DISABLED-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 4
+; DISABLED-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
+; DISABLED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 [[TMP9]]
+; DISABLED-NEXT:    call void [[TMP2]](ptr [[TMP10]])
+; DISABLED-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP4]], align 4
+; DISABLED-NEXT:    [[TMP12:%.*]] = add nsw i32 [[TMP11]], -1
+; DISABLED-NEXT:    call void @func(ptr [[TMP0]], i32 [[TMP12]], ptr [[TMP2]])
+; DISABLED-NEXT:    br label %[[BB13]]
+; DISABLED:       [[BB13]]:
+; DISABLED-NEXT:    ret void
+;
+;
+; DISABLED-LABEL: define internal void @increment(
+; DISABLED-SAME: ptr nocapture [[TMP0:%.*]]) {
+; DISABLED-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP0]], align 4
+; DISABLED-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], 1
+; DISABLED-NEXT:    store i32 [[TMP3]], ptr [[TMP0]], align 4
+; DISABLED-NEXT:    ret void
+;
+;
+; DISABLED-LABEL: define internal void @decrement(
+; DISABLED-SAME: ptr nocapture [[TMP0:%.*]]) {
+; DISABLED-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP0]], align 4
+; DISABLED-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], -1
+; DISABLED-NEXT:    store i32 [[TMP3]], ptr [[TMP0]], align 4
+; DISABLED-NEXT:    ret void
+;
+;
+; DISABLED-LABEL: define i32 @main(
+; DISABLED-SAME: ptr [[TMP0:%.*]], i32 [[TMP1:%.*]]) {
+; DISABLED-NEXT:    call void @func(ptr [[TMP0]], i32 [[TMP1]], ptr nonnull @increment)
+; DISABLED-NEXT:    call void @func(ptr [[TMP0]], i32 0, ptr nonnull @decrement)
+; DISABLED-NEXT:    ret i32 0
+;
diff --git a/llvm/test/Transforms/FunctionSpecialization/identical-specializations.ll b/llvm/test/Transforms/FunctionSpecialization/identical-specializations.ll
index 368f3b0440344..930ed6627f7f1 100644
--- a/llvm/test/Transforms/FunctionSpecialization/identical-specializations.ll
+++ b/llvm/test/Transforms/FunctionSpecialization/identical-specializations.ll
@@ -1,21 +1,7 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --include-generated-funcs --version 5
 ; RUN: opt -passes="ipsccp<func-spec>" -force-specialization -S < %s | FileCheck %s
 
 define i64 @main(i64 %x, i64 %y, i1 %flag) {
-; CHECK-LABEL: @main(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[FLAG:%.*]], label [[PLUS:%.*]], label [[MINUS:%.*]]
-; CHECK:       plus:
-; CHECK-NEXT:    [[CMP0:%.*]] = call i64 @compute.specialized.2(i64 [[X:%.*]], i64 [[Y:%.*]], ptr @plus, ptr @minus)
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       minus:
-; CHECK-NEXT:    [[CMP1:%.*]] = call i64 @compute.specialized.3(i64 [[X]], i64 [[Y]], ptr @minus, ptr @plus)
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    [[PH:%.*]] = phi i64 [ [[CMP0]], [[PLUS]] ], [ [[CMP1]], [[MINUS]] ]
-; CHECK-NEXT:    [[CMP2:%.*]] = call i64 @compute.specialized.2(i64 [[PH]], i64 42, ptr @plus, ptr @minus)
-; CHECK-NEXT:    ret i64 [[CMP2]]
-;
 entry:
   br i1 %flag, label %plus, label %minus
 
@@ -47,33 +33,87 @@ entry:
 }
 
 define internal i64 @plus(i64 %x, i64 %y) {
-; CHECK-LABEL: @plus(
 entry:
   %add = add i64 %x, %y
   ret i64 %add
 }
 
 define internal i64 @minus(i64 %x, i64 %y) {
-; CHECK-LABEL: @minus(
 entry:
   %sub = sub i64 %x, %y
   ret i64 %sub
 }
 
-; CHECK-LABEL: @compute.specialized.1
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP0:%.*]] = call i64 %binop1(i64 [[X:%.*]], i64 [[Y:%.*]])
-; CHECK-NEXT:    [[CMP1:%.*]] = call i64 @plus(i64 [[X]], i64 [[Y]])
-; CHECK-NEXT:    [[CMP2:%.*]] = call i64 @compute.specialized.1(i64 [[X]], i64 [[Y]], ptr %binop1, ptr @plus)
 
-; CHECK-LABEL: @compute.specialized.2
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP0:%.*]] = call i64 @plus(i64 [[X:%.*]], i64 [[Y:%.*]])
-; CHECK-NEXT:    [[CMP1:%.*]] = call i64 @minus(i64 [[X]], i64 [[Y]])
-; CHECK-NEXT:    [[CMP2:%.*]] = call i64 @compute.specialized.1(i64 [[X]], i64 [[Y]], ptr @plus, ptr @plus)
 
-; CHECK-LABEL: @compute.specialized.3
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP0:%.*]] = call i64 @minus(i64 [[X:%.*]], i64 [[Y:%.*]])
-; CHECK-NEXT:    [[CMP1:%.*]] = call i64 @plus(i64 [[X]], i64 [[Y]])
-; CHECK-NEXT:    [[CMP2:%.*]] = call i64 @compute.specialized.3(i64 [[X]], i64 [[Y]], ptr @minus, ptr @plus)
+; CHECK-LABEL: define i64 @main(
+; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]], i1 [[FLAG:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[FLAG]], label %[[PLUS:.*]], label %[[MINUS:.*]]
+; CHECK:       [[PLUS]]:
+; CHECK-NEXT:    [[CMP0:%.*]] = call i64 @compute.specialized.2(i64 [[X]], i64 [[Y]], ptr @plus, ptr @minus)
+; CHECK-NEXT:    br label %[[MERGE:.*]]
+; CHECK:       [[MINUS]]:
+; CHECK-NEXT:    [[CMP1:%.*]] = call i64 @compute.specialized.3(i64 [[X]], i64 [[Y]], ptr @minus, ptr @plus)
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    [[PH:%.*]] = phi i64 [ [[CMP0]], %[[PLUS]] ], [ [[CMP1]], %[[MINUS]] ]
+; CHECK-NEXT:    [[CMP2:%.*]] = call i64 @compute.specialized.2(i64 [[PH]], i64 42, ptr @plus, ptr @minus)
+; CHECK-NEXT:    ret i64 [[CMP2]]
+;
+;
+; CHECK-LABEL: define internal i64 @plus(
+; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[X]], [[Y]]
+; CHECK-NEXT:    ret i64 [[ADD]]
+;
+;
+; CHECK-LABEL: define internal i64 @minus(
+; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i64 [[X]], [[Y]]
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+;
+; CHECK-LABEL: define internal i64 @compute.specialized.1(
+; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]], ptr [[BINOP1:%.*]], ptr [[BINOP2:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OP1:%.*]] = call i64 [[BINOP1]](i64 [[X]], i64 [[Y]])
+; CHECK-NEXT:    [[OP0:%.*]] = call i64 @plus(i64 [[X]], i64 [[Y]])
+; CHECK-NEXT:    [[OP2:%.*]] = call i64 @compute.specialized.1(i64 [[X]], i64 [[Y]], ptr [[BINOP1]], ptr @plus)
+; CHECK-NEXT:    [[ADD0:%.*]] = add i64 [[OP1]], [[OP0]]
+; CHECK-NEXT:    [[ADD1:%.*]] = add i64 [[ADD0]], [[OP2]]
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i64 [[ADD1]], [[X]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i64 [[DIV]], [[Y]]
+; CHECK-NEXT:    [[MUL:%.*]] = mul i64 [[SUB]], 2
+; CHECK-NEXT:    ret i64 [[MUL]]
+;
+;
+; CHECK-LABEL: define internal i64 @compute.specialized.2(
+; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]], ptr [[BINOP1:%.*]], ptr [[BINOP2:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OP0:%.*]] = call i64 @plus(i64 [[X]], i64 [[Y]])
+; CHECK-NEXT:    [[OP1:%.*]] = call i64 @minus(i64 [[X]], i64 [[Y]])
+; CHECK-NEXT:    [[OP2:%.*]] = call i64 @compute.specialized.1(i64 [[X]], i64 [[Y]], ptr @plus, ptr @plus)
+; CHECK-NEXT:    [[ADD0:%.*]] = add i64 [[OP0]], [[OP1]]
+; CHECK-NEXT:    [[ADD1:%.*]] = add i64 [[ADD0]], [[OP2]]
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i64 [[ADD1]], [[X]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i64 [[DIV]], [[Y]]
+; CHECK-NEXT:    [[MUL:%.*]] = mul i64 [[SUB]], 2
+; CHECK-NEXT:    ret i64 [[MUL]]
+;
+;
+; CHECK-LABEL: define internal i64 @compute.specialized.3(
+; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]], ptr [[BINOP1:%.*]], ptr [[BINOP2:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OP0:%.*]] = call i64 @minus(i64 [[X]], i64 [[Y]])
+; CHECK-NEXT:    [[OP1:%.*]] = call i64 @plus(i64 [[X]], i64 [[Y]])
+; CHECK-NEXT:    [[OP2:%.*]] = call i64 @compute.specialized.3(i64 [[X]], i64 [[Y]], ptr @minus, ptr @plus)
+; CHECK-NEXT:    [[ADD0:%.*]] = add i64 [[OP0]], [[OP1]]
+; CHECK-NEXT:    [[ADD1:%.*]] = add i64 [[ADD0]], [[OP2]]
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i64 [[ADD1]], [[X]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i64 [[DIV]], [[Y]]
+; CHECK-NEXT:    [[MUL:%.*]] = mul i64 [[SUB]], 2
+; CHECK-NEXT:    ret i64 [[MUL]]
+;
diff --git a/llvm/test/Transforms/IndVarSimplify/D108043.ll b/llvm/test/Transforms/IndVarSimplify/D108043.ll
index cc553e205ad35..85028e77dfb02 100644
--- a/llvm/test/Transforms/IndVarSimplify/D108043.ll
+++ b/llvm/test/Transforms/IndVarSimplify/D108043.ll
@@ -9,7 +9,7 @@ define internal fastcc void @func_2() unnamed_addr {
 ; CHECK-NEXT:  lbl_2898.preheader:
 ; CHECK-NEXT:    br label [[LBL_2898:%.*]]
 ; CHECK:       lbl_2898.loopexit:
-; CHECK-NEXT:    store ptr getelementptr inbounds (i8, ptr @g_2168, i64 76), ptr @g_1150, align 8
+; CHECK-NEXT:    store ptr getelementptr inbounds nuw (i8, ptr @g_2168, i64 76), ptr @g_1150, align 8
 ; CHECK-NEXT:    br label [[LBL_2898]]
 ; CHECK:       lbl_2898:
 ; CHECK-NEXT:    br label [[FOR_COND884:%.*]]
diff --git a/llvm/test/Transforms/IndVarSimplify/lftr-dead-ivs.ll b/llvm/test/Transforms/IndVarSimplify/lftr-dead-ivs.ll
index 6c15eb4af4f13..fd6f2b694a729 100644
--- a/llvm/test/Transforms/IndVarSimplify/lftr-dead-ivs.ll
+++ b/llvm/test/Transforms/IndVarSimplify/lftr-dead-ivs.ll
@@ -112,7 +112,7 @@ define void @dom_store_preinc() #0 {
 ; CHECK-NEXT:    [[P_0:%.*]] = phi ptr [ @data, [[ENTRY:%.*]] ], [ [[TMP3:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    store volatile i8 0, ptr [[P_0]], align 1
 ; CHECK-NEXT:    [[TMP3]] = getelementptr inbounds i8, ptr [[P_0]], i64 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne ptr [[P_0]], getelementptr (i8, ptr @data, i64 245)
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne ptr [[P_0]], getelementptr nuw (i8, ptr @data, i64 245)
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
@@ -141,7 +141,7 @@ define void @dom_store_postinc() #0 {
 ; CHECK-NEXT:    [[P_0:%.*]] = phi ptr [ @data, [[ENTRY:%.*]] ], [ [[TMP3:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[TMP3]] = getelementptr inbounds i8, ptr [[P_0]], i64 1
 ; CHECK-NEXT:    store volatile i8 0, ptr [[TMP3]], align 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne ptr [[TMP3]], getelementptr (i8, ptr @data, i64 246)
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne ptr [[TMP3]], getelementptr nuw (i8, ptr @data, i64 246)
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
@@ -170,7 +170,7 @@ define i8 @dom_load() #0 {
 ; CHECK-NEXT:    [[P_0:%.*]] = phi ptr [ @data, [[ENTRY:%.*]] ], [ [[TMP3:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[TMP3]] = getelementptr inbounds i8, ptr [[P_0]], i64 1
 ; CHECK-NEXT:    [[V:%.*]] = load i8, ptr [[TMP3]], align 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne ptr [[TMP3]], getelementptr (i8, ptr @data, i64 246)
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne ptr [[TMP3]], getelementptr nuw (i8, ptr @data, i64 246)
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    [[V_LCSSA:%.*]] = phi i8 [ [[V]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/IndVarSimplify/lftr.ll b/llvm/test/Transforms/IndVarSimplify/lftr.ll
index e37a34019ccd1..def2cd9b8c679 100644
--- a/llvm/test/Transforms/IndVarSimplify/lftr.ll
+++ b/llvm/test/Transforms/IndVarSimplify/lftr.ll
@@ -196,7 +196,7 @@ define void @test_zext(ptr %a) #0 {
 ; CHECK-NEXT:    [[T2:%.*]] = load i8, ptr [[DOT0]], align 1
 ; CHECK-NEXT:    [[T3]] = getelementptr inbounds i8, ptr [[P_0]], i64 1
 ; CHECK-NEXT:    store i8 [[T2]], ptr [[P_0]], align 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne ptr [[P_0]], getelementptr inbounds (i8, ptr @data, i64 239)
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne ptr [[P_0]], getelementptr inbounds nuw (i8, ptr @data, i64 239)
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/Inline/memprof_inline2.ll b/llvm/test/Transforms/Inline/memprof_inline2.ll
index 625971c5fa37f..21448f142ed07 100644
--- a/llvm/test/Transforms/Inline/memprof_inline2.ll
+++ b/llvm/test/Transforms/Inline/memprof_inline2.ll
@@ -90,10 +90,18 @@ entry:
 ; CHECK-LABEL: define dso_local noundef ptr @notprofiled
 define dso_local noundef ptr @notprofiled() #0 !dbg !66 {
 entry:
+  ;; When foo is inlined, both the memprof and callsite metadata should be
+  ;; stripped from the inlined call to new, as there is no callsite metadata on
+  ;; the call.
   ; CHECK: call {{.*}} @_Znam
   ; CHECK-NOT: !memprof
   ; CHECK-NOT: !callsite
   %call = call noundef ptr @_Z3foov(), !dbg !67
+  ;; When baz is inlined, the callsite metadata should be stripped from the
+  ;; inlined call to foo2, as there is no callsite metadata on the call.
+  ; CHECK: call {{.*}} @_Z4foo2v
+  ; CHECK-NOT: !callsite
+  %call2 = call noundef ptr @_Z3bazv()
   ; CHECK-NEXT: ret
   ret ptr %call, !dbg !68
 }
diff --git a/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-comb-all-active-lanes-cvt.ll b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-comb-all-active-lanes-cvt.ll
index 04550156be30b..930fdfc136565 100644
--- a/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-comb-all-active-lanes-cvt.ll
+++ b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-comb-all-active-lanes-cvt.ll
@@ -5,36 +5,36 @@ target triple = "aarch64-unknown-linux-gnu"
 define <vscale x 8 x bfloat> @test_fcvt_bf16_f32_undef(<vscale x 8 x bfloat> %a, <vscale x 4 x float> %b) {
 ; CHECK-LABEL: define <vscale x 8 x bfloat> @test_fcvt_bf16_f32_undef(
 ; CHECK-SAME: <vscale x 8 x bfloat> [[A:%.*]], <vscale x 4 x float> [[B:%.*]]) {
-; CHECK-NEXT:    [[PG:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-; CHECK-NEXT:    [[OUT:%.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> [[PG]], <vscale x 4 x float> [[B]])
+; CHECK-NEXT:    [[PG:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+; CHECK-NEXT:    [[OUT:%.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32.v2(<vscale x 8 x bfloat> undef, <vscale x 4 x i1> [[PG]], <vscale x 4 x float> [[B]])
 ; CHECK-NEXT:    ret <vscale x 8 x bfloat> [[OUT]]
 ;
-  %pg = tail call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> %pg, <vscale x 4 x float> %b)
+  %pg = tail call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32.v2(<vscale x 8 x bfloat> undef, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b)
   ret <vscale x 8 x bfloat> %out
 }
 
 define <vscale x 8 x bfloat> @test_fcvt_bf16_f32_poison(<vscale x 8 x bfloat> %a, <vscale x 4 x float> %b) {
 ; CHECK-LABEL: define <vscale x 8 x bfloat> @test_fcvt_bf16_f32_poison(
 ; CHECK-SAME: <vscale x 8 x bfloat> [[A:%.*]], <vscale x 4 x float> [[B:%.*]]) {
-; CHECK-NEXT:    [[PG:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-; CHECK-NEXT:    [[OUT:%.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32(<vscale x 8 x bfloat> poison, <vscale x 8 x i1> [[PG]], <vscale x 4 x float> [[B]])
+; CHECK-NEXT:    [[PG:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+; CHECK-NEXT:    [[OUT:%.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32.v2(<vscale x 8 x bfloat> poison, <vscale x 4 x i1> [[PG]], <vscale x 4 x float> [[B]])
 ; CHECK-NEXT:    ret <vscale x 8 x bfloat> [[OUT]]
 ;
-  %pg = tail call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32(<vscale x 8 x bfloat> poison, <vscale x 8 x i1> %pg, <vscale x 4 x float> %b)
+  %pg = tail call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32.v2(<vscale x 8 x bfloat> poison, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b)
   ret <vscale x 8 x bfloat> %out
 }
 
 define <vscale x 8 x bfloat> @test_fcvt_bf16_f32(<vscale x 8 x bfloat> %a, <vscale x 4 x float> %b) {
 ; CHECK-LABEL: define <vscale x 8 x bfloat> @test_fcvt_bf16_f32(
 ; CHECK-SAME: <vscale x 8 x bfloat> [[A:%.*]], <vscale x 4 x float> [[B:%.*]]) {
-; CHECK-NEXT:    [[PG:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-; CHECK-NEXT:    [[OUT:%.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> [[PG]], <vscale x 4 x float> [[B]])
+; CHECK-NEXT:    [[PG:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+; CHECK-NEXT:    [[OUT:%.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32.v2(<vscale x 8 x bfloat> undef, <vscale x 4 x i1> [[PG]], <vscale x 4 x float> [[B]])
 ; CHECK-NEXT:    ret <vscale x 8 x bfloat> [[OUT]]
 ;
-  %pg = tail call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32(<vscale x 8 x bfloat> %a, <vscale x 8 x i1> %pg, <vscale x 4 x float> %b)
+  %pg = tail call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32.v2(<vscale x 8 x bfloat> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b)
   ret <vscale x 8 x bfloat> %out
 }
 
@@ -137,12 +137,12 @@ define <vscale x 2 x double> @test_fcvtlt_f64_f32(<vscale x 2 x double> %a, <vsc
 define <vscale x 8 x bfloat> @test_fcvtnt_bf16_f32(<vscale x 8 x bfloat> %a, <vscale x 4 x float> %b) {
 ; CHECK-LABEL: define <vscale x 8 x bfloat> @test_fcvtnt_bf16_f32(
 ; CHECK-SAME: <vscale x 8 x bfloat> [[A:%.*]], <vscale x 4 x float> [[B:%.*]]) {
-; CHECK-NEXT:    [[PG:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-; CHECK-NEXT:    [[OUT:%.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32(<vscale x 8 x bfloat> [[A]], <vscale x 8 x i1> [[PG]], <vscale x 4 x float> [[B]])
+; CHECK-NEXT:    [[PG:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+; CHECK-NEXT:    [[OUT:%.*]] = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32.v2(<vscale x 8 x bfloat> [[A]], <vscale x 4 x i1> [[PG]], <vscale x 4 x float> [[B]])
 ; CHECK-NEXT:    ret <vscale x 8 x bfloat> [[OUT]]
 ;
-  %pg = tail call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32(<vscale x 8 x bfloat> %a, <vscale x 8 x i1> %pg, <vscale x 4 x float> %b)
+  %pg = tail call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32.v2(<vscale x 8 x bfloat> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b)
   ret <vscale x 8 x bfloat> %out
 }
 
diff --git a/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-comb-no-active-lanes-cvt.ll b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-comb-no-active-lanes-cvt.ll
index 9b1528eda8ffd..4f6f839b3d64f 100644
--- a/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-comb-no-active-lanes-cvt.ll
+++ b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-comb-no-active-lanes-cvt.ll
@@ -7,7 +7,7 @@ define <vscale x 8 x bfloat> @test_fcvt_bf16_f32(<vscale x 8 x bfloat> %a, <vsca
 ; CHECK-SAME: <vscale x 8 x bfloat> [[A:%.*]], <vscale x 4 x float> [[B:%.*]]) {
 ; CHECK-NEXT:    ret <vscale x 8 x bfloat> [[A]]
 ;
-  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32(<vscale x 8 x bfloat> %a, <vscale x 8 x i1> zeroinitializer, <vscale x 4 x float> %b)
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvt.bf16f32.v2(<vscale x 8 x bfloat> %a, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x float> %b)
   ret <vscale x 8 x bfloat> %out
 }
 
@@ -88,7 +88,7 @@ define <vscale x 8 x bfloat> @test_fcvtnt_bf16_f32(<vscale x 8 x bfloat> %a, <vs
 ; CHECK-SAME: <vscale x 8 x bfloat> [[A:%.*]], <vscale x 4 x float> [[B:%.*]]) {
 ; CHECK-NEXT:    ret <vscale x 8 x bfloat> [[A]]
 ;
-  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32(<vscale x 8 x bfloat> %a, <vscale x 8 x i1> zeroinitializer, <vscale x 4 x float> %b)
+  %out = call <vscale x 8 x bfloat> @llvm.aarch64.sve.fcvtnt.bf16f32.v2(<vscale x 8 x bfloat> %a, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x float> %b)
   ret <vscale x 8 x bfloat> %out
 }
 
diff --git a/llvm/test/Transforms/InstCombine/and-or-icmps.ll b/llvm/test/Transforms/InstCombine/and-or-icmps.ll
index 74ef365db8d22..9ddc628aa4769 100644
--- a/llvm/test/Transforms/InstCombine/and-or-icmps.ll
+++ b/llvm/test/Transforms/InstCombine/and-or-icmps.ll
@@ -3332,3 +3332,86 @@ define i1 @icmp_eq_or_z_or_pow2orz_fail_bad_pred2(i8 %x, i8 %y) {
   %r = or i1 %c0, %cp2
   ret i1 %r
 }
+
+define i1 @and_slt_to_mask(i8 %x) {
+; CHECK-LABEL: @and_slt_to_mask(
+; CHECK-NEXT:    [[AND2:%.*]] = icmp slt i8 [[X:%.*]], -126
+; CHECK-NEXT:    ret i1 [[AND2]]
+;
+  %cmp = icmp slt i8 %x, -124
+  %and = and i8 %x, 2
+  %cmp2 = icmp eq i8 %and, 0
+  %and2 = and i1 %cmp, %cmp2
+  ret i1 %and2
+}
+
+define i1 @and_slt_to_mask_off_by_one(i8 %x) {
+; CHECK-LABEL: @and_slt_to_mask_off_by_one(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], -123
+; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X]], 2
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i8 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i1 [[CMP]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND2]]
+;
+  %cmp = icmp slt i8 %x, -123
+  %and = and i8 %x, 2
+  %cmp2 = icmp eq i8 %and, 0
+  %and2 = and i1 %cmp, %cmp2
+  ret i1 %and2
+}
+
+define i1 @and_sgt_to_mask(i8 %x) {
+; CHECK-LABEL: @and_sgt_to_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X:%.*]], -2
+; CHECK-NEXT:    [[AND2:%.*]] = icmp eq i8 [[TMP1]], 124
+; CHECK-NEXT:    ret i1 [[AND2]]
+;
+  %cmp = icmp sgt i8 %x, 123
+  %and = and i8 %x, 2
+  %cmp2 = icmp eq i8 %and, 0
+  %and2 = and i1 %cmp, %cmp2
+  ret i1 %and2
+}
+
+define i1 @and_sgt_to_mask_off_by_one(i8 %x) {
+; CHECK-LABEL: @and_sgt_to_mask_off_by_one(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[X:%.*]], 124
+; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X]], 2
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i8 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i1 [[CMP]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND2]]
+;
+  %cmp = icmp sgt i8 %x, 124
+  %and = and i8 %x, 2
+  %cmp2 = icmp eq i8 %and, 0
+  %and2 = and i1 %cmp, %cmp2
+  ret i1 %and2
+}
+
+define i1 @and_ugt_to_mask(i8 %x) {
+; CHECK-LABEL: @and_ugt_to_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X:%.*]], -2
+; CHECK-NEXT:    [[AND2:%.*]] = icmp eq i8 [[TMP1]], -4
+; CHECK-NEXT:    ret i1 [[AND2]]
+;
+  %cmp = icmp ugt i8 %x, -5
+  %and = and i8 %x, 2
+  %cmp2 = icmp eq i8 %and, 0
+  %and2 = and i1 %cmp, %cmp2
+  ret i1 %and2
+}
+
+define i1 @and_ugt_to_mask_off_by_one(i8 %x) {
+; CHECK-LABEL: @and_ugt_to_mask_off_by_one(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[X:%.*]], -6
+; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X]], 2
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i8 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i1 [[CMP]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND2]]
+;
+  %cmp = icmp ugt i8 %x, -6
+  %and = and i8 %x, 2
+  %cmp2 = icmp eq i8 %and, 0
+  %and2 = and i1 %cmp, %cmp2
+  ret i1 %and2
+}
diff --git a/llvm/test/Transforms/InstCombine/ashr-lshr.ll b/llvm/test/Transforms/InstCombine/ashr-lshr.ll
index 49041906680b3..1abf1be2cbedd 100644
--- a/llvm/test/Transforms/InstCombine/ashr-lshr.ll
+++ b/llvm/test/Transforms/InstCombine/ashr-lshr.ll
@@ -741,7 +741,7 @@ define i32 @lshr_mul_times_5_div_4_exact_2(i32 %x) {
 
 define i32 @ashr_mul_times_3_div_2(i32 %0) {
 ; CHECK-LABEL: @ashr_mul_times_3_div_2(
-; CHECK-NEXT:    [[TMP2:%.*]] = ashr i32 [[TMP0:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP0:%.*]], 1
 ; CHECK-NEXT:    [[ASHR:%.*]] = add nuw nsw i32 [[TMP0]], [[TMP2]]
 ; CHECK-NEXT:    ret i32 [[ASHR]]
 ;
@@ -815,7 +815,7 @@ define i32 @ashr_mul_times_3_div_2_exact_2(i32 %x) {
 
 define i32 @ashr_mul_times_5_div_4(i32 %0) {
 ; CHECK-LABEL: @ashr_mul_times_5_div_4(
-; CHECK-NEXT:    [[TMP2:%.*]] = ashr i32 [[TMP0:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP0:%.*]], 2
 ; CHECK-NEXT:    [[ASHR:%.*]] = add nuw nsw i32 [[TMP0]], [[TMP2]]
 ; CHECK-NEXT:    ret i32 [[ASHR]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/cast_phi.ll b/llvm/test/Transforms/InstCombine/cast_phi.ll
index 7dfe60539138d..99da3ac1c7c81 100644
--- a/llvm/test/Transforms/InstCombine/cast_phi.ll
+++ b/llvm/test/Transforms/InstCombine/cast_phi.ll
@@ -29,7 +29,7 @@ define void @MainKernel(i32 %iNumSteps, i32 %tid, i32 %base) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = phi float [ [[TMP11:%.*]], [[DOTBB12]] ], [ [[CONV_I]], [[DOTBB2]] ]
 ; CHECK-NEXT:    [[I12_06:%.*]] = phi i32 [ [[SUB:%.*]], [[DOTBB12]] ], [ [[INUMSTEPS]], [[DOTBB2]] ]
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i32 [[I12_06]], [[BASE:%.*]]
-; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[I12_06]], 1
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[I12_06]], 1
 ; CHECK-NEXT:    [[CONV_I9:%.*]] = sext i32 [[ADD]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds [258 x float], ptr [[CALLA]], i64 0, i64 [[CONV_I9]]
 ; CHECK-NEXT:    [[ARRAYIDX24:%.*]] = getelementptr inbounds [258 x float], ptr [[CALLB]], i64 0, i64 [[CONV_I9]]
@@ -70,8 +70,8 @@ define void @MainKernel(i32 %iNumSteps, i32 %tid, i32 %base) {
 ; CHECK-NEXT:    store float [[TMP10]], ptr [[ARRAYIDX6]], align 4
 ; CHECK-NEXT:    br label [[DOTBB12]]
 ; CHECK:       .bb12:
-; CHECK-NEXT:    [[SUB]] = add i32 [[I12_06]], -4
-; CHECK-NEXT:    [[CMP13:%.*]] = icmp sgt i32 [[SUB]], 0
+; CHECK-NEXT:    [[SUB]] = add nsw i32 [[I12_06]], -4
+; CHECK-NEXT:    [[CMP13:%.*]] = icmp sgt i32 [[I12_06]], 4
 ; CHECK-NEXT:    br i1 [[CMP13]], label [[DOTBB3]], label [[DOTBB8]]
 ;
   %callA = alloca [258 x float], align 4
diff --git a/llvm/test/Transforms/InstCombine/icmp-equality-test.ll b/llvm/test/Transforms/InstCombine/icmp-equality-test.ll
new file mode 100644
index 0000000000000..c2740ca7fe8aa
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-equality-test.ll
@@ -0,0 +1,229 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+define i1 @icmp_equality_test(i64 %X, i64 %Y, i64 %Z) {
+; CHECK-LABEL: @icmp_equality_test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[XEQY:%.*]] = icmp eq i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[XEQY]]
+;
+entry:
+  %XeqZ = icmp eq i64 %X, %Z
+  %YeqZ = icmp eq i64 %Y, %Z
+  %XeqY = icmp eq i64 %X, %Y
+  %not.YeqZ = xor i1 %YeqZ, true
+  %and = select i1 %not.YeqZ, i1 %XeqY, i1 false
+  %equal = select i1 %XeqZ, i1 %YeqZ, i1 %and
+  ret i1 %equal
+}
+
+define i1 @icmp_equality_test_constant(i42 %X, i42 %Y) {
+; CHECK-LABEL: @icmp_equality_test_constant(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[XEQY:%.*]] = icmp eq i42 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[XEQY]]
+;
+entry:
+  %XeqC = icmp eq i42 %X, -42
+  %YeqC = icmp eq i42 %Y, -42
+  %XeqY = icmp eq i42 %X, %Y
+  %not.YeqC = xor i1 %YeqC, true
+  %and = select i1 %not.YeqC, i1 %XeqY, i1 false
+  %equal = select i1 %XeqC, i1 %YeqC, i1 %and
+  ret i1 %equal
+}
+
+define i1 @icmp_equality_test_swift_optional_pointers(i64 %X, i64 %Y) {
+; CHECK-LABEL: @icmp_equality_test_swift_optional_pointers(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[XEQY:%.*]] = icmp eq i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[XEQY]]
+;
+entry:
+  %XeqC = icmp eq i64 %X, 0
+  %YeqC = icmp eq i64 %Y, 0
+  %either = select i1 %XeqC, i1 true, i1 %YeqC
+  %both = select i1 %XeqC, i1 %YeqC, i1 false
+  %XeqY = icmp eq i64 %X, %Y
+  %equal = select i1 %either, i1 %both, i1 %XeqY
+  ret i1 %equal
+}
+
+define <2 x i1> @icmp_equality_test_vector(<2 x i64> %X, <2 x i64> %Y) {
+; CHECK-LABEL: @icmp_equality_test_vector(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[XEQY:%.*]] = icmp eq <2 x i64> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[XEQY]]
+;
+entry:
+  %XeqC = icmp eq <2 x i64> %X, <i64 123, i64 456>
+  %YeqC = icmp eq <2 x i64> %Y, <i64 123, i64 456>
+  %XeqY = icmp eq <2 x i64> %X, %Y
+  %not.YeqC = xor <2 x i1> %YeqC, <i1 true, i1 true>
+  %and = select <2 x i1> %not.YeqC, <2 x i1> %XeqY, <2 x i1> <i1 false, i1 false>
+  %equal = select <2 x i1> %XeqC, <2 x i1> %YeqC, <2 x i1> %and
+  ret <2 x i1> %equal
+}
+
+define i1 @icmp_equality_test_commute_icmp1(i64 %X, i64 %Y, i64 %Z) {
+; CHECK-LABEL: @icmp_equality_test_commute_icmp1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[XEQY:%.*]] = icmp eq i64 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[XEQY]]
+;
+entry:
+  %XeqZ = icmp eq i64 %Z, %X
+  %YeqZ = icmp eq i64 %Z, %Y
+  %XeqY = icmp eq i64 %Y, %X
+  %not.YeqZ = xor i1 %YeqZ, true
+  %and = select i1 %not.YeqZ, i1 %XeqY, i1 false
+  %equal = select i1 %XeqZ, i1 %YeqZ, i1 %and
+  ret i1 %equal
+}
+
+define i1 @icmp_equality_test_commute_icmp2(i64 %X, i64 %Y, i64 %Z) {
+; CHECK-LABEL: @icmp_equality_test_commute_icmp2(
+; CHECK-NEXT:    [[XEQY:%.*]] = icmp eq i64 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[XEQY]]
+;
+  %XeqZ = icmp eq i64 %Z, %X
+  %YeqZ = icmp eq i64 %Y, %Z
+  %XeqY = icmp eq i64 %Y, %X
+  %not.YeqZ = xor i1 %YeqZ, true
+  %and = select i1 %not.YeqZ, i1 %XeqY, i1 false
+  %equal = select i1 %XeqZ, i1 %YeqZ, i1 %and
+  ret i1 %equal
+}
+
+define i1 @icmp_equality_test_commute_select1(i64 %X, i64 %Y, i64 %Z) {
+; CHECK-LABEL: @icmp_equality_test_commute_select1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[XEQY:%.*]] = icmp eq i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[XEQY]]
+;
+entry:
+  %XeqZ = icmp eq i64 %X, %Z
+  %YeqZ = icmp eq i64 %Y, %Z
+  %XeqY = icmp eq i64 %X, %Y
+  %and = select i1 %YeqZ, i1 false, i1 %XeqY
+  %equal = select i1 %XeqZ, i1 %YeqZ, i1 %and
+  ret i1 %equal
+}
+
+define i1 @icmp_equality_test_commute_select2(i64 %X, i64 %Y, i64 %Z) {
+; CHECK-LABEL: @icmp_equality_test_commute_select2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[XEQY:%.*]] = icmp eq i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[XEQY]]
+;
+entry:
+  %XeqZ = icmp eq i64 %X, %Z
+  %YeqZ = icmp eq i64 %Y, %Z
+  %XeqY = icmp eq i64 %X, %Y
+  %not.XeqZ = xor i1 %XeqZ, true
+  %and = select i1 %YeqZ, i1 false, i1 %XeqY
+  %equal = select i1 %not.XeqZ, i1 %and, i1 %YeqZ
+  ret i1 %equal
+}
+
+; Negative tests below
+
+define i1 @icmp_equality_test_wrong_constant(i64 %X, i64 %Y) {
+; CHECK-LABEL: @icmp_equality_test_wrong_constant(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[XEQC:%.*]] = icmp eq i64 [[X:%.*]], 0
+; CHECK-NEXT:    [[YEQC:%.*]] = icmp eq i64 [[Y:%.*]], 999
+; CHECK-NEXT:    [[XEQY:%.*]] = icmp eq i64 [[X]], [[Y]]
+; CHECK-NEXT:    [[NOT_YEQC:%.*]] = xor i1 [[YEQC]], true
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[NOT_YEQC]], i1 [[XEQY]], i1 false
+; CHECK-NEXT:    [[EQUAL:%.*]] = select i1 [[XEQC]], i1 [[YEQC]], i1 [[AND]]
+; CHECK-NEXT:    ret i1 [[EQUAL]]
+;
+entry:
+  %XeqC = icmp eq i64 %X, 0
+  %YeqC = icmp eq i64 %Y, 999
+  %XeqY = icmp eq i64 %X, %Y
+  %not.YeqC = xor i1 %YeqC, true
+  %and = select i1 %not.YeqC, i1 %XeqY, i1 false
+  %equal = select i1 %XeqC, i1 %YeqC, i1 %and
+  ret i1 %equal
+}
+
+define i1 @icmp_equality_test_missing_not(i64 %X, i64 %Y, i64 %Z) {
+; CHECK-LABEL: @icmp_equality_test_missing_not(
+; CHECK-NEXT:    [[XEQZ:%.*]] = icmp eq i64 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[YEQZ:%.*]] = icmp eq i64 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[XEQY:%.*]] = icmp eq i64 [[X]], [[Y]]
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[YEQZ]], i1 [[XEQY]], i1 false
+; CHECK-NEXT:    [[EQUAL:%.*]] = select i1 [[XEQZ]], i1 [[YEQZ]], i1 [[AND]]
+; CHECK-NEXT:    ret i1 [[EQUAL]]
+;
+  %XeqZ = icmp eq i64 %X, %Z
+  %YeqZ = icmp eq i64 %Y, %Z
+  %XeqY = icmp eq i64 %X, %Y
+  %and = select i1 %YeqZ, i1 %XeqY, i1 false
+  %equal = select i1 %XeqZ, i1 %YeqZ, i1 %and
+  ret i1 %equal
+}
+
+define i1 @icmp_equality_test_wrong_and(i64 %X, i64 %Y, i64 %Z) {
+; CHECK-LABEL: @icmp_equality_test_wrong_and(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[XEQZ:%.*]] = icmp eq i64 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[YEQZ:%.*]] = icmp eq i64 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[XEQY:%.*]] = icmp eq i64 [[X]], [[Y]]
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[YEQZ]], i1 [[XEQY]], i1 false
+; CHECK-NEXT:    [[EQUAL:%.*]] = select i1 [[XEQZ]], i1 [[YEQZ]], i1 [[AND]]
+; CHECK-NEXT:    ret i1 [[EQUAL]]
+;
+entry:
+  %XeqZ = icmp eq i64 %X, %Z
+  %YeqZ = icmp eq i64 %Y, %Z
+  %XeqY = icmp eq i64 %X, %Y
+  %not.YeqZ = xor i1 %YeqZ, true
+  %and = select i1 %not.YeqZ, i1 false, i1 %XeqY
+  %equal = select i1 %XeqZ, i1 %YeqZ, i1 %and
+  ret i1 %equal
+}
+
+define i1 @icmp_equality_test_wrong_cmp(i64 %X, i64 %Y, i64 %Z) {
+; CHECK-LABEL: @icmp_equality_test_wrong_cmp(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[XEQZ:%.*]] = icmp eq i64 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[YEQZ:%.*]] = icmp eq i64 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[XEQY:%.*]] = icmp eq i64 [[X]], 999
+; CHECK-NEXT:    [[NOT_YEQZ:%.*]] = xor i1 [[YEQZ]], true
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[NOT_YEQZ]], i1 [[XEQY]], i1 false
+; CHECK-NEXT:    [[EQUAL:%.*]] = select i1 [[XEQZ]], i1 [[YEQZ]], i1 [[AND]]
+; CHECK-NEXT:    ret i1 [[EQUAL]]
+;
+entry:
+  %XeqZ = icmp eq i64 %X, %Z
+  %YeqZ = icmp eq i64 %Y, %Z
+  %XeqY = icmp eq i64 %X, 999
+  %not.YeqZ = xor i1 %YeqZ, true
+  %and = select i1 %not.YeqZ, i1 %XeqY, i1 false
+  %equal = select i1 %XeqZ, i1 %YeqZ, i1 %and
+  ret i1 %equal
+}
+
+define i1 @icmp_equality_test_wrong_equal(i64 %X, i64 %Y, i64 %Z) {
+; CHECK-LABEL: @icmp_equality_test_wrong_equal(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[XEQZ:%.*]] = icmp eq i64 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[YEQZ:%.*]] = icmp eq i64 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[XEQY:%.*]] = icmp eq i64 [[X]], [[Y]]
+; CHECK-NEXT:    [[NOT_YEQZ:%.*]] = xor i1 [[YEQZ]], true
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[NOT_YEQZ]], i1 [[XEQY]], i1 false
+; CHECK-NEXT:    [[EQUAL:%.*]] = select i1 [[XEQZ]], i1 [[AND]], i1 [[YEQZ]]
+; CHECK-NEXT:    ret i1 [[EQUAL]]
+;
+entry:
+  %XeqZ = icmp eq i64 %X, %Z
+  %YeqZ = icmp eq i64 %Y, %Z
+  %XeqY = icmp eq i64 %X, %Y
+  %not.YeqZ = xor i1 %YeqZ, true
+  %and = select i1 %not.YeqZ, i1 %XeqY, i1 false
+  %equal = select i1 %XeqZ, i1 %and, i1 %YeqZ
+  ret i1 %equal
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-signmask.ll b/llvm/test/Transforms/InstCombine/icmp-signmask.ll
new file mode 100644
index 0000000000000..5424f7d7e8021
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-signmask.ll
@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+define i1 @cmp_x_and_negp2_with_eq(i8 %x) {
+; CHECK-LABEL: @cmp_x_and_negp2_with_eq(
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[X:%.*]], -126
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %andx = and i8 %x, -2
+  %r = icmp eq i8 %andx, 128
+  ret i1 %r
+}
+
+define i1 @cmp_x_and_negp2_with_eq_fail_not_signmask(i8 %x) {
+; CHECK-LABEL: @cmp_x_and_negp2_with_eq_fail_not_signmask(
+; CHECK-NEXT:    [[ANDX:%.*]] = and i8 [[X:%.*]], -2
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[ANDX]], -124
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %andx = and i8 %x, -2
+  %r = icmp eq i8 %andx, 132
+  ret i1 %r
+}
+
+define <2 x i1> @cmp_x_and_negp2_with_ne(<2 x i8> %x) {
+; CHECK-LABEL: @cmp_x_and_negp2_with_ne(
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt <2 x i8> [[X:%.*]], <i8 -121, i8 -113>
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %andx = and <2 x i8> %x, <i8 -8, i8 -16>
+  %r = icmp ne <2 x i8> %andx, <i8 128, i8 128>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @cmp_x_and_negp2_with_ne_or_z(<2 x i8> %x) {
+; CHECK-LABEL: @cmp_x_and_negp2_with_ne_or_z(
+; CHECK-NEXT:    [[R:%.*]] = icmp sge <2 x i8> [[X:%.*]], <i8 -128, i8 -112>
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %andx = and <2 x i8> %x, <i8 0, i8 -16>
+  %r = icmp ne <2 x i8> %andx, <i8 128, i8 128>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @cmp_x_and_negp2_with_ne_fail_not_p2(<2 x i8> %x) {
+; CHECK-LABEL: @cmp_x_and_negp2_with_ne_fail_not_p2(
+; CHECK-NEXT:    [[ANDX:%.*]] = and <2 x i8> [[X:%.*]], <i8 -8, i8 -15>
+; CHECK-NEXT:    [[R:%.*]] = icmp ne <2 x i8> [[ANDX]], <i8 -128, i8 -128>
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %andx = and <2 x i8> %x, <i8 -8, i8 -15>
+  %r = icmp ne <2 x i8> %andx, <i8 128, i8 128>
+  ret <2 x i1> %r
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp.ll b/llvm/test/Transforms/InstCombine/icmp.ll
index ecf21b8a42cf5..17ed64b6e04d9 100644
--- a/llvm/test/Transforms/InstCombine/icmp.ll
+++ b/llvm/test/Transforms/InstCombine/icmp.ll
@@ -1116,8 +1116,7 @@ define i1 @test53(i32 %a, i32 %b) {
 
 define i1 @test54(i8 %a) {
 ; CHECK-LABEL: @test54(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[A:%.*]], -64
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP1]], -128
+; CHECK-NEXT:    [[RET:%.*]] = icmp slt i8 [[A:%.*]], -64
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %ext = zext i8 %a to i32
diff --git a/llvm/test/Transforms/InstCombine/indexed-gep-compares.ll b/llvm/test/Transforms/InstCombine/indexed-gep-compares.ll
index ef8f5fea4ff7c..79511cf9e666f 100644
--- a/llvm/test/Transforms/InstCombine/indexed-gep-compares.ll
+++ b/llvm/test/Transforms/InstCombine/indexed-gep-compares.ll
@@ -32,6 +32,64 @@ bb2:
   ret ptr %RHS
 }
 
+define ptr @test1_nuw(ptr %A, i32 %Offset) {
+; CHECK-LABEL: @test1_nuw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP_IDX:%.*]] = shl nuw nsw i32 [[OFFSET:%.*]], 2
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[RHS_IDX:%.*]] = phi i32 [ [[RHS_ADD:%.*]], [[BB]] ], [ [[TMP_IDX]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[RHS_ADD]] = add nuw nsw i32 [[RHS_IDX]], 4
+; CHECK-NEXT:    [[COND:%.*]] = icmp ugt i32 [[RHS_IDX]], 400
+; CHECK-NEXT:    br i1 [[COND]], label [[BB2:%.*]], label [[BB]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[RHS_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i32 [[RHS_IDX]]
+; CHECK-NEXT:    ret ptr [[RHS_PTR]]
+;
+entry:
+  %tmp = getelementptr inbounds nuw i32, ptr %A, i32 %Offset
+  br label %bb
+
+bb:
+  %RHS = phi ptr [ %RHS.next, %bb ], [ %tmp, %entry ]
+  %LHS = getelementptr inbounds nuw i32, ptr %A, i32 100
+  %RHS.next = getelementptr inbounds nuw i32, ptr %RHS, i64 1
+  %cond = icmp ult ptr %LHS, %RHS
+  br i1 %cond, label %bb2, label %bb
+
+bb2:
+  ret ptr %RHS
+}
+
+define ptr @test1_not_all_nuw(ptr %A, i32 %Offset) {
+; CHECK-LABEL: @test1_not_all_nuw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP_IDX:%.*]] = shl nsw i32 [[OFFSET:%.*]], 2
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[RHS_IDX:%.*]] = phi i32 [ [[RHS_ADD:%.*]], [[BB]] ], [ [[TMP_IDX]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[RHS_ADD]] = add nsw i32 [[RHS_IDX]], 4
+; CHECK-NEXT:    [[COND:%.*]] = icmp sgt i32 [[RHS_IDX]], 400
+; CHECK-NEXT:    br i1 [[COND]], label [[BB2:%.*]], label [[BB]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[RHS_PTR:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i32 [[RHS_IDX]]
+; CHECK-NEXT:    ret ptr [[RHS_PTR]]
+;
+entry:
+  %tmp = getelementptr inbounds i32, ptr %A, i32 %Offset
+  br label %bb
+
+bb:
+  %RHS = phi ptr [ %RHS.next, %bb ], [ %tmp, %entry ]
+  %LHS = getelementptr inbounds nuw i32, ptr %A, i32 100
+  %RHS.next = getelementptr inbounds nuw i32, ptr %RHS, i64 1
+  %cond = icmp ult ptr %LHS, %RHS
+  br i1 %cond, label %bb2, label %bb
+
+bb2:
+  ret ptr %RHS
+}
+
 define ptr@test2(i32 %A, i32 %Offset) {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:  entry:
diff --git a/llvm/test/Transforms/InstCombine/known-bits.ll b/llvm/test/Transforms/InstCombine/known-bits.ll
index 8cfb987e422f3..aa24ec3905f13 100644
--- a/llvm/test/Transforms/InstCombine/known-bits.ll
+++ b/llvm/test/Transforms/InstCombine/known-bits.ll
@@ -2051,6 +2051,71 @@ exit:
   ret i16 %conv
 }
 
+define i1 @mul_nuw_nsw_nonneg_const(i8 %x) {
+; CHECK-LABEL: @mul_nuw_nsw_nonneg_const(
+; CHECK-NEXT:    ret i1 true
+;
+  %mul = mul nuw nsw i8 %x, 3
+  %cmp = icmp sgt i8 %mul, -1
+  ret i1 %cmp
+}
+
+define i1 @mul_nuw_nsw_nonneg_const_missing_nuw(i8 %x) {
+; CHECK-LABEL: @mul_nuw_nsw_nonneg_const_missing_nuw(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[X:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %mul = mul nsw i8 %x, 3
+  %cmp = icmp sgt i8 %mul, -1
+  ret i1 %cmp
+}
+
+define i1 @mul_nuw_nsw_nonneg_const_missing_nsw(i8 %x) {
+; CHECK-LABEL: @mul_nuw_nsw_nonneg_const_missing_nsw(
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw i8 [[X:%.*]], 3
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[MUL]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %mul = mul nuw i8 %x, 3
+  %cmp = icmp sgt i8 %mul, -1
+  ret i1 %cmp
+}
+
+define i1 @mul_nuw_nsw_nonneg_can_be_one(i8 %x, i8 %y) {
+; CHECK-LABEL: @mul_nuw_nsw_nonneg_can_be_one(
+; CHECK-NEXT:    [[Y_NNEG:%.*]] = and i8 [[Y:%.*]], 127
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i8 [[X:%.*]], [[Y_NNEG]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[MUL]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %y.nneg = and i8 %y, 127
+  %mul = mul nuw nsw i8 %x, %y.nneg
+  %cmp = icmp sgt i8 %mul, -1
+  ret i1 %cmp
+}
+
+define i1 @mul_nuw_nsw_nonneg_cant_be_one(i8 %x, i8 %y) {
+; CHECK-LABEL: @mul_nuw_nsw_nonneg_cant_be_one(
+; CHECK-NEXT:    ret i1 true
+;
+  %y.nneg = and i8 %y, 127
+  %y.nneg.not.one = or i8 %y.nneg, 2
+  %mul = mul nuw nsw i8 %x, %y.nneg.not.one
+  %cmp = icmp sgt i8 %mul, -1
+  ret i1 %cmp
+}
+
+define i1 @mul_nuw_nsw_nonneg_cant_be_one_commuted(i8 %x, i8 %y) {
+; CHECK-LABEL: @mul_nuw_nsw_nonneg_cant_be_one_commuted(
+; CHECK-NEXT:    ret i1 true
+;
+  %y.nneg = and i8 %y, 127
+  %y.nneg.not.one = or i8 %y.nneg, 2
+  %mul = mul nuw nsw i8 %y.nneg.not.one, %x
+  %cmp = icmp sgt i8 %mul, -1
+  ret i1 %cmp
+}
+
 declare void @dummy()
 declare void @use(i1)
 declare void @sink(i8)
diff --git a/llvm/test/Transforms/InstCombine/known-non-zero.ll b/llvm/test/Transforms/InstCombine/known-non-zero.ll
index b77c04eb81475..9bfec0d7289dd 100644
--- a/llvm/test/Transforms/InstCombine/known-non-zero.ll
+++ b/llvm/test/Transforms/InstCombine/known-non-zero.ll
@@ -92,7 +92,8 @@ exit:
   ret <8 x i64> %res
 }
 
-; Test that exposed a bug in the PHI handling after D60846. No folding should happen here!
+; Test that exposed a bug in the PHI handling after D60846.
+; Check that folding happens correctly.
 define void @D60846_miscompile(ptr %p) {
 ; CHECK-LABEL: @D60846_miscompile(
 ; CHECK-NEXT:  entry:
@@ -102,12 +103,11 @@ define void @D60846_miscompile(ptr %p) {
 ; CHECK-NEXT:    [[IS_ZERO:%.*]] = icmp eq i16 [[I]], 0
 ; CHECK-NEXT:    br i1 [[IS_ZERO]], label [[COMMON]], label [[NON_ZERO:%.*]]
 ; CHECK:       non_zero:
-; CHECK-NEXT:    [[IS_ONE:%.*]] = icmp eq i16 [[I]], 1
-; CHECK-NEXT:    store i1 [[IS_ONE]], ptr [[P:%.*]], align 1
+; CHECK-NEXT:    store i1 true, ptr [[P:%.*]], align 1
 ; CHECK-NEXT:    br label [[COMMON]]
 ; CHECK:       common:
-; CHECK-NEXT:    [[I_INC]] = add i16 [[I]], 1
-; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp ult i16 [[I_INC]], 2
+; CHECK-NEXT:    [[I_INC]] = add nuw nsw i16 [[I]], 1
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp eq i16 [[I]], 0
 ; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/InstCombine/new-inst-dbgloc-overwrite.ll b/llvm/test/Transforms/InstCombine/new-inst-dbgloc-overwrite.ll
new file mode 100644
index 0000000000000..2b933a2f9e075
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/new-inst-dbgloc-overwrite.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals smart
+;; Tests that when InstCombine sets a DILocation on the new instruction when it
+;; is created, we do not try to overwrite that DILocation when we later insert
+;; the new instruction.
+;; In this test, InstCombine replaces two loads joined by a PHI into a PHI of
+;; the addresses followed by a load, and gives the new load a merge of the two
+;; incoming load DILocations. This test verifies that the new load keeps that
+;; DILocation, and doesn't have it overwritten by the DILocation of the original
+;; PHI. It will, however, receive the !annotation attached to the original PHI.
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+define void @test(ptr %xfA, ptr %xfB, i1 %cmp5) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[CMP5:%.*]], label [[IF_ELSE:%.*]], label [[IF_THEN6:%.*]]
+; CHECK:       if.then6:
+; CHECK-NEXT:    br label [[IF_END11:%.*]]
+; CHECK:       if.else:
+; CHECK-NEXT:    br label [[IF_END11]]
+; CHECK:       if.end11:
+; CHECK-NEXT:    [[XFA_PN:%.*]] = phi ptr [ [[XFA:%.*]], [[IF_ELSE]] ], [ [[XFB:%.*]], [[IF_THEN6]] ]
+; CHECK-NEXT:    [[XF1_SROA_8_0_IN:%.*]] = getelementptr i8, ptr [[XFA_PN]], i64 4
+; CHECK-NEXT:    [[XF1_SROA_8_0:%.*]] = load float, ptr [[XF1_SROA_8_0_IN]], align 4, !dbg [[DBG3:![0-9]+]], !annotation [[META7:![0-9]+]]
+; CHECK-NEXT:    [[CMP_I:%.*]] = fcmp ugt float [[XF1_SROA_8_0]], 0.000000e+00
+; CHECK-NEXT:    br i1 [[CMP_I]], label [[IF_END_I:%.*]], label [[IF_THEN_I:%.*]]
+; CHECK:       if.then.i:
+; CHECK-NEXT:    br label [[IF_END_I]]
+; CHECK:       if.end.i:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %cmp5, label %if.else, label %if.then6
+
+if.then6:                                         ; preds = %entry
+  %xf1.sroa.8.0.xfB.sroa_idx = getelementptr i8, ptr %xfB, i64 4
+  %xf1.sroa.8.0.copyload = load float, ptr %xf1.sroa.8.0.xfB.sroa_idx, align 4, !dbg !3
+  br label %if.end11
+
+if.else:                                          ; preds = %entry
+  %xf1.sroa.8.0.xfA.sroa_idx = getelementptr i8, ptr %xfA, i64 4
+  %xf1.sroa.8.0.copyload494 = load float, ptr %xf1.sroa.8.0.xfA.sroa_idx, align 4, !dbg !7
+  br label %if.end11
+
+if.end11:                                         ; preds = %if.else, %if.then6
+  %xf1.sroa.8.0 = phi float [ %xf1.sroa.8.0.copyload494, %if.else ], [ %xf1.sroa.8.0.copyload, %if.then6 ], !dbg !8, !annotation !9
+  %cmp.i = fcmp ugt float %xf1.sroa.8.0, 0.000000e+00
+  br i1 %cmp.i, label %if.end.i, label %if.then.i
+
+if.then.i:                                        ; preds = %if.end11
+  br label %if.end.i
+
+if.end.i:                                         ; preds = %if.then.i, %if.end11
+  ret void
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 20.0.0git", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "test.cpp", directory: "/tmp")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = !DILocation(line: 13, column: 12, scope: !4)
+!4 = distinct !DISubprogram(name: "operator=", linkageName: "_ZN11btMatrix3x3aSERKS_", scope: null, file: !1, line: 61, type: !5, scopeLine: 62, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !6)
+!5 = distinct !DISubroutineType(types: !6)
+!6 = !{}
+!7 = !DILocation(line: 13, column: 15, scope: !4)
+!8 = !DILocation(line: 100, scope: !4)
+!9 = !{!"Test String"}
+;.
+; CHECK: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: [[META1:![0-9]+]], producer: "{{.*}}clang version {{.*}}", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+; CHECK: [[META1]] = !DIFile(filename: "test.cpp", directory: {{.*}})
+; CHECK: [[DBG3]] = !DILocation(line: 13, scope: [[META4:![0-9]+]])
+; CHECK: [[META4]] = distinct !DISubprogram(name: "operator=", linkageName: "_ZN11btMatrix3x3aSERKS_", scope: null, file: [[META1]], line: 61, type: [[META5:![0-9]+]], scopeLine: 62, spFlags: DISPFlagDefinition, unit: [[META0]], retainedNodes: [[META6:![0-9]+]])
+; CHECK: [[META5]] = distinct !DISubroutineType(types: [[META6]])
+; CHECK: [[META6]] = !{}
+; CHECK: [[META7]] = !{!"Test String"}
+;.
diff --git a/llvm/test/Transforms/InstCombine/rem-mul-shl.ll b/llvm/test/Transforms/InstCombine/rem-mul-shl.ll
index 9e2df157c2c85..45db2cf675852 100644
--- a/llvm/test/Transforms/InstCombine/rem-mul-shl.ll
+++ b/llvm/test/Transforms/InstCombine/rem-mul-shl.ll
@@ -8,7 +8,7 @@ define i8 @srem_non_matching(i8 %X, i8 %Y) {
 ; CHECK-LABEL: @srem_non_matching(
 ; CHECK-NEXT:    [[BO0:%.*]] = mul nuw nsw i8 [[X:%.*]], 15
 ; CHECK-NEXT:    [[BO1:%.*]] = mul nuw nsw i8 [[Y:%.*]], 5
-; CHECK-NEXT:    [[R:%.*]] = srem i8 [[BO0]], [[BO1]]
+; CHECK-NEXT:    [[R:%.*]] = urem i8 [[BO0]], [[BO1]]
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
   %BO0 = mul nsw nuw i8 %X, 15
diff --git a/llvm/test/Transforms/InstCombine/remove-loop-phi-multiply-by-zero.ll b/llvm/test/Transforms/InstCombine/remove-loop-phi-multiply-by-zero.ll
index d431055f0c21b..4123bc5557899 100644
--- a/llvm/test/Transforms/InstCombine/remove-loop-phi-multiply-by-zero.ll
+++ b/llvm/test/Transforms/InstCombine/remove-loop-phi-multiply-by-zero.ll
@@ -6,8 +6,8 @@ define double @test_mul_fast_flags(ptr %arr_d) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_02:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[INC]] = add i64 [[I_02]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[INC]], 1000
+; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_02]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[I_02]], 999
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[END:%.*]]
 ; CHECK:       end:
 ; CHECK-NEXT:    ret double 0.000000e+00
@@ -36,8 +36,8 @@ define double @test_nsz_nnan_flags_enabled(ptr %arr_d) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_02:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[INC]] = add i64 [[I_02]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[INC]], 1000
+; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_02]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[I_02]], 999
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[END:%.*]]
 ; CHECK:       end:
 ; CHECK-NEXT:    ret double 0.000000e+00
@@ -70,8 +70,8 @@ define double @test_nnan_flag_enabled(ptr %arr_d) {
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1000 x double], ptr [[ARR_D:%.*]], i64 0, i64 [[I_02]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[MUL]] = fmul nnan double [[F_PROD_01]], [[TMP0]]
-; CHECK-NEXT:    [[INC]] = add i64 [[I_02]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[INC]], 1000
+; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_02]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[I_02]], 999
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[END:%.*]]
 ; CHECK:       end:
 ; CHECK-NEXT:    ret double [[MUL]]
@@ -104,8 +104,8 @@ define double @test_ninf_flag_enabled(ptr %arr_d) {
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1000 x double], ptr [[ARR_D:%.*]], i64 0, i64 [[I_02]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[MUL]] = fmul ninf double [[F_PROD_01]], [[TMP0]]
-; CHECK-NEXT:    [[INC]] = add i64 [[I_02]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[INC]], 1000
+; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_02]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[I_02]], 999
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[END:%.*]]
 ; CHECK:       end:
 ; CHECK-NEXT:    ret double [[MUL]]
@@ -138,8 +138,8 @@ define double @test_nsz_flag_enabled(ptr %arr_d) {
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1000 x double], ptr [[ARR_D:%.*]], i64 0, i64 [[I_02]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[MUL]] = fmul nsz double [[F_PROD_01]], [[TMP0]]
-; CHECK-NEXT:    [[INC]] = add i64 [[I_02]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[INC]], 1000
+; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_02]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[I_02]], 999
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[END:%.*]]
 ; CHECK:       end:
 ; CHECK-NEXT:    ret double [[MUL]]
@@ -172,8 +172,8 @@ define double @test_phi_initalise_to_non_zero(ptr %arr_d) {
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1000 x double], ptr [[ARR_D:%.*]], i64 0, i64 [[I_02]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[MUL]] = fmul fast double [[F_PROD_01]], [[TMP0]]
-; CHECK-NEXT:    [[INC]] = add i64 [[I_02]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[INC]], 1000
+; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_02]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[I_02]], 999
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[END:%.*]]
 ; CHECK:       end:
 ; CHECK-NEXT:    ret double [[MUL]]
@@ -284,8 +284,8 @@ define i32 @test_int_phi_operands(ptr %arr_d) {
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[ARR_D:%.*]], i64 [[I_02]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[MUL]] = mul nsw i32 [[F_PROD_01]], [[TMP0]]
-; CHECK-NEXT:    [[INC]] = add i64 [[I_02]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[INC]], 1000
+; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_02]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[I_02]], 999
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[END:%.*]]
 ; CHECK:       end:
 ; CHECK-NEXT:    ret i32 [[MUL]]
@@ -318,8 +318,8 @@ define i32 @test_int_phi_operands_initalise_to_non_zero(ptr %arr_d) {
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[ARR_D:%.*]], i64 [[I_02]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[MUL]] = mul i32 [[F_PROD_01]], [[TMP0]]
-; CHECK-NEXT:    [[INC]] = add i64 [[I_02]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[INC]], 1000
+; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_02]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[I_02]], 999
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[END:%.*]]
 ; CHECK:       end:
 ; CHECK-NEXT:    ret i32 [[MUL]]
diff --git a/llvm/test/Transforms/InstCombine/simplify-demanded-fpclass.ll b/llvm/test/Transforms/InstCombine/simplify-demanded-fpclass.ll
index 403f3bacf34d8..f1f1708333901 100644
--- a/llvm/test/Transforms/InstCombine/simplify-demanded-fpclass.ll
+++ b/llvm/test/Transforms/InstCombine/simplify-demanded-fpclass.ll
@@ -91,6 +91,126 @@ define nofpclass(inf) float @ret_nofpclass_inf__ninf() {
   ret float 0xFFF0000000000000
 }
 
+; Basic aggregate tests to ensure this does not crash.
+define nofpclass(nan) { float } @ret_nofpclass_struct_ty() {
+; CHECK-LABEL: define nofpclass(nan) { float } @ret_nofpclass_struct_ty() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret { float } zeroinitializer
+;
+entry:
+  ret { float } zeroinitializer
+}
+
+define nofpclass(nan) { float, float } @ret_nofpclass_multiple_elems_struct_ty() {
+; CHECK-LABEL: define nofpclass(nan) { float, float } @ret_nofpclass_multiple_elems_struct_ty() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret { float, float } zeroinitializer
+;
+entry:
+  ret { float, float } zeroinitializer
+}
+
+define nofpclass(nan) { <4 x float>, <4 x float> } @ret_nofpclass_vector_elems_struct_ty() {
+; CHECK-LABEL: define nofpclass(nan) { <4 x float>, <4 x float> } @ret_nofpclass_vector_elems_struct_ty() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret { <4 x float>, <4 x float> } zeroinitializer
+;
+entry:
+  ret { <4 x float>, <4 x float> } zeroinitializer
+}
+
+define nofpclass(nan) [ 5 x float ] @ret_nofpclass_array_ty() {
+; CHECK-LABEL: define nofpclass(nan) [5 x float] @ret_nofpclass_array_ty() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret [5 x float] zeroinitializer
+;
+entry:
+  ret [ 5 x float ] zeroinitializer
+}
+
+define nofpclass(nan) [ 2 x [ 5 x float ]] @ret_nofpclass_nested_array_ty() {
+; CHECK-LABEL: define nofpclass(nan) [2 x [5 x float]] @ret_nofpclass_nested_array_ty() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret [2 x [5 x float]] zeroinitializer
+;
+entry:
+  ret [ 2 x [ 5 x float ]] zeroinitializer
+}
+
+define nofpclass(pinf) { float } @ret_nofpclass_struct_ty_pinf__ninf() {
+; CHECK-LABEL: define nofpclass(pinf) { float } @ret_nofpclass_struct_ty_pinf__ninf() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret { float } { float 0xFFF0000000000000 }
+;
+entry:
+  ret { float } { float 0xFFF0000000000000 }
+}
+
+define nofpclass(pinf) { float, float } @ret_nofpclass_multiple_elems_struct_ty_pinf__ninf() {
+; CHECK-LABEL: define nofpclass(pinf) { float, float } @ret_nofpclass_multiple_elems_struct_ty_pinf__ninf() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret { float, float } { float 0xFFF0000000000000, float 0xFFF0000000000000 }
+;
+entry:
+  ret { float, float } { float 0xFFF0000000000000, float 0xFFF0000000000000 }
+}
+
+define nofpclass(pinf) { <2 x float> } @ret_nofpclass_vector_elems_struct_ty_pinf__ninf() {
+; CHECK-LABEL: define nofpclass(pinf) { <2 x float> } @ret_nofpclass_vector_elems_struct_ty_pinf__ninf() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret { <2 x float> } { <2 x float> <float 0xFFF0000000000000, float 0xFFF0000000000000> }
+;
+entry:
+  ret { <2 x float>} { <2 x float> <float 0xFFF0000000000000, float 0xFFF0000000000000> }
+}
+
+define nofpclass(pinf) [ 1 x [ 1 x float ]] @ret_nofpclass_nested_array_ty_pinf__ninf() {
+; CHECK-LABEL: @ret_nofpclass_nested_array_ty_pinf__ninf() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret {{.*}}float 0xFFF0000000000000
+;
+entry:
+  ret [ 1 x [ 1 x float ]] [[ 1 x float ] [float 0xFFF0000000000000]]
+}
+
+define nofpclass(pzero) { float, float } @ret_nofpclass_multiple_elems_struct_ty_pzero__nzero() {
+; CHECK-LABEL: define nofpclass(pzero) { float, float } @ret_nofpclass_multiple_elems_struct_ty_pzero__nzero() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret { float, float } { float -0.000000e+00, float -0.000000e+00 }
+;
+entry:
+  ret { float, float } { float -0.0, float -0.0 }
+}
+
+define nofpclass(ninf) { float, float } @ret_nofpclass_multiple_elems_struct_ty_ninf__npinf() {
+; CHECK-LABEL: define nofpclass(ninf) { float, float } @ret_nofpclass_multiple_elems_struct_ty_ninf__npinf() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret { float, float } { float 0x7FF0000000000000, float 0x7FF0000000000000 }
+;
+entry:
+  ret { float, float } { float 0x7FF0000000000000, float 0x7FF0000000000000 }
+}
+
+; FIXME (should be poison): Support computeKnownFPClass() for non-zero aggregates.
+define nofpclass(inf) { float, float } @ret_nofpclass_multiple_elems_struct_ty_inf__npinf() {
+; CHECK-LABEL: define nofpclass(inf) { float, float } @ret_nofpclass_multiple_elems_struct_ty_inf__npinf() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret { float, float } { float 0x7FF0000000000000, float 0x7FF0000000000000 }
+;
+entry:
+  ret { float, float } { float 0x7FF0000000000000, float 0x7FF0000000000000 }
+}
+
+; FIXME (should be poison): Support computeKnownFPClass() for non-zero aggregates.
+define nofpclass(nzero) [ 1 x float ] @ret_nofpclass_multiple_elems_struct_ty_nzero_nzero() {
+; CHECK-LABEL: define nofpclass(nzero) [1 x float] @ret_nofpclass_multiple_elems_struct_ty_nzero_nzero() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret [1 x float] [float -0.000000e+00]
+;
+entry:
+  ret [ 1 x float ] [ float -0.0 ]
+}
+
 ; Negative test, do nothing
 define nofpclass(inf) float @ret_nofpclass_inf__select_nofpclass_inf_lhs(i1 %cond, float nofpclass(inf) %x, float %y) {
 ; CHECK-LABEL: define nofpclass(inf) float @ret_nofpclass_inf__select_nofpclass_inf_lhs
diff --git a/llvm/test/Transforms/InstCombine/sub.ll b/llvm/test/Transforms/InstCombine/sub.ll
index ec88984c49cca..ba596e10e8b3f 100644
--- a/llvm/test/Transforms/InstCombine/sub.ll
+++ b/llvm/test/Transforms/InstCombine/sub.ll
@@ -2797,3 +2797,71 @@ if.then:
 if.else:
   ret i32 0
 }
+
+define i32 @fold_sub_and_into_andn(i32 %x) {
+; CHECK-LABEL: @fold_sub_and_into_andn(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[X:%.*]], 63
+; CHECK-NEXT:    [[AND:%.*]] = xor i32 [[TMP0]], 63
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+entry:
+  %sub = sub i32 63, %x
+  %and = and i32 %sub, 63
+  ret i32 %and
+}
+
+define i1 @fold_sub_and_into_andn_icmp(i32 %x) {
+; CHECK-LABEL: @fold_sub_and_into_andn_icmp(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[X:%.*]], 63
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP0]], 63
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %sub = sub i32 63, %x
+  %and = and i32 %sub, 63
+  %cmp = icmp eq i32 %and, 0
+  ret i1 %cmp
+}
+
+define i32 @fold_sub_and_into_andn_subset(i32 %x) {
+; CHECK-LABEL: @fold_sub_and_into_andn_subset(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[X:%.*]], 31
+; CHECK-NEXT:    [[AND:%.*]] = xor i32 [[TMP0]], 31
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+entry:
+  %sub = sub i32 63, %x
+  %and = and i32 %sub, 31
+  ret i32 %and
+}
+
+; Negative tests
+
+define i32 @fold_sub_and_into_andn_nonmask(i32 %x, i32 %y) {
+; CHECK-LABEL: @fold_sub_and_into_andn_nonmask(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SUB]], 63
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+entry:
+  %sub = sub i32 %y, %x
+  %and = and i32 %sub, 63
+  ret i32 %and
+}
+
+define i32 @fold_sub_and_into_andn_superset(i32 %x) {
+; CHECK-LABEL: @fold_sub_and_into_andn_superset(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 63, [[X:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SUB]], 127
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+entry:
+  %sub = sub i32 63, %x
+  %and = and i32 %sub, 127
+  ret i32 %and
+}
diff --git a/llvm/test/Transforms/LICM/hoist-binop.ll b/llvm/test/Transforms/LICM/hoist-binop.ll
index 74e2b7a2caf4a..ea7d96c07d5ff 100644
--- a/llvm/test/Transforms/LICM/hoist-binop.ll
+++ b/llvm/test/Transforms/LICM/hoist-binop.ll
@@ -681,6 +681,69 @@ loop:
   br label %loop
 }
 
+; Trivially hoist and.
+define void @and(i64 %c1, i64 %c2) {
+; CHECK-LABEL: @and(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = and i64 [[C1:%.*]], [[C2:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = and i64 [[INDEX]], [[INVARIANT_OP]]
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
+  %step.add = and i64 %index, %c1
+  %index.next = and i64 %step.add, %c2
+  br label %loop
+}
+
+; Trivially hoist or.
+define void @or(i64 %c1, i64 %c2) {
+; CHECK-LABEL: @or(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = or i64 [[C1:%.*]], [[C2:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = or i64 [[INDEX]], [[INVARIANT_OP]]
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
+  %step.add = or i64 %index, %c1
+  %index.next = or i64 %c2, %step.add
+  br label %loop
+}
+
+; Trivially hoist xor.
+define void @xor(i64 %c1, i64 %c2) {
+; CHECK-LABEL: @xor(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = xor i64 [[C1:%.*]], [[C2:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = xor i64 [[INDEX]], [[INVARIANT_OP]]
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
+  %step.add = xor i64 %c1, %index
+  %index.next = xor i64 %step.add, %c2
+  br label %loop
+}
+
 ; Don't hoist if the intermediate op has more than two uses. This is an
 ; heuristic that can be adjusted if warranted. Currently we are being
 ; conservative to minimise potential impact in code size.
diff --git a/llvm/test/Transforms/LoopIdiom/basic-address-space.ll b/llvm/test/Transforms/LoopIdiom/basic-address-space.ll
index de55a46a164e6..9945aeb2021a9 100644
--- a/llvm/test/Transforms/LoopIdiom/basic-address-space.ll
+++ b/llvm/test/Transforms/LoopIdiom/basic-address-space.ll
@@ -13,7 +13,7 @@ define void @test10(ptr addrspace(2) %X) nounwind ssp {
 ; CHECK:       bb.nph:
 ; CHECK-NEXT:    [[I_04:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[INC12:%.*]], [[FOR_INC10:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = mul nuw nsw i16 [[I_04]], 100
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr addrspace(2) [[X]], i16 [[TMP0]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr addrspace(2) [[X]], i16 [[TMP0]]
 ; CHECK-NEXT:    br label [[FOR_BODY5:%.*]]
 ; CHECK:       for.body5:
 ; CHECK-NEXT:    [[J_02:%.*]] = phi i16 [ 0, [[BB_NPH]] ], [ [[INC:%.*]], [[FOR_BODY5]] ]
diff --git a/llvm/test/Transforms/LoopIdiom/basic.ll b/llvm/test/Transforms/LoopIdiom/basic.ll
index 912b1542a267b..e6fc11625317b 100644
--- a/llvm/test/Transforms/LoopIdiom/basic.ll
+++ b/llvm/test/Transforms/LoopIdiom/basic.ll
@@ -481,7 +481,7 @@ define void @test10(ptr %X) nounwind ssp {
 ; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[FOR_INC10:%.*]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC12:%.*]], [[FOR_INC10]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = mul nuw nsw i64 [[INDVAR]], 100
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[X]], i64 [[TMP0]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[X]], i64 [[TMP0]]
 ; CHECK-NEXT:    br label [[FOR_BODY5:%.*]]
 ; CHECK:       for.body5:
 ; CHECK-NEXT:    [[J_02:%.*]] = phi i32 [ 0, [[BB_NPH]] ], [ [[INC:%.*]], [[FOR_BODY5]] ]
@@ -682,7 +682,7 @@ define void @PR14241(ptr %s, i64 %size) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[END_IDX:%.*]] = add i64 [[SIZE:%.*]], -1
 ; CHECK-NEXT:    [[END_PTR:%.*]] = getelementptr inbounds i32, ptr [[S:%.*]], i64 [[END_IDX]]
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[S]], i64 4
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[S]], i64 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[SIZE]], 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], -8
 ; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 2
diff --git a/llvm/test/Transforms/LoopIdiom/memset-pr52104.ll b/llvm/test/Transforms/LoopIdiom/memset-pr52104.ll
index 0c7d1ec59b2cd..0cacb74bf01be 100644
--- a/llvm/test/Transforms/LoopIdiom/memset-pr52104.ll
+++ b/llvm/test/Transforms/LoopIdiom/memset-pr52104.ll
@@ -6,12 +6,12 @@ define void @f(ptr nocapture nonnull align 4 dereferenceable(20) %0, i32 %1) loc
 ; CHECK-NEXT:    [[TMP3:%.*]] = trunc i32 [[TMP1:%.*]] to i2
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext i2 [[TMP3]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
-; CHECK-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, ptr [[TMP0:%.*]], i64 [[TMP5]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[TMP0:%.*]], i64 [[TMP5]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = sub i2 -1, [[TMP3]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = zext i2 [[TMP6]] to i64
 ; CHECK-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2
 ; CHECK-NEXT:    [[TMP9:%.*]] = add nuw nsw i64 [[TMP8]], 4
-; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[UGLYGEP]], i8 0, i64 [[TMP9]], i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[SCEVGEP]], i8 0, i64 [[TMP9]], i1 false)
 ; CHECK-NEXT:    br label [[TMP10:%.*]]
 ; CHECK:       10:
 ; CHECK-NEXT:    [[TMP11:%.*]] = phi i32 [ [[TMP15:%.*]], [[TMP10]] ], [ [[TMP1]], [[TMP2:%.*]] ]
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2011-12-19-PostincQuadratic.ll b/llvm/test/Transforms/LoopStrengthReduce/2011-12-19-PostincQuadratic.ll
index 616e3ae1b0368..621a4e8797d57 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/2011-12-19-PostincQuadratic.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/2011-12-19-PostincQuadratic.ll
@@ -16,7 +16,7 @@ define void @vb() nounwind {
 ; CHECK-NEXT:  for.cond.preheader:
 ; CHECK-NEXT:    br label [[FOR_BODY7:%.*]]
 ; CHECK:       for.body7:
-; CHECK-NEXT:    [[LSR_IV1:%.*]] = phi ptr [ [[SCEVGEP:%.*]], [[FOR_BODY7]] ], [ getelementptr inbounds (i8, ptr @b, i32 4), [[FOR_COND_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[LSR_IV1:%.*]] = phi ptr [ [[SCEVGEP:%.*]], [[FOR_BODY7]] ], [ getelementptr inbounds nuw (i8, ptr @b, i32 4), [[FOR_COND_PREHEADER:%.*]] ]
 ; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i32 [ [[LSR_IV_NEXT:%.*]], [[FOR_BODY7]] ], [ 8, [[FOR_COND_PREHEADER]] ]
 ; CHECK-NEXT:    [[INDVARS_IV77:%.*]] = phi i32 [ [[INDVARS_IV_NEXT78:%.*]], [[FOR_BODY7]] ], [ 1, [[FOR_COND_PREHEADER]] ]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT78]] = add i32 [[INDVARS_IV77]], 1
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/2011-11-29-postincphi.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/2011-11-29-postincphi.ll
index 841836c7d2dd8..7ae78ae6a1fd4 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/X86/2011-11-29-postincphi.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/2011-11-29-postincphi.ll
@@ -17,7 +17,7 @@ define i64 @sqlite3DropTriggerPtr() nounwind {
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    movl $1, %ebx
 ; CHECK-NEXT:    callq check@PLT
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %bb1
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    testb %al, %al
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/2012-01-13-phielim.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/2012-01-13-phielim.ll
index c4aa6c7725d41..492746615d846 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/X86/2012-01-13-phielim.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/2012-01-13-phielim.ll
@@ -96,7 +96,7 @@ define void @test2(i32 %n) nounwind uwtable {
 ; CHECK-NEXT:    br label [[FOR_COND468:%.*]]
 ; CHECK:       for.cond468:
 ; CHECK-NEXT:    [[LSR_IV1:%.*]] = phi i32 [ 1, [[FOR_COND468_PREHEADER]] ], [ [[LSR_IV_NEXT:%.*]], [[IF_THEN477:%.*]] ]
-; CHECK-NEXT:    [[LSR_IV:%.*]] = phi ptr [ getelementptr inbounds (i8, ptr @tags, i64 8), [[FOR_COND468_PREHEADER]] ], [ [[SCEVGEP:%.*]], [[IF_THEN477]] ]
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi ptr [ getelementptr inbounds nuw (i8, ptr @tags, i64 8), [[FOR_COND468_PREHEADER]] ], [ [[SCEVGEP:%.*]], [[IF_THEN477]] ]
 ; CHECK-NEXT:    [[K_0:%.*]] = load i32, ptr [[LSR_IV]], align 4
 ; CHECK-NEXT:    [[CMP469:%.*]] = icmp slt i32 [[LSR_IV1]], [[N:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP469]], label [[FOR_BODY471:%.*]], label [[FOR_INC498_PREHEADER:%.*]]
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-X86.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-X86.ll
index 39e2d6f1acca4..93203ef6e17f5 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-X86.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-X86.ll
@@ -20,7 +20,7 @@ define i32 @simple(ptr %a, ptr %b, i32 %x) nounwind {
 ; X64-NEXT:    shlq $2, %rcx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    leaq (%rcx,%rcx), %rdx
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB0_1: # %loop
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    addl (%rdi), %eax
@@ -47,7 +47,7 @@ define i32 @simple(ptr %a, ptr %b, i32 %x) nounwind {
 ; X32-NEXT:    shll $2, %edx
 ; X32-NEXT:    xorl %eax, %eax
 ; X32-NEXT:    leal (%edx,%edx), %esi
-; X32-NEXT:    .p2align 4, 0x90
+; X32-NEXT:    .p2align 4
 ; X32-NEXT:  .LBB0_1: # %loop
 ; X32-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X32-NEXT:    addl (%edi), %eax
@@ -101,7 +101,7 @@ define i32 @user(ptr %a, ptr %b, i32 %x) nounwind {
 ; X64-NEXT:    leaq (,%rcx,4), %rax
 ; X64-NEXT:    leaq (%rax,%rax,2), %r8
 ; X64-NEXT:    xorl %eax, %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB1_1: # %loop
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    addl (%rdi), %eax
@@ -128,7 +128,7 @@ define i32 @user(ptr %a, ptr %b, i32 %x) nounwind {
 ; X32-NEXT:    leal (,%ecx,4), %eax
 ; X32-NEXT:    leal (%eax,%eax,2), %ebx
 ; X32-NEXT:    xorl %eax, %eax
-; X32-NEXT:    .p2align 4, 0x90
+; X32-NEXT:    .p2align 4
 ; X32-NEXT:  .LBB1_1: # %loop
 ; X32-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X32-NEXT:    addl (%esi), %eax
@@ -200,7 +200,7 @@ define void @extrastride(ptr nocapture %main, i32 %main_stride, ptr nocapture %r
 ; X64-NEXT:    movslq %r8d, %r8
 ; X64-NEXT:    shlq $2, %r8
 ; X64-NEXT:    movslq %ebx, %r11
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB2_2: # %for.body
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl (%rdi,%rsi), %ebx
@@ -235,7 +235,7 @@ define void @extrastride(ptr nocapture %main, i32 %main_stride, ptr nocapture %r
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X32-NEXT:    addl %esi, %edi
 ; X32-NEXT:    shll $2, %ecx
-; X32-NEXT:    .p2align 4, 0x90
+; X32-NEXT:    .p2align 4
 ; X32-NEXT:  .LBB2_2: # %for.body
 ; X32-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X32-NEXT:    movl (%ebx,%esi), %ebp
@@ -309,7 +309,7 @@ define void @foldedidx(ptr nocapture %a, ptr nocapture %b, ptr nocapture %c) nou
 ; X64-LABEL: foldedidx:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movl $3, %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB3_1: # %for.body
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movzbl -3(%rdi,%rax), %ecx
@@ -343,7 +343,7 @@ define void @foldedidx(ptr nocapture %a, ptr nocapture %b, ptr nocapture %c) nou
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    .p2align 4, 0x90
+; X32-NEXT:    .p2align 4
 ; X32-NEXT:  .LBB3_1: # %for.body
 ; X32-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X32-NEXT:    movzbl -3(%esi,%eax), %edi
@@ -433,7 +433,7 @@ define void @multioper(ptr %a, i32 %n) nounwind {
 ; X64-LABEL: multioper:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    xorl %eax, %eax
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB4_1: # %for.body
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, (%rdi,%rax,4)
@@ -455,7 +455,7 @@ define void @multioper(ptr %a, i32 %n) nounwind {
 ; X32-NEXT:    xorl %eax, %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    .p2align 4, 0x90
+; X32-NEXT:    .p2align 4
 ; X32-NEXT:  .LBB4_1: # %for.body
 ; X32-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X32-NEXT:    movl %eax, (%edx,%eax,4)
@@ -511,7 +511,7 @@ define void @testCmpZero(ptr %src, ptr %dst, i32 %srcidx, i32 %dstidx, i32 %len)
 ; X64-NEXT:    movslq %r8d, %rcx
 ; X64-NEXT:    subq %rdx, %rcx
 ; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:    .p2align 4
 ; X64-NEXT:  .LBB5_1: # %for.body82.us
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movzbl (%rax,%rdx,4), %esi
@@ -532,7 +532,7 @@ define void @testCmpZero(ptr %src, ptr %dst, i32 %srcidx, i32 %dstidx, i32 %len)
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    addl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    xorl %esi, %esi
-; X32-NEXT:    .p2align 4, 0x90
+; X32-NEXT:    .p2align 4
 ; X32-NEXT:  .LBB5_1: # %for.body82.us
 ; X32-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X32-NEXT:    movzbl (%edx,%esi,4), %ebx
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-1.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-1.ll
index 753aaed7d77c0..1c953515a3108 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-1.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-1.ll
@@ -65,7 +65,7 @@ define void @foo(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr nocap
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
-; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  .LBB0_1: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movl 4096(%rsi,%rax), %ecx
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/macro-fuse-cmp.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/macro-fuse-cmp.ll
index feff010300c1e..ef2ed1f0e87e2 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/X86/macro-fuse-cmp.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/macro-fuse-cmp.ll
@@ -79,7 +79,7 @@ define void @maxArray(ptr noalias nocapture %x, ptr noalias nocapture readonly %
 ; BASE-LABEL: maxArray:
 ; BASE:       # %bb.0: # %entry
 ; BASE-NEXT:    movq $-524288, %rax # imm = 0xFFF80000
-; BASE-NEXT:    .p2align 4, 0x90
+; BASE-NEXT:    .p2align 4
 ; BASE-NEXT:  .LBB0_1: # %vector.body
 ; BASE-NEXT:    # =>This Inner Loop Header: Depth=1
 ; BASE-NEXT:    movupd 524288(%rdi,%rax), %xmm0
@@ -93,7 +93,7 @@ define void @maxArray(ptr noalias nocapture %x, ptr noalias nocapture readonly %
 ; FUSE-LABEL: maxArray:
 ; FUSE:       # %bb.0: # %entry
 ; FUSE-NEXT:    xorl %eax, %eax
-; FUSE-NEXT:    .p2align 4, 0x90
+; FUSE-NEXT:    .p2align 4
 ; FUSE-NEXT:  .LBB0_1: # %vector.body
 ; FUSE-NEXT:    # =>This Inner Loop Header: Depth=1
 ; FUSE-NEXT:    movupd (%rdi,%rax,8), %xmm0
diff --git a/llvm/test/Transforms/LoopStrengthReduce/debuginfo-scev-salvage-large-width.ll b/llvm/test/Transforms/LoopStrengthReduce/debuginfo-scev-salvage-large-width.ll
new file mode 100644
index 0000000000000..dcd17cae59f4d
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/debuginfo-scev-salvage-large-width.ll
@@ -0,0 +1,40 @@
+; RUN: opt -S -loop-reduce %s | FileCheck %s
+
+;; We (currently?) can't salvage an IV if the offset is wider than 64 bits.
+;; Check we poison it instead.
+
+; CHECK: #dbg_value(i[[#]] poison, ![[#]], !DIExpression(), ![[#]])
+
+define i16 @main() {
+entry:
+  br label %for.cond29
+
+for.cond29:                                       ; preds = %for.body32, %entry
+  %il_1000.0 = phi i128 [ 0, %entry ], [ %inc72, %for.body32 ]
+  %l_995.0 = phi i128 [ 4704496199548239085565, %entry ], [ %inc70, %for.body32 ]
+    #dbg_value(i128 %l_995.0, !4, !DIExpression(), !9)
+  %cmp30 = icmp slt i128 %il_1000.0, 0
+  br i1 %cmp30, label %for.body32, label %for.cond.cleanup31
+
+for.cond.cleanup31:                               ; preds = %for.cond29
+  ret i16 0
+
+for.body32:                                       ; preds = %for.cond29
+  %inc70 = add i128 %l_995.0, 1
+  %inc72 = add i128 %il_1000.0, 1
+  br label %for.cond29
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !2, globals: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "foo.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !DILocalVariable(name: "l_995", scope: !5, file: !1, line: 414, type: !7)
+!5 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 397, type: !6, scopeLine: 398, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!6 = !DISubroutineType(types: !2)
+!7 = !DIDerivedType(tag: DW_TAG_typedef, name: "__uint128_t", file: !1, baseType: !8)
+!8 = !DIBasicType(name: "unsigned __int128", size: 128, encoding: DW_ATE_unsigned)
+!9 = !DILocation(line: 0, scope: !5)
diff --git a/llvm/test/Transforms/LoopStrengthReduce/post-inc-icmpzero.ll b/llvm/test/Transforms/LoopStrengthReduce/post-inc-icmpzero.ll
index f73f92246fff2..d0a910a0758a9 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/post-inc-icmpzero.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/post-inc-icmpzero.ll
@@ -18,7 +18,7 @@ define void @_Z15IntegerToStringjjR7Vector2(i32 %i, i32 %radix, ptr nocapture %r
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr [33 x i16], ptr [[BUFFER]], i64 0, i64 33
 ; CHECK-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[ADD_PTR]] to i64
 ; CHECK-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[ADD_PTR]] to i64
-; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[BUFFER]], i64 64
+; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr nuw i8, ptr [[BUFFER]], i64 64
 ; CHECK-NEXT:    br label [[DO_BODY:%.*]]
 ; CHECK:       do.body:
 ; CHECK-NEXT:    [[LSR_IV10:%.*]] = phi i64 [ [[LSR_IV_NEXT11:%.*]], [[DO_BODY]] ], [ -1, [[ENTRY:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
index 0959d913fd0cd..6a709c9c316f9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
@@ -1446,7 +1446,7 @@ define void @PR34743(ptr %a, ptr %b, i64 %n) #1 {
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], -4
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP5]]
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[TMP6]], i64 4
-; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 2
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr nuw i8, ptr [[A]], i64 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
 ; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[TMP7]], i64 6
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP2]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-function-calls.ll b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-function-calls.ll
index 41ccb3c404dd7..05bf841345ca2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-function-calls.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-function-calls.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "call.*(cos|sin|tan|cbrt|erf|exp[^e]|gamma|log|sqrt|copysign|dim|min|mod|hypot|nextafter|pow|fma)" --version 2
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --scrub-attributes --filter "call.*(cos|sin|tan|cbrt|erf|exp[^e]|gamma|log|sqrt|copysign|dim|min|mod|hypot|nextafter|pow|fma)" --version 2
 ; RUN: opt -mattr=+neon -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -S < %s | FileCheck %s -check-prefix=SLEEF-NEON
 ; RUN: opt -mattr=+sve -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S < %s | FileCheck %s -check-prefix=SLEEF-SVE
 ; RUN: opt -mattr=+sve -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -S < %s | FileCheck %s -check-prefixes=SLEEF-SVE-NOPRED
@@ -25,12 +25,12 @@ define void @acos_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @acos_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0:[0-9]+]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acos(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acos(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @acos_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0:[0-9]+]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acos(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @acos(double [[IN:%.*]]) #[[ATTR2:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @acos(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @acos_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0:[0-9]+]] {
@@ -38,12 +38,12 @@ define void @acos_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @acos_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0:[0-9]+]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svacos_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svacos_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @acos_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0:[0-9]+]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svacos_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @acos(double [[IN:%.*]]) #[[ATTR2:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @acos(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -70,12 +70,12 @@ define void @acos_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @acos_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acosf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acosf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @acos_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acosf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @acosf(float [[IN:%.*]]) #[[ATTR3:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @acosf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @acos_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -83,12 +83,12 @@ define void @acos_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @acos_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svacos_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svacos_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @acos_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svacos_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @acosf(float [[IN:%.*]]) #[[ATTR3:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @acosf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -118,12 +118,12 @@ define void @acosh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @acosh_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acosh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acosh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @acosh_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acosh(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @acosh(double [[IN:%.*]]) #[[ATTR4:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @acosh(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @acosh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -131,12 +131,12 @@ define void @acosh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @acosh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svacosh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svacosh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @acosh_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svacosh_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @acosh(double [[IN:%.*]]) #[[ATTR4:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @acosh(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -163,12 +163,12 @@ define void @acosh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @acosh_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acoshf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acoshf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @acosh_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acoshf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @acoshf(float [[IN:%.*]]) #[[ATTR5:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @acoshf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @acosh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -176,12 +176,12 @@ define void @acosh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @acosh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svacosh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svacosh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @acosh_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svacosh_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @acoshf(float [[IN:%.*]]) #[[ATTR5:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @acoshf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -211,12 +211,12 @@ define void @asin_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @asin_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @asin_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asin(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @asin(double [[IN:%.*]]) #[[ATTR6:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @asin(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @asin_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -224,12 +224,12 @@ define void @asin_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @asin_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svasin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svasin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @asin_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svasin_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @asin(double [[IN:%.*]]) #[[ATTR6:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @asin(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -256,12 +256,12 @@ define void @asin_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @asin_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @asin_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @asinf(float [[IN:%.*]]) #[[ATTR7:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @asinf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @asin_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -269,12 +269,12 @@ define void @asin_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @asin_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svasin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svasin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @asin_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svasin_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @asinf(float [[IN:%.*]]) #[[ATTR7:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @asinf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -304,12 +304,12 @@ define void @asinh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @asinh_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asinh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asinh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @asinh_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asinh(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @asinh(double [[IN:%.*]]) #[[ATTR8:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @asinh(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @asinh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -317,12 +317,12 @@ define void @asinh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @asinh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svasinh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svasinh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @asinh_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svasinh_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @asinh(double [[IN:%.*]]) #[[ATTR8:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @asinh(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -349,12 +349,12 @@ define void @asinh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @asinh_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @asinh_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinhf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @asinhf(float [[IN:%.*]]) #[[ATTR9:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @asinhf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @asinh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -362,12 +362,12 @@ define void @asinh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @asinh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svasinh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svasinh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @asinh_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svasinh_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @asinhf(float [[IN:%.*]]) #[[ATTR9:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @asinhf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -397,12 +397,12 @@ define void @atan_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @atan_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atan(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atan(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @atan_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atan(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @atan(double [[IN:%.*]]) #[[ATTR10:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @atan(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @atan_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -410,12 +410,12 @@ define void @atan_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @atan_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svatan_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svatan_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @atan_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svatan_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @atan(double [[IN:%.*]]) #[[ATTR10:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @atan(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -442,12 +442,12 @@ define void @atan_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @atan_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @atan_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @atanf(float [[IN:%.*]]) #[[ATTR11:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @atanf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @atan_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -455,12 +455,12 @@ define void @atan_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @atan_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svatan_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svatan_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @atan_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svatan_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @atanf(float [[IN:%.*]]) #[[ATTR11:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @atanf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -490,12 +490,12 @@ define void @atan2_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @atan2_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_atan2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_atan2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @atan2_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_atan2(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @atan2(double [[IN:%.*]], double [[IN]]) #[[ATTR12:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @atan2(double [[IN:%.*]], double [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @atan2_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -503,12 +503,12 @@ define void @atan2_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @atan2_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svatan2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svatan2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @atan2_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svatan2_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @atan2(double [[IN:%.*]], double [[IN]]) #[[ATTR12:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @atan2(double [[IN:%.*]], double [[IN]])
 ;
   entry:
   br label %for.body
@@ -535,12 +535,12 @@ define void @atan2_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @atan2_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_atan2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_atan2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @atan2_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_atan2f(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @atan2f(float [[IN:%.*]], float [[IN]]) #[[ATTR13:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @atan2f(float [[IN:%.*]], float [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @atan2_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -548,12 +548,12 @@ define void @atan2_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @atan2_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svatan2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svatan2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @atan2_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svatan2_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @atan2f(float [[IN:%.*]], float [[IN]]) #[[ATTR13:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @atan2f(float [[IN:%.*]], float [[IN]])
 ;
   entry:
   br label %for.body
@@ -583,12 +583,12 @@ define void @atanh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @atanh_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atanh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atanh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @atanh_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atanh(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @atanh(double [[IN:%.*]]) #[[ATTR14:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @atanh(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @atanh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -596,12 +596,12 @@ define void @atanh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @atanh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svatanh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svatanh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @atanh_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svatanh_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @atanh(double [[IN:%.*]]) #[[ATTR14:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @atanh(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -628,12 +628,12 @@ define void @atanh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @atanh_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @atanh_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanhf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @atanhf(float [[IN:%.*]]) #[[ATTR15:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @atanhf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @atanh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -641,12 +641,12 @@ define void @atanh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @atanh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svatanh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svatanh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @atanh_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svatanh_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @atanhf(float [[IN:%.*]]) #[[ATTR15:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @atanhf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -676,12 +676,12 @@ define void @cbrt_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @cbrt_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cbrt(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cbrt(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @cbrt_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cbrt(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cbrt(double [[IN:%.*]]) #[[ATTR16:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cbrt(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cbrt_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -689,12 +689,12 @@ define void @cbrt_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @cbrt_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svcbrt_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svcbrt_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @cbrt_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svcbrt_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cbrt(double [[IN:%.*]]) #[[ATTR16:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cbrt(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -721,12 +721,12 @@ define void @cbrt_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @cbrt_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cbrtf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cbrtf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @cbrt_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cbrtf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @cbrtf(float [[IN:%.*]]) #[[ATTR17:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @cbrtf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cbrt_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -734,12 +734,12 @@ define void @cbrt_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @cbrt_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svcbrt_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svcbrt_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @cbrt_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svcbrt_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @cbrtf(float [[IN:%.*]]) #[[ATTR17:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @cbrtf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -769,12 +769,12 @@ define void @copysign_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @copysign_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_copysign(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_copysign(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @copysign_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_copysign(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @copysign(double [[IN:%.*]], double [[IN]]) #[[ATTR18:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @copysign(double [[IN:%.*]], double [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @copysign_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -782,12 +782,12 @@ define void @copysign_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @copysign_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svcopysign_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svcopysign_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @copysign_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svcopysign_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @copysign(double [[IN:%.*]], double [[IN]]) #[[ATTR18:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @copysign(double [[IN:%.*]], double [[IN]])
 ;
   entry:
   br label %for.body
@@ -814,12 +814,12 @@ define void @copysign_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @copysign_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_copysignf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_copysignf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @copysign_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_copysignf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @copysignf(float [[IN:%.*]], float [[IN]]) #[[ATTR19:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @copysignf(float [[IN:%.*]], float [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @copysign_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -827,12 +827,12 @@ define void @copysign_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @copysign_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svcopysign_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svcopysign_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @copysign_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svcopysign_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @copysignf(float [[IN:%.*]], float [[IN]]) #[[ATTR19:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @copysignf(float [[IN:%.*]], float [[IN]])
 ;
   entry:
   br label %for.body
@@ -862,12 +862,12 @@ define void @cos_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @cos_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cos(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cos(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @cos_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cos(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cos(double [[IN:%.*]]) #[[ATTR20:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cos(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cos_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -875,12 +875,12 @@ define void @cos_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @cos_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svcos_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svcos_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @cos_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svcos_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cos(double [[IN:%.*]]) #[[ATTR20:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cos(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -907,12 +907,12 @@ define void @cos_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @cos_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cosf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cosf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @cos_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cosf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @cosf(float [[IN:%.*]]) #[[ATTR21:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @cosf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cos_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -920,12 +920,12 @@ define void @cos_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @cos_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svcos_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svcos_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @cos_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svcos_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @cosf(float [[IN:%.*]]) #[[ATTR21:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @cosf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -955,12 +955,12 @@ define void @cosh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @cosh_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cosh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cosh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @cosh_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cosh(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cosh(double [[IN:%.*]]) #[[ATTR22:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cosh(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cosh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -968,12 +968,12 @@ define void @cosh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @cosh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svcosh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svcosh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @cosh_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svcosh_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cosh(double [[IN:%.*]]) #[[ATTR22:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cosh(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -1000,12 +1000,12 @@ define void @cosh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @cosh_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_coshf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_coshf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @cosh_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_coshf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @coshf(float [[IN:%.*]]) #[[ATTR23:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @coshf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cosh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1013,12 +1013,12 @@ define void @cosh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @cosh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svcosh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svcosh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @cosh_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svcosh_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @coshf(float [[IN:%.*]]) #[[ATTR23:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @coshf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -1048,12 +1048,12 @@ define void @cospi_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @cospi_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cospi(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cospi(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @cospi_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cospi(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cospi(double [[IN:%.*]]) #[[ATTR24:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cospi(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cospi_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1061,12 +1061,12 @@ define void @cospi_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @cospi_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svcospi_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svcospi_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @cospi_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svcospi_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cospi(double [[IN:%.*]]) #[[ATTR24:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @cospi(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -1093,12 +1093,12 @@ define void @cospi_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @cospi_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cospif(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cospif(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @cospi_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cospif(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @cospif(float [[IN:%.*]]) #[[ATTR25:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @cospif(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cospi_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1106,12 +1106,12 @@ define void @cospi_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @cospi_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svcospi_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svcospi_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @cospi_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svcospi_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @cospif(float [[IN:%.*]]) #[[ATTR25:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @cospif(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -1141,12 +1141,12 @@ define void @erf_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @erf_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_erf(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_erf(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @erf_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_erf(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @erf(double [[IN:%.*]]) #[[ATTR26:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @erf(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @erf_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1154,12 +1154,12 @@ define void @erf_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @erf_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_sverf_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_sverf_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @erf_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_sverf_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @erf(double [[IN:%.*]]) #[[ATTR26:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @erf(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -1186,12 +1186,12 @@ define void @erf_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @erf_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_erff(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_erff(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @erf_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_erff(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @erff(float [[IN:%.*]]) #[[ATTR27:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @erff(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @erf_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1199,12 +1199,12 @@ define void @erf_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @erf_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_sverf_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_sverf_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @erf_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_sverf_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @erff(float [[IN:%.*]]) #[[ATTR27:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @erff(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -1234,12 +1234,12 @@ define void @erfc_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @erfc_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_erfc(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_erfc(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @erfc_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_erfc(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @erfc(double [[IN:%.*]]) #[[ATTR28:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @erfc(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @erfc_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1247,12 +1247,12 @@ define void @erfc_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @erfc_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_sverfc_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_sverfc_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @erfc_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_sverfc_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @erfc(double [[IN:%.*]]) #[[ATTR28:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @erfc(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -1279,12 +1279,12 @@ define void @erfc_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @erfc_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_erfcf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_erfcf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @erfc_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_erfcf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @erfcf(float [[IN:%.*]]) #[[ATTR29:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @erfcf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @erfc_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1292,12 +1292,12 @@ define void @erfc_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @erfc_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_sverfc_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_sverfc_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @erfc_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_sverfc_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @erfcf(float [[IN:%.*]]) #[[ATTR29:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @erfcf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -1327,12 +1327,12 @@ define void @exp_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @exp_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @exp_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @exp(double [[IN:%.*]]) #[[ATTR30:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @exp(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1340,12 +1340,12 @@ define void @exp_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @exp_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svexp_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svexp_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @exp_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svexp_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @exp(double [[IN:%.*]]) #[[ATTR30:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @exp(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -1372,12 +1372,12 @@ define void @exp_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @exp_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @exp_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @expf(float [[IN:%.*]]) #[[ATTR31:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @expf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1385,12 +1385,12 @@ define void @exp_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @exp_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svexp_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svexp_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @exp_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svexp_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @expf(float [[IN:%.*]]) #[[ATTR31:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @expf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -1420,12 +1420,12 @@ define void @exp10_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @exp10_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp10(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp10(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @exp10_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp10(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @exp10(double [[IN:%.*]]) #[[ATTR32:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @exp10(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp10_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1433,12 +1433,12 @@ define void @exp10_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @exp10_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svexp10_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svexp10_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @exp10_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svexp10_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @exp10(double [[IN:%.*]]) #[[ATTR32:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @exp10(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -1465,12 +1465,12 @@ define void @exp10_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @exp10_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp10f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp10f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @exp10_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp10f(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @exp10f(float [[IN:%.*]]) #[[ATTR33:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @exp10f(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp10_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1478,12 +1478,12 @@ define void @exp10_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @exp10_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svexp10_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svexp10_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @exp10_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svexp10_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @exp10f(float [[IN:%.*]]) #[[ATTR33:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @exp10f(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -1513,12 +1513,12 @@ define void @exp2_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @exp2_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @exp2_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp2(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @exp2(double [[IN:%.*]]) #[[ATTR34:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @exp2(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp2_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1526,12 +1526,12 @@ define void @exp2_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @exp2_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svexp2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svexp2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @exp2_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svexp2_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @exp2(double [[IN:%.*]]) #[[ATTR34:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @exp2(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -1558,12 +1558,12 @@ define void @exp2_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @exp2_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @exp2_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp2f(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @exp2f(float [[IN:%.*]]) #[[ATTR35:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @exp2f(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp2_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1571,12 +1571,12 @@ define void @exp2_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @exp2_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svexp2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svexp2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @exp2_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svexp2_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @exp2f(float [[IN:%.*]]) #[[ATTR35:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @exp2f(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -1606,12 +1606,12 @@ define void @expm1_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @expm1_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_expm1(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_expm1(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @expm1_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_expm1(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @expm1(double [[IN:%.*]]) #[[ATTR36:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @expm1(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @expm1_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1619,12 +1619,12 @@ define void @expm1_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @expm1_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svexpm1_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svexpm1_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @expm1_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svexpm1_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @expm1(double [[IN:%.*]]) #[[ATTR36:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @expm1(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -1651,12 +1651,12 @@ define void @expm1_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @expm1_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expm1f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expm1f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @expm1_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expm1f(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @expm1f(float [[IN:%.*]]) #[[ATTR37:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @expm1f(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @expm1_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1664,12 +1664,12 @@ define void @expm1_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @expm1_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svexpm1_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svexpm1_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @expm1_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svexpm1_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @expm1f(float [[IN:%.*]]) #[[ATTR37:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @expm1f(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -1699,12 +1699,12 @@ define void @fdim_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @fdim_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fdim(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fdim(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @fdim_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fdim(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fdim(double [[IN:%.*]], double [[IN]]) #[[ATTR38:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fdim(double [[IN:%.*]], double [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @fdim_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1712,12 +1712,12 @@ define void @fdim_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @fdim_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svfdim_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svfdim_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @fdim_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svfdim_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fdim(double [[IN:%.*]], double [[IN]]) #[[ATTR38:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fdim(double [[IN:%.*]], double [[IN]])
 ;
   entry:
   br label %for.body
@@ -1744,12 +1744,12 @@ define void @fdim_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @fdim_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fdimf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fdimf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @fdim_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fdimf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fdimf(float [[IN:%.*]], float [[IN]]) #[[ATTR39:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fdimf(float [[IN:%.*]], float [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @fdim_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1757,12 +1757,12 @@ define void @fdim_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @fdim_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svfdim_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svfdim_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @fdim_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svfdim_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fdimf(float [[IN:%.*]], float [[IN]]) #[[ATTR39:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fdimf(float [[IN:%.*]], float [[IN]])
 ;
   entry:
   br label %for.body
@@ -1792,12 +1792,12 @@ define void @fma_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @fma_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxvvv_fma(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvvv_fma(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @fma_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxvvv_fma(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fma(double [[IN:%.*]], double [[IN]], double [[IN]]) #[[ATTR40:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fma(double [[IN:%.*]], double [[IN]], double [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @fma_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1805,12 +1805,12 @@ define void @fma_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @fma_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svfma_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svfma_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @fma_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svfma_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fma(double [[IN:%.*]], double [[IN]], double [[IN]]) #[[ATTR40:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fma(double [[IN:%.*]], double [[IN]], double [[IN]])
 ;
   entry:
   br label %for.body
@@ -1837,12 +1837,12 @@ define void @fma_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @fma_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxvvv_fmaf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvvv_fmaf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @fma_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxvvv_fmaf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fmaf(float [[IN:%.*]], float [[IN]], float [[IN]]) #[[ATTR41:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fmaf(float [[IN:%.*]], float [[IN]], float [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @fma_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1850,12 +1850,12 @@ define void @fma_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @fma_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svfma_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svfma_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @fma_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svfma_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fmaf(float [[IN:%.*]], float [[IN]], float [[IN]]) #[[ATTR41:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fmaf(float [[IN:%.*]], float [[IN]], float [[IN]])
 ;
   entry:
   br label %for.body
@@ -1885,12 +1885,12 @@ define void @fmax_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @fmax_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmax(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmax(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @fmax_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmax(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fmax(double [[IN:%.*]], double [[IN]]) #[[ATTR42:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fmax(double [[IN:%.*]], double [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @fmax_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1898,12 +1898,12 @@ define void @fmax_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @fmax_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svfmax_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svfmax_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @fmax_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svfmax_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fmax(double [[IN:%.*]], double [[IN]]) #[[ATTR42:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fmax(double [[IN:%.*]], double [[IN]])
 ;
   entry:
   br label %for.body
@@ -1930,12 +1930,12 @@ define void @fmax_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @fmax_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fmaxf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fmaxf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @fmax_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fmaxf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fmaxf(float [[IN:%.*]], float [[IN]]) #[[ATTR43:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fmaxf(float [[IN:%.*]], float [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @fmax_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1943,12 +1943,12 @@ define void @fmax_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @fmax_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svfmax_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svfmax_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @fmax_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svfmax_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fmaxf(float [[IN:%.*]], float [[IN]]) #[[ATTR43:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fmaxf(float [[IN:%.*]], float [[IN]])
 ;
   entry:
   br label %for.body
@@ -1978,12 +1978,12 @@ define void @fmin_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @fmin_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @fmin_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmin(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fmin(double [[IN:%.*]], double [[IN]]) #[[ATTR44:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fmin(double [[IN:%.*]], double [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @fmin_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -1991,12 +1991,12 @@ define void @fmin_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @fmin_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svfmin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svfmin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @fmin_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svfmin_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fmin(double [[IN:%.*]], double [[IN]]) #[[ATTR44:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fmin(double [[IN:%.*]], double [[IN]])
 ;
   entry:
   br label %for.body
@@ -2023,12 +2023,12 @@ define void @fmin_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @fmin_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fminf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fminf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @fmin_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fminf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fminf(float [[IN:%.*]], float [[IN]]) #[[ATTR45:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fminf(float [[IN:%.*]], float [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @fmin_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2036,12 +2036,12 @@ define void @fmin_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @fmin_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svfmin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svfmin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @fmin_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svfmin_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fminf(float [[IN:%.*]], float [[IN]]) #[[ATTR45:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fminf(float [[IN:%.*]], float [[IN]])
 ;
   entry:
   br label %for.body
@@ -2071,12 +2071,12 @@ define void @fmod_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @fmod_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmod(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmod(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @fmod_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmod(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fmod(double [[IN:%.*]], double [[IN]]) #[[ATTR46:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fmod(double [[IN:%.*]], double [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @fmod_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2084,12 +2084,12 @@ define void @fmod_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @fmod_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svfmod_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svfmod_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @fmod_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svfmod_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fmod(double [[IN:%.*]], double [[IN]]) #[[ATTR46:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @fmod(double [[IN:%.*]], double [[IN]])
 ;
   entry:
   br label %for.body
@@ -2116,12 +2116,12 @@ define void @fmod_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @fmod_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fmodf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fmodf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @fmod_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fmodf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fmodf(float [[IN:%.*]], float [[IN]]) #[[ATTR47:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fmodf(float [[IN:%.*]], float [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @fmod_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2129,12 +2129,12 @@ define void @fmod_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @fmod_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svfmod_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svfmod_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @fmod_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svfmod_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fmodf(float [[IN:%.*]], float [[IN]]) #[[ATTR47:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @fmodf(float [[IN:%.*]], float [[IN]])
 ;
   entry:
   br label %for.body
@@ -2164,12 +2164,12 @@ define void @hypot_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @hypot_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_hypot(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_hypot(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @hypot_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_hypot(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @hypot(double [[IN:%.*]], double [[IN]]) #[[ATTR48:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @hypot(double [[IN:%.*]], double [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @hypot_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2177,12 +2177,12 @@ define void @hypot_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @hypot_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svhypot_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svhypot_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @hypot_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svhypot_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @hypot(double [[IN:%.*]], double [[IN]]) #[[ATTR48:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @hypot(double [[IN:%.*]], double [[IN]])
 ;
   entry:
   br label %for.body
@@ -2209,12 +2209,12 @@ define void @hypot_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @hypot_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_hypotf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_hypotf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @hypot_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_hypotf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @hypotf(float [[IN:%.*]], float [[IN]]) #[[ATTR49:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @hypotf(float [[IN:%.*]], float [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @hypot_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2222,12 +2222,12 @@ define void @hypot_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @hypot_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svhypot_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svhypot_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @hypot_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svhypot_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @hypotf(float [[IN:%.*]], float [[IN]]) #[[ATTR49:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @hypotf(float [[IN:%.*]], float [[IN]])
 ;
   entry:
   br label %for.body
@@ -2257,12 +2257,12 @@ define void @ilogb_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @ilogb_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x i32> @_ZGVsMxv_ilogb(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x i32> @_ZGVsMxv_ilogb(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @ilogb_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x i32> @_ZGVsMxv_ilogb(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call i32 @ilogb(double [[IN:%.*]]) #[[ATTR50:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call i32 @ilogb(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @ilogb_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2270,12 +2270,12 @@ define void @ilogb_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @ilogb_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x i32> @armpl_svilogb_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x i32> @armpl_svilogb_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @ilogb_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x i32> @armpl_svilogb_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call i32 @ilogb(double [[IN:%.*]]) #[[ATTR50:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call i32 @ilogb(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -2302,12 +2302,12 @@ define void @ilogb_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @ilogb_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x i32> @_ZGVsMxv_ilogbf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x i32> @_ZGVsMxv_ilogbf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @ilogb_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x i32> @_ZGVsMxv_ilogbf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call i32 @ilogbf(float [[IN:%.*]]) #[[ATTR51:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call i32 @ilogbf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @ilogb_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2315,12 +2315,12 @@ define void @ilogb_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @ilogb_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x i32> @armpl_svilogb_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x i32> @armpl_svilogb_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @ilogb_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x i32> @armpl_svilogb_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call i32 @ilogbf(float [[IN:%.*]]) #[[ATTR51:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call i32 @ilogbf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -2350,12 +2350,12 @@ define void @ldexp_f64(ptr noalias %in1.ptr, ptr noalias %in2.ptr, ptr noalias %
 ;
 ; SLEEF-SVE-LABEL: define void @ldexp_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP17:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_ldexp(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i32> [[WIDE_MASKED_LOAD1:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_ldexp(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i32> [[WIDE_MASKED_LOAD1:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @ldexp_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP11:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_ldexp(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i32> [[WIDE_LOAD1:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @ldexp(double [[IN1:%.*]], i32 [[IN2:%.*]]) #[[ATTR52:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @ldexp(double [[IN1:%.*]], i32 [[IN2:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @ldexp_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2363,12 +2363,12 @@ define void @ldexp_f64(ptr noalias %in1.ptr, ptr noalias %in2.ptr, ptr noalias %
 ;
 ; ARMPL-SVE-LABEL: define void @ldexp_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP17:%.*]] = call <vscale x 2 x double> @armpl_svldexp_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i32> [[WIDE_MASKED_LOAD1:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svldexp_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i32> [[WIDE_MASKED_LOAD1:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @ldexp_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP11:%.*]] = call <vscale x 2 x double> @armpl_svldexp_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i32> [[WIDE_LOAD1:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @ldexp(double [[IN1:%.*]], i32 [[IN2:%.*]]) #[[ATTR52:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @ldexp(double [[IN1:%.*]], i32 [[IN2:%.*]])
 ;
   entry:
   br label %for.body
@@ -2397,12 +2397,12 @@ define void @ldexp_f32(ptr noalias %in1.ptr, ptr noalias %in2.ptr, ptr noalias %
 ;
 ; SLEEF-SVE-LABEL: define void @ldexp_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP17:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_ldexpf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD1:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_ldexpf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD1:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @ldexp_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP11:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_ldexpf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i32> [[WIDE_LOAD1:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @ldexpf(float [[IN1:%.*]], i32 [[IN2:%.*]]) #[[ATTR53:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @ldexpf(float [[IN1:%.*]], i32 [[IN2:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @ldexp_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2410,12 +2410,12 @@ define void @ldexp_f32(ptr noalias %in1.ptr, ptr noalias %in2.ptr, ptr noalias %
 ;
 ; ARMPL-SVE-LABEL: define void @ldexp_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP17:%.*]] = call <vscale x 4 x float> @armpl_svldexp_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD1:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svldexp_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD1:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @ldexp_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN1_PTR:%.*]], ptr noalias [[IN2_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP11:%.*]] = call <vscale x 4 x float> @armpl_svldexp_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i32> [[WIDE_LOAD1:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @ldexpf(float [[IN1:%.*]], i32 [[IN2:%.*]]) #[[ATTR53:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @ldexpf(float [[IN1:%.*]], i32 [[IN2:%.*]])
 ;
   entry:
   br label %for.body
@@ -2447,12 +2447,12 @@ define void @lgamma_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @lgamma_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_lgamma(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_lgamma(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @lgamma_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_lgamma(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @lgamma(double [[IN:%.*]]) #[[ATTR54:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @lgamma(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @lgamma_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2460,12 +2460,12 @@ define void @lgamma_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @lgamma_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svlgamma_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svlgamma_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @lgamma_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svlgamma_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @lgamma(double [[IN:%.*]]) #[[ATTR54:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @lgamma(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -2492,12 +2492,12 @@ define void @lgamma_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @lgamma_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_lgammaf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_lgammaf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @lgamma_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_lgammaf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @lgammaf(float [[IN:%.*]]) #[[ATTR55:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @lgammaf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @lgamma_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2505,12 +2505,12 @@ define void @lgamma_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @lgamma_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svlgamma_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svlgamma_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @lgamma_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svlgamma_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @lgammaf(float [[IN:%.*]]) #[[ATTR55:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @lgammaf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -2540,12 +2540,12 @@ define void @log_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @log_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @log_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log(double [[IN:%.*]]) #[[ATTR56:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2553,12 +2553,12 @@ define void @log_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @log_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svlog_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svlog_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @log_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svlog_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log(double [[IN:%.*]]) #[[ATTR56:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -2585,12 +2585,12 @@ define void @log_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @log_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_logf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_logf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @log_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_logf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @logf(float [[IN:%.*]]) #[[ATTR57:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @logf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2598,12 +2598,12 @@ define void @log_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @log_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svlog_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svlog_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @log_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svlog_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @logf(float [[IN:%.*]]) #[[ATTR57:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @logf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -2633,12 +2633,12 @@ define void @log10_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @log10_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log10(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log10(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @log10_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log10(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log10(double [[IN:%.*]]) #[[ATTR58:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log10(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log10_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2646,12 +2646,12 @@ define void @log10_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @log10_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svlog10_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svlog10_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @log10_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svlog10_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log10(double [[IN:%.*]]) #[[ATTR58:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log10(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -2678,12 +2678,12 @@ define void @log10_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @log10_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log10f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log10f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @log10_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log10f(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @log10f(float [[IN:%.*]]) #[[ATTR59:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @log10f(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log10_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2691,12 +2691,12 @@ define void @log10_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @log10_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svlog10_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svlog10_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @log10_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svlog10_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @log10f(float [[IN:%.*]]) #[[ATTR59:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @log10f(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -2726,12 +2726,12 @@ define void @log1p_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @log1p_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log1p(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log1p(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @log1p_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log1p(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log1p(double [[IN:%.*]]) #[[ATTR60:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log1p(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log1p_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2739,12 +2739,12 @@ define void @log1p_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @log1p_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svlog1p_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svlog1p_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @log1p_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svlog1p_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log1p(double [[IN:%.*]]) #[[ATTR60:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log1p(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -2771,12 +2771,12 @@ define void @log1p_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @log1p_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log1pf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log1pf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @log1p_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log1pf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @log1pf(float [[IN:%.*]]) #[[ATTR61:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @log1pf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log1p_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2784,12 +2784,12 @@ define void @log1p_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @log1p_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svlog1p_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svlog1p_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @log1p_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svlog1p_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @log1pf(float [[IN:%.*]]) #[[ATTR61:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @log1pf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -2819,12 +2819,12 @@ define void @log2_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @log2_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @log2_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log2(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log2(double [[IN:%.*]]) #[[ATTR62:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log2(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log2_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2832,12 +2832,12 @@ define void @log2_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @log2_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svlog2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svlog2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @log2_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svlog2_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log2(double [[IN:%.*]]) #[[ATTR62:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @log2(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -2864,12 +2864,12 @@ define void @log2_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @log2_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @log2_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log2f(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @log2f(float [[IN:%.*]]) #[[ATTR63:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @log2f(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log2_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -2877,12 +2877,12 @@ define void @log2_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @log2_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svlog2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svlog2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @log2_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svlog2_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @log2f(float [[IN:%.*]]) #[[ATTR63:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @log2f(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -3006,12 +3006,12 @@ define void @nextafter_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @nextafter_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_nextafter(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_nextafter(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @nextafter_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_nextafter(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @nextafter(double [[IN:%.*]], double [[IN]]) #[[ATTR66:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @nextafter(double [[IN:%.*]], double [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @nextafter_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3019,12 +3019,12 @@ define void @nextafter_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @nextafter_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svnextafter_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svnextafter_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @nextafter_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svnextafter_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @nextafter(double [[IN:%.*]], double [[IN]]) #[[ATTR66:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @nextafter(double [[IN:%.*]], double [[IN]])
 ;
   entry:
   br label %for.body
@@ -3051,12 +3051,12 @@ define void @nextafter_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @nextafter_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_nextafterf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_nextafterf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @nextafter_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_nextafterf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @nextafterf(float [[IN:%.*]], float [[IN]]) #[[ATTR67:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @nextafterf(float [[IN:%.*]], float [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @nextafter_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3064,12 +3064,12 @@ define void @nextafter_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @nextafter_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svnextafter_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svnextafter_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @nextafter_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svnextafter_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @nextafterf(float [[IN:%.*]], float [[IN]]) #[[ATTR67:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @nextafterf(float [[IN:%.*]], float [[IN]])
 ;
   entry:
   br label %for.body
@@ -3099,12 +3099,12 @@ define void @pow_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @pow_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_pow(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_pow(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @pow_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_pow(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @pow(double [[IN:%.*]], double [[IN]]) #[[ATTR68:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @pow(double [[IN:%.*]], double [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @pow_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3112,12 +3112,12 @@ define void @pow_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @pow_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svpow_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svpow_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @pow_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svpow_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @pow(double [[IN:%.*]], double [[IN]]) #[[ATTR68:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @pow(double [[IN:%.*]], double [[IN]])
 ;
   entry:
   br label %for.body
@@ -3144,12 +3144,12 @@ define void @pow_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @pow_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_powf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_powf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @pow_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_powf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @powf(float [[IN:%.*]], float [[IN]]) #[[ATTR69:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @powf(float [[IN:%.*]], float [[IN]])
 ;
 ; ARMPL-NEON-LABEL: define void @pow_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3157,12 +3157,12 @@ define void @pow_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @pow_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svpow_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svpow_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @pow_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svpow_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @powf(float [[IN:%.*]], float [[IN]]) #[[ATTR69:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @powf(float [[IN:%.*]], float [[IN]])
 ;
   entry:
   br label %for.body
@@ -3192,12 +3192,12 @@ define void @sin_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @sin_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @sin_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sin(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sin(double [[IN:%.*]]) #[[ATTR70:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sin(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sin_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3205,12 +3205,12 @@ define void @sin_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @sin_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svsin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svsin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @sin_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svsin_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sin(double [[IN:%.*]]) #[[ATTR70:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sin(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -3237,12 +3237,12 @@ define void @sin_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @sin_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @sin_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sinf(float [[IN:%.*]]) #[[ATTR71:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sinf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sin_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3250,12 +3250,12 @@ define void @sin_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @sin_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svsin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svsin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @sin_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svsin_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sinf(float [[IN:%.*]]) #[[ATTR71:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sinf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -3469,12 +3469,12 @@ define void @sinh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @sinh_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @sinh_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinh(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sinh(double [[IN:%.*]]) #[[ATTR76:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sinh(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sinh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3482,12 +3482,12 @@ define void @sinh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @sinh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svsinh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svsinh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @sinh_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svsinh_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sinh(double [[IN:%.*]]) #[[ATTR76:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sinh(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -3514,12 +3514,12 @@ define void @sinh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @sinh_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @sinh_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinhf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sinhf(float [[IN:%.*]]) #[[ATTR77:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sinhf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sinh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3527,12 +3527,12 @@ define void @sinh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @sinh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svsinh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svsinh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @sinh_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svsinh_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sinhf(float [[IN:%.*]]) #[[ATTR77:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sinhf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -3562,12 +3562,12 @@ define void @sinpi_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @sinpi_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinpi(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinpi(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @sinpi_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinpi(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sinpi(double [[IN:%.*]]) #[[ATTR78:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sinpi(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sinpi_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3575,12 +3575,12 @@ define void @sinpi_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @sinpi_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svsinpi_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svsinpi_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @sinpi_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svsinpi_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sinpi(double [[IN:%.*]]) #[[ATTR78:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sinpi(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -3607,12 +3607,12 @@ define void @sinpi_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @sinpi_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinpif(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinpif(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @sinpi_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinpif(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sinpif(float [[IN:%.*]]) #[[ATTR79:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sinpif(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sinpi_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3620,12 +3620,12 @@ define void @sinpi_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @sinpi_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svsinpi_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svsinpi_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @sinpi_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svsinpi_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sinpif(float [[IN:%.*]]) #[[ATTR79:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sinpif(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -3655,12 +3655,12 @@ define void @sqrt_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @sqrt_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sqrt(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sqrt(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @sqrt_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sqrt(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sqrt(double [[IN:%.*]]) #[[ATTR80:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sqrt(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sqrt_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3668,12 +3668,12 @@ define void @sqrt_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @sqrt_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svsqrt_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svsqrt_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @sqrt_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svsqrt_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sqrt(double [[IN:%.*]]) #[[ATTR80:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @sqrt(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -3700,12 +3700,12 @@ define void @sqrt_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @sqrt_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sqrtf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sqrtf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @sqrt_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sqrtf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sqrtf(float [[IN:%.*]]) #[[ATTR81:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sqrtf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sqrt_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3713,12 +3713,12 @@ define void @sqrt_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @sqrt_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svsqrt_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svsqrt_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @sqrt_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svsqrt_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sqrtf(float [[IN:%.*]]) #[[ATTR81:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @sqrtf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -3748,12 +3748,12 @@ define void @tan_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @tan_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tan(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tan(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @tan_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tan(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @tan(double [[IN:%.*]]) #[[ATTR82:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @tan(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @tan_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3761,12 +3761,12 @@ define void @tan_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @tan_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svtan_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svtan_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @tan_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svtan_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @tan(double [[IN:%.*]]) #[[ATTR82:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @tan(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -3793,12 +3793,12 @@ define void @tan_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @tan_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @tan_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @tanf(float [[IN:%.*]]) #[[ATTR83:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @tanf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @tan_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3806,12 +3806,12 @@ define void @tan_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @tan_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svtan_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svtan_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @tan_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svtan_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @tanf(float [[IN:%.*]]) #[[ATTR83:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @tanf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -3841,12 +3841,12 @@ define void @tanh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @tanh_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tanh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tanh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @tanh_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tanh(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @tanh(double [[IN:%.*]]) #[[ATTR84:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @tanh(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @tanh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3854,12 +3854,12 @@ define void @tanh_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @tanh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svtanh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svtanh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @tanh_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svtanh_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @tanh(double [[IN:%.*]]) #[[ATTR84:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @tanh(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -3886,12 +3886,12 @@ define void @tanh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @tanh_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @tanh_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanhf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @tanhf(float [[IN:%.*]]) #[[ATTR85:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @tanhf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @tanh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3899,12 +3899,12 @@ define void @tanh_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @tanh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svtanh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svtanh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @tanh_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svtanh_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @tanhf(float [[IN:%.*]]) #[[ATTR85:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @tanhf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -3934,12 +3934,12 @@ define void @tgamma_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @tgamma_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tgamma(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tgamma(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @tgamma_f64
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tgamma(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @tgamma(double [[IN:%.*]]) #[[ATTR86:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call double @tgamma(double [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @tgamma_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3947,12 +3947,12 @@ define void @tgamma_f64(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @tgamma_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svtgamma_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svtgamma_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @tgamma_f64
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 2 x double> @armpl_svtgamma_f64_x(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @tgamma(double [[IN:%.*]]) #[[ATTR86:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call double @tgamma(double [[IN:%.*]])
 ;
   entry:
   br label %for.body
@@ -3979,12 +3979,12 @@ define void @tgamma_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @tgamma_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tgammaf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tgammaf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; SLEEF-SVE-NOPRED-LABEL: define void @tgamma_f32
 ; SLEEF-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; SLEEF-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tgammaf(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @tgammaf(float [[IN:%.*]]) #[[ATTR87:[0-9]+]]
+; SLEEF-SVE-NOPRED:    [[CALL:%.*]] = tail call float @tgammaf(float [[IN:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @tgamma_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
@@ -3992,12 +3992,12 @@ define void @tgamma_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @tgamma_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svtgamma_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svtgamma_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-SVE-NOPRED-LABEL: define void @tgamma_f32
 ; ARMPL-SVE-NOPRED-SAME: (ptr noalias [[IN_PTR:%.*]], ptr noalias [[OUT_PTR:%.*]]) #[[ATTR0]] {
 ; ARMPL-SVE-NOPRED:    [[TMP9:%.*]] = call <vscale x 4 x float> @armpl_svtgamma_f32_x(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @tgammaf(float [[IN:%.*]]) #[[ATTR87:[0-9]+]]
+; ARMPL-SVE-NOPRED:    [[CALL:%.*]] = tail call float @tgammaf(float [[IN:%.*]])
 ;
   entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
index a107013124a71..f3b47d04d216f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "call.*(acos|asin|atan|cos|cosh|exp|log|sin|sinh|pow|ceil|copysign|fabs|floor|fma|m..num|nearbyint|rint|round|sqrt|tan|tanh|trunc)" --version 2
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --scrub-attributes --filter "call.*(acos|asin|atan|cos|cosh|exp|log|sin|sinh|pow|ceil|copysign|fabs|floor|fma|m..num|nearbyint|rint|round|sqrt|tan|tanh|trunc)" --version 2
 
 ; RUN: opt -mattr=+neon -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -S < %s | FileCheck %s --check-prefix=SLEEF-NEON
 ; RUN: opt -mattr=+sve -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S < %s | FileCheck %s --check-prefix=SLEEF-SVE
@@ -22,7 +22,7 @@ define void @acos_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @acos_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1:[0-9]+]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acos(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acos(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @acos_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1:[0-9]+]] {
@@ -30,7 +30,7 @@ define void @acos_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @acos_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1:[0-9]+]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svacos_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svacos_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -57,7 +57,7 @@ define void @acos_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @acos_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acosf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acosf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @acos_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -65,7 +65,7 @@ define void @acos_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @acos_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svacos_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svacos_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -95,7 +95,7 @@ define void @asin_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @asin_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @asin_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -103,7 +103,7 @@ define void @asin_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @asin_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svasin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svasin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -130,7 +130,7 @@ define void @asin_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @asin_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @asin_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -138,7 +138,7 @@ define void @asin_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @asin_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svasin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svasin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -168,7 +168,7 @@ define void @atan_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @atan_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atan(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atan(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @atan_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -176,7 +176,7 @@ define void @atan_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @atan_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svatan_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svatan_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -203,7 +203,7 @@ define void @atan_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @atan_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @atan_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -211,7 +211,7 @@ define void @atan_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @atan_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svatan_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svatan_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -241,7 +241,7 @@ define void @ceil_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @ceil_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @ceil_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -249,7 +249,7 @@ define void @ceil_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @ceil_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -276,7 +276,7 @@ define void @ceil_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @ceil_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.ceil.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.ceil.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @ceil_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -284,7 +284,7 @@ define void @ceil_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @ceil_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.ceil.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.ceil.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -314,7 +314,7 @@ define void @copysign_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @copysign_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.copysign.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.copysign.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
 ;
 ; ARMPL-NEON-LABEL: define void @copysign_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -322,7 +322,7 @@ define void @copysign_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @copysign_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.copysign.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.copysign.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
 ;
   entry:
   br label %for.body
@@ -349,7 +349,7 @@ define void @copysign_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @copysign_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.copysign.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.copysign.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
 ;
 ; ARMPL-NEON-LABEL: define void @copysign_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -357,7 +357,7 @@ define void @copysign_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @copysign_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.copysign.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.copysign.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
 ;
   entry:
   br label %for.body
@@ -387,7 +387,7 @@ define void @cos_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @cos_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cos(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cos(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cos_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -395,7 +395,7 @@ define void @cos_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @cos_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svcos_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svcos_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -422,7 +422,7 @@ define void @cos_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @cos_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cosf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cosf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cos_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -430,7 +430,7 @@ define void @cos_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @cos_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svcos_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svcos_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -460,7 +460,7 @@ define void @cosh_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @cosh_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cosh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cosh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cosh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -468,7 +468,7 @@ define void @cosh_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @cosh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svcosh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svcosh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -495,7 +495,7 @@ define void @cosh_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @cosh_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_coshf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_coshf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @cosh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -503,7 +503,7 @@ define void @cosh_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @cosh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svcosh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svcosh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -533,7 +533,7 @@ define void @exp_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @exp_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -541,7 +541,7 @@ define void @exp_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @exp_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svexp_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svexp_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -568,7 +568,7 @@ define void @exp_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @exp_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -576,7 +576,7 @@ define void @exp_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @exp_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svexp_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svexp_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -606,7 +606,7 @@ define void @exp10_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @exp10_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp10(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp10(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp10_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -614,7 +614,7 @@ define void @exp10_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @exp10_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svexp10_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svexp10_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -641,7 +641,7 @@ define void @exp10_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @exp10_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp10f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp10f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp10_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -649,7 +649,7 @@ define void @exp10_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @exp10_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svexp10_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svexp10_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -679,7 +679,7 @@ define void @exp2_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @exp2_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp2_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -687,7 +687,7 @@ define void @exp2_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @exp2_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svexp2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svexp2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -714,7 +714,7 @@ define void @exp2_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @exp2_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @exp2_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -722,7 +722,7 @@ define void @exp2_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @exp2_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svexp2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svexp2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -752,7 +752,7 @@ define void @fabs_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @fabs_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.fabs.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.fabs.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @fabs_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -760,7 +760,7 @@ define void @fabs_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @fabs_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.fabs.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.fabs.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -787,7 +787,7 @@ define void @fabs_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @fabs_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @fabs_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -795,7 +795,7 @@ define void @fabs_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @fabs_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -825,7 +825,7 @@ define void @floor_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @floor_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.floor.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.floor.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @floor_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -833,7 +833,7 @@ define void @floor_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @floor_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.floor.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.floor.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -860,7 +860,7 @@ define void @floor_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @floor_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.floor.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.floor.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @floor_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -868,7 +868,7 @@ define void @floor_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @floor_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.floor.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.floor.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -898,7 +898,7 @@ define void @fma_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @fma_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.fma.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.fma.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
 ;
 ; ARMPL-NEON-LABEL: define void @fma_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -906,7 +906,7 @@ define void @fma_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @fma_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.fma.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.fma.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
 ;
   entry:
   br label %for.body
@@ -933,7 +933,7 @@ define void @fma_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @fma_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
 ;
 ; ARMPL-NEON-LABEL: define void @fma_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -941,7 +941,7 @@ define void @fma_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @fma_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
 ;
   entry:
   br label %for.body
@@ -971,7 +971,7 @@ define void @log_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @log_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -979,7 +979,7 @@ define void @log_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @log_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svlog_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svlog_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1006,7 +1006,7 @@ define void @log_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @log_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_logf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_logf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1014,7 +1014,7 @@ define void @log_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @log_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svlog_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svlog_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1044,7 +1044,7 @@ define void @log10_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @log10_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log10(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log10(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log10_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1052,7 +1052,7 @@ define void @log10_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @log10_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svlog10_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svlog10_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1079,7 +1079,7 @@ define void @log10_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @log10_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log10f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log10f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log10_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1087,7 +1087,7 @@ define void @log10_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @log10_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svlog10_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svlog10_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1117,7 +1117,7 @@ define void @log2_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @log2_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log2(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log2_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1125,7 +1125,7 @@ define void @log2_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @log2_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svlog2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svlog2_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1152,7 +1152,7 @@ define void @log2_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @log2_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log2f(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @log2_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1160,7 +1160,7 @@ define void @log2_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @log2_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svlog2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svlog2_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1190,7 +1190,7 @@ define void @maxnum_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @maxnum_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.maxnum.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.maxnum.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
 ;
 ; ARMPL-NEON-LABEL: define void @maxnum_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1198,7 +1198,7 @@ define void @maxnum_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @maxnum_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.maxnum.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.maxnum.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
 ;
   entry:
   br label %for.body
@@ -1225,7 +1225,7 @@ define void @maxnum_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @maxnum_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
 ;
 ; ARMPL-NEON-LABEL: define void @maxnum_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1233,7 +1233,7 @@ define void @maxnum_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @maxnum_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
 ;
   entry:
   br label %for.body
@@ -1263,7 +1263,7 @@ define void @minnum_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @minnum_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.minnum.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.minnum.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
 ;
 ; ARMPL-NEON-LABEL: define void @minnum_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1271,7 +1271,7 @@ define void @minnum_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @minnum_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.minnum.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.minnum.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]])
 ;
   entry:
   br label %for.body
@@ -1298,7 +1298,7 @@ define void @minnum_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @minnum_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.minnum.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.minnum.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
 ;
 ; ARMPL-NEON-LABEL: define void @minnum_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1306,7 +1306,7 @@ define void @minnum_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @minnum_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.minnum.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.minnum.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]])
 ;
   entry:
   br label %for.body
@@ -1336,7 +1336,7 @@ define void @nearbyint_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @nearbyint_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.nearbyint.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.nearbyint.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @nearbyint_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1344,7 +1344,7 @@ define void @nearbyint_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @nearbyint_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.nearbyint.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.nearbyint.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -1371,7 +1371,7 @@ define void @nearbyint_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @nearbyint_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.nearbyint.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.nearbyint.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @nearbyint_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1379,7 +1379,7 @@ define void @nearbyint_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @nearbyint_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.nearbyint.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.nearbyint.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -1409,7 +1409,7 @@ define void @pow_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @pow_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_pow(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_pow(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @pow_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1417,7 +1417,7 @@ define void @pow_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @pow_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svpow_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svpow_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x double> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1444,7 +1444,7 @@ define void @pow_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @pow_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_powf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_powf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @pow_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1452,7 +1452,7 @@ define void @pow_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @pow_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svpow_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svpow_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1482,7 +1482,7 @@ define void @rint_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @rint_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.rint.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.rint.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @rint_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1490,7 +1490,7 @@ define void @rint_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @rint_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.rint.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.rint.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -1517,7 +1517,7 @@ define void @rint_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @rint_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.rint.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.rint.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @rint_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1525,7 +1525,7 @@ define void @rint_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @rint_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.rint.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.rint.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -1555,7 +1555,7 @@ define void @round_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @round_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.round.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.round.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @round_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1563,7 +1563,7 @@ define void @round_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @round_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.round.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.round.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -1590,7 +1590,7 @@ define void @round_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @round_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.round.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.round.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @round_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1598,7 +1598,7 @@ define void @round_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @round_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.round.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.round.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -1628,7 +1628,7 @@ define void @sin_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @sin_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sin(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sin_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1636,7 +1636,7 @@ define void @sin_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @sin_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svsin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svsin_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1663,7 +1663,7 @@ define void @sin_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @sin_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sin_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1671,7 +1671,7 @@ define void @sin_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @sin_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svsin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svsin_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1701,7 +1701,7 @@ define void @sinh_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @sinh_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sinh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1709,7 +1709,7 @@ define void @sinh_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @sinh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svsinh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svsinh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1736,7 +1736,7 @@ define void @sinh_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @sinh_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sinh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1744,7 +1744,7 @@ define void @sinh_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @sinh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svsinh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svsinh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1774,7 +1774,7 @@ define void @sqrt_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @sqrt_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sqrt_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1782,7 +1782,7 @@ define void @sqrt_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @sqrt_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -1809,7 +1809,7 @@ define void @sqrt_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @sqrt_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.sqrt.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.sqrt.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @sqrt_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1817,7 +1817,7 @@ define void @sqrt_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @sqrt_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.sqrt.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.sqrt.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -1847,7 +1847,7 @@ define void @tan_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @tan_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tan(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tan(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @tan_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1855,7 +1855,7 @@ define void @tan_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @tan_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svtan_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svtan_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1882,7 +1882,7 @@ define void @tan_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @tan_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @tan_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1890,7 +1890,7 @@ define void @tan_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @tan_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svtan_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svtan_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1920,7 +1920,7 @@ define void @tanh_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @tanh_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tanh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tanh(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @tanh_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1928,7 +1928,7 @@ define void @tanh_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @tanh_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @armpl_svtanh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @armpl_svtanh_f64_x(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1955,7 +1955,7 @@ define void @tanh_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @tanh_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanhf(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @tanh_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1963,7 +1963,7 @@ define void @tanh_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @tanh_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @armpl_svtanh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @armpl_svtanh_f32_x(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
 ;
   entry:
   br label %for.body
@@ -1993,7 +1993,7 @@ define void @trunc_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @trunc_f64
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @trunc_f64
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -2001,7 +2001,7 @@ define void @trunc_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @trunc_f64
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
@@ -2028,7 +2028,7 @@ define void @trunc_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; SLEEF-SVE-LABEL: define void @trunc_f32
 ; SLEEF-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; SLEEF-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; SLEEF-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
 ; ARMPL-NEON-LABEL: define void @trunc_f32
 ; ARMPL-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -2036,7 +2036,7 @@ define void @trunc_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; ARMPL-SVE-LABEL: define void @trunc_f32
 ; ARMPL-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; ARMPL-SVE:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; ARMPL-SVE:    [[TMP13:%.*]] = call <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
 ;
   entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index b6a9fed507acd..1d5e6c117a2ea 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -53,13 +53,13 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: Scalarizing: %cmp = icmp ugt i64 %indvars.iv, 1
 ; CHECK-NEXT:  LV: Scalarizing: %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:  VPlan 'Initial VPlan for VF={vscale x 4},UF>=1' {
-; CHECK-NEXT:  Live-in vp<%0> = VF * UF
-; CHECK-NEXT:  Live-in vp<%1> = vector-trip-count
-; CHECK-NEXT:  vp<%2> = original trip-count
+; CHECK-NEXT:  Live-in vp<[[VFxUF:%.+]]> = VF * UF
+; CHECK-NEXT:  Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT:  vp<[[TC:%.+]]> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body.preheader>:
 ; CHECK-NEXT:    IR %0 = zext i32 %n to i64
-; CHECK-NEXT:    EMIT vp<%2> = EXPAND SCEV (zext i32 %n to i64)
+; CHECK-NEXT:    EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64)
 ; CHECK-NEXT:  No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  vector.ph:
@@ -67,27 +67,27 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  <x1> vector loop: {
 ; CHECK-NEXT:    vector.body:
-; CHECK-NEXT:      EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%8>
-; CHECK-NEXT:      vp<%4> = DERIVED-IV ir<%n> + vp<%3> * ir<-1>
-; CHECK-NEXT:      vp<%5> = SCALAR-STEPS vp<%4>, ir<-1>
-; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<%5>, ir<-1>
+; CHECK-NEXT:      EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:      vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
+; CHECK-NEXT:      vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>
+; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
 ; CHECK-NEXT:      CLONE ir<%idxprom> = zext ir<%i.0>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT:      vp<%6> = vector-pointer (reverse) ir<%arrayidx>
-; CHECK-NEXT:      WIDEN ir<%1> = load vp<%6>
+; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx>
+; CHECK-NEXT:      WIDEN ir<%1> = load vp<[[VEC_PTR]]>
 ; CHECK-NEXT:      WIDEN ir<%add9> = add ir<%1>, ir<1>
 ; CHECK-NEXT:      CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT:      vp<%7> = vector-pointer (reverse) ir<%arrayidx3>
-; CHECK-NEXT:      WIDEN store vp<%7>, ir<%add9>
-; CHECK-NEXT:      EMIT vp<%8> = add nuw vp<%3>, vp<%0>
-; CHECK-NEXT:      EMIT branch-on-count vp<%8>, vp<%1>
+; CHECK-NEXT:      vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) ir<%arrayidx3>
+; CHECK-NEXT:      WIDEN store vp<[[VEC_PTR2]]>, ir<%add9>
+; CHECK-NEXT:      EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:      EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT:    No successors
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  middle.block:
-; CHECK-NEXT:    EMIT vp<%10> = icmp eq vp<%2>, vp<%1>
-; CHECK-NEXT:    EMIT branch-on-cond vp<%10>
+; CHECK-NEXT:    EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VEC_TC]]>
+; CHECK-NEXT:    EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT:  Successor(s): ir-bb<for.cond.cleanup.loopexit>, scalar.ph
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
@@ -137,13 +137,13 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LEV: Epilogue vectorization is not profitable for this loop
 ; CHECK-NEXT:  Executing best plan with VF=vscale x 4, UF=1
 ; CHECK-NEXT:  VPlan 'Final VPlan for VF={vscale x 4},UF={1}' {
-; CHECK-NEXT:  Live-in vp<%0> = VF * UF
-; CHECK-NEXT:  Live-in vp<%1> = vector-trip-count
-; CHECK-NEXT:  vp<%2> = original trip-count
+; CHECK-NEXT:  Live-in vp<[[VFxUF:%.+]]> = VF * UF
+; CHECK-NEXT:  Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT:  vp<[[TC:%.+]]> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body.preheader>:
 ; CHECK-NEXT:    IR %0 = zext i32 %n to i64
-; CHECK-NEXT:    EMIT vp<%2> = EXPAND SCEV (zext i32 %n to i64)
+; CHECK-NEXT:    EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64)
 ; CHECK-NEXT:  No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  vector.ph:
@@ -151,27 +151,27 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  <x1> vector loop: {
 ; CHECK-NEXT:    vector.body:
-; CHECK-NEXT:      EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%8>
-; CHECK-NEXT:      vp<%4> = DERIVED-IV ir<%n> + vp<%3> * ir<-1>
-; CHECK-NEXT:      vp<%5> = SCALAR-STEPS vp<%4>, ir<-1>
-; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<%5>, ir<-1>
+; CHECK-NEXT:      EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:      vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
+; CHECK-NEXT:      vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>
+; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
 ; CHECK-NEXT:      CLONE ir<%idxprom> = zext ir<%i.0>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT:      vp<%6> = vector-pointer (reverse) ir<%arrayidx>
-; CHECK-NEXT:      WIDEN ir<%13> = load vp<%6>
+; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx>
+; CHECK-NEXT:      WIDEN ir<%13> = load vp<[[VEC_PTR]]>
 ; CHECK-NEXT:      WIDEN ir<%add9> = add ir<%13>, ir<1>
 ; CHECK-NEXT:      CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT:      vp<%7> = vector-pointer (reverse) ir<%arrayidx3>
-; CHECK-NEXT:      WIDEN store vp<%7>, ir<%add9>
-; CHECK-NEXT:      EMIT vp<%8> = add nuw vp<%3>, vp<%0>
-; CHECK-NEXT:      EMIT branch-on-count vp<%8>, vp<%1>
+; CHECK-NEXT:      vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) ir<%arrayidx3>
+; CHECK-NEXT:      WIDEN store vp<[[VEC_PTR2]]>, ir<%add9>
+; CHECK-NEXT:      EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:      EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT:    No successors
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  middle.block:
-; CHECK-NEXT:    EMIT vp<%10> = icmp eq vp<%2>, vp<%1>
-; CHECK-NEXT:    EMIT branch-on-cond vp<%10>
+; CHECK-NEXT:    EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VEC_TC]]>
+; CHECK-NEXT:    EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT:  Successor(s): ir-bb<for.cond.cleanup.loopexit>, scalar.ph
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
@@ -257,13 +257,13 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: Scalarizing: %cmp = icmp ugt i64 %indvars.iv, 1
 ; CHECK-NEXT:  LV: Scalarizing: %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:  VPlan 'Initial VPlan for VF={vscale x 4},UF>=1' {
-; CHECK-NEXT:  Live-in vp<%0> = VF * UF
-; CHECK-NEXT:  Live-in vp<%1> = vector-trip-count
-; CHECK-NEXT:  vp<%2> = original trip-count
+; CHECK-NEXT:  Live-in vp<[[VFxUF:%.+]]> = VF * UF
+; CHECK-NEXT:  Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT:  vp<[[TC:%.+]]> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body.preheader>:
 ; CHECK-NEXT:    IR %0 = zext i32 %n to i64
-; CHECK-NEXT:    EMIT vp<%2> = EXPAND SCEV (zext i32 %n to i64)
+; CHECK-NEXT:    EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64)
 ; CHECK-NEXT:  No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  vector.ph:
@@ -271,27 +271,27 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  <x1> vector loop: {
 ; CHECK-NEXT:    vector.body:
-; CHECK-NEXT:      EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%8>
-; CHECK-NEXT:      vp<%4> = DERIVED-IV ir<%n> + vp<%3> * ir<-1>
-; CHECK-NEXT:      vp<%5> = SCALAR-STEPS vp<%4>, ir<-1>
-; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<%5>, ir<-1>
+; CHECK-NEXT:      EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:      vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
+; CHECK-NEXT:      vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>
+; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
 ; CHECK-NEXT:      CLONE ir<%idxprom> = zext ir<%i.0>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT:      vp<%6> = vector-pointer (reverse) ir<%arrayidx>
-; CHECK-NEXT:      WIDEN ir<%1> = load vp<%6>
+; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx>
+; CHECK-NEXT:      WIDEN ir<%1> = load vp<[[VEC_PTR]]>
 ; CHECK-NEXT:      WIDEN ir<%conv1> = fadd ir<%1>, ir<1.000000e+00>
 ; CHECK-NEXT:      CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT:      vp<%7> = vector-pointer (reverse) ir<%arrayidx3>
-; CHECK-NEXT:      WIDEN store vp<%7>, ir<%conv1>
-; CHECK-NEXT:      EMIT vp<%8> = add nuw vp<%3>, vp<%0>
-; CHECK-NEXT:      EMIT branch-on-count vp<%8>, vp<%1>
+; CHECK-NEXT:      vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) ir<%arrayidx3>
+; CHECK-NEXT:      WIDEN store vp<[[VEC_PTR2]]>, ir<%conv1>
+; CHECK-NEXT:      EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:      EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT:    No successors
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  middle.block:
-; CHECK-NEXT:    EMIT vp<%10> = icmp eq vp<%2>, vp<%1>
-; CHECK-NEXT:    EMIT branch-on-cond vp<%10>
+; CHECK-NEXT:    EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VEC_TC]]>
+; CHECK-NEXT:    EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT:  Successor(s): ir-bb<for.cond.cleanup.loopexit>, scalar.ph
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
@@ -341,13 +341,13 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LEV: Epilogue vectorization is not profitable for this loop
 ; CHECK-NEXT:  Executing best plan with VF=vscale x 4, UF=1
 ; CHECK-NEXT:  VPlan 'Final VPlan for VF={vscale x 4},UF={1}' {
-; CHECK-NEXT:  Live-in vp<%0> = VF * UF
-; CHECK-NEXT:  Live-in vp<%1> = vector-trip-count
-; CHECK-NEXT:  vp<%2> = original trip-count
+; CHECK-NEXT:  Live-in vp<[[VFxUF:%.+]]> = VF * UF
+; CHECK-NEXT:  Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT:  vp<[[TC:%.+]]> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.body.preheader>:
 ; CHECK-NEXT:    IR %0 = zext i32 %n to i64
-; CHECK-NEXT:    EMIT vp<%2> = EXPAND SCEV (zext i32 %n to i64)
+; CHECK-NEXT:    EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64)
 ; CHECK-NEXT:  No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  vector.ph:
@@ -355,27 +355,27 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  <x1> vector loop: {
 ; CHECK-NEXT:    vector.body:
-; CHECK-NEXT:      EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%8>
-; CHECK-NEXT:      vp<%4> = DERIVED-IV ir<%n> + vp<%3> * ir<-1>
-; CHECK-NEXT:      vp<%5> = SCALAR-STEPS vp<%4>, ir<-1>
-; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<%5>, ir<-1>
+; CHECK-NEXT:      EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:      vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
+; CHECK-NEXT:      vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>
+; CHECK-NEXT:      CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
 ; CHECK-NEXT:      CLONE ir<%idxprom> = zext ir<%i.0>
 ; CHECK-NEXT:      CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT:      vp<%6> = vector-pointer (reverse) ir<%arrayidx>
-; CHECK-NEXT:      WIDEN ir<%13> = load vp<%6>
+; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx>
+; CHECK-NEXT:      WIDEN ir<%13> = load vp<[[VEC_PTR]]>
 ; CHECK-NEXT:      WIDEN ir<%conv1> = fadd ir<%13>, ir<1.000000e+00>
 ; CHECK-NEXT:      CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT:      vp<%7> = vector-pointer (reverse) ir<%arrayidx3>
-; CHECK-NEXT:      WIDEN store vp<%7>, ir<%conv1>
-; CHECK-NEXT:      EMIT vp<%8> = add nuw vp<%3>, vp<%0>
-; CHECK-NEXT:      EMIT branch-on-count vp<%8>, vp<%1>
+; CHECK-NEXT:      vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx3>
+; CHECK-NEXT:      WIDEN store vp<[[VEC_PTR]]>, ir<%conv1>
+; CHECK-NEXT:      EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:      EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT:    No successors
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  middle.block:
-; CHECK-NEXT:    EMIT vp<%10> = icmp eq vp<%2>, vp<%1>
-; CHECK-NEXT:    EMIT branch-on-cond vp<%10>
+; CHECK-NEXT:    EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VEC_TC]]>
+; CHECK-NEXT:    EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT:  Successor(s): ir-bb<for.cond.cleanup.loopexit>, scalar.ph
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<for.cond.cleanup.loopexit>:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-constant-known-via-scev.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-constant-known-via-scev.ll
index 4c10ac3cd519e..6060a7318deb6 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-constant-known-via-scev.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-constant-known-via-scev.ll
@@ -58,9 +58,93 @@ exit:
   %ret = phi i64 [ %mul, %loop ]
   ret i64 %ret
 }
+
+; Test case for https://github.com/llvm/llvm-project/issues/109528.
+define i64 @second_lshr_operand_zero_via_scev() {
+; CHECK-LABEL: define i64 @second_lshr_operand_zero_via_scev() {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[EXT_0:%.*]] = sext i8 0 to i32
+; CHECK-NEXT:    br i1 true, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[STEP_ADD4:%.*]] = add <2 x i32> [[VEC_IND2]], <i32 2, i32 2>
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq <2 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i64> [[STEP_ADD]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = and <2 x i64> [[STEP_ADD]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr <2 x i32> [[VEC_IND2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr <2 x i32> [[STEP_ADD4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = zext <2 x i32> [[TMP4]] to <2 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = zext <2 x i32> [[TMP5]] to <2 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <2 x i1> [[TMP0]], <2 x i64> [[TMP2]], <2 x i64> [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <2 x i1> [[TMP1]], <2 x i64> [[TMP3]], <2 x i64> [[TMP7]]
+; CHECK-NEXT:    [[TMP10]] = or <2 x i64> [[TMP8]], [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP11]] = or <2 x i64> [[TMP9]], [[VEC_PHI1]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[STEP_ADD]], <i64 2, i64 2>
+; CHECK-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i32> [[STEP_ADD4]], <i32 2, i32 2>
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 0
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = or <2 x i64> [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[BIN_RDX]])
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOPS:.*]]
+; CHECK:       [[LOOPS]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOPS]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOPS]] ]
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i64 [[IV]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[TMP14]], [[EXT_0]]
+; CHECK-NEXT:    [[CONV_1:%.*]] = zext i32 [[SHR]] to i64
+; CHECK-NEXT:    [[RED_NEXT_V:%.*]] = select i1 [[C]], i64 [[AND]], i64 [[CONV_1]]
+; CHECK-NEXT:    [[RED_NEXT]] = or i64 [[RED_NEXT_V]], [[RED]]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 0
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOPS]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[RED_NEXT]], %[[LOOPS]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  %ext.0 = sext i8 0 to i32
+  br label %loops
+
+loops:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loops ]
+  %red = phi i64 [ 0, %entry ], [ %red.next, %loops ]
+  %c = icmp eq i64 %iv, 0
+  %and = and i64 %iv, 0
+  %0 = trunc i64 %iv to i32
+  %shr = lshr i32 %0, %ext.0
+  %conv.1 = zext i32 %shr to i64
+  %red.next.v = select i1 %c, i64 %and, i64 %conv.1
+  %red.next = or i64 %red.next.v, %red
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 0
+  br i1 %ec, label %exit, label %loops
+
+exit:
+  %res = phi i64 [ %red.next, %loops ]
+  ret i64 %res
+}
+
 ;.
 ; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
 ; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
 ; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
index 0072dd95bd098..7ae20a07290c3 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
@@ -52,3 +52,52 @@ loop.latch:
 exit:
   ret void
 }
+
+; Test case for https://github.com/llvm/llvm-project/issues/108697.
+define void @test_wide_shift_uses_predicated_invariant_instruction(i32 %d, i1 %c, ptr %dst) {
+; CHECK-LABEL: define void @test_wide_shift_uses_predicated_invariant_instruction(
+; CHECK-SAME: i32 [[D:%.*]], i1 [[C:%.*]], ptr [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP_LATCH]], label %[[ELSE:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 100, [[D]]
+; CHECK-NEXT:    [[SEXT:%.*]] = shl i32 [[REM]], 12
+; CHECK-NEXT:    [[SHL_I:%.*]] = shl i32 999, [[SEXT]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ [[SHL_I]], %[[ELSE]] ], [ 0, %[[LOOP_HEADER]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[IV]]
+; CHECK-NEXT:    store i32 [[P]], ptr [[GEP]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 100
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  br i1 %c, label %loop.latch, label %else
+
+else:
+  %rem = urem i32 100, %d
+  %sext = shl i32 %rem, 12
+  %shl.i = shl i32 999, %sext
+  br label %loop.latch
+
+loop.latch:
+  %p = phi i32 [ %shl.i, %else ], [ 0, %loop.header ]
+  %gep = getelementptr inbounds i32, ptr %dst, i32 %iv
+  store i32 %p, ptr %gep
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 100
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll
index e3820db0643a3..b061cf526b780 100644
--- a/llvm/test/Transforms/LoopVectorize/induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction.ll
@@ -1552,7 +1552,7 @@ define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) {
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; CHECK:       vector.memcheck:
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 4
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[P:%.*]], i64 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[N]], -1
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP3]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 3
@@ -1617,7 +1617,7 @@ define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) {
 ; IND-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp eq i32 [[TMP0]], 0
 ; IND-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; IND:       vector.memcheck:
-; IND-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 4
+; IND-NEXT:    [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[P:%.*]], i64 4
 ; IND-NEXT:    [[TMP3:%.*]] = add i32 [[N]], -1
 ; IND-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP3]] to i64
 ; IND-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 3
@@ -1680,7 +1680,7 @@ define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) {
 ; UNROLL-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 3
 ; UNROLL-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; UNROLL:       vector.memcheck:
-; UNROLL-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 4
+; UNROLL-NEXT:    [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[P:%.*]], i64 4
 ; UNROLL-NEXT:    [[TMP3:%.*]] = add i32 [[N]], -1
 ; UNROLL-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP3]] to i64
 ; UNROLL-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 3
@@ -1757,7 +1757,7 @@ define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) {
 ; UNROLL-NO-IC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
 ; UNROLL-NO-IC-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; UNROLL-NO-IC:       vector.memcheck:
-; UNROLL-NO-IC-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 4
+; UNROLL-NO-IC-NEXT:    [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[P:%.*]], i64 4
 ; UNROLL-NO-IC-NEXT:    [[TMP3:%.*]] = add i32 [[N]], -1
 ; UNROLL-NO-IC-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP3]] to i64
 ; UNROLL-NO-IC-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 3
@@ -1836,7 +1836,7 @@ define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) {
 ; INTERLEAVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 8
 ; INTERLEAVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; INTERLEAVE:       vector.memcheck:
-; INTERLEAVE-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 4
+; INTERLEAVE-NEXT:    [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[P:%.*]], i64 4
 ; INTERLEAVE-NEXT:    [[TMP3:%.*]] = add i32 [[N]], -1
 ; INTERLEAVE-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP3]] to i64
 ; INTERLEAVE-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 3
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
index 12d0078617d93..36ccbb14f986f 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
@@ -1478,7 +1478,7 @@ define void @PR34743(ptr %a, ptr %b, i64 %n) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = and i64 [[TMP2]], -4
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP3]]
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[TMP4]], i64 4
-; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 2
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr nuw i8, ptr [[A]], i64 2
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP3]]
 ; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[TMP5]], i64 6
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP2]]
diff --git a/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll b/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll
index 66a0c5fd807f2..a0cd3c64f2d77 100644
--- a/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll
@@ -38,12 +38,12 @@ define void @Test(ptr nocapture %obj, i64 %z) #0 {
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[I_NEXT:%.*]], [[DOTOUTER:%.*]] ]
 ; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[I]], 7
 ; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP3]], 256
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP4]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[OBJ]], i64 [[TMP4]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP2]], [[TMP3]]
 ; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP5]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw nsw i64 [[I]], 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP6]], 128
-; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP7]]
+; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr nuw i8, ptr [[OBJ]], i64 [[TMP7]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP6]], 132
 ; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP8]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[OBJ]], i64 0, i32 1, i64 [[I]]
@@ -107,12 +107,12 @@ define void @Test(ptr nocapture %obj, i64 %z) #0 {
 ; CHECK-NEXT:    br i1 [[EXITCOND_INNER]], label [[DOTOUTER]], label [[DOTINNER]], !llvm.loop [[LOOP11:![0-9]+]]
 ;
 ; CHECK-HOIST-LABEL: @Test(
-; CHECK-HOIST-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[OBJ:%.*]], i64 256
+; CHECK-HOIST-NEXT:    [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[OBJ:%.*]], i64 256
 ; CHECK-HOIST-NEXT:    [[TMP1:%.*]] = shl i64 [[Z:%.*]], 2
 ; CHECK-HOIST-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 4224
 ; CHECK-HOIST-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP2]]
 ; CHECK-HOIST-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[OBJ]], i64 [[TMP1]]
-; CHECK-HOIST-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[OBJ]], i64 128
+; CHECK-HOIST-NEXT:    [[SCEVGEP3:%.*]] = getelementptr nuw i8, ptr [[OBJ]], i64 128
 ; CHECK-HOIST-NEXT:    br label [[DOTOUTER_PREHEADER:%.*]]
 ; CHECK-HOIST:       .outer.preheader:
 ; CHECK-HOIST-NEXT:    [[I:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[I_NEXT:%.*]], [[DOTOUTER:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_early_exit.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_early_exit.ll
new file mode 100644
index 0000000000000..9dc76c6bf0c43
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/outer_loop_early_exit.ll
@@ -0,0 +1,49 @@
+; REQUIRES: asserts
+; RUN: opt -S -passes=loop-vectorize -enable-vplan-native-path -disable-output -debug 2>&1 < %s | FileCheck %s
+
+; CHECK-LABEL: LV: Found a loop: for.body
+; CHECK: LV: Not vectorizing: Unsupported conditional branch.
+; CHECK: loop not vectorized: loop control flow is not understood by vectorizer
+; CHECK: LV: Not vectorizing: Unsupported outer loop.
+
+@arr2 = external global [8 x i32], align 16
+@arr = external global [8 x [8 x i32]], align 16
+
+define i32 @foo(i32 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %iv.outer = phi i64 [ 0, %entry ], [%iv.outer.next, %for.inc ]
+  %arrayidx = getelementptr inbounds [8 x i32], ptr @arr2, i64 0, i64 %iv.outer
+  %ld1 = load i32, ptr %arrayidx, align 4
+  %0 = trunc i64 %iv.outer to i32
+  store i32 %0, ptr %arrayidx, align 4
+  %1 = trunc i64 %iv.outer to i32
+  %add = add nsw i32 %1, %n
+  %cmp.early = icmp eq i32 %ld1, 3
+  br i1 %cmp.early, label %for.early, label %for.body.inner
+
+for.body.inner:
+  %iv.inner = phi i64 [ 0, %for.body ], [ %iv.inner.next, %for.body.inner ]
+  %arrayidx7 = getelementptr inbounds [8 x [8 x i32]], ptr @arr, i64 0, i64 %iv.inner, i64 %iv.outer
+  store i32 %add, ptr %arrayidx7, align 4
+  %iv.inner.next = add nuw nsw i64 %iv.inner, 1
+  %cmp.inner = icmp eq i64 %iv.inner.next, 8
+  br i1 %cmp.inner, label %for.inc, label %for.body.inner
+
+for.inc:
+  %iv.outer.next = add nuw nsw i64 %iv.outer, 1
+  %cmp.outer = icmp eq i64%iv.outer.next, 8
+  br i1 %cmp.outer, label %for.end, label %for.body, !llvm.loop !1
+
+for.early:
+  ret i32 1
+
+for.end:
+  ret i32 0
+}
+
+!1 = distinct !{!1, !2, !3}
+!2 = !{!"llvm.loop.vectorize.width", i32 4}
+!3 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/pr47343-expander-lcssa-after-cfg-update.ll b/llvm/test/Transforms/LoopVectorize/pr47343-expander-lcssa-after-cfg-update.ll
index aebe47c128797..6aaa44344ae46 100644
--- a/llvm/test/Transforms/LoopVectorize/pr47343-expander-lcssa-after-cfg-update.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr47343-expander-lcssa-after-cfg-update.ll
@@ -39,7 +39,7 @@ define void @f() {
 ; CHECK:       vector.memcheck:
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[TMP1]], i64 1
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr @f.e, [[SCEVGEP]]
-; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[TMP1]], getelementptr inbounds (i8, ptr @f.e, i64 4)
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[TMP1]], getelementptr inbounds nuw (i8, ptr @f.e, i64 4)
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll
index 3d044fde2d257..ecdc4ed416d47 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll
@@ -183,7 +183,7 @@ define void @nested_loop_outer_iv_addrec_invariant_in_inner1(ptr %a, ptr %b, i64
 
 ; CHECK:       outer.header:
 ; CHECK:         [[OUTER_IV_SHL_2:%.]] = shl i64 %outer.iv, 2
-; CHECK-NEXT:    [[A_GEP_UPPER:%.*]] = getelementptr i8, ptr %a, i64 [[OUTER_IV_SHL_2]]
+; CHECK-NEXT:    [[A_GEP_UPPER:%.*]] = getelementptr nuw i8, ptr %a, i64 [[OUTER_IV_SHL_2]]
 ; CHECK-NEXT:    [[OUTER_IV_4:%.]] = add i64 [[OUTER_IV_SHL_2]], 4
 ; CHECK-NEXT:    [[A_GEP_UPPER_4:%.*]] = getelementptr i8, ptr %a, i64 [[OUTER_IV_4]]
 ; CHECK:         [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4
@@ -233,7 +233,7 @@ define void @nested_loop_outer_iv_addrec_invariant_in_inner2(ptr %a, ptr %b, i64
 
 ; CHECK:       outer.header:
 ; CHECK:         [[OUTER_IV_SHL_2:%.]] = shl i64 %outer.iv, 2
-; CHECK-NEXT:    [[A_GEP_UPPER:%.*]] = getelementptr i8, ptr %a, i64 [[OUTER_IV_SHL_2]]
+; CHECK-NEXT:    [[A_GEP_UPPER:%.*]] = getelementptr nuw i8, ptr %a, i64 [[OUTER_IV_SHL_2]]
 ; CHECK-NEXT:    [[OUTER_IV_4:%.]] = add i64 [[OUTER_IV_SHL_2]], 4
 ; CHECK-NEXT:    [[A_GEP_UPPER_4:%.*]] = getelementptr i8, ptr %a, i64 [[OUTER_IV_4]]
 ; CHECK:         [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4
diff --git a/llvm/test/Transforms/LoopVersioning/add-phi-update-users.ll b/llvm/test/Transforms/LoopVersioning/add-phi-update-users.ll
index e326064175d18..16ad4bfed0fd3 100644
--- a/llvm/test/Transforms/LoopVersioning/add-phi-update-users.ll
+++ b/llvm/test/Transforms/LoopVersioning/add-phi-update-users.ll
@@ -27,7 +27,7 @@ define void @f1() {
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[T0]], i64 2
 ; CHECK-NEXT:    br label [[FOR_BODY_LVER_CHECK:%.*]]
 ; CHECK:       for.body.lver.check:
-; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[T0]], getelementptr inbounds (i8, ptr @b, i64 2)
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[T0]], getelementptr inbounds nuw (i8, ptr @b, i64 2)
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr @b, [[SCEVGEP]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[FOR_BODY_PH_LVER_ORIG:%.*]], label [[FOR_BODY_PH:%.*]]
diff --git a/llvm/test/Transforms/LoopVersioning/bound-check-partially-known.ll b/llvm/test/Transforms/LoopVersioning/bound-check-partially-known.ll
index 2fb58f5980ec8..22ea70b1f6162 100644
--- a/llvm/test/Transforms/LoopVersioning/bound-check-partially-known.ll
+++ b/llvm/test/Transforms/LoopVersioning/bound-check-partially-known.ll
@@ -11,14 +11,14 @@ define void @bound_check_partially_known_1(i32 %N) {
 ; CHECK-NEXT:  loop.lver.check:
 ; CHECK-NEXT:    [[N_EXT:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    [[TMP0:%.*]] = shl nuw nsw i64 [[N_EXT]], 3
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr @global, i64 [[TMP0]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr @global, i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[N_EXT]], 4
 ; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr @global, i64 [[TMP1]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP0]], 256000
 ; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr @global, i64 [[TMP2]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr @global, [[SCEVGEP1]]
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP2]]
-; CHECK-NEXT:    [[BOUND13:%.*]] = icmp ult ptr getelementptr inbounds (i8, ptr @global, i64 256000), [[SCEVGEP1]]
+; CHECK-NEXT:    [[BOUND13:%.*]] = icmp ult ptr getelementptr inbounds nuw (i8, ptr @global, i64 256000), [[SCEVGEP1]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND13]]
 ; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[LOOP_PH_LVER_ORIG:%.*]], label [[LOOP_PH:%.*]]
 ; CHECK:       loop.ph.lver.orig:
diff --git a/llvm/test/Transforms/PGOProfile/icp_vtable_cmp.ll b/llvm/test/Transforms/PGOProfile/icp_vtable_cmp.ll
index b6afce3d7c6d5..84bb7a5830af2 100644
--- a/llvm/test/Transforms/PGOProfile/icp_vtable_cmp.ll
+++ b/llvm/test/Transforms/PGOProfile/icp_vtable_cmp.ll
@@ -1,7 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 
+; Tests that ICP compares vtables by checking IR.
 ; RUN: opt < %s -passes='pgo-icall-prom' -pass-remarks=pgo-icall-prom -enable-vtable-profile-use -icp-max-num-vtable-last-candidate=2 -S 2>&1 | FileCheck %s --check-prefixes=VTABLE-COMMON,VTABLE-CMP
+; Require exactly one vtable candidate for each function candidate. Tests that ICP compares function by checking IR.
 ; RUN: opt < %s -passes='pgo-icall-prom' -pass-remarks=pgo-icall-prom -enable-vtable-profile-use -icp-max-num-vtable-last-candidate=1 -S 2>&1 | FileCheck %s --check-prefixes=VTABLE-COMMON,FUNC-CMP
+; On top of line 4, ignore 'Base1' and its derived types for vtable-based comparison. Tests that ICP compares functions.
+; RUN: opt < %s -passes='pgo-icall-prom' -pass-remarks=pgo-icall-prom -enable-vtable-profile-use -icp-max-num-vtable-last-candidate=2 -icp-ignored-base-types='Base1' -S 2>&1 | FileCheck %s --check-prefixes=VTABLE-COMMON,FUNC-CMP
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vec-shift.ll b/llvm/test/Transforms/PhaseOrdering/X86/vec-shift.ll
index 7ee9812b3e74c..841096d226f75 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vec-shift.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vec-shift.ll
@@ -17,13 +17,13 @@ define noundef i64 @foo(i64 noundef %0) {
 ; CHECK-NEXT:    ret i64 [[TMP3]]
 ;
 ; SSE-LABEL: @foo(
-; SSE-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP0:%.*]], 44
-; SSE-NEXT:    [[TMP3:%.*]] = sub nuw nsw i64 -17592186044416, [[TMP2]]
+; SSE-NEXT:    [[TMP2:%.*]] = xor i64 [[TMP0:%.*]], -1
+; SSE-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 44
 ; SSE-NEXT:    ret i64 [[TMP3]]
 ;
 ; AVX-LABEL: @foo(
-; AVX-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP0:%.*]], 44
-; AVX-NEXT:    [[TMP3:%.*]] = sub nuw nsw i64 -17592186044416, [[TMP2]]
+; AVX-NEXT:    [[TMP2:%.*]] = xor i64 [[TMP0:%.*]], -1
+; AVX-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 44
 ; AVX-NEXT:    ret i64 [[TMP3]]
 ;
   %2 = sub i64 1048575, %0
@@ -34,35 +34,35 @@ define noundef i64 @foo(i64 noundef %0) {
 define void @bar(ptr noundef %0) {
 ; SSE-LABEL: @bar(
 ; SSE-NEXT:    [[TMP2:%.*]] = load <2 x i64>, ptr [[TMP0:%.*]], align 8
-; SSE-NEXT:    [[TMP3:%.*]] = shl <2 x i64> [[TMP2]], <i64 44, i64 44>
-; SSE-NEXT:    [[TMP4:%.*]] = sub nuw nsw <2 x i64> <i64 -17592186044416, i64 -17592186044416>, [[TMP3]]
+; SSE-NEXT:    [[TMP3:%.*]] = xor <2 x i64> [[TMP2]], <i64 -1, i64 -1>
+; SSE-NEXT:    [[TMP4:%.*]] = shl <2 x i64> [[TMP3]], <i64 44, i64 44>
 ; SSE-NEXT:    store <2 x i64> [[TMP4]], ptr [[TMP0]], align 8
 ; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16
 ; SSE-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr [[TMP5]], align 8
-; SSE-NEXT:    [[TMP7:%.*]] = shl <2 x i64> [[TMP6]], <i64 44, i64 44>
-; SSE-NEXT:    [[TMP8:%.*]] = sub nuw nsw <2 x i64> <i64 -17592186044416, i64 -17592186044416>, [[TMP7]]
+; SSE-NEXT:    [[TMP7:%.*]] = xor <2 x i64> [[TMP6]], <i64 -1, i64 -1>
+; SSE-NEXT:    [[TMP8:%.*]] = shl <2 x i64> [[TMP7]], <i64 44, i64 44>
 ; SSE-NEXT:    store <2 x i64> [[TMP8]], ptr [[TMP5]], align 8
 ; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 32
 ; SSE-NEXT:    [[TMP10:%.*]] = load <2 x i64>, ptr [[TMP9]], align 8
-; SSE-NEXT:    [[TMP11:%.*]] = shl <2 x i64> [[TMP10]], <i64 44, i64 44>
-; SSE-NEXT:    [[TMP12:%.*]] = sub nuw nsw <2 x i64> <i64 -17592186044416, i64 -17592186044416>, [[TMP11]]
+; SSE-NEXT:    [[TMP11:%.*]] = xor <2 x i64> [[TMP10]], <i64 -1, i64 -1>
+; SSE-NEXT:    [[TMP12:%.*]] = shl <2 x i64> [[TMP11]], <i64 44, i64 44>
 ; SSE-NEXT:    store <2 x i64> [[TMP12]], ptr [[TMP9]], align 8
 ; SSE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 48
 ; SSE-NEXT:    [[TMP14:%.*]] = load <2 x i64>, ptr [[TMP13]], align 8
-; SSE-NEXT:    [[TMP15:%.*]] = shl <2 x i64> [[TMP14]], <i64 44, i64 44>
-; SSE-NEXT:    [[TMP16:%.*]] = sub nuw nsw <2 x i64> <i64 -17592186044416, i64 -17592186044416>, [[TMP15]]
+; SSE-NEXT:    [[TMP15:%.*]] = xor <2 x i64> [[TMP14]], <i64 -1, i64 -1>
+; SSE-NEXT:    [[TMP16:%.*]] = shl <2 x i64> [[TMP15]], <i64 44, i64 44>
 ; SSE-NEXT:    store <2 x i64> [[TMP16]], ptr [[TMP13]], align 8
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @bar(
 ; AVX-NEXT:    [[TMP2:%.*]] = load <4 x i64>, ptr [[TMP0:%.*]], align 8
-; AVX-NEXT:    [[TMP3:%.*]] = shl <4 x i64> [[TMP2]], <i64 44, i64 44, i64 44, i64 44>
-; AVX-NEXT:    [[TMP4:%.*]] = sub nuw nsw <4 x i64> <i64 -17592186044416, i64 -17592186044416, i64 -17592186044416, i64 -17592186044416>, [[TMP3]]
+; AVX-NEXT:    [[TMP3:%.*]] = xor <4 x i64> [[TMP2]], <i64 -1, i64 -1, i64 -1, i64 -1>
+; AVX-NEXT:    [[TMP4:%.*]] = shl <4 x i64> [[TMP3]], <i64 44, i64 44, i64 44, i64 44>
 ; AVX-NEXT:    store <4 x i64> [[TMP4]], ptr [[TMP0]], align 8
 ; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 32
 ; AVX-NEXT:    [[TMP6:%.*]] = load <4 x i64>, ptr [[TMP5]], align 8
-; AVX-NEXT:    [[TMP7:%.*]] = shl <4 x i64> [[TMP6]], <i64 44, i64 44, i64 44, i64 44>
-; AVX-NEXT:    [[TMP8:%.*]] = sub nuw nsw <4 x i64> <i64 -17592186044416, i64 -17592186044416, i64 -17592186044416, i64 -17592186044416>, [[TMP7]]
+; AVX-NEXT:    [[TMP7:%.*]] = xor <4 x i64> [[TMP6]], <i64 -1, i64 -1, i64 -1, i64 -1>
+; AVX-NEXT:    [[TMP8:%.*]] = shl <4 x i64> [[TMP7]], <i64 44, i64 44, i64 44, i64 44>
 ; AVX-NEXT:    store <4 x i64> [[TMP8]], ptr [[TMP5]], align 8
 ; AVX-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/reused-scalar-repeated-in-node.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/reused-scalar-repeated-in-node.ll
index dbc4f3d59d4f9..d6073ea4bbbae 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/reused-scalar-repeated-in-node.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/reused-scalar-repeated-in-node.ll
@@ -33,40 +33,40 @@ define void @test() {
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <16 x float> [[TMP6]], float [[I68]], i32 6
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <16 x float> [[TMP7]], float [[I66]], i32 7
 ; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <16 x float> [[TMP8]], float [[I72]], i32 13
-; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <16 x float> [[TMP9]], float [[I69]], i32 14
-; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <16 x float> [[TMP10]], float [[I67]], i32 15
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <16 x float> [[TMP9]], float [[I67]], i32 14
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <16 x float> [[TMP10]], float [[I69]], i32 15
 ; CHECK-NEXT:    br i1 poison, label %[[BB167:.*]], label %[[BB77:.*]]
 ; CHECK:       [[BB77]]:
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <16 x float> [[TMP11]], <16 x float> poison, <8 x i32> <i32 poison, i32 5, i32 6, i32 7, i32 15, i32 15, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <16 x float> [[TMP11]], <16 x float> poison, <8 x i32> <i32 poison, i32 5, i32 6, i32 7, i32 14, i32 14, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x float> [[TMP0]], <2 x float> poison, <16 x i32> <i32 poison, i32 poison, i32 1, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x float> [[TMP0]], <2 x float> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    br label %[[BB78:.*]]
 ; CHECK:       [[BB78]]:
 ; CHECK-NEXT:    [[TMP15:%.*]] = phi <8 x float> [ [[TMP12]], %[[BB77]] ], [ [[TMP30:%.*]], %[[BB78]] ]
 ; CHECK-NEXT:    [[TMP16:%.*]] = phi <2 x float> [ poison, %[[BB77]] ], [ [[TMP31:%.*]], %[[BB78]] ]
-; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <8 x float> [[TMP15]], <8 x float> poison, <16 x i32> <i32 0, i32 3, i32 1, i32 2, i32 3, i32 0, i32 2, i32 3, i32 2, i32 7, i32 2, i32 3, i32 0, i32 6, i32 7, i32 7>
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <8 x float> [[TMP15]], <8 x float> poison, <16 x i32> <i32 0, i32 3, i32 1, i32 2, i32 3, i32 0, i32 2, i32 3, i32 2, i32 6, i32 2, i32 3, i32 0, i32 7, i32 6, i32 6>
 ; CHECK-NEXT:    [[TMP18:%.*]] = fmul fast <16 x float> [[TMP17]], [[TMP13]]
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <8 x float> [[TMP15]], <8 x float> poison, <16 x i32> <i32 1, i32 poison, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 7, i32 6, i32 6>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <8 x float> [[TMP15]], <8 x float> poison, <16 x i32> <i32 1, i32 poison, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 6, i32 7, i32 7>
 ; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <2 x float> [[TMP16]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <16 x float> [[TMP19]], <16 x float> [[TMP20]], <16 x i32> <i32 0, i32 17, i32 2, i32 16, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <8 x float> [[TMP15]], <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <16 x float> [[TMP21]], <16 x float> [[TMP22]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 17, i32 6, i32 7, i32 8, i32 22, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <16 x float> [[TMP21]], <16 x float> [[TMP22]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 17, i32 6, i32 7, i32 8, i32 23, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <16 x float> [[TMP23]], <16 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 1, i32 5, i32 3, i32 1, i32 3, i32 9, i32 3, i32 1, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP25:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v2f32(<16 x float> [[TMP14]], <2 x float> [[TMP0]], i64 2)
 ; CHECK-NEXT:    [[TMP26:%.*]] = fmul fast <16 x float> [[TMP24]], [[TMP25]]
 ; CHECK-NEXT:    [[TMP27:%.*]] = fadd fast <16 x float> [[TMP26]], [[TMP18]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = fadd fast <16 x float> [[TMP27]], poison
 ; CHECK-NEXT:    [[TMP29:%.*]] = fadd fast <16 x float> [[TMP28]], poison
-; CHECK-NEXT:    [[TMP30]] = shufflevector <16 x float> [[TMP29]], <16 x float> poison, <8 x i32> <i32 12, i32 5, i32 6, i32 7, i32 15, i32 15, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP30]] = shufflevector <16 x float> [[TMP29]], <16 x float> poison, <8 x i32> <i32 12, i32 5, i32 6, i32 7, i32 14, i32 14, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP31]] = shufflevector <16 x float> [[TMP29]], <16 x float> poison, <2 x i32> <i32 10, i32 11>
 ; CHECK-NEXT:    br i1 poison, label %[[BB78]], label %[[BB167]]
 ; CHECK:       [[BB167]]:
 ; CHECK-NEXT:    [[TMP32:%.*]] = phi <16 x float> [ [[TMP11]], %[[BB64]] ], [ [[TMP29]], %[[BB78]] ]
-; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <16 x float> [[TMP32]], i32 15
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <16 x float> [[TMP32]], i32 14
 ; CHECK-NEXT:    store float [[TMP33]], ptr poison, align 1
 ; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <16 x float> [[TMP32]], i32 13
 ; CHECK-NEXT:    store float [[TMP34]], ptr poison, align 1
-; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <16 x float> [[TMP32]], i32 14
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <16 x float> [[TMP32]], i32 15
 ; CHECK-NEXT:    br i1 poison, label %[[BB186:.*]], label %[[BB184:.*]]
 ; CHECK:       [[BB184]]:
 ; CHECK-NEXT:    br label %[[BB185:.*]]
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
index 803af4d166b21..b38c636ccaf5d 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
@@ -54,38 +54,43 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; CHECK-NEXT:    [[ARRAYIDX13_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 5
 ; CHECK-NEXT:    [[ARRAYIDX15_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 5
 ; CHECK-NEXT:    [[TMP19:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ADD_PTR_1]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP192:%.*]] = zext <2 x i8> [[TMP19]] to <2 x i32>
-; CHECK-NEXT:    [[TMP21:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ADD_PTR64_1]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP22:%.*]] = zext <2 x i8> [[TMP21]] to <2 x i32>
-; CHECK-NEXT:    [[TMP23:%.*]] = sub <2 x i32> [[TMP192]], [[TMP22]]
-; CHECK-NEXT:    [[TMP29:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX3_2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP40:%.*]] = zext <2 x i8> [[TMP29]] to <2 x i32>
-; CHECK-NEXT:    [[TMP26:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX5_2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP27:%.*]] = zext <2 x i8> [[TMP26]] to <2 x i32>
-; CHECK-NEXT:    [[TMP24:%.*]] = sub <2 x i32> [[TMP40]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = load i8, ptr [[ADD_PTR_1]], align 1
+; CHECK-NEXT:    [[TMP21:%.*]] = zext <2 x i8> [[TMP19]] to <2 x i32>
+; CHECK-NEXT:    [[TMP84:%.*]] = zext i8 [[TMP29]] to i32
+; CHECK-NEXT:    [[TMP22:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ADD_PTR64_1]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP23:%.*]] = zext <2 x i8> [[TMP22]] to <2 x i32>
+; CHECK-NEXT:    [[TMP30:%.*]] = sub <2 x i32> [[TMP21]], [[TMP23]]
+; CHECK-NEXT:    [[TMP42:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX3_2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP26:%.*]] = zext <2 x i8> [[TMP42]] to <2 x i32>
+; CHECK-NEXT:    [[TMP27:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX5_2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP49:%.*]] = zext <2 x i8> [[TMP27]] to <2 x i32>
+; CHECK-NEXT:    [[TMP24:%.*]] = sub <2 x i32> [[TMP26]], [[TMP49]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = shl <2 x i32> [[TMP24]], <i32 16, i32 16>
-; CHECK-NEXT:    [[TMP30:%.*]] = add <2 x i32> [[TMP25]], [[TMP23]]
-; CHECK-NEXT:    [[TMP31:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX8_2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP32:%.*]] = zext <2 x i8> [[TMP31]] to <2 x i32>
-; CHECK-NEXT:    [[TMP33:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX10_2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP49:%.*]] = zext <2 x i8> [[TMP33]] to <2 x i32>
-; CHECK-NEXT:    [[TMP35:%.*]] = sub <2 x i32> [[TMP32]], [[TMP49]]
-; CHECK-NEXT:    [[TMP50:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX13_2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP51:%.*]] = zext <2 x i8> [[TMP50]] to <2 x i32>
-; CHECK-NEXT:    [[TMP38:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX15_2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP31:%.*]] = add <2 x i32> [[TMP25]], [[TMP30]]
+; CHECK-NEXT:    [[TMP32:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX8_2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP33:%.*]] = load i8, ptr [[ARRAYIDX8_2]], align 1
+; CHECK-NEXT:    [[TMP50:%.*]] = zext <2 x i8> [[TMP32]] to <2 x i32>
+; CHECK-NEXT:    [[TMP83:%.*]] = zext i8 [[TMP33]] to i32
+; CHECK-NEXT:    [[TMP35:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX10_2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP51:%.*]] = zext <2 x i8> [[TMP35]] to <2 x i32>
+; CHECK-NEXT:    [[TMP52:%.*]] = sub <2 x i32> [[TMP50]], [[TMP51]]
+; CHECK-NEXT:    [[TMP38:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX13_2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
 ; CHECK-NEXT:    [[TMP39:%.*]] = zext <2 x i8> [[TMP38]] to <2 x i32>
-; CHECK-NEXT:    [[TMP36:%.*]] = sub <2 x i32> [[TMP51]], [[TMP39]]
+; CHECK-NEXT:    [[TMP40:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX15_2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP56:%.*]] = zext <2 x i8> [[TMP40]] to <2 x i32>
+; CHECK-NEXT:    [[TMP36:%.*]] = sub <2 x i32> [[TMP39]], [[TMP56]]
 ; CHECK-NEXT:    [[TMP37:%.*]] = shl <2 x i32> [[TMP36]], <i32 16, i32 16>
-; CHECK-NEXT:    [[TMP42:%.*]] = add <2 x i32> [[TMP37]], [[TMP35]]
-; CHECK-NEXT:    [[TMP56:%.*]] = add <2 x i32> [[TMP42]], [[TMP30]]
-; CHECK-NEXT:    [[TMP60:%.*]] = sub <2 x i32> [[TMP30]], [[TMP42]]
-; CHECK-NEXT:    [[TMP73:%.*]] = extractelement <2 x i32> [[TMP56]], i32 0
-; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <2 x i32> [[TMP56]], i32 1
+; CHECK-NEXT:    [[TMP59:%.*]] = add <2 x i32> [[TMP37]], [[TMP52]]
+; CHECK-NEXT:    [[TMP63:%.*]] = add <2 x i32> [[TMP59]], [[TMP31]]
+; CHECK-NEXT:    [[TMP72:%.*]] = sub <2 x i32> [[TMP31]], [[TMP59]]
+; CHECK-NEXT:    [[TMP73:%.*]] = extractelement <2 x i32> [[TMP63]], i32 0
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <2 x i32> [[TMP63]], i32 1
 ; CHECK-NEXT:    [[ADD48_2:%.*]] = add i32 [[TMP34]], [[TMP73]]
 ; CHECK-NEXT:    [[SUB51_2:%.*]] = sub i32 [[TMP73]], [[TMP34]]
-; CHECK-NEXT:    [[TMP47:%.*]] = extractelement <2 x i32> [[TMP60]], i32 0
-; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <2 x i32> [[TMP60]], i32 1
-; CHECK-NEXT:    [[TMP68:%.*]] = sub i32 [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP47:%.*]] = extractelement <2 x i32> [[TMP72]], i32 0
+; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <2 x i32> [[TMP72]], i32 1
+; CHECK-NEXT:    [[ADD55_2:%.*]] = add i32 [[TMP48]], [[TMP47]]
+; CHECK-NEXT:    [[SUB59_2:%.*]] = sub i32 [[TMP47]], [[TMP48]]
 ; CHECK-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr i8, ptr null, i64 4
 ; CHECK-NEXT:    [[ARRAYIDX5_3:%.*]] = getelementptr i8, ptr null, i64 4
 ; CHECK-NEXT:    [[ARRAYIDX8_3:%.*]] = getelementptr i8, ptr null, i64 1
@@ -94,52 +99,55 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; CHECK-NEXT:    [[ARRAYIDX15_3:%.*]] = getelementptr i8, ptr null, i64 5
 ; CHECK-NEXT:    [[TMP43:%.*]] = load i8, ptr null, align 1
 ; CHECK-NEXT:    [[TMP53:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 null, i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP52:%.*]] = zext <2 x i8> [[TMP53]] to <2 x i32>
+; CHECK-NEXT:    [[TMP76:%.*]] = load i8, ptr null, align 1
+; CHECK-NEXT:    [[TMP55:%.*]] = zext <2 x i8> [[TMP53]] to <2 x i32>
+; CHECK-NEXT:    [[TMP77:%.*]] = zext i8 [[TMP76]] to i32
 ; CHECK-NEXT:    [[TMP54:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 null, i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP61:%.*]] = zext <2 x i8> [[TMP54]] to <2 x i32>
-; CHECK-NEXT:    [[TMP55:%.*]] = sub <2 x i32> [[TMP52]], [[TMP61]]
+; CHECK-NEXT:    [[TMP57:%.*]] = zext <2 x i8> [[TMP54]] to <2 x i32>
+; CHECK-NEXT:    [[TMP58:%.*]] = sub <2 x i32> [[TMP55]], [[TMP57]]
 ; CHECK-NEXT:    [[TMP41:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX3_3]], i64 -4, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP57:%.*]] = zext <2 x i8> [[TMP41]] to <2 x i32>
-; CHECK-NEXT:    [[TMP58:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX5_3]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP59:%.*]] = zext <2 x i8> [[TMP58]] to <2 x i32>
-; CHECK-NEXT:    [[TMP45:%.*]] = sub <2 x i32> [[TMP57]], [[TMP59]]
+; CHECK-NEXT:    [[TMP60:%.*]] = zext <2 x i8> [[TMP41]] to <2 x i32>
+; CHECK-NEXT:    [[TMP61:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX5_3]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP62:%.*]] = zext <2 x i8> [[TMP61]] to <2 x i32>
+; CHECK-NEXT:    [[TMP45:%.*]] = sub <2 x i32> [[TMP60]], [[TMP62]]
 ; CHECK-NEXT:    [[TMP46:%.*]] = shl <2 x i32> [[TMP45]], <i32 16, i32 16>
-; CHECK-NEXT:    [[TMP62:%.*]] = add <2 x i32> [[TMP46]], [[TMP55]]
-; CHECK-NEXT:    [[TMP63:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX8_3]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP74:%.*]] = zext <2 x i8> [[TMP63]] to <2 x i32>
-; CHECK-NEXT:    [[TMP79:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX10_3]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP82:%.*]] = zext <2 x i8> [[TMP79]] to <2 x i32>
-; CHECK-NEXT:    [[TMP85:%.*]] = sub <2 x i32> [[TMP74]], [[TMP82]]
+; CHECK-NEXT:    [[TMP82:%.*]] = add <2 x i32> [[TMP46]], [[TMP58]]
+; CHECK-NEXT:    [[TMP85:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX8_3]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP94:%.*]] = zext <2 x i8> [[TMP85]] to <2 x i32>
+; CHECK-NEXT:    [[TMP68:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX10_3]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP103:%.*]] = zext <2 x i8> [[TMP68]] to <2 x i32>
+; CHECK-NEXT:    [[TMP106:%.*]] = sub <2 x i32> [[TMP94]], [[TMP103]]
 ; CHECK-NEXT:    [[TMP64:%.*]] = insertelement <2 x i8> poison, i8 [[TMP44]], i32 0
 ; CHECK-NEXT:    [[TMP65:%.*]] = insertelement <2 x i8> [[TMP64]], i8 [[TMP43]], i32 1
-; CHECK-NEXT:    [[TMP94:%.*]] = zext <2 x i8> [[TMP65]] to <2 x i32>
-; CHECK-NEXT:    [[TMP103:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX15_3]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP72:%.*]] = zext <2 x i8> [[TMP103]] to <2 x i32>
-; CHECK-NEXT:    [[TMP69:%.*]] = sub <2 x i32> [[TMP94]], [[TMP72]]
+; CHECK-NEXT:    [[TMP108:%.*]] = zext <2 x i8> [[TMP65]] to <2 x i32>
+; CHECK-NEXT:    [[TMP74:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX15_3]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP75:%.*]] = zext <2 x i8> [[TMP74]] to <2 x i32>
+; CHECK-NEXT:    [[TMP69:%.*]] = sub <2 x i32> [[TMP108]], [[TMP75]]
 ; CHECK-NEXT:    [[TMP70:%.*]] = shl <2 x i32> [[TMP69]], <i32 16, i32 16>
-; CHECK-NEXT:    [[TMP75:%.*]] = add <2 x i32> [[TMP70]], [[TMP85]]
-; CHECK-NEXT:    [[TMP76:%.*]] = add <2 x i32> [[TMP75]], [[TMP62]]
-; CHECK-NEXT:    [[TMP106:%.*]] = sub <2 x i32> [[TMP62]], [[TMP75]]
-; CHECK-NEXT:    [[TMP78:%.*]] = extractelement <2 x i32> [[TMP76]], i32 0
-; CHECK-NEXT:    [[TMP71:%.*]] = extractelement <2 x i32> [[TMP76]], i32 1
+; CHECK-NEXT:    [[TMP109:%.*]] = add <2 x i32> [[TMP70]], [[TMP106]]
+; CHECK-NEXT:    [[TMP79:%.*]] = add <2 x i32> [[TMP109]], [[TMP82]]
+; CHECK-NEXT:    [[TMP115:%.*]] = sub <2 x i32> [[TMP82]], [[TMP109]]
+; CHECK-NEXT:    [[TMP78:%.*]] = extractelement <2 x i32> [[TMP79]], i32 0
+; CHECK-NEXT:    [[TMP71:%.*]] = extractelement <2 x i32> [[TMP79]], i32 1
 ; CHECK-NEXT:    [[ADD48_3:%.*]] = add i32 [[TMP71]], [[TMP78]]
 ; CHECK-NEXT:    [[SUB51_3:%.*]] = sub i32 [[TMP78]], [[TMP71]]
-; CHECK-NEXT:    [[TMP80:%.*]] = extractelement <2 x i32> [[TMP106]], i32 0
-; CHECK-NEXT:    [[TMP81:%.*]] = extractelement <2 x i32> [[TMP106]], i32 1
+; CHECK-NEXT:    [[TMP80:%.*]] = extractelement <2 x i32> [[TMP115]], i32 0
+; CHECK-NEXT:    [[TMP81:%.*]] = extractelement <2 x i32> [[TMP115]], i32 1
 ; CHECK-NEXT:    [[ADD55_3:%.*]] = add i32 [[TMP81]], [[TMP80]]
-; CHECK-NEXT:    [[TMP107:%.*]] = sub i32 [[TMP80]], [[TMP81]]
-; CHECK-NEXT:    [[TMP77:%.*]] = extractelement <2 x i32> [[TMP52]], i32 0
+; CHECK-NEXT:    [[SUB59_3:%.*]] = sub i32 [[TMP80]], [[TMP81]]
+; CHECK-NEXT:    [[ADD94:%.*]] = add i32 [[ADD48_3]], [[ADD48_2]]
+; CHECK-NEXT:    [[SUB102:%.*]] = sub i32 [[ADD48_2]], [[ADD48_3]]
 ; CHECK-NEXT:    [[SHR_I49_3:%.*]] = lshr i32 [[TMP77]], 15
 ; CHECK-NEXT:    [[AND_I50_3:%.*]] = and i32 [[SHR_I49_3]], 65537
 ; CHECK-NEXT:    [[MUL_I51_3:%.*]] = mul i32 [[AND_I50_3]], 65535
 ; CHECK-NEXT:    [[SHR_I_1:%.*]] = lshr i32 [[TMP34]], 15
 ; CHECK-NEXT:    [[AND_I_1:%.*]] = and i32 [[SHR_I_1]], 65537
 ; CHECK-NEXT:    [[MUL_I_1:%.*]] = mul i32 [[AND_I_1]], 65535
-; CHECK-NEXT:    [[TMP83:%.*]] = extractelement <2 x i32> [[TMP32]], i32 0
+; CHECK-NEXT:    [[ADD94_1:%.*]] = add i32 [[ADD55_3]], [[ADD55_2]]
+; CHECK-NEXT:    [[SUB102_1:%.*]] = sub i32 [[ADD55_2]], [[ADD55_3]]
 ; CHECK-NEXT:    [[SHR_I_2:%.*]] = lshr i32 [[TMP83]], 15
 ; CHECK-NEXT:    [[AND_I_2:%.*]] = and i32 [[SHR_I_2]], 65537
 ; CHECK-NEXT:    [[MUL_I_2:%.*]] = mul i32 [[AND_I_2]], 65535
-; CHECK-NEXT:    [[TMP84:%.*]] = extractelement <2 x i32> [[TMP192]], i32 0
 ; CHECK-NEXT:    [[SHR_I49_1:%.*]] = lshr i32 [[TMP84]], 15
 ; CHECK-NEXT:    [[AND_I50_1:%.*]] = and i32 [[SHR_I49_1]], 65537
 ; CHECK-NEXT:    [[ADD94_2:%.*]] = mul i32 [[AND_I50_1]], 65535
@@ -147,60 +155,62 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; CHECK-NEXT:    [[SHR_I49_2:%.*]] = lshr i32 [[CONV_1]], 15
 ; CHECK-NEXT:    [[AND_I50_2:%.*]] = and i32 [[SHR_I49_2]], 65537
 ; CHECK-NEXT:    [[MUL_I51_2:%.*]] = mul i32 [[AND_I50_2]], 65535
-; CHECK-NEXT:    [[ADD94_5:%.*]] = add i32 [[TMP107]], [[TMP68]]
-; CHECK-NEXT:    [[SUB102_3:%.*]] = sub i32 [[TMP68]], [[TMP107]]
+; CHECK-NEXT:    [[ADD94_5:%.*]] = add i32 [[SUB59_3]], [[SUB59_2]]
+; CHECK-NEXT:    [[SUB102_3:%.*]] = sub i32 [[SUB59_2]], [[SUB59_3]]
 ; CHECK-NEXT:    [[SHR_I49_4:%.*]] = lshr i32 [[CONV1]], 15
 ; CHECK-NEXT:    [[AND_I50_4:%.*]] = and i32 [[SHR_I49_4]], 65537
 ; CHECK-NEXT:    [[MUL_I51_4:%.*]] = mul i32 [[AND_I50_4]], 65535
 ; CHECK-NEXT:    [[TMP66:%.*]] = load <2 x i8>, ptr [[ARRAYIDX8]], align 1
 ; CHECK-NEXT:    [[TMP102:%.*]] = zext <2 x i8> [[TMP66]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP67:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[PIX2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP108:%.*]] = zext <2 x i8> [[TMP67]] to <2 x i32>
+; CHECK-NEXT:    [[TMP112:%.*]] = zext <2 x i8> [[TMP67]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP89:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[TMP1]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
 ; CHECK-NEXT:    [[TMP90:%.*]] = zext <2 x i8> [[TMP89]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP91:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX5]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP109:%.*]] = zext <2 x i8> [[TMP91]] to <2 x i32>
-; CHECK-NEXT:    [[TMP87:%.*]] = sub <2 x i32> [[TMP90]], [[TMP109]]
+; CHECK-NEXT:    [[TMP117:%.*]] = zext <2 x i8> [[TMP91]] to <2 x i32>
+; CHECK-NEXT:    [[TMP87:%.*]] = sub <2 x i32> [[TMP90]], [[TMP117]]
 ; CHECK-NEXT:    [[TMP88:%.*]] = shl <2 x i32> [[TMP87]], <i32 16, i32 16>
-; CHECK-NEXT:    [[TMP111:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX22]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP112:%.*]] = zext <2 x i8> [[TMP111]] to <2 x i32>
-; CHECK-NEXT:    [[TMP120:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX25]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP121:%.*]] = zext <2 x i8> [[TMP120]] to <2 x i32>
-; CHECK-NEXT:    [[TMP131:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX27]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT:    [[TMP100:%.*]] = zext <2 x i8> [[TMP131]] to <2 x i32>
-; CHECK-NEXT:    [[TMP95:%.*]] = sub <2 x i32> [[TMP121]], [[TMP100]]
+; CHECK-NEXT:    [[TMP119:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX22]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP121:%.*]] = zext <2 x i8> [[TMP119]] to <2 x i32>
+; CHECK-NEXT:    [[TMP128:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX25]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP131:%.*]] = zext <2 x i8> [[TMP128]] to <2 x i32>
+; CHECK-NEXT:    [[TMP132:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX27]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
+; CHECK-NEXT:    [[TMP100:%.*]] = zext <2 x i8> [[TMP132]] to <2 x i32>
+; CHECK-NEXT:    [[TMP95:%.*]] = sub <2 x i32> [[TMP131]], [[TMP100]]
 ; CHECK-NEXT:    [[TMP96:%.*]] = shl <2 x i32> [[TMP95]], <i32 16, i32 16>
 ; CHECK-NEXT:    [[TMP97:%.*]] = insertelement <2 x i32> [[TMP102]], i32 [[CONV33]], i32 1
-; CHECK-NEXT:    [[TMP132:%.*]] = sub <2 x i32> [[TMP97]], [[TMP112]]
-; CHECK-NEXT:    [[TMP105:%.*]] = add <2 x i32> [[TMP96]], [[TMP132]]
+; CHECK-NEXT:    [[TMP133:%.*]] = sub <2 x i32> [[TMP97]], [[TMP121]]
+; CHECK-NEXT:    [[TMP105:%.*]] = add <2 x i32> [[TMP96]], [[TMP133]]
 ; CHECK-NEXT:    [[TMP86:%.*]] = insertelement <2 x i32> [[TMP102]], i32 [[CONV1]], i32 0
-; CHECK-NEXT:    [[TMP133:%.*]] = sub <2 x i32> [[TMP86]], [[TMP108]]
-; CHECK-NEXT:    [[TMP92:%.*]] = add <2 x i32> [[TMP88]], [[TMP133]]
+; CHECK-NEXT:    [[TMP107:%.*]] = sub <2 x i32> [[TMP86]], [[TMP112]]
+; CHECK-NEXT:    [[TMP92:%.*]] = add <2 x i32> [[TMP88]], [[TMP107]]
 ; CHECK-NEXT:    [[TMP93:%.*]] = shufflevector <2 x i32> [[TMP105]], <2 x i32> [[TMP92]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP101:%.*]] = sub <2 x i32> [[TMP92]], [[TMP105]]
+; CHECK-NEXT:    [[TMP111:%.*]] = extractelement <2 x i32> [[TMP101]], i32 0
 ; CHECK-NEXT:    [[TMP99:%.*]] = extractelement <2 x i32> [[TMP101]], i32 1
+; CHECK-NEXT:    [[ADD55:%.*]] = add i32 [[TMP99]], [[TMP111]]
 ; CHECK-NEXT:    [[SHR_I59_1:%.*]] = lshr i32 [[TMP99]], 15
 ; CHECK-NEXT:    [[AND_I60_1:%.*]] = and i32 [[SHR_I59_1]], 65537
 ; CHECK-NEXT:    [[MUL_I61_1:%.*]] = mul i32 [[AND_I60_1]], 65535
 ; CHECK-NEXT:    [[TMP104:%.*]] = load <2 x i8>, ptr [[ARRAYIDX8_1]], align 1
 ; CHECK-NEXT:    [[TMP113:%.*]] = zext <2 x i8> [[TMP104]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP114:%.*]] = load <2 x i8>, ptr [[ADD_PTR644]], align 1
-; CHECK-NEXT:    [[TMP115:%.*]] = zext <2 x i8> [[TMP114]] to <2 x i32>
+; CHECK-NEXT:    [[TMP134:%.*]] = zext <2 x i8> [[TMP114]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP116:%.*]] = load <2 x i8>, ptr [[ARRAYIDX3_1]], align 1
-; CHECK-NEXT:    [[TMP117:%.*]] = zext <2 x i8> [[TMP116]] to <2 x i32>
+; CHECK-NEXT:    [[TMP145:%.*]] = zext <2 x i8> [[TMP116]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP118:%.*]] = load <2 x i8>, ptr [[ARRAYIDX5_1]], align 1
-; CHECK-NEXT:    [[TMP119:%.*]] = zext <2 x i8> [[TMP118]] to <2 x i32>
-; CHECK-NEXT:    [[TMP124:%.*]] = sub <2 x i32> [[TMP117]], [[TMP119]]
+; CHECK-NEXT:    [[TMP120:%.*]] = zext <2 x i8> [[TMP118]] to <2 x i32>
+; CHECK-NEXT:    [[TMP124:%.*]] = sub <2 x i32> [[TMP145]], [[TMP120]]
 ; CHECK-NEXT:    [[TMP125:%.*]] = shl <2 x i32> [[TMP124]], <i32 16, i32 16>
 ; CHECK-NEXT:    [[TMP122:%.*]] = shufflevector <2 x i32> [[TMP113]], <2 x i32> poison, <2 x i32> <i32 poison, i32 0>
 ; CHECK-NEXT:    [[TMP123:%.*]] = insertelement <2 x i32> [[TMP122]], i32 [[CONV_1]], i32 0
-; CHECK-NEXT:    [[TMP134:%.*]] = sub <2 x i32> [[TMP123]], [[TMP115]]
-; CHECK-NEXT:    [[TMP156:%.*]] = add <2 x i32> [[TMP125]], [[TMP134]]
+; CHECK-NEXT:    [[TMP157:%.*]] = sub <2 x i32> [[TMP123]], [[TMP134]]
+; CHECK-NEXT:    [[TMP156:%.*]] = add <2 x i32> [[TMP125]], [[TMP157]]
 ; CHECK-NEXT:    [[TMP126:%.*]] = shufflevector <2 x i8> [[TMP7]], <2 x i8> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP127:%.*]] = insertelement <2 x i8> [[TMP126]], i8 [[TMP14]], i32 1
-; CHECK-NEXT:    [[TMP128:%.*]] = zext <2 x i8> [[TMP127]] to <2 x i32>
+; CHECK-NEXT:    [[TMP158:%.*]] = zext <2 x i8> [[TMP127]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP129:%.*]] = insertelement <2 x i32> [[TMP113]], i32 [[TMP9]], i32 0
-; CHECK-NEXT:    [[TMP130:%.*]] = sub <2 x i32> [[TMP129]], [[TMP128]]
+; CHECK-NEXT:    [[TMP130:%.*]] = sub <2 x i32> [[TMP129]], [[TMP158]]
 ; CHECK-NEXT:    [[TMP135:%.*]] = insertelement <2 x i32> [[TMP130]], i32 [[TMP16]], i32 1
 ; CHECK-NEXT:    [[TMP136:%.*]] = shl <2 x i32> [[TMP135]], <i32 16, i32 16>
 ; CHECK-NEXT:    [[TMP110:%.*]] = shufflevector <2 x i32> [[TMP130]], <2 x i32> poison, <2 x i32> <i32 1, i32 poison>
@@ -214,23 +224,25 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; CHECK-NEXT:    [[SUB47_1:%.*]] = sub i32 [[TMP138]], [[TMP171]]
 ; CHECK-NEXT:    [[TMP140:%.*]] = shufflevector <2 x i32> [[TMP156]], <2 x i32> [[TMP105]], <2 x i32> <i32 1, i32 2>
 ; CHECK-NEXT:    [[TMP153:%.*]] = shufflevector <2 x i32> [[TMP156]], <2 x i32> [[TMP92]], <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:    [[TMP157:%.*]] = add <2 x i32> [[TMP140]], [[TMP153]]
+; CHECK-NEXT:    [[TMP163:%.*]] = add <2 x i32> [[TMP140]], [[TMP153]]
 ; CHECK-NEXT:    [[TMP143:%.*]] = shufflevector <2 x i32> [[TMP105]], <2 x i32> [[TMP155]], <2 x i32> <i32 3, i32 1>
 ; CHECK-NEXT:    [[TMP144:%.*]] = shufflevector <2 x i32> [[TMP92]], <2 x i32> [[TMP155]], <2 x i32> <i32 2, i32 1>
-; CHECK-NEXT:    [[TMP145:%.*]] = add <2 x i32> [[TMP143]], [[TMP144]]
-; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <2 x i32> [[TMP157]], i32 1
-; CHECK-NEXT:    [[TMP146:%.*]] = extractelement <2 x i32> [[TMP145]], i32 1
+; CHECK-NEXT:    [[TMP165:%.*]] = add <2 x i32> [[TMP143]], [[TMP144]]
+; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <2 x i32> [[TMP163]], i32 1
+; CHECK-NEXT:    [[TMP146:%.*]] = extractelement <2 x i32> [[TMP165]], i32 1
+; CHECK-NEXT:    [[ADD48:%.*]] = add i32 [[TMP146]], [[TMP98]]
 ; CHECK-NEXT:    [[SHR_I54_1:%.*]] = lshr i32 [[TMP146]], 15
 ; CHECK-NEXT:    [[AND_I55_1:%.*]] = and i32 [[SHR_I54_1]], 65537
 ; CHECK-NEXT:    [[MUL_I56_1:%.*]] = mul i32 [[AND_I55_1]], 65535
-; CHECK-NEXT:    [[TMP164:%.*]] = add <2 x i32> [[TMP145]], [[TMP157]]
-; CHECK-NEXT:    [[TMP141:%.*]] = sub <2 x i32> [[TMP157]], [[TMP145]]
-; CHECK-NEXT:    [[TMP165:%.*]] = insertelement <2 x i32> [[TMP101]], i32 [[SUB47_1]], i32 0
+; CHECK-NEXT:    [[TMP167:%.*]] = extractelement <2 x i32> [[TMP163]], i32 0
+; CHECK-NEXT:    [[TMP166:%.*]] = extractelement <2 x i32> [[TMP165]], i32 0
+; CHECK-NEXT:    [[ADD48_1:%.*]] = add i32 [[TMP166]], [[TMP167]]
+; CHECK-NEXT:    [[TMP141:%.*]] = sub <2 x i32> [[TMP163]], [[TMP165]]
+; CHECK-NEXT:    [[ADD55_1:%.*]] = add i32 [[SUB47_1]], [[SUB45_1]]
 ; CHECK-NEXT:    [[TMP151:%.*]] = shufflevector <2 x i32> [[TMP101]], <2 x i32> poison, <2 x i32> <i32 poison, i32 0>
 ; CHECK-NEXT:    [[TMP152:%.*]] = insertelement <2 x i32> [[TMP151]], i32 [[SUB45_1]], i32 0
-; CHECK-NEXT:    [[TMP180:%.*]] = add <2 x i32> [[TMP165]], [[TMP152]]
-; CHECK-NEXT:    [[TMP154:%.*]] = sub <2 x i32> [[TMP152]], [[TMP165]]
-; CHECK-NEXT:    [[TMP166:%.*]] = extractelement <2 x i32> [[TMP145]], i32 0
+; CHECK-NEXT:    [[TMP154:%.*]] = insertelement <2 x i32> [[TMP101]], i32 [[SUB47_1]], i32 0
+; CHECK-NEXT:    [[TMP168:%.*]] = sub <2 x i32> [[TMP152]], [[TMP154]]
 ; CHECK-NEXT:    [[SHR_I54:%.*]] = lshr i32 [[TMP166]], 15
 ; CHECK-NEXT:    [[AND_I55:%.*]] = and i32 [[SHR_I54]], 65537
 ; CHECK-NEXT:    [[MUL_I56:%.*]] = mul i32 [[AND_I55]], 65535
@@ -240,74 +252,48 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; CHECK-NEXT:    [[TMP147:%.*]] = lshr <2 x i32> [[TMP113]], <i32 15, i32 15>
 ; CHECK-NEXT:    [[TMP148:%.*]] = and <2 x i32> [[TMP147]], <i32 65537, i32 65537>
 ; CHECK-NEXT:    [[TMP149:%.*]] = mul <2 x i32> [[TMP148]], <i32 65535, i32 65535>
-; CHECK-NEXT:    [[TMP167:%.*]] = shufflevector <2 x i32> [[TMP164]], <2 x i32> poison, <2 x i32> <i32 poison, i32 0>
-; CHECK-NEXT:    [[TMP160:%.*]] = insertelement <2 x i32> [[TMP167]], i32 [[ADD48_3]], i32 0
-; CHECK-NEXT:    [[TMP161:%.*]] = insertelement <2 x i32> [[TMP164]], i32 [[ADD48_2]], i32 0
-; CHECK-NEXT:    [[TMP162:%.*]] = add <2 x i32> [[TMP160]], [[TMP161]]
-; CHECK-NEXT:    [[TMP163:%.*]] = sub <2 x i32> [[TMP161]], [[TMP160]]
-; CHECK-NEXT:    [[ADD95:%.*]] = extractelement <2 x i32> [[TMP162]], i32 0
-; CHECK-NEXT:    [[ADD79:%.*]] = extractelement <2 x i32> [[TMP162]], i32 1
-; CHECK-NEXT:    [[ADD103:%.*]] = add i32 [[ADD95]], [[ADD79]]
-; CHECK-NEXT:    [[SUB105:%.*]] = sub i32 [[ADD79]], [[ADD95]]
-; CHECK-NEXT:    [[ADD94:%.*]] = extractelement <2 x i32> [[TMP163]], i32 0
-; CHECK-NEXT:    [[ADD78:%.*]] = extractelement <2 x i32> [[TMP163]], i32 1
-; CHECK-NEXT:    [[ADD105:%.*]] = add i32 [[ADD94]], [[ADD78]]
+; CHECK-NEXT:    [[ADD78:%.*]] = add i32 [[ADD48_1]], [[ADD48]]
+; CHECK-NEXT:    [[SUB86:%.*]] = sub i32 [[ADD48]], [[ADD48_1]]
+; CHECK-NEXT:    [[ADD103:%.*]] = add i32 [[ADD94]], [[ADD78]]
 ; CHECK-NEXT:    [[SUB104:%.*]] = sub i32 [[ADD78]], [[ADD94]]
+; CHECK-NEXT:    [[ADD105:%.*]] = add i32 [[SUB102]], [[SUB86]]
+; CHECK-NEXT:    [[SUB106:%.*]] = sub i32 [[SUB86]], [[SUB102]]
 ; CHECK-NEXT:    [[ADD_I:%.*]] = add i32 [[MUL_I51_3]], [[ADD103]]
 ; CHECK-NEXT:    [[XOR_I:%.*]] = xor i32 [[ADD_I]], [[TMP77]]
 ; CHECK-NEXT:    [[ADD_I52:%.*]] = add i32 [[MUL_I_1]], [[ADD105]]
 ; CHECK-NEXT:    [[XOR_I53:%.*]] = xor i32 [[ADD_I52]], [[TMP34]]
-; CHECK-NEXT:    [[ADD_I57:%.*]] = add i32 [[MUL_I56]], [[SUB105]]
+; CHECK-NEXT:    [[ADD_I57:%.*]] = add i32 [[MUL_I56]], [[SUB104]]
 ; CHECK-NEXT:    [[XOR_I58:%.*]] = xor i32 [[ADD_I57]], [[TMP166]]
-; CHECK-NEXT:    [[ADD_I62:%.*]] = add i32 [[MUL_I56_1]], [[SUB104]]
+; CHECK-NEXT:    [[ADD_I62:%.*]] = add i32 [[MUL_I56_1]], [[SUB106]]
 ; CHECK-NEXT:    [[XOR_I63:%.*]] = xor i32 [[ADD_I62]], [[TMP146]]
 ; CHECK-NEXT:    [[ADD110:%.*]] = add i32 [[XOR_I53]], [[XOR_I]]
 ; CHECK-NEXT:    [[ADD112:%.*]] = add i32 [[ADD110]], [[XOR_I58]]
 ; CHECK-NEXT:    [[ADD113:%.*]] = add i32 [[ADD112]], [[XOR_I63]]
-; CHECK-NEXT:    [[TMP168:%.*]] = extractelement <2 x i32> [[TMP180]], i32 0
-; CHECK-NEXT:    [[TMP169:%.*]] = extractelement <2 x i32> [[TMP180]], i32 1
-; CHECK-NEXT:    [[SUB102:%.*]] = add i32 [[TMP168]], [[TMP169]]
-; CHECK-NEXT:    [[ADD55_2:%.*]] = add i32 [[TMP48]], [[TMP47]]
-; CHECK-NEXT:    [[TMP170:%.*]] = insertelement <2 x i32> [[TMP180]], i32 [[ADD55_2]], i32 0
-; CHECK-NEXT:    [[TMP173:%.*]] = shufflevector <2 x i32> [[TMP180]], <2 x i32> poison, <2 x i32> <i32 poison, i32 0>
-; CHECK-NEXT:    [[TMP172:%.*]] = insertelement <2 x i32> [[TMP173]], i32 [[ADD55_3]], i32 0
-; CHECK-NEXT:    [[TMP175:%.*]] = sub <2 x i32> [[TMP170]], [[TMP172]]
-; CHECK-NEXT:    [[TMP174:%.*]] = extractelement <2 x i32> [[TMP175]], i32 0
-; CHECK-NEXT:    [[TMP159:%.*]] = extractelement <2 x i32> [[TMP175]], i32 1
-; CHECK-NEXT:    [[ADD94_4:%.*]] = add i32 [[TMP174]], [[TMP159]]
-; CHECK-NEXT:    [[ADD94_1:%.*]] = add i32 [[ADD55_3]], [[ADD55_2]]
-; CHECK-NEXT:    [[TMP244:%.*]] = insertelement <2 x i32> poison, i32 [[ADD94_2]], i32 0
-; CHECK-NEXT:    [[TMP177:%.*]] = insertelement <2 x i32> [[TMP244]], i32 [[ADD55_3]], i32 1
-; CHECK-NEXT:    [[TMP197:%.*]] = insertelement <2 x i32> poison, i32 [[ADD94_4]], i32 0
-; CHECK-NEXT:    [[TMP179:%.*]] = insertelement <2 x i32> [[TMP197]], i32 [[ADD55_2]], i32 1
-; CHECK-NEXT:    [[TMP181:%.*]] = add <2 x i32> [[TMP177]], [[TMP179]]
-; CHECK-NEXT:    [[SUB104_1:%.*]] = sub i32 [[SUB102]], [[ADD94_1]]
-; CHECK-NEXT:    [[SUB106_1:%.*]] = sub i32 [[TMP159]], [[TMP174]]
-; CHECK-NEXT:    [[TMP193:%.*]] = insertelement <2 x i32> [[TMP192]], i32 [[SUB102]], i32 1
-; CHECK-NEXT:    [[TMP182:%.*]] = xor <2 x i32> [[TMP181]], [[TMP193]]
-; CHECK-NEXT:    [[TMP183:%.*]] = add <2 x i32> [[TMP181]], [[TMP193]]
-; CHECK-NEXT:    [[TMP184:%.*]] = shufflevector <2 x i32> [[TMP182]], <2 x i32> [[TMP183]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:    [[TMP185:%.*]] = insertelement <2 x i32> poison, i32 [[ADD113]], i32 0
-; CHECK-NEXT:    [[TMP186:%.*]] = insertelement <2 x i32> [[TMP185]], i32 [[MUL_I_2]], i32 1
-; CHECK-NEXT:    [[TMP202:%.*]] = add <2 x i32> [[TMP184]], [[TMP186]]
-; CHECK-NEXT:    [[TMP203:%.*]] = extractelement <2 x i32> [[TMP202]], i32 1
-; CHECK-NEXT:    [[TMP189:%.*]] = shufflevector <2 x i32> [[TMP202]], <2 x i32> [[TMP32]], <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT:    [[ADD78_1:%.*]] = add i32 [[ADD55_1]], [[ADD55]]
+; CHECK-NEXT:    [[SUB86_1:%.*]] = sub i32 [[ADD55]], [[ADD55_1]]
+; CHECK-NEXT:    [[ADD103_1:%.*]] = add i32 [[ADD94_1]], [[ADD78_1]]
+; CHECK-NEXT:    [[SUB104_1:%.*]] = sub i32 [[ADD78_1]], [[ADD94_1]]
+; CHECK-NEXT:    [[ADD105_1:%.*]] = add i32 [[SUB102_1]], [[SUB86_1]]
+; CHECK-NEXT:    [[SUB106_1:%.*]] = sub i32 [[SUB86_1]], [[SUB102_1]]
+; CHECK-NEXT:    [[TMP203:%.*]] = add i32 [[MUL_I_2]], [[ADD103_1]]
 ; CHECK-NEXT:    [[XOR_I_1:%.*]] = xor i32 [[TMP203]], [[TMP83]]
+; CHECK-NEXT:    [[ADD_I52_1:%.*]] = add i32 [[ADD94_2]], [[ADD105_1]]
+; CHECK-NEXT:    [[XOR_I53_1:%.*]] = xor i32 [[ADD_I52_1]], [[TMP84]]
 ; CHECK-NEXT:    [[ADD_I57_1:%.*]] = add i32 [[MUL_I56_2]], [[SUB104_1]]
 ; CHECK-NEXT:    [[XOR_I58_1:%.*]] = xor i32 [[ADD_I57_1]], [[SUB47_1]]
 ; CHECK-NEXT:    [[ADD_I62_1:%.*]] = add i32 [[MUL_I61_1]], [[SUB106_1]]
 ; CHECK-NEXT:    [[XOR_I63_1:%.*]] = xor i32 [[ADD_I62_1]], [[TMP99]]
-; CHECK-NEXT:    [[TMP190:%.*]] = extractelement <2 x i32> [[TMP202]], i32 0
-; CHECK-NEXT:    [[ADD110_1:%.*]] = add i32 [[TMP190]], [[XOR_I_1]]
+; CHECK-NEXT:    [[ADD108_1:%.*]] = add i32 [[XOR_I53_1]], [[ADD113]]
+; CHECK-NEXT:    [[ADD110_1:%.*]] = add i32 [[ADD108_1]], [[XOR_I_1]]
 ; CHECK-NEXT:    [[ADD112_1:%.*]] = add i32 [[ADD110_1]], [[XOR_I58_1]]
 ; CHECK-NEXT:    [[ADD113_1:%.*]] = add i32 [[ADD112_1]], [[XOR_I63_1]]
-; CHECK-NEXT:    [[TMP191:%.*]] = extractelement <2 x i32> [[TMP141]], i32 0
-; CHECK-NEXT:    [[TMP205:%.*]] = extractelement <2 x i32> [[TMP141]], i32 1
-; CHECK-NEXT:    [[ADD78_2:%.*]] = add i32 [[TMP191]], [[TMP205]]
+; CHECK-NEXT:    [[TMP169:%.*]] = extractelement <2 x i32> [[TMP141]], i32 0
+; CHECK-NEXT:    [[TMP160:%.*]] = extractelement <2 x i32> [[TMP141]], i32 1
+; CHECK-NEXT:    [[ADD78_2:%.*]] = add i32 [[TMP169]], [[TMP160]]
 ; CHECK-NEXT:    [[TMP196:%.*]] = insertelement <2 x i32> [[TMP141]], i32 [[SUB51_2]], i32 0
 ; CHECK-NEXT:    [[TMP194:%.*]] = shufflevector <2 x i32> [[TMP141]], <2 x i32> poison, <2 x i32> <i32 poison, i32 0>
 ; CHECK-NEXT:    [[TMP195:%.*]] = insertelement <2 x i32> [[TMP194]], i32 [[SUB51_3]], i32 0
-; CHECK-NEXT:    [[TMP206:%.*]] = sub <2 x i32> [[TMP196]], [[TMP195]]
+; CHECK-NEXT:    [[TMP164:%.*]] = sub <2 x i32> [[TMP196]], [[TMP195]]
 ; CHECK-NEXT:    [[TMP201:%.*]] = insertelement <2 x i32> poison, i32 [[ADD78_2]], i32 0
 ; CHECK-NEXT:    [[TMP198:%.*]] = shufflevector <2 x i32> [[TMP201]], <2 x i32> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP199:%.*]] = insertelement <2 x i32> poison, i32 [[ADD94_6]], i32 0
@@ -315,8 +301,8 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; CHECK-NEXT:    [[TMP225:%.*]] = add <2 x i32> [[TMP198]], [[TMP200]]
 ; CHECK-NEXT:    [[TMP226:%.*]] = sub <2 x i32> [[TMP198]], [[TMP200]]
 ; CHECK-NEXT:    [[TMP227:%.*]] = shufflevector <2 x i32> [[TMP225]], <2 x i32> [[TMP226]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:    [[TMP204:%.*]] = extractelement <2 x i32> [[TMP206]], i32 0
-; CHECK-NEXT:    [[TMP212:%.*]] = extractelement <2 x i32> [[TMP206]], i32 1
+; CHECK-NEXT:    [[TMP204:%.*]] = extractelement <2 x i32> [[TMP164]], i32 0
+; CHECK-NEXT:    [[TMP212:%.*]] = extractelement <2 x i32> [[TMP164]], i32 1
 ; CHECK-NEXT:    [[ADD105_2:%.*]] = add i32 [[TMP204]], [[TMP212]]
 ; CHECK-NEXT:    [[SUB106_2:%.*]] = sub i32 [[TMP212]], [[TMP204]]
 ; CHECK-NEXT:    [[ADD_I52_2:%.*]] = add i32 [[MUL_I51_2]], [[ADD105_2]]
@@ -329,13 +315,13 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; CHECK-NEXT:    [[ADD_I62_2:%.*]] = add i32 [[MUL_I61_2]], [[SUB106_2]]
 ; CHECK-NEXT:    [[XOR_I63_2:%.*]] = xor i32 [[ADD_I62_2]], [[TMP98]]
 ; CHECK-NEXT:    [[ADD108_2:%.*]] = add i32 [[XOR_I53_2]], [[ADD113_1]]
-; CHECK-NEXT:    [[TMP208:%.*]] = extractelement <2 x i32> [[TMP213]], i32 0
-; CHECK-NEXT:    [[ADD110_2:%.*]] = add i32 [[ADD108_2]], [[TMP208]]
-; CHECK-NEXT:    [[TMP209:%.*]] = extractelement <2 x i32> [[TMP213]], i32 1
-; CHECK-NEXT:    [[ADD112_2:%.*]] = add i32 [[ADD110_2]], [[TMP209]]
+; CHECK-NEXT:    [[TMP176:%.*]] = extractelement <2 x i32> [[TMP213]], i32 0
+; CHECK-NEXT:    [[ADD110_2:%.*]] = add i32 [[ADD108_2]], [[TMP176]]
+; CHECK-NEXT:    [[TMP177:%.*]] = extractelement <2 x i32> [[TMP213]], i32 1
+; CHECK-NEXT:    [[ADD112_2:%.*]] = add i32 [[ADD110_2]], [[TMP177]]
 ; CHECK-NEXT:    [[ADD113_2:%.*]] = add i32 [[ADD112_2]], [[XOR_I63_2]]
-; CHECK-NEXT:    [[SUB59_1:%.*]] = extractelement <2 x i32> [[TMP154]], i32 0
-; CHECK-NEXT:    [[SUB59:%.*]] = extractelement <2 x i32> [[TMP154]], i32 1
+; CHECK-NEXT:    [[SUB59_1:%.*]] = extractelement <2 x i32> [[TMP168]], i32 0
+; CHECK-NEXT:    [[SUB59:%.*]] = extractelement <2 x i32> [[TMP168]], i32 1
 ; CHECK-NEXT:    [[ADD94_3:%.*]] = add i32 [[SUB59_1]], [[SUB59]]
 ; CHECK-NEXT:    [[SUB86_3:%.*]] = sub i32 [[SUB59]], [[SUB59_1]]
 ; CHECK-NEXT:    [[TMP223:%.*]] = insertelement <2 x i32> poison, i32 [[ADD94_3]], i32 0
@@ -360,10 +346,10 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; CHECK-NEXT:    [[ADD_I62_3:%.*]] = add i32 [[MUL_I61_3]], [[SUB106_3]]
 ; CHECK-NEXT:    [[XOR_I63_3:%.*]] = xor i32 [[ADD_I62_3]], [[CONV33]]
 ; CHECK-NEXT:    [[ADD108_3:%.*]] = add i32 [[XOR_I53_3]], [[ADD113_2]]
-; CHECK-NEXT:    [[TMP228:%.*]] = extractelement <2 x i32> [[TMP234]], i32 0
-; CHECK-NEXT:    [[ADD110_3:%.*]] = add i32 [[ADD108_3]], [[TMP228]]
-; CHECK-NEXT:    [[TMP229:%.*]] = extractelement <2 x i32> [[TMP234]], i32 1
-; CHECK-NEXT:    [[ADD112_3:%.*]] = add i32 [[ADD110_3]], [[TMP229]]
+; CHECK-NEXT:    [[TMP192:%.*]] = extractelement <2 x i32> [[TMP234]], i32 0
+; CHECK-NEXT:    [[ADD110_3:%.*]] = add i32 [[ADD108_3]], [[TMP192]]
+; CHECK-NEXT:    [[TMP193:%.*]] = extractelement <2 x i32> [[TMP234]], i32 1
+; CHECK-NEXT:    [[ADD112_3:%.*]] = add i32 [[ADD110_3]], [[TMP193]]
 ; CHECK-NEXT:    [[ADD113_3:%.*]] = add i32 [[ADD112_3]], [[XOR_I63_3]]
 ; CHECK-NEXT:    ret i32 [[ADD113_3]]
 ;
@@ -422,35 +408,35 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; THR15-NEXT:    [[TMP23:%.*]] = sub <2 x i32> [[TMP66]], [[TMP22]]
 ; THR15-NEXT:    [[TMP24:%.*]] = load <2 x i8>, ptr [[ARRAYIDX3_2]], align 1
 ; THR15-NEXT:    [[TMP28:%.*]] = zext <2 x i8> [[TMP24]] to <2 x i32>
-; THR15-NEXT:    [[TMP30:%.*]] = load <2 x i8>, ptr [[ARRAYIDX5_2]], align 1
-; THR15-NEXT:    [[TMP47:%.*]] = zext <2 x i8> [[TMP30]] to <2 x i32>
-; THR15-NEXT:    [[TMP13:%.*]] = sub <2 x i32> [[TMP28]], [[TMP47]]
+; THR15-NEXT:    [[TMP29:%.*]] = load <2 x i8>, ptr [[ARRAYIDX5_2]], align 1
+; THR15-NEXT:    [[TMP30:%.*]] = zext <2 x i8> [[TMP29]] to <2 x i32>
+; THR15-NEXT:    [[TMP13:%.*]] = sub <2 x i32> [[TMP28]], [[TMP30]]
 ; THR15-NEXT:    [[TMP14:%.*]] = shl <2 x i32> [[TMP13]], <i32 16, i32 16>
 ; THR15-NEXT:    [[TMP15:%.*]] = add <2 x i32> [[TMP14]], [[TMP23]]
 ; THR15-NEXT:    [[ARRAYIDX20_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 2
 ; THR15-NEXT:    [[ARRAYIDX22_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 2
 ; THR15-NEXT:    [[ARRAYIDX25_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 6
 ; THR15-NEXT:    [[ARRAYIDX27_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 6
-; THR15-NEXT:    [[TMP49:%.*]] = load <2 x i8>, ptr [[ARRAYIDX20_2]], align 1
-; THR15-NEXT:    [[TMP59:%.*]] = zext <2 x i8> [[TMP49]] to <2 x i32>
+; THR15-NEXT:    [[TMP31:%.*]] = load <2 x i8>, ptr [[ARRAYIDX20_2]], align 1
+; THR15-NEXT:    [[TMP47:%.*]] = zext <2 x i8> [[TMP31]] to <2 x i32>
 ; THR15-NEXT:    [[TMP33:%.*]] = load <2 x i8>, ptr [[ARRAYIDX22_2]], align 1
-; THR15-NEXT:    [[TMP78:%.*]] = zext <2 x i8> [[TMP33]] to <2 x i32>
-; THR15-NEXT:    [[TMP35:%.*]] = sub <2 x i32> [[TMP59]], [[TMP78]]
+; THR15-NEXT:    [[TMP50:%.*]] = zext <2 x i8> [[TMP33]] to <2 x i32>
+; THR15-NEXT:    [[TMP35:%.*]] = sub <2 x i32> [[TMP47]], [[TMP50]]
 ; THR15-NEXT:    [[TMP36:%.*]] = load <2 x i8>, ptr [[ARRAYIDX25_2]], align 1
-; THR15-NEXT:    [[TMP80:%.*]] = zext <2 x i8> [[TMP36]] to <2 x i32>
+; THR15-NEXT:    [[TMP53:%.*]] = zext <2 x i8> [[TMP36]] to <2 x i32>
 ; THR15-NEXT:    [[TMP38:%.*]] = load <2 x i8>, ptr [[ARRAYIDX27_2]], align 1
 ; THR15-NEXT:    [[TMP39:%.*]] = zext <2 x i8> [[TMP38]] to <2 x i32>
-; THR15-NEXT:    [[TMP25:%.*]] = sub <2 x i32> [[TMP80]], [[TMP39]]
+; THR15-NEXT:    [[TMP25:%.*]] = sub <2 x i32> [[TMP53]], [[TMP39]]
 ; THR15-NEXT:    [[TMP26:%.*]] = shl <2 x i32> [[TMP25]], <i32 16, i32 16>
 ; THR15-NEXT:    [[TMP27:%.*]] = add <2 x i32> [[TMP26]], [[TMP35]]
-; THR15-NEXT:    [[TMP83:%.*]] = extractelement <2 x i32> [[TMP15]], i32 0
-; THR15-NEXT:    [[TMP29:%.*]] = extractelement <2 x i32> [[TMP15]], i32 1
-; THR15-NEXT:    [[ADD44_2:%.*]] = add i32 [[TMP29]], [[TMP83]]
-; THR15-NEXT:    [[SUB45_2:%.*]] = sub i32 [[TMP83]], [[TMP29]]
-; THR15-NEXT:    [[TMP87:%.*]] = extractelement <2 x i32> [[TMP27]], i32 0
-; THR15-NEXT:    [[TMP31:%.*]] = extractelement <2 x i32> [[TMP27]], i32 1
-; THR15-NEXT:    [[ADD46_2:%.*]] = add i32 [[TMP31]], [[TMP87]]
-; THR15-NEXT:    [[SUB47_2:%.*]] = sub i32 [[TMP87]], [[TMP31]]
+; THR15-NEXT:    [[TMP68:%.*]] = extractelement <2 x i32> [[TMP15]], i32 0
+; THR15-NEXT:    [[TMP59:%.*]] = extractelement <2 x i32> [[TMP15]], i32 1
+; THR15-NEXT:    [[ADD44_2:%.*]] = add i32 [[TMP59]], [[TMP68]]
+; THR15-NEXT:    [[SUB45_2:%.*]] = sub i32 [[TMP68]], [[TMP59]]
+; THR15-NEXT:    [[TMP76:%.*]] = extractelement <2 x i32> [[TMP27]], i32 0
+; THR15-NEXT:    [[TMP60:%.*]] = extractelement <2 x i32> [[TMP27]], i32 1
+; THR15-NEXT:    [[ADD46_2:%.*]] = add i32 [[TMP60]], [[TMP76]]
+; THR15-NEXT:    [[SUB47_2:%.*]] = sub i32 [[TMP76]], [[TMP60]]
 ; THR15-NEXT:    [[ADD48_2:%.*]] = add i32 [[ADD46_2]], [[ADD44_2]]
 ; THR15-NEXT:    [[SUB51_2:%.*]] = sub i32 [[ADD44_2]], [[ADD46_2]]
 ; THR15-NEXT:    [[ADD55_2:%.*]] = add i32 [[SUB47_2]], [[SUB45_2]]
@@ -458,15 +444,17 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; THR15-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr i8, ptr null, i64 4
 ; THR15-NEXT:    [[ARRAYIDX5_3:%.*]] = getelementptr i8, ptr null, i64 4
 ; THR15-NEXT:    [[TMP32:%.*]] = load <2 x i8>, ptr null, align 1
-; THR15-NEXT:    [[TMP48:%.*]] = zext <2 x i8> [[TMP32]] to <2 x i32>
+; THR15-NEXT:    [[TMP48:%.*]] = load i8, ptr null, align 1
+; THR15-NEXT:    [[TMP49:%.*]] = zext <2 x i8> [[TMP32]] to <2 x i32>
+; THR15-NEXT:    [[TMP63:%.*]] = zext i8 [[TMP48]] to i32
 ; THR15-NEXT:    [[TMP34:%.*]] = load <2 x i8>, ptr null, align 1
-; THR15-NEXT:    [[TMP50:%.*]] = zext <2 x i8> [[TMP34]] to <2 x i32>
-; THR15-NEXT:    [[TMP93:%.*]] = sub <2 x i32> [[TMP48]], [[TMP50]]
+; THR15-NEXT:    [[TMP61:%.*]] = zext <2 x i8> [[TMP34]] to <2 x i32>
+; THR15-NEXT:    [[TMP93:%.*]] = sub <2 x i32> [[TMP49]], [[TMP61]]
 ; THR15-NEXT:    [[TMP37:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX3_3]], i64 -4, <2 x i1> <i1 true, i1 true>, i32 2)
-; THR15-NEXT:    [[TMP53:%.*]] = zext <2 x i8> [[TMP37]] to <2 x i32>
-; THR15-NEXT:    [[TMP54:%.*]] = load <2 x i8>, ptr [[ARRAYIDX5_3]], align 1
-; THR15-NEXT:    [[TMP55:%.*]] = zext <2 x i8> [[TMP54]] to <2 x i32>
-; THR15-NEXT:    [[TMP41:%.*]] = sub <2 x i32> [[TMP53]], [[TMP55]]
+; THR15-NEXT:    [[TMP54:%.*]] = zext <2 x i8> [[TMP37]] to <2 x i32>
+; THR15-NEXT:    [[TMP55:%.*]] = load <2 x i8>, ptr [[ARRAYIDX5_3]], align 1
+; THR15-NEXT:    [[TMP80:%.*]] = zext <2 x i8> [[TMP55]] to <2 x i32>
+; THR15-NEXT:    [[TMP41:%.*]] = sub <2 x i32> [[TMP54]], [[TMP80]]
 ; THR15-NEXT:    [[TMP42:%.*]] = shl <2 x i32> [[TMP41]], <i32 16, i32 16>
 ; THR15-NEXT:    [[TMP43:%.*]] = add <2 x i32> [[TMP42]], [[TMP93]]
 ; THR15-NEXT:    [[ARRAYIDX20_3:%.*]] = getelementptr i8, ptr null, i64 2
@@ -474,34 +462,33 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; THR15-NEXT:    [[TMP44:%.*]] = load i8, ptr null, align 1
 ; THR15-NEXT:    [[ARRAYIDX27_3:%.*]] = getelementptr i8, ptr null, i64 6
 ; THR15-NEXT:    [[TMP45:%.*]] = load i8, ptr null, align 1
-; THR15-NEXT:    [[TMP61:%.*]] = load <2 x i8>, ptr [[ARRAYIDX20_3]], align 1
-; THR15-NEXT:    [[TMP98:%.*]] = zext <2 x i8> [[TMP61]] to <2 x i32>
-; THR15-NEXT:    [[TMP99:%.*]] = load <2 x i8>, ptr [[ARRAYIDX22_3]], align 1
-; THR15-NEXT:    [[TMP101:%.*]] = zext <2 x i8> [[TMP99]] to <2 x i32>
-; THR15-NEXT:    [[TMP65:%.*]] = sub <2 x i32> [[TMP98]], [[TMP101]]
+; THR15-NEXT:    [[TMP62:%.*]] = load <2 x i8>, ptr [[ARRAYIDX20_3]], align 1
+; THR15-NEXT:    [[TMP83:%.*]] = zext <2 x i8> [[TMP62]] to <2 x i32>
+; THR15-NEXT:    [[TMP87:%.*]] = load <2 x i8>, ptr [[ARRAYIDX22_3]], align 1
+; THR15-NEXT:    [[TMP98:%.*]] = zext <2 x i8> [[TMP87]] to <2 x i32>
+; THR15-NEXT:    [[TMP65:%.*]] = sub <2 x i32> [[TMP83]], [[TMP98]]
 ; THR15-NEXT:    [[TMP51:%.*]] = insertelement <2 x i8> poison, i8 [[TMP44]], i32 0
 ; THR15-NEXT:    [[TMP52:%.*]] = insertelement <2 x i8> [[TMP51]], i8 [[TMP45]], i32 1
-; THR15-NEXT:    [[TMP68:%.*]] = zext <2 x i8> [[TMP52]] to <2 x i32>
-; THR15-NEXT:    [[TMP102:%.*]] = load <2 x i8>, ptr [[ARRAYIDX27_3]], align 1
-; THR15-NEXT:    [[TMP70:%.*]] = zext <2 x i8> [[TMP102]] to <2 x i32>
-; THR15-NEXT:    [[TMP56:%.*]] = sub <2 x i32> [[TMP68]], [[TMP70]]
+; THR15-NEXT:    [[TMP99:%.*]] = zext <2 x i8> [[TMP52]] to <2 x i32>
+; THR15-NEXT:    [[TMP70:%.*]] = load <2 x i8>, ptr [[ARRAYIDX27_3]], align 1
+; THR15-NEXT:    [[TMP101:%.*]] = zext <2 x i8> [[TMP70]] to <2 x i32>
+; THR15-NEXT:    [[TMP56:%.*]] = sub <2 x i32> [[TMP99]], [[TMP101]]
 ; THR15-NEXT:    [[TMP57:%.*]] = shl <2 x i32> [[TMP56]], <i32 16, i32 16>
 ; THR15-NEXT:    [[TMP58:%.*]] = add <2 x i32> [[TMP57]], [[TMP65]]
 ; THR15-NEXT:    [[TMP104:%.*]] = extractelement <2 x i32> [[TMP43]], i32 0
-; THR15-NEXT:    [[TMP60:%.*]] = extractelement <2 x i32> [[TMP43]], i32 1
-; THR15-NEXT:    [[ADD44_3:%.*]] = add i32 [[TMP60]], [[TMP104]]
-; THR15-NEXT:    [[SUB45_3:%.*]] = sub i32 [[TMP104]], [[TMP60]]
-; THR15-NEXT:    [[TMP76:%.*]] = extractelement <2 x i32> [[TMP58]], i32 0
-; THR15-NEXT:    [[TMP62:%.*]] = extractelement <2 x i32> [[TMP58]], i32 1
-; THR15-NEXT:    [[ADD46_3:%.*]] = add i32 [[TMP62]], [[TMP76]]
-; THR15-NEXT:    [[SUB47_3:%.*]] = sub i32 [[TMP76]], [[TMP62]]
+; THR15-NEXT:    [[TMP102:%.*]] = extractelement <2 x i32> [[TMP43]], i32 1
+; THR15-NEXT:    [[ADD44_3:%.*]] = add i32 [[TMP102]], [[TMP104]]
+; THR15-NEXT:    [[SUB45_3:%.*]] = sub i32 [[TMP104]], [[TMP102]]
+; THR15-NEXT:    [[TMP107:%.*]] = extractelement <2 x i32> [[TMP58]], i32 0
+; THR15-NEXT:    [[TMP78:%.*]] = extractelement <2 x i32> [[TMP58]], i32 1
+; THR15-NEXT:    [[ADD46_3:%.*]] = add i32 [[TMP78]], [[TMP107]]
+; THR15-NEXT:    [[SUB47_3:%.*]] = sub i32 [[TMP107]], [[TMP78]]
 ; THR15-NEXT:    [[ADD48_3:%.*]] = add i32 [[ADD46_3]], [[ADD44_3]]
 ; THR15-NEXT:    [[SUB51_3:%.*]] = sub i32 [[ADD44_3]], [[ADD46_3]]
 ; THR15-NEXT:    [[ADD55_3:%.*]] = add i32 [[SUB47_3]], [[SUB45_3]]
 ; THR15-NEXT:    [[SUB59_3:%.*]] = sub i32 [[SUB45_3]], [[SUB47_3]]
 ; THR15-NEXT:    [[ADD94:%.*]] = add i32 [[ADD48_3]], [[ADD48_2]]
 ; THR15-NEXT:    [[SUB102:%.*]] = sub i32 [[ADD48_2]], [[ADD48_3]]
-; THR15-NEXT:    [[TMP63:%.*]] = extractelement <2 x i32> [[TMP48]], i32 0
 ; THR15-NEXT:    [[SHR_I:%.*]] = lshr i32 [[TMP63]], 15
 ; THR15-NEXT:    [[AND_I:%.*]] = and i32 [[SHR_I]], 65537
 ; THR15-NEXT:    [[MUL_I:%.*]] = mul i32 [[AND_I]], 65535
@@ -531,15 +518,15 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; THR15-NEXT:    [[TMP81:%.*]] = load <2 x i8>, ptr [[ARRAYIDX8]], align 1
 ; THR15-NEXT:    [[TMP74:%.*]] = zext <2 x i8> [[TMP81]] to <2 x i32>
 ; THR15-NEXT:    [[TMP67:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[PIX2]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; THR15-NEXT:    [[TMP107:%.*]] = zext <2 x i8> [[TMP67]] to <2 x i32>
+; THR15-NEXT:    [[TMP108:%.*]] = zext <2 x i8> [[TMP67]] to <2 x i32>
 ; THR15-NEXT:    [[TMP69:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX3]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; THR15-NEXT:    [[TMP108:%.*]] = zext <2 x i8> [[TMP69]] to <2 x i32>
+; THR15-NEXT:    [[TMP109:%.*]] = zext <2 x i8> [[TMP69]] to <2 x i32>
 ; THR15-NEXT:    [[TMP71:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX5]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; THR15-NEXT:    [[TMP109:%.*]] = zext <2 x i8> [[TMP71]] to <2 x i32>
-; THR15-NEXT:    [[TMP72:%.*]] = sub <2 x i32> [[TMP108]], [[TMP109]]
+; THR15-NEXT:    [[TMP111:%.*]] = zext <2 x i8> [[TMP71]] to <2 x i32>
+; THR15-NEXT:    [[TMP72:%.*]] = sub <2 x i32> [[TMP109]], [[TMP111]]
 ; THR15-NEXT:    [[TMP73:%.*]] = shl <2 x i32> [[TMP72]], <i32 16, i32 16>
 ; THR15-NEXT:    [[TMP75:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX22]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
-; THR15-NEXT:    [[TMP111:%.*]] = zext <2 x i8> [[TMP75]] to <2 x i32>
+; THR15-NEXT:    [[TMP125:%.*]] = zext <2 x i8> [[TMP75]] to <2 x i32>
 ; THR15-NEXT:    [[TMP82:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX25]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
 ; THR15-NEXT:    [[TMP94:%.*]] = zext <2 x i8> [[TMP82]] to <2 x i32>
 ; THR15-NEXT:    [[TMP79:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX27]], i64 2, <2 x i1> <i1 true, i1 true>, i32 2)
@@ -547,10 +534,10 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; THR15-NEXT:    [[TMP84:%.*]] = sub <2 x i32> [[TMP94]], [[TMP96]]
 ; THR15-NEXT:    [[TMP85:%.*]] = shl <2 x i32> [[TMP84]], <i32 16, i32 16>
 ; THR15-NEXT:    [[TMP86:%.*]] = insertelement <2 x i32> [[TMP74]], i32 [[CONV33]], i32 1
-; THR15-NEXT:    [[TMP100:%.*]] = sub <2 x i32> [[TMP86]], [[TMP111]]
+; THR15-NEXT:    [[TMP100:%.*]] = sub <2 x i32> [[TMP86]], [[TMP125]]
 ; THR15-NEXT:    [[TMP88:%.*]] = add <2 x i32> [[TMP85]], [[TMP100]]
 ; THR15-NEXT:    [[TMP92:%.*]] = insertelement <2 x i32> [[TMP74]], i32 [[CONV]], i32 0
-; THR15-NEXT:    [[TMP120:%.*]] = sub <2 x i32> [[TMP92]], [[TMP107]]
+; THR15-NEXT:    [[TMP120:%.*]] = sub <2 x i32> [[TMP92]], [[TMP108]]
 ; THR15-NEXT:    [[TMP95:%.*]] = add <2 x i32> [[TMP73]], [[TMP120]]
 ; THR15-NEXT:    [[TMP97:%.*]] = shufflevector <2 x i32> [[TMP88]], <2 x i32> [[TMP95]], <2 x i32> <i32 0, i32 2>
 ; THR15-NEXT:    [[TMP77:%.*]] = add <2 x i32> [[TMP88]], [[TMP95]]
@@ -575,8 +562,8 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; THR15-NEXT:    [[TMP116:%.*]] = load <2 x i8>, ptr [[ARRAYIDX3_1]], align 1
 ; THR15-NEXT:    [[TMP117:%.*]] = zext <2 x i8> [[TMP116]] to <2 x i32>
 ; THR15-NEXT:    [[TMP131:%.*]] = load <2 x i8>, ptr [[ARRAYIDX5_1]], align 1
-; THR15-NEXT:    [[TMP132:%.*]] = zext <2 x i8> [[TMP131]] to <2 x i32>
-; THR15-NEXT:    [[TMP113:%.*]] = sub <2 x i32> [[TMP117]], [[TMP132]]
+; THR15-NEXT:    [[TMP119:%.*]] = zext <2 x i8> [[TMP131]] to <2 x i32>
+; THR15-NEXT:    [[TMP113:%.*]] = sub <2 x i32> [[TMP117]], [[TMP119]]
 ; THR15-NEXT:    [[TMP114:%.*]] = shl <2 x i32> [[TMP113]], <i32 16, i32 16>
 ; THR15-NEXT:    [[TMP103:%.*]] = shufflevector <2 x i32> [[TMP130]], <2 x i32> poison, <2 x i32> <i32 poison, i32 0>
 ; THR15-NEXT:    [[TMP126:%.*]] = insertelement <2 x i32> [[TMP103]], i32 [[CONV_1]], i32 0
@@ -589,11 +576,11 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; THR15-NEXT:    [[TMP106:%.*]] = sub <2 x i32> [[TMP146]], [[TMP128]]
 ; THR15-NEXT:    [[TMP118:%.*]] = extractelement <2 x i32> [[TMP106]], i32 0
 ; THR15-NEXT:    [[SHL30_1:%.*]] = shl i32 [[TMP118]], 16
-; THR15-NEXT:    [[TMP119:%.*]] = extractelement <2 x i32> [[TMP106]], i32 1
-; THR15-NEXT:    [[ADD31_1:%.*]] = add i32 [[SHL30_1]], [[TMP119]]
+; THR15-NEXT:    [[TMP132:%.*]] = extractelement <2 x i32> [[TMP106]], i32 1
+; THR15-NEXT:    [[ADD31_1:%.*]] = add i32 [[SHL30_1]], [[TMP132]]
 ; THR15-NEXT:    [[TMP133:%.*]] = extractelement <2 x i32> [[TMP121]], i32 0
-; THR15-NEXT:    [[TMP125:%.*]] = extractelement <2 x i32> [[TMP121]], i32 1
-; THR15-NEXT:    [[SUB45_1:%.*]] = sub i32 [[TMP133]], [[TMP125]]
+; THR15-NEXT:    [[TMP147:%.*]] = extractelement <2 x i32> [[TMP121]], i32 1
+; THR15-NEXT:    [[SUB45_1:%.*]] = sub i32 [[TMP133]], [[TMP147]]
 ; THR15-NEXT:    [[TMP135:%.*]] = shufflevector <2 x i32> [[TMP121]], <2 x i32> poison, <2 x i32> <i32 1, i32 poison>
 ; THR15-NEXT:    [[TMP136:%.*]] = insertelement <2 x i32> [[TMP135]], i32 [[ADD43_1]], i32 1
 ; THR15-NEXT:    [[TMP137:%.*]] = insertelement <2 x i32> [[TMP121]], i32 [[ADD31_1]], i32 1
@@ -679,9 +666,9 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; THR15-NEXT:    [[ADD112_2:%.*]] = add i32 [[ADD110_2]], [[TMP162]]
 ; THR15-NEXT:    [[ADD113_2:%.*]] = add i32 [[ADD112_2]], [[XOR_I63_2]]
 ; THR15-NEXT:    [[TMP159:%.*]] = extractelement <2 x i32> [[TMP144]], i32 0
-; THR15-NEXT:    [[TMP178:%.*]] = extractelement <2 x i32> [[TMP144]], i32 1
-; THR15-NEXT:    [[ADD78_3:%.*]] = add i32 [[TMP159]], [[TMP178]]
-; THR15-NEXT:    [[SUB86_3:%.*]] = sub i32 [[TMP178]], [[TMP159]]
+; THR15-NEXT:    [[TMP176:%.*]] = extractelement <2 x i32> [[TMP144]], i32 1
+; THR15-NEXT:    [[ADD78_3:%.*]] = add i32 [[TMP159]], [[TMP176]]
+; THR15-NEXT:    [[SUB86_3:%.*]] = sub i32 [[TMP176]], [[TMP159]]
 ; THR15-NEXT:    [[TMP163:%.*]] = insertelement <2 x i32> poison, i32 [[ADD78_3]], i32 0
 ; THR15-NEXT:    [[TMP164:%.*]] = shufflevector <2 x i32> [[TMP163]], <2 x i32> poison, <2 x i32> zeroinitializer
 ; THR15-NEXT:    [[TMP165:%.*]] = insertelement <2 x i32> poison, i32 [[ADD94_3]], i32 0
@@ -706,8 +693,8 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
 ; THR15-NEXT:    [[ADD108_3:%.*]] = add i32 [[XOR_I53_3]], [[ADD113_2]]
 ; THR15-NEXT:    [[TMP175:%.*]] = extractelement <2 x i32> [[TMP174]], i32 0
 ; THR15-NEXT:    [[ADD110_3:%.*]] = add i32 [[ADD108_3]], [[TMP175]]
-; THR15-NEXT:    [[TMP176:%.*]] = extractelement <2 x i32> [[TMP174]], i32 1
-; THR15-NEXT:    [[ADD112_3:%.*]] = add i32 [[ADD110_3]], [[TMP176]]
+; THR15-NEXT:    [[TMP178:%.*]] = extractelement <2 x i32> [[TMP174]], i32 1
+; THR15-NEXT:    [[ADD112_3:%.*]] = add i32 [[ADD110_3]], [[TMP178]]
 ; THR15-NEXT:    [[ADD113_3:%.*]] = add i32 [[ADD112_3]], [[XOR_I63_3]]
 ; THR15-NEXT:    ret i32 [[ADD113_3]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cast-operand-extracted.ll b/llvm/test/Transforms/SLPVectorizer/X86/cast-operand-extracted.ll
index 12fcaa1c87a9c..e8548e0467381 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cast-operand-extracted.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cast-operand-extracted.ll
@@ -23,13 +23,16 @@ define void @test(ptr %0, i32 %add651) {
 ; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> <i32 1, i32 poison>, <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[TMP10:%.*]] = add <2 x i32> [[TMP8]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[ADD651]], i32 0
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> [[TMP12]], <4 x i32> <i32 0, i32 5, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP2]], i32 1
 ; CHECK-NEXT:    [[TMP14:%.*]] = call <4 x i32> @llvm.vector.insert.v4i32.v2i32(<4 x i32> [[TMP13]], <2 x i32> [[TMP10]], i64 2)
 ; CHECK-NEXT:    [[TMP15:%.*]] = lshr <4 x i32> [[TMP14]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[SHR685:%.*]] = lshr i32 [[TMP2]], 1
 ; CHECK-NEXT:    [[TMP16:%.*]] = trunc <4 x i32> [[TMP15]] to <4 x i16>
-; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <4 x i16> [[TMP16]], <4 x i16> poison, <2 x i32> <i32 1, i32 2>
-; CHECK-NEXT:    store <2 x i16> [[TMP17]], ptr [[ARRAYIDX689]], align 8
+; CHECK-NEXT:    [[CONV686:%.*]] = trunc i32 [[SHR685]] to i16
+; CHECK-NEXT:    store i16 [[CONV686]], ptr [[ARRAYIDX689]], align 8
+; CHECK-NEXT:    [[ARRAYIDX727:%.*]] = getelementptr i8, ptr [[TMP4]], i64 7818
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x i16> [[TMP16]], i32 2
+; CHECK-NEXT:    store i16 [[TMP17]], ptr [[ARRAYIDX727]], align 2
 ; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <4 x i16> [[TMP16]], i32 3
 ; CHECK-NEXT:    store i16 [[TMP18]], ptr [[TMP4]], align 8
 ; CHECK-NEXT:    store <4 x i16> [[TMP16]], ptr [[ARRAYIDX660]], align 8
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/debug-counter.ll b/llvm/test/Transforms/SLPVectorizer/X86/debug-counter.ll
new file mode 100644
index 0000000000000..d203888e0c6ee
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/debug-counter.ll
@@ -0,0 +1,182 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -S -passes=slp-vectorizer -mtriple=x86_64-unknown-linux -debug-counter=slp-vectorized=0 -slp-threshold=-99999 < %s | FileCheck %s --check-prefix=COUNT0
+; RUN: opt -S -passes=slp-vectorizer -mtriple=x86_64-unknown-linux -debug-counter=slp-vectorized=1 -slp-threshold=-99999 < %s | FileCheck %s --check-prefix=COUNT1
+; RUN: opt -S -passes=slp-vectorizer -mtriple=x86_64-unknown-linux -debug-counter=slp-vectorized=2 -slp-threshold=-99999 < %s | FileCheck %s --check-prefix=COUNT2
+; RUN: opt -S -passes=slp-vectorizer -mtriple=x86_64-unknown-linux -debug-counter=slp-vectorized=0-1 -slp-threshold=-99999 < %s | FileCheck %s --check-prefix=COUNT-1
+; REQUIRES: asserts
+
+define void @blam(ptr %arg, double %load2, i1 %fcmp3) {
+; COUNT0-LABEL: define void @blam
+; COUNT0-SAME: (ptr [[ARG:%.*]], double [[LOAD2:%.*]], i1 [[FCMP3:%.*]]) {
+; COUNT0-NEXT:  bb:
+; COUNT0-NEXT:    [[GETELEMENTPTR13:%.*]] = getelementptr double, ptr [[ARG]], i64 3
+; COUNT0-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[ARG]], align 8
+; COUNT0-NEXT:    [[TMP1:%.*]] = insertelement <2 x i1> poison, i1 [[FCMP3]], i32 0
+; COUNT0-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i1> [[TMP1]], <2 x i1> poison, <2 x i32> zeroinitializer
+; COUNT0-NEXT:    [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x double> zeroinitializer, <2 x double> [[TMP0]]
+; COUNT0-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[TMP0]], double [[LOAD2]], i32 0
+; COUNT0-NEXT:    [[TMP5:%.*]] = fcmp olt <2 x double> [[TMP4]], zeroinitializer
+; COUNT0-NEXT:    [[TMP6:%.*]] = select <2 x i1> [[TMP5]], <2 x double> zeroinitializer, <2 x double> [[TMP0]]
+; COUNT0-NEXT:    [[TMP7:%.*]] = fcmp olt <2 x double> [[TMP3]], zeroinitializer
+; COUNT0-NEXT:    [[TMP8:%.*]] = select <2 x i1> [[TMP7]], <2 x double> <double 0.000000e+00, double 1.000000e+00>, <2 x double> <double 1.000000e+00, double 0.000000e+00>
+; COUNT0-NEXT:    [[TMP9:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
+; COUNT0-NEXT:    [[TMP10:%.*]] = fcmp olt <2 x double> [[TMP9]], [[TMP6]]
+; COUNT0-NEXT:    [[TMP11:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> <double poison, double 0.000000e+00>, <2 x i32> <i32 0, i32 3>
+; COUNT0-NEXT:    [[TMP12:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> <double 0.000000e+00, double poison>, <2 x i32> <i32 2, i32 0>
+; COUNT0-NEXT:    [[TMP13:%.*]] = select <2 x i1> [[TMP10]], <2 x double> [[TMP11]], <2 x double> [[TMP12]]
+; COUNT0-NEXT:    [[TMP14:%.*]] = fcmp olt <2 x double> [[TMP13]], zeroinitializer
+; COUNT0-NEXT:    [[TMP15:%.*]] = select <2 x i1> [[TMP14]], <2 x double> zeroinitializer, <2 x double> <double 1.000000e+00, double 1.000000e+00>
+; COUNT0-NEXT:    [[TMP16:%.*]] = fcmp ogt <2 x double> [[TMP15]], zeroinitializer
+; COUNT0-NEXT:    [[TMP17:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> poison, <2 x i32> zeroinitializer
+; COUNT0-NEXT:    [[TMP18:%.*]] = select <2 x i1> [[TMP16]], <2 x double> zeroinitializer, <2 x double> [[TMP17]]
+; COUNT0-NEXT:    [[TMP19:%.*]] = fcmp olt <2 x double> [[TMP18]], zeroinitializer
+; COUNT0-NEXT:    [[TMP20:%.*]] = select <2 x i1> [[TMP19]], <2 x double> <double 1.000000e+00, double 1.000000e+00>, <2 x double> zeroinitializer
+; COUNT0-NEXT:    store <2 x double> [[TMP20]], ptr [[GETELEMENTPTR13]], align 8
+; COUNT0-NEXT:    ret void
+;
+; COUNT1-LABEL: define void @blam
+; COUNT1-SAME: (ptr [[ARG:%.*]], double [[LOAD2:%.*]], i1 [[FCMP3:%.*]]) {
+; COUNT1-NEXT:  bb:
+; COUNT1-NEXT:    [[GETELEMENTPTR:%.*]] = getelementptr double, ptr [[ARG]], i64 1
+; COUNT1-NEXT:    [[LOAD:%.*]] = load double, ptr [[GETELEMENTPTR]], align 8
+; COUNT1-NEXT:    [[FCMP:%.*]] = fcmp olt double [[LOAD]], 0.000000e+00
+; COUNT1-NEXT:    [[SELECT3:%.*]] = select i1 [[FCMP]], double 0.000000e+00, double [[LOAD]]
+; COUNT1-NEXT:    [[SELECT4:%.*]] = select i1 [[FCMP3]], double 0.000000e+00, double [[LOAD]]
+; COUNT1-NEXT:    [[LOAD7:%.*]] = load double, ptr [[ARG]], align 8
+; COUNT1-NEXT:    [[SELECT10:%.*]] = select i1 [[FCMP3]], double 0.000000e+00, double [[LOAD7]]
+; COUNT1-NEXT:    [[GETELEMENTPTR13:%.*]] = getelementptr double, ptr [[ARG]], i64 3
+; COUNT1-NEXT:    [[GETELEMENTPTR21:%.*]] = getelementptr double, ptr [[ARG]], i64 4
+; COUNT1-NEXT:    [[FCMP23:%.*]] = fcmp olt double [[SELECT10]], 0.000000e+00
+; COUNT1-NEXT:    [[SELECT24:%.*]] = select i1 [[FCMP23]], double 0.000000e+00, double 1.000000e+00
+; COUNT1-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[LOAD2]], i32 1
+; COUNT1-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[SELECT4]], i32 0
+; COUNT1-NEXT:    [[TMP2:%.*]] = fcmp olt <2 x double> [[TMP1]], zeroinitializer
+; COUNT1-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> <double 0.000000e+00, double poison>, double [[LOAD7]], i32 1
+; COUNT1-NEXT:    [[TMP4:%.*]] = select <2 x i1> [[TMP2]], <2 x double> <double 1.000000e+00, double 0.000000e+00>, <2 x double> [[TMP3]]
+; COUNT1-NEXT:    [[FCMP33:%.*]] = fcmp olt double [[SELECT24]], [[SELECT3]]
+; COUNT1-NEXT:    [[SELECT34:%.*]] = select i1 [[FCMP33]], double 0.000000e+00, double [[LOAD2]]
+; COUNT1-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
+; COUNT1-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[TMP4]], i32 1
+; COUNT1-NEXT:    [[FCMP39:%.*]] = fcmp olt double [[TMP5]], [[TMP6]]
+; COUNT1-NEXT:    [[SELECT40:%.*]] = select i1 [[FCMP39]], double [[LOAD2]], double 0.000000e+00
+; COUNT1-NEXT:    [[FCMP62:%.*]] = fcmp olt double [[SELECT34]], 0.000000e+00
+; COUNT1-NEXT:    [[SELECT639:%.*]] = select i1 [[FCMP62]], double 0.000000e+00, double 1.000000e+00
+; COUNT1-NEXT:    [[FCMP76:%.*]] = fcmp olt double [[SELECT40]], 0.000000e+00
+; COUNT1-NEXT:    [[SELECT77:%.*]] = select i1 [[FCMP76]], double 0.000000e+00, double 1.000000e+00
+; COUNT1-NEXT:    [[FCMP90:%.*]] = fcmp ogt double [[SELECT639]], 0.000000e+00
+; COUNT1-NEXT:    [[SELECT91:%.*]] = select i1 [[FCMP90]], double 0.000000e+00, double [[LOAD2]]
+; COUNT1-NEXT:    [[FCMP92:%.*]] = fcmp ogt double [[SELECT77]], 0.000000e+00
+; COUNT1-NEXT:    [[SELECT93:%.*]] = select i1 [[FCMP92]], double 0.000000e+00, double [[LOAD2]]
+; COUNT1-NEXT:    [[FCMP108:%.*]] = fcmp olt double [[SELECT93]], 0.000000e+00
+; COUNT1-NEXT:    [[SELECT109:%.*]] = select i1 [[FCMP108]], double 1.000000e+00, double 0.000000e+00
+; COUNT1-NEXT:    [[FCMP110:%.*]] = fcmp olt double [[SELECT91]], 0.000000e+00
+; COUNT1-NEXT:    [[SELECT111:%.*]] = select i1 [[FCMP110]], double 1.000000e+00, double 0.000000e+00
+; COUNT1-NEXT:    store double [[SELECT111]], ptr [[GETELEMENTPTR21]], align 8
+; COUNT1-NEXT:    store double [[SELECT109]], ptr [[GETELEMENTPTR13]], align 8
+; COUNT1-NEXT:    ret void
+;
+; COUNT2-LABEL: define void @blam
+; COUNT2-SAME: (ptr [[ARG:%.*]], double [[LOAD2:%.*]], i1 [[FCMP3:%.*]]) {
+; COUNT2-NEXT:  bb:
+; COUNT2-NEXT:    [[GETELEMENTPTR:%.*]] = getelementptr double, ptr [[ARG]], i64 1
+; COUNT2-NEXT:    [[LOAD:%.*]] = load double, ptr [[GETELEMENTPTR]], align 8
+; COUNT2-NEXT:    [[SELECT4:%.*]] = select i1 [[FCMP3]], double 0.000000e+00, double [[LOAD]]
+; COUNT2-NEXT:    [[LOAD7:%.*]] = load double, ptr [[ARG]], align 8
+; COUNT2-NEXT:    [[SELECT10:%.*]] = select i1 [[FCMP3]], double 0.000000e+00, double [[LOAD7]]
+; COUNT2-NEXT:    [[FCMP11:%.*]] = fcmp olt double [[LOAD2]], 0.000000e+00
+; COUNT2-NEXT:    [[SELECT128:%.*]] = select i1 [[FCMP11]], double 0.000000e+00, double [[LOAD7]]
+; COUNT2-NEXT:    [[GETELEMENTPTR13:%.*]] = getelementptr double, ptr [[ARG]], i64 3
+; COUNT2-NEXT:    [[GETELEMENTPTR21:%.*]] = getelementptr double, ptr [[ARG]], i64 4
+; COUNT2-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[SELECT10]], i32 0
+; COUNT2-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[LOAD]], i32 1
+; COUNT2-NEXT:    [[TMP2:%.*]] = fcmp olt <2 x double> [[TMP1]], zeroinitializer
+; COUNT2-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> <double 1.000000e+00, double poison>, <2 x i32> <i32 2, i32 1>
+; COUNT2-NEXT:    [[TMP4:%.*]] = select <2 x i1> [[TMP2]], <2 x double> zeroinitializer, <2 x double> [[TMP3]]
+; COUNT2-NEXT:    [[FCMP29:%.*]] = fcmp olt double [[SELECT4]], 0.000000e+00
+; COUNT2-NEXT:    [[SELECT30:%.*]] = select i1 [[FCMP29]], double 1.000000e+00, double 0.000000e+00
+; COUNT2-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
+; COUNT2-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[TMP4]], i32 1
+; COUNT2-NEXT:    [[FCMP33:%.*]] = fcmp olt double [[TMP5]], [[TMP6]]
+; COUNT2-NEXT:    [[SELECT34:%.*]] = select i1 [[FCMP33]], double 0.000000e+00, double [[LOAD2]]
+; COUNT2-NEXT:    [[FCMP39:%.*]] = fcmp olt double [[SELECT30]], [[SELECT128]]
+; COUNT2-NEXT:    [[SELECT40:%.*]] = select i1 [[FCMP39]], double [[LOAD2]], double 0.000000e+00
+; COUNT2-NEXT:    [[FCMP62:%.*]] = fcmp olt double [[SELECT34]], 0.000000e+00
+; COUNT2-NEXT:    [[SELECT639:%.*]] = select i1 [[FCMP62]], double 0.000000e+00, double 1.000000e+00
+; COUNT2-NEXT:    [[FCMP76:%.*]] = fcmp olt double [[SELECT40]], 0.000000e+00
+; COUNT2-NEXT:    [[SELECT77:%.*]] = select i1 [[FCMP76]], double 0.000000e+00, double 1.000000e+00
+; COUNT2-NEXT:    [[FCMP90:%.*]] = fcmp ogt double [[SELECT639]], 0.000000e+00
+; COUNT2-NEXT:    [[SELECT91:%.*]] = select i1 [[FCMP90]], double 0.000000e+00, double [[LOAD2]]
+; COUNT2-NEXT:    [[FCMP92:%.*]] = fcmp ogt double [[SELECT77]], 0.000000e+00
+; COUNT2-NEXT:    [[SELECT93:%.*]] = select i1 [[FCMP92]], double 0.000000e+00, double [[LOAD2]]
+; COUNT2-NEXT:    [[FCMP108:%.*]] = fcmp olt double [[SELECT93]], 0.000000e+00
+; COUNT2-NEXT:    [[SELECT109:%.*]] = select i1 [[FCMP108]], double 1.000000e+00, double 0.000000e+00
+; COUNT2-NEXT:    [[FCMP110:%.*]] = fcmp olt double [[SELECT91]], 0.000000e+00
+; COUNT2-NEXT:    [[SELECT111:%.*]] = select i1 [[FCMP110]], double 1.000000e+00, double 0.000000e+00
+; COUNT2-NEXT:    store double [[SELECT111]], ptr [[GETELEMENTPTR21]], align 8
+; COUNT2-NEXT:    store double [[SELECT109]], ptr [[GETELEMENTPTR13]], align 8
+; COUNT2-NEXT:    ret void
+;
+; COUNT-1-LABEL: define void @blam
+; COUNT-1-SAME: (ptr [[ARG:%.*]], double [[LOAD2:%.*]], i1 [[FCMP3:%.*]]) {
+; COUNT-1-NEXT:  bb:
+; COUNT-1-NEXT:    [[GETELEMENTPTR13:%.*]] = getelementptr double, ptr [[ARG]], i64 3
+; COUNT-1-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[ARG]], align 8
+; COUNT-1-NEXT:    [[TMP1:%.*]] = insertelement <2 x i1> poison, i1 [[FCMP3]], i32 0
+; COUNT-1-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i1> [[TMP1]], <2 x i1> poison, <2 x i32> zeroinitializer
+; COUNT-1-NEXT:    [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x double> zeroinitializer, <2 x double> [[TMP0]]
+; COUNT-1-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[TMP0]], double [[LOAD2]], i32 0
+; COUNT-1-NEXT:    [[TMP5:%.*]] = fcmp olt <2 x double> [[TMP4]], zeroinitializer
+; COUNT-1-NEXT:    [[TMP6:%.*]] = select <2 x i1> [[TMP5]], <2 x double> zeroinitializer, <2 x double> [[TMP0]]
+; COUNT-1-NEXT:    [[TMP7:%.*]] = fcmp olt <2 x double> [[TMP3]], zeroinitializer
+; COUNT-1-NEXT:    [[TMP8:%.*]] = select <2 x i1> [[TMP7]], <2 x double> <double 0.000000e+00, double 1.000000e+00>, <2 x double> <double 1.000000e+00, double 0.000000e+00>
+; COUNT-1-NEXT:    [[TMP9:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
+; COUNT-1-NEXT:    [[TMP10:%.*]] = fcmp olt <2 x double> [[TMP9]], [[TMP6]]
+; COUNT-1-NEXT:    [[TMP11:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> <double poison, double 0.000000e+00>, <2 x i32> <i32 0, i32 3>
+; COUNT-1-NEXT:    [[TMP12:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> <double 0.000000e+00, double poison>, <2 x i32> <i32 2, i32 0>
+; COUNT-1-NEXT:    [[TMP13:%.*]] = select <2 x i1> [[TMP10]], <2 x double> [[TMP11]], <2 x double> [[TMP12]]
+; COUNT-1-NEXT:    [[TMP14:%.*]] = fcmp olt <2 x double> [[TMP13]], zeroinitializer
+; COUNT-1-NEXT:    [[TMP15:%.*]] = select <2 x i1> [[TMP14]], <2 x double> zeroinitializer, <2 x double> <double 1.000000e+00, double 1.000000e+00>
+; COUNT-1-NEXT:    [[TMP16:%.*]] = fcmp ogt <2 x double> [[TMP15]], zeroinitializer
+; COUNT-1-NEXT:    [[TMP17:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> poison, <2 x i32> zeroinitializer
+; COUNT-1-NEXT:    [[TMP18:%.*]] = select <2 x i1> [[TMP16]], <2 x double> zeroinitializer, <2 x double> [[TMP17]]
+; COUNT-1-NEXT:    [[TMP19:%.*]] = fcmp olt <2 x double> [[TMP18]], zeroinitializer
+; COUNT-1-NEXT:    [[TMP20:%.*]] = select <2 x i1> [[TMP19]], <2 x double> <double 1.000000e+00, double 1.000000e+00>, <2 x double> zeroinitializer
+; COUNT-1-NEXT:    store <2 x double> [[TMP20]], ptr [[GETELEMENTPTR13]], align 8
+; COUNT-1-NEXT:    ret void
+;
+bb:
+  %getelementptr = getelementptr double, ptr %arg, i64 1
+  %load = load double, ptr %getelementptr, align 8
+  %fcmp = fcmp olt double %load, 0.000000e+00
+  %select3 = select i1 %fcmp, double 0.000000e+00, double %load
+  %select4 = select i1 %fcmp3, double 0.000000e+00, double %load
+  %load7 = load double, ptr %arg, align 8
+  %select10 = select i1 %fcmp3, double 0.000000e+00, double %load7
+  %fcmp11 = fcmp olt double %load2, 0.000000e+00
+  %select128 = select i1 %fcmp11, double 0.000000e+00, double %load7
+  %getelementptr13 = getelementptr double, ptr %arg, i64 3
+  %getelementptr21 = getelementptr double, ptr %arg, i64 4
+  %fcmp23 = fcmp olt double %select10, 0.000000e+00
+  %select24 = select i1 %fcmp23, double 0.000000e+00, double 1.000000e+00
+  %fcmp29 = fcmp olt double %select4, 0.000000e+00
+  %select30 = select i1 %fcmp29, double 1.000000e+00, double 0.000000e+00
+  %fcmp33 = fcmp olt double %select24, %select3
+  %select34 = select i1 %fcmp33, double 0.000000e+00, double %load2
+  %fcmp39 = fcmp olt double %select30, %select128
+  %select40 = select i1 %fcmp39, double %load2, double 0.000000e+00
+  %fcmp62 = fcmp olt double %select34, 0.000000e+00
+  %select639 = select i1 %fcmp62, double 0.000000e+00, double 1.000000e+00
+  %fcmp76 = fcmp olt double %select40, 0.000000e+00
+  %select77 = select i1 %fcmp76, double 0.000000e+00, double 1.000000e+00
+  %fcmp90 = fcmp ogt double %select639, 0.000000e+00
+  %select91 = select i1 %fcmp90, double 0.000000e+00, double %load2
+  %fcmp92 = fcmp ogt double %select77, 0.000000e+00
+  %select93 = select i1 %fcmp92, double 0.000000e+00, double %load2
+  %fcmp108 = fcmp olt double %select93, 0.000000e+00
+  %select109 = select i1 %fcmp108, double 1.000000e+00, double 0.000000e+00
+  %fcmp110 = fcmp olt double %select91, 0.000000e+00
+  %select111 = select i1 %fcmp110, double 1.000000e+00, double 0.000000e+00
+  store double %select111, ptr %getelementptr21, align 8
+  store double %select109, ptr %getelementptr13, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/long-full-reg-stores.ll b/llvm/test/Transforms/SLPVectorizer/X86/long-full-reg-stores.ll
new file mode 100644
index 0000000000000..b5f993f986c7c
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/long-full-reg-stores.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define void @test(ptr noalias %0, ptr noalias %1) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr noalias [[TMP0:%.*]], ptr noalias [[TMP1:%.*]]) {
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i64 24
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[TMP1]], i64 48
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[TMP1]], i64 8
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[TMP1]], i64 16
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP0]], i64 24
+; CHECK-NEXT:    [[TMP8:%.*]] = load double, ptr [[TMP7]], align 8
+; CHECK-NEXT:    store double [[TMP8]], ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP0]], i64 48
+; CHECK-NEXT:    [[TMP10:%.*]] = load double, ptr [[TMP9]], align 16
+; CHECK-NEXT:    store double [[TMP10]], ptr [[TMP6]], align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP0]], i64 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load double, ptr [[TMP11]], align 8
+; CHECK-NEXT:    store double [[TMP12]], ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[TMP0]], i64 32
+; CHECK-NEXT:    [[TMP14:%.*]] = load double, ptr [[TMP13]], align 16
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[TMP1]], i64 32
+; CHECK-NEXT:    store double [[TMP14]], ptr [[TMP15]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[TMP0]], i64 56
+; CHECK-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr i8, ptr [[TMP1]], i64 40
+; CHECK-NEXT:    store double [[TMP17]], ptr [[TMP18]], align 8
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[TMP0]], i64 16
+; CHECK-NEXT:    [[TMP20:%.*]] = load double, ptr [[TMP19]], align 16
+; CHECK-NEXT:    store double [[TMP20]], ptr [[TMP4]], align 16
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr i8, ptr [[TMP0]], i64 40
+; CHECK-NEXT:    [[TMP22:%.*]] = load double, ptr [[TMP21]], align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr i8, ptr [[TMP1]], i64 56
+; CHECK-NEXT:    store double [[TMP22]], ptr [[TMP23]], align 8
+; CHECK-NEXT:    ret void
+;
+  %3 = getelementptr i8, ptr %1, i64 24
+  %4 = getelementptr i8, ptr %1, i64 48
+  %5 = getelementptr i8, ptr %1, i64 8
+  %6 = getelementptr i8, ptr %1, i64 16
+  %8 = getelementptr i8, ptr %0, i64 24
+  %9 = load double, ptr %8, align 8
+  store double %9, ptr %5, align 8
+  %10 = getelementptr i8, ptr %0, i64 48
+  %11 = load double, ptr %10, align 16
+  store double %11, ptr %6, align 16
+  %12 = getelementptr i8, ptr %0, i64 8
+  %13 = load double, ptr %12, align 8
+  store double %13, ptr %3, align 8
+  %14 = getelementptr i8, ptr %0, i64 32
+  %15 = load double, ptr %14, align 16
+  %16 = getelementptr i8, ptr %1, i64 32
+  store double %15, ptr %16, align 16
+  %17 = getelementptr i8, ptr %0, i64 56
+  %18 = load double, ptr %17, align 8
+  %19 = getelementptr i8, ptr %1, i64 40
+  store double %18, ptr %19, align 8
+  %20 = getelementptr i8, ptr %0, i64 16
+  %21 = load double, ptr %20, align 16
+  store double %21, ptr %4, align 16
+  %22 = getelementptr i8, ptr %0, i64 40
+  %23 = load double, ptr %22, align 8
+  %24 = getelementptr i8, ptr %1, i64 56
+  store double %23, ptr %24, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduced-val-vectorized-in-transform.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduced-val-vectorized-in-transform.ll
new file mode 100644
index 0000000000000..81f3bf99f3fd8
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduced-val-vectorized-in-transform.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-10 < %s | FileCheck %s
+
+define i32 @test(i1 %cond) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BB:.*]]
+; CHECK:       [[BB]]:
+; CHECK-NEXT:    [[P1:%.*]] = phi i32 [ [[OR92:%.*]], %[[BB]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x i32> [ [[TMP8:%.*]], %[[BB]] ], [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 1, 0
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> <i32 poison, i32 poison, i32 0, i32 0>, <4 x i32> <i32 poison, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[P1]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i32> zeroinitializer, [[TMP4]]
+; CHECK-NEXT:    [[OR92]] = or i32 1, 0
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> [[TMP5]])
+; CHECK-NEXT:    [[OP_RDX:%.*]] = xor i32 [[TMP6]], [[OR92]]
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[OP_RDX]], i32 0
+; CHECK-NEXT:    [[TMP8]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP1]], i32 1
+; CHECK-NEXT:    br i1 [[COND]], label %[[EXIT:.*]], label %[[BB]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i32 [[OP_RDX]]
+;
+entry:
+  br label %bb
+
+bb:
+  %p1 = phi i32 [ %or92, %bb ], [ 0, %entry ]
+  %p2 = phi i32 [ %0, %bb ], [ 0, %entry ]
+  %p3 = phi i32 [ %4, %bb ], [ 0, %entry ]
+  %0 = or i32 1, 0
+  %or8.i = or i32 0, 0
+  %or9.i = or i32 0, 0
+  %or91 = or i32 %p1, 0
+  %or12.i = or i32 %p2, 0
+  %or92 = or i32 1, 0
+  %1 = xor i32 %or91, %or12.i
+  %2 = xor i32 %1, %or9.i
+  %3 = xor i32 %2, %or8.i
+  %4 = xor i32 %3, %or92
+  br i1 %cond, label %exit, label %bb
+
+exit:
+  ret i32 %4
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reused-mask-with-poison-index.ll b/llvm/test/Transforms/SLPVectorizer/X86/reused-mask-with-poison-index.ll
new file mode 100644
index 0000000000000..ea6d96147c951
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reused-mask-with-poison-index.ll
@@ -0,0 +1,186 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux < %s | FileCheck %s
+
+define fastcc i32 @test(ptr %0, <2 x float> %1, i1 %2, float %3, float %4) {
+; CHECK-LABEL: define fastcc i32 @test(
+; CHECK-SAME: ptr [[TMP0:%.*]], <2 x float> [[TMP1:%.*]], i1 [[TMP2:%.*]], float [[TMP3:%.*]], float [[TMP4:%.*]]) {
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x float> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x float> [[TMP1]], i64 1
+; CHECK-NEXT:    br label %[[BB8:.*]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[TMP9:%.*]] = phi float [ 0.000000e+00, [[TMP5:%.*]] ], [ [[TMP58:%.*]], %[[TMP56:.*]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = phi float [ 0.000000e+00, [[TMP5]] ], [ [[TMP59:%.*]], %[[TMP56]] ]
+; CHECK-NEXT:    [[TMP11:%.*]] = phi float [ [[TMP4]], [[TMP5]] ], [ [[TMP60:%.*]], %[[TMP56]] ]
+; CHECK-NEXT:    [[TMP12:%.*]] = phi float [ [[TMP7]], [[TMP5]] ], [ [[TMP61:%.*]], %[[TMP56]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = phi float [ [[TMP6]], [[TMP5]] ], [ [[TMP62:%.*]], %[[TMP56]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = phi float [ 0.000000e+00, [[TMP5]] ], [ [[TMP63:%.*]], %[[TMP56]] ]
+; CHECK-NEXT:    [[TMP15:%.*]] = phi float [ 0.000000e+00, [[TMP5]] ], [ [[TMP64:%.*]], %[[TMP56]] ]
+; CHECK-NEXT:    [[TMP16:%.*]] = phi float [ 0.000000e+00, [[TMP5]] ], [ [[TMP65:%.*]], %[[TMP56]] ]
+; CHECK-NEXT:    [[TMP17:%.*]] = phi float [ undef, [[TMP5]] ], [ [[TMP66:%.*]], %[[TMP56]] ]
+; CHECK-NEXT:    [[TMP18:%.*]] = phi float [ 0.000000e+00, [[TMP5]] ], [ [[TMP67:%.*]], %[[TMP56]] ]
+; CHECK-NEXT:    [[TMP19:%.*]] = phi float [ [[TMP4]], [[TMP5]] ], [ [[TMP68:%.*]], %[[TMP56]] ]
+; CHECK-NEXT:    [[TMP20:%.*]] = phi float [ [[TMP4]], [[TMP5]] ], [ [[TMP69:%.*]], %[[TMP56]] ]
+; CHECK-NEXT:    [[TMP21:%.*]] = phi float [ [[TMP4]], [[TMP5]] ], [ [[TMP70:%.*]], %[[TMP56]] ]
+; CHECK-NEXT:    [[TMP22:%.*]] = phi float [ [[TMP4]], [[TMP5]] ], [ [[TMP71:%.*]], %[[TMP56]] ]
+; CHECK-NEXT:    [[TMP23:%.*]] = phi float [ 0.000000e+00, [[TMP5]] ], [ [[TMP73:%.*]], %[[TMP56]] ]
+; CHECK-NEXT:    [[TMP24:%.*]] = phi float [ 0.000000e+00, [[TMP5]] ], [ [[TMP72:%.*]], %[[TMP56]] ]
+; CHECK-NEXT:    [[TMP25:%.*]] = phi <4 x float> [ zeroinitializer, [[TMP5]] ], [ poison, %[[TMP56]] ]
+; CHECK-NEXT:    [[TMP26:%.*]] = phi <2 x float> [ zeroinitializer, [[TMP5]] ], [ poison, %[[TMP56]] ]
+; CHECK-NEXT:    br i1 false, label %[[BB57:.*]], label %[[BB27:.*]]
+; CHECK:       [[BB27]]:
+; CHECK-NEXT:    [[TMP28:%.*]] = fcmp olt float [[TMP22]], 0.000000e+00
+; CHECK-NEXT:    br i1 [[TMP28]], label %[[BB57]], label %[[BB29:.*]]
+; CHECK:       [[BB29]]:
+; CHECK-NEXT:    [[TMP30:%.*]] = fcmp olt float [[TMP21]], 0.000000e+00
+; CHECK-NEXT:    br i1 [[TMP30]], label %[[BB57]], label %[[BB31:.*]]
+; CHECK:       [[BB31]]:
+; CHECK-NEXT:    [[TMP32:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP33:%.*]] = shufflevector <4 x float> [[TMP32]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP33]], <4 x float> zeroinitializer, <4 x float> zeroinitializer)
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <4 x float> [[TMP25]], i32 0
+; CHECK-NEXT:    [[TMP36:%.*]] = fsub float [[TMP17]], [[TMP35]]
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <4 x float> [[TMP25]], i32 1
+; CHECK-NEXT:    [[TMP38:%.*]] = fsub float [[TMP15]], [[TMP37]]
+; CHECK-NEXT:    [[TMP39:%.*]] = insertelement <2 x float> zeroinitializer, float [[TMP14]], i64 0
+; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <4 x float> [[TMP25]], <4 x float> poison, <2 x i32> <i32 poison, i32 2>
+; CHECK-NEXT:    [[TMP41:%.*]] = shufflevector <2 x float> [[TMP39]], <2 x float> [[TMP40]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP42:%.*]] = insertelement <2 x float> zeroinitializer, float [[TMP38]], i64 0
+; CHECK-NEXT:    [[TMP43:%.*]] = insertelement <2 x float> zeroinitializer, float [[TMP36]], i64 0
+; CHECK-NEXT:    [[TMP44:%.*]] = fmul <2 x float> [[TMP42]], [[TMP43]]
+; CHECK-NEXT:    [[TMP45:%.*]] = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP41]], <2 x float> [[TMP26]], <2 x float> [[TMP44]])
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <2 x float> [[TMP45]], i64 0
+; CHECK-NEXT:    [[TMP47:%.*]] = fcmp ogt float [[TMP46]], 0.000000e+00
+; CHECK-NEXT:    br i1 [[TMP47]], label %[[BB48:.*]], label %[[BB50:.*]]
+; CHECK:       [[BB48]]:
+; CHECK-NEXT:    br label %[[BB50]]
+; CHECK:       [[BB49:.*]]:
+; CHECK-NEXT:    br label %[[BB50]]
+; CHECK:       [[BB50]]:
+; CHECK-NEXT:    [[TMP51:%.*]] = phi float [ [[TMP12]], %[[BB49]] ], [ [[TMP19]], %[[BB48]] ], [ 0.000000e+00, %[[BB31]] ]
+; CHECK-NEXT:    [[TMP52:%.*]] = phi float [ [[TMP13]], %[[BB49]] ], [ [[TMP20]], %[[BB48]] ], [ 0.000000e+00, %[[BB31]] ]
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[BB57]], label %[[BB53:.*]]
+; CHECK:       [[BB53]]:
+; CHECK-NEXT:    [[TMP54:%.*]] = extractelement <2 x float> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP55:%.*]] = extractelement <4 x float> [[TMP25]], i32 3
+; CHECK-NEXT:    br label %[[BB57]]
+; CHECK:       [[TMP56]]:
+; CHECK-NEXT:    br label %[[BB8]]
+; CHECK:       [[BB57]]:
+; CHECK-NEXT:    [[TMP58]] = phi float [ [[TMP9]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP3]], %[[BB50]] ]
+; CHECK-NEXT:    [[TMP59]] = phi float [ [[TMP10]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ [[TMP55]], %[[BB53]] ], [ 0.000000e+00, %[[BB50]] ]
+; CHECK-NEXT:    [[TMP60]] = phi float [ [[TMP11]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP3]], %[[BB50]] ]
+; CHECK-NEXT:    [[TMP61]] = phi float [ [[TMP12]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ 0.000000e+00, %[[BB50]] ]
+; CHECK-NEXT:    [[TMP62]] = phi float [ [[TMP13]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ [[TMP54]], %[[BB53]] ], [ 0.000000e+00, %[[BB50]] ]
+; CHECK-NEXT:    [[TMP63]] = phi float [ [[TMP14]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP9]], %[[BB50]] ]
+; CHECK-NEXT:    [[TMP64]] = phi float [ [[TMP15]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ [[TMP55]], %[[BB53]] ], [ [[TMP10]], %[[BB50]] ]
+; CHECK-NEXT:    [[TMP65]] = phi float [ [[TMP16]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP11]], %[[BB50]] ]
+; CHECK-NEXT:    [[TMP66]] = phi float [ [[TMP17]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ 0.000000e+00, %[[BB50]] ]
+; CHECK-NEXT:    [[TMP67]] = phi float [ [[TMP18]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ 0.000000e+00, %[[BB50]] ]
+; CHECK-NEXT:    [[TMP68]] = phi float [ [[TMP19]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP3]], %[[BB50]] ]
+; CHECK-NEXT:    [[TMP69]] = phi float [ [[TMP20]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ [[TMP54]], %[[BB53]] ], [ 0.000000e+00, %[[BB50]] ]
+; CHECK-NEXT:    [[TMP70]] = phi float [ [[TMP21]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP51]], %[[BB50]] ]
+; CHECK-NEXT:    [[TMP71]] = phi float [ [[TMP22]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ [[TMP54]], %[[BB53]] ], [ [[TMP52]], %[[BB50]] ]
+; CHECK-NEXT:    [[TMP72]] = phi float [ [[TMP24]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP24]], %[[BB50]] ]
+; CHECK-NEXT:    [[TMP73]] = phi float [ [[TMP23]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP23]], %[[BB50]] ]
+; CHECK-NEXT:    [[TMP74:%.*]] = phi <4 x float> [ [[TMP25]], %[[BB29]] ], [ [[TMP25]], %[[BB27]] ], [ zeroinitializer, %[[BB8]] ], [ [[TMP34]], %[[BB53]] ], [ [[TMP34]], %[[BB50]] ]
+; CHECK-NEXT:    [[TMP75:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP73]], float [[TMP72]], float 0.000000e+00)
+; CHECK-NEXT:    store float [[TMP75]], ptr [[TMP0]], align 4
+; CHECK-NEXT:    ret i32 0
+;
+  %6 = extractelement <2 x float> %1, i64 0
+  %7 = extractelement <2 x float> %1, i64 1
+  br label %8
+
+8:
+  %9 = phi float [ 0.000000e+00, %5 ], [ %59, %57 ]
+  %10 = phi float [ 0.000000e+00, %5 ], [ %60, %57 ]
+  %11 = phi float [ 0.000000e+00, %5 ], [ %61, %57 ]
+  %12 = phi float [ %4, %5 ], [ %62, %57 ]
+  %13 = phi float [ %7, %5 ], [ %63, %57 ]
+  %14 = phi float [ %6, %5 ], [ %64, %57 ]
+  %15 = phi float [ 0.000000e+00, %5 ], [ %65, %57 ]
+  %16 = phi float [ 0.000000e+00, %5 ], [ %66, %57 ]
+  %17 = phi float [ 0.000000e+00, %5 ], [ %67, %57 ]
+  %18 = phi float [ 0.000000e+00, %5 ], [ %68, %57 ]
+  %19 = phi float [ undef, %5 ], [ %69, %57 ]
+  %20 = phi float [ 0.000000e+00, %5 ], [ %70, %57 ]
+  %21 = phi float [ 0.000000e+00, %5 ], [ %71, %57 ]
+  %22 = phi float [ %4, %5 ], [ %72, %57 ]
+  %23 = phi float [ %4, %5 ], [ %73, %57 ]
+  %24 = phi float [ %4, %5 ], [ %74, %57 ]
+  %25 = phi float [ %4, %5 ], [ %75, %57 ]
+  %26 = phi float [ 0.000000e+00, %5 ], [ %76, %57 ]
+  %27 = phi float [ 0.000000e+00, %5 ], [ %78, %57 ]
+  %28 = phi float [ 0.000000e+00, %5 ], [ %77, %57 ]
+  br i1 false, label %58, label %29
+
+29:
+  %30 = fcmp olt float %25, 0.000000e+00
+  br i1 %30, label %58, label %31
+
+31:
+  %32 = fcmp olt float %24, 0.000000e+00
+  br i1 %32, label %58, label %33
+
+33:
+  %34 = tail call float @llvm.fmuladd.f32(float %3, float 0.000000e+00, float 0.000000e+00)
+  %35 = tail call float @llvm.fmuladd.f32(float %3, float 0.000000e+00, float 0.000000e+00)
+  %36 = tail call float @llvm.fmuladd.f32(float %3, float 0.000000e+00, float 0.000000e+00)
+  %37 = tail call float @llvm.fmuladd.f32(float %3, float 0.000000e+00, float 0.000000e+00)
+  %38 = insertelement <2 x float> zeroinitializer, float %17, i64 0
+  %39 = insertelement <2 x float> %38, float %21, i64 1
+  %40 = fsub float %19, %18
+  %41 = fsub float %16, %20
+  %42 = insertelement <2 x float> zeroinitializer, float %15, i64 0
+  %43 = insertelement <2 x float> %42, float %26, i64 1
+  %44 = insertelement <2 x float> zeroinitializer, float %41, i64 0
+  %45 = insertelement <2 x float> zeroinitializer, float %40, i64 0
+  %46 = fmul <2 x float> %44, %45
+  %47 = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %43, <2 x float> %39, <2 x float> %46)
+  %48 = extractelement <2 x float> %47, i64 0
+  %49 = fcmp ogt float %48, 0.000000e+00
+  br i1 %49, label %50, label %52
+
+50:
+  br label %52
+
+51:
+  br label %52
+
+52:
+  %53 = phi float [ %13, %51 ], [ %22, %50 ], [ 0.000000e+00, %33 ]
+  %54 = phi float [ %14, %51 ], [ %23, %50 ], [ 0.000000e+00, %33 ]
+  br i1 %2, label %58, label %55
+
+55:
+  %56 = extractelement <2 x float> %1, i64 0
+  br label %58
+
+57:
+  br label %8
+
+58:
+  %59 = phi float [ %9, %31 ], [ %9, %29 ], [ 0.000000e+00, %8 ], [ %36, %55 ], [ %36, %52 ]
+  %60 = phi float [ %10, %31 ], [ 0.000000e+00, %29 ], [ 0.000000e+00, %8 ], [ 0.000000e+00, %55 ], [ %3, %52 ]
+  %61 = phi float [ %11, %31 ], [ 0.000000e+00, %29 ], [ 0.000000e+00, %8 ], [ %9, %55 ], [ 0.000000e+00, %52 ]
+  %62 = phi float [ %12, %31 ], [ 0.000000e+00, %29 ], [ 0.000000e+00, %8 ], [ 0.000000e+00, %55 ], [ %3, %52 ]
+  %63 = phi float [ %13, %31 ], [ 0.000000e+00, %29 ], [ 0.000000e+00, %8 ], [ 0.000000e+00, %55 ], [ 0.000000e+00, %52 ]
+  %64 = phi float [ %14, %31 ], [ 0.000000e+00, %29 ], [ 0.000000e+00, %8 ], [ %56, %55 ], [ 0.000000e+00, %52 ]
+  %65 = phi float [ %15, %31 ], [ 0.000000e+00, %29 ], [ 0.000000e+00, %8 ], [ 0.000000e+00, %55 ], [ %10, %52 ]
+  %66 = phi float [ %16, %31 ], [ 0.000000e+00, %29 ], [ 0.000000e+00, %8 ], [ %9, %55 ], [ %11, %52 ]
+  %67 = phi float [ %17, %31 ], [ 0.000000e+00, %29 ], [ 0.000000e+00, %8 ], [ 0.000000e+00, %55 ], [ %12, %52 ]
+  %68 = phi float [ %18, %31 ], [ %18, %29 ], [ 0.000000e+00, %8 ], [ %35, %55 ], [ %35, %52 ]
+  %69 = phi float [ %19, %31 ], [ 0.000000e+00, %29 ], [ 0.000000e+00, %8 ], [ 0.000000e+00, %55 ], [ 0.000000e+00, %52 ]
+  %70 = phi float [ %20, %31 ], [ %20, %29 ], [ 0.000000e+00, %8 ], [ %34, %55 ], [ %34, %52 ]
+  %71 = phi float [ %21, %31 ], [ 0.000000e+00, %29 ], [ 0.000000e+00, %8 ], [ 0.000000e+00, %55 ], [ 0.000000e+00, %52 ]
+  %72 = phi float [ %22, %31 ], [ 0.000000e+00, %29 ], [ 0.000000e+00, %8 ], [ 0.000000e+00, %55 ], [ %3, %52 ]
+  %73 = phi float [ %23, %31 ], [ 0.000000e+00, %29 ], [ 0.000000e+00, %8 ], [ %56, %55 ], [ 0.000000e+00, %52 ]
+  %74 = phi float [ %24, %31 ], [ 0.000000e+00, %29 ], [ 0.000000e+00, %8 ], [ 0.000000e+00, %55 ], [ %53, %52 ]
+  %75 = phi float [ %25, %31 ], [ 0.000000e+00, %29 ], [ 0.000000e+00, %8 ], [ %56, %55 ], [ %54, %52 ]
+  %76 = phi float [ %26, %31 ], [ %26, %29 ], [ 0.000000e+00, %8 ], [ %37, %55 ], [ %37, %52 ]
+  %77 = phi float [ %28, %31 ], [ 0.000000e+00, %29 ], [ 0.000000e+00, %8 ], [ 0.000000e+00, %55 ], [ %28, %52 ]
+  %78 = phi float [ %27, %31 ], [ 0.000000e+00, %29 ], [ 0.000000e+00, %8 ], [ 0.000000e+00, %55 ], [ %27, %52 ]
+  %79 = tail call float @llvm.fmuladd.f32(float %78, float %77, float 0.000000e+00)
+  store float %79, ptr %0, align 4
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/subvector-minbitwidth-unsigned-value.ll b/llvm/test/Transforms/SLPVectorizer/X86/subvector-minbitwidth-unsigned-value.ll
new file mode 100644
index 0000000000000..2f126df81f230
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/subvector-minbitwidth-unsigned-value.ll
@@ -0,0 +1,97 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define i1 @test(i64 %v1, ptr %v2, i32 %v3, i1 %v4) {
+; CHECK-LABEL: define i1 @test(
+; CHECK-SAME: i64 [[V1:%.*]], ptr [[V2:%.*]], i32 [[V3:%.*]], i1 [[V4:%.*]]) {
+; CHECK-NEXT:  [[NEWFUNCROOT:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i64> poison, i64 [[V1]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <2 x i64> [[TMP1]], <i64 32, i64 40>
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc <2 x i64> [[TMP2]] to <2 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = and <2 x i8> [[TMP3]], <i8 1, i8 -1>
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <2 x i8> [[TMP4]] to <2 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq <2 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[V3]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> poison, <4 x i32> <i32 poison, i32 poison, i32 0, i32 0>
+; CHECK-NEXT:    [[TMP9:%.*]] = zext <2 x i8> [[TMP4]] to <2 x i32>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.vector.insert.v4i32.v2i32(<4 x i32> [[TMP8]], <2 x i32> [[TMP9]], i64 0)
+; CHECK-NEXT:    [[TMP11:%.*]] = uitofp <4 x i32> [[TMP10]] to <4 x float>
+; CHECK-NEXT:    [[TMP12:%.*]] = fdiv <4 x float> zeroinitializer, [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x i1> poison, i1 [[V4]], i32 0
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i1> [[TMP13]], <4 x i1> poison, <4 x i32> <i32 poison, i32 poison, i32 0, i32 0>
+; CHECK-NEXT:    [[TMP15:%.*]] = call <4 x i1> @llvm.vector.insert.v4i1.v2i1(<4 x i1> [[TMP14]], <2 x i1> [[TMP6]], i64 0)
+; CHECK-NEXT:    [[TMP16:%.*]] = select <4 x i1> [[TMP15]], <4 x float> zeroinitializer, <4 x float> [[TMP12]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x float> [[TMP16]], i32 3
+; CHECK-NEXT:    [[CONV_I_I1743_3:%.*]] = fptoui float [[TMP17]] to i32
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp ne i32 [[CONV_I_I1743_3]], 0
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <4 x float> [[TMP16]] to <4 x i32>
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp ult <4 x i32> [[TMP19]], <i32 1333788672, i32 1333788672, i32 1333788672, i32 1333788672>
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x i1> [[TMP20]], i32 3
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP21]], i1 [[TMP18]], i1 false
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x float> [[TMP16]], i32 2
+; CHECK-NEXT:    [[CONV_I_I1743_2:%.*]] = fptoui float [[TMP22]] to i32
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <4 x i1> [[TMP20]], i32 2
+; CHECK-NEXT:    [[NARROW1:%.*]] = select i1 [[TMP23]], i32 [[CONV_I_I1743_2]], i32 0
+; CHECK-NEXT:    [[TMP24:%.*]] = zext i1 [[NARROW]] to i32
+; CHECK-NEXT:    [[TMP25:%.*]] = or i32 [[NARROW1]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <4 x float> [[TMP16]], i32 1
+; CHECK-NEXT:    [[CONV_I_I1743_1:%.*]] = fptoui float [[TMP26]] to i32
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <4 x i1> [[TMP20]], i32 1
+; CHECK-NEXT:    [[NARROW2:%.*]] = select i1 [[TMP27]], i32 [[CONV_I_I1743_1]], i32 0
+; CHECK-NEXT:    [[RV3:%.*]] = or i32 [[TMP25]], [[NARROW2]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <4 x float> [[TMP16]], i32 0
+; CHECK-NEXT:    [[CONV_I_I1743:%.*]] = fptoui float [[TMP28]] to i32
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <4 x i1> [[TMP20]], i32 0
+; CHECK-NEXT:    [[NARROW4:%.*]] = select i1 [[TMP29]], i32 [[CONV_I_I1743]], i32 0
+; CHECK-NEXT:    [[RT5:%.*]] = or i32 [[RV3]], [[NARROW4]]
+; CHECK-NEXT:    [[RT:%.*]] = zext i32 [[RT5]] to i64
+; CHECK-NEXT:    store i64 [[RT]], ptr [[V2]], align 1
+; CHECK-NEXT:    ret i1 false
+;
+newFuncRoot:
+  %conv.i147.i1756.3 = uitofp i32 %v3 to float
+  %div.i.i.i1749.3 = fdiv float 0.000000e+00, %conv.i147.i1756.3
+  %cond.i.i.i1751.3 = select i1 %v4, float 0.000000e+00, float %div.i.i.i1749.3
+  %conv.i147.i1756.2 = uitofp i32 %v3 to float
+  %div.i.i.i1749.2 = fdiv float 0.000000e+00, %conv.i147.i1756.2
+  %cond.i.i.i1751.2 = select i1 %v4, float 0.000000e+00, float %div.i.i.i1749.2
+  %0 = lshr i64 %v1, 40
+  %1 = trunc i64 %0 to i32
+  %tt2 = and i32 %1, 255
+  %cmp1.i.i.i1746.1 = icmp eq i32 %tt2, 0
+  %conv.i147.i1756.1 = uitofp i32 %tt2 to float
+  %div.i.i.i1749.1 = fdiv float 0.000000e+00, %conv.i147.i1756.1
+  %cond.i.i.i1751.1 = select i1 %cmp1.i.i.i1746.1, float 0.000000e+00, float %div.i.i.i1749.1
+  %tt3 = lshr i64 %v1, 32
+  %2 = trunc i64 %tt3 to i32
+  %tt1 = and i32 %2, 1
+  %cmp1.i.i.i1746 = icmp eq i32 %tt1, 0
+  %conv.i147.i1756 = uitofp i32 %tt1 to float
+  %div.i.i.i1749 = fdiv float 0.000000e+00, %conv.i147.i1756
+  %cond.i.i.i1751 = select i1 %cmp1.i.i.i1746, float 0.000000e+00, float %div.i.i.i1749
+  %3 = bitcast float %cond.i.i.i1751.3 to i32
+  %cmp.i99.i1736.3 = icmp ult i32 %3, 1333788672
+  %conv.i.i1743.3 = fptoui float %cond.i.i.i1751.3 to i32
+  %4 = icmp ne i32 %conv.i.i1743.3, 0
+  %narrow = select i1 %cmp.i99.i1736.3, i1 %4, i1 false
+  %5 = bitcast float %cond.i.i.i1751.2 to i32
+  %cmp.i99.i1736.2 = icmp ult i32 %5, 1333788672
+  %conv.i.i1743.2 = fptoui float %cond.i.i.i1751.2 to i32
+  %narrow1 = select i1 %cmp.i99.i1736.2, i32 %conv.i.i1743.2, i32 0
+  %6 = zext i1 %narrow to i32
+  %7 = or i32 %narrow1, %6
+  %8 = bitcast float %cond.i.i.i1751.1 to i32
+  %cmp.i99.i1736.1 = icmp ult i32 %8, 1333788672
+  %conv.i.i1743.1 = fptoui float %cond.i.i.i1751.1 to i32
+  %narrow2 = select i1 %cmp.i99.i1736.1, i32 %conv.i.i1743.1, i32 0
+  %rv3 = or i32 %7, %narrow2
+  %9 = bitcast float %cond.i.i.i1751 to i32
+  %cmp.i99.i1736 = icmp ult i32 %9, 1333788672
+  %conv.i.i1743 = fptoui float %cond.i.i.i1751 to i32
+  %narrow4 = select i1 %cmp.i99.i1736, i32 %conv.i.i1743, i32 0
+  %rt5 = or i32 %rv3, %narrow4
+  %rt = zext i32 %rt5 to i64
+  store i64 %rt, ptr %v2, align 1
+  ret i1 false
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/addsub.ll b/llvm/test/Transforms/SLPVectorizer/addsub.ll
similarity index 98%
rename from llvm/test/Transforms/SLPVectorizer/X86/addsub.ll
rename to llvm/test/Transforms/SLPVectorizer/addsub.ll
index 96b498ced7d0f..3961250d56451 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/addsub.ll
+++ b/llvm/test/Transforms/SLPVectorizer/addsub.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer -S | FileCheck %s
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
+; RUN: %if x86-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=aarch64-unknown-linux-gnu | FileCheck %s %}
 
 @b = common global [4 x i32] zeroinitializer, align 16
 @c = common global [4 x i32] zeroinitializer, align 16
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/consecutive-access.ll b/llvm/test/Transforms/SLPVectorizer/consecutive-access.ll
similarity index 86%
rename from llvm/test/Transforms/SLPVectorizer/X86/consecutive-access.ll
rename to llvm/test/Transforms/SLPVectorizer/consecutive-access.ll
index 2e733930650ea..369ca28ece55b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/consecutive-access.ll
+++ b/llvm/test/Transforms/SLPVectorizer/consecutive-access.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer -S | FileCheck %s
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-apple-macosx10.9.0"
+; RUN: %if x86-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-apple-macosx10.9.0 | FileCheck %s --check-prefixes=CHECK-X86 %}
+; RUN: %if aarch64-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=aarch64-unknown-linux-gnu | FileCheck %s --check-prefixes=CHECK-AARCH64 %}
 
 @A = common global [2000 x double] zeroinitializer, align 16
 @B = common global [2000 x double] zeroinitializer, align 16
@@ -440,27 +439,55 @@ for.end:                                          ; preds = %for.cond.for.end_cr
 ; Make sure we are able to vectorize this from now on:
 ;
 define double @bar(ptr nocapture readonly %a, i32 %n) local_unnamed_addr #0 {
-; CHECK-LABEL: @bar(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP15:%.*]] = icmp eq i32 [[N:%.*]], 0
-; CHECK-NEXT:    br i1 [[CMP15]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY:%.*]]
-; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x double> [ zeroinitializer, [[ENTRY:%.*]] ], [ [[TMP5:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[TMP0]], i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[TMP0]], i32 1
-; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    ret double [[MUL]]
-; CHECK:       for.body:
-; CHECK-NEXT:    [[I_018:%.*]] = phi i32 [ [[ADD5:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = phi <2 x double> [ [[TMP5]], [[FOR_BODY]] ], [ zeroinitializer, [[ENTRY]] ]
-; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[I_018]] to i64
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 [[IDXPROM]]
-; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x double>, ptr [[ARRAYIDX]], align 8
-; CHECK-NEXT:    [[TMP5]] = fadd <2 x double> [[TMP3]], [[TMP4]]
-; CHECK-NEXT:    [[ADD5]] = add i32 [[I_018]], 2
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[ADD5]], [[N]]
-; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP]]
+; CHECK-X86-LABEL: @bar(
+; CHECK-X86-NEXT:  entry:
+; CHECK-X86-NEXT:    [[CMP15:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-X86-NEXT:    br i1 [[CMP15]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY:%.*]]
+; CHECK-X86:       for.cond.cleanup:
+; CHECK-X86-NEXT:    [[TMP0:%.*]] = phi <2 x double> [ zeroinitializer, [[ENTRY:%.*]] ], [ [[TMP5:%.*]], [[FOR_BODY]] ]
+; CHECK-X86-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[TMP0]], i32 0
+; CHECK-X86-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[TMP0]], i32 1
+; CHECK-X86-NEXT:    [[MUL:%.*]] = fmul double [[TMP1]], [[TMP2]]
+; CHECK-X86-NEXT:    ret double [[MUL]]
+; CHECK-X86:       for.body:
+; CHECK-X86-NEXT:    [[I_018:%.*]] = phi i32 [ [[ADD5:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
+; CHECK-X86-NEXT:    [[TMP3:%.*]] = phi <2 x double> [ [[TMP5]], [[FOR_BODY]] ], [ zeroinitializer, [[ENTRY]] ]
+; CHECK-X86-NEXT:    [[IDXPROM:%.*]] = zext i32 [[I_018]] to i64
+; CHECK-X86-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 [[IDXPROM]]
+; CHECK-X86-NEXT:    [[TMP4:%.*]] = load <2 x double>, ptr [[ARRAYIDX]], align 8
+; CHECK-X86-NEXT:    [[TMP5]] = fadd <2 x double> [[TMP3]], [[TMP4]]
+; CHECK-X86-NEXT:    [[ADD5]] = add i32 [[I_018]], 2
+; CHECK-X86-NEXT:    [[CMP:%.*]] = icmp ult i32 [[ADD5]], [[N]]
+; CHECK-X86-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP]]
 ;
+
+; CHECK-AARCH64-LABEL: @bar(
+; CHECK-AARCH64-NEXT:  entry:
+; CHECK-AARCH64-NEXT:    [[CMP15:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-AARCH64-NEXT:    br i1 [[CMP15]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY:%.*]]
+; CHECK-AARCH64:       for.cond.cleanup:
+; CHECK-AARCH64-NEXT:    [[X_0_LCSSA:%.*]] = phi double [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-AARCH64-NEXT:    [[Y_0_LCSSA:%.*]] = phi double [ 0.000000e+00, [[ENTRY]] ], [ [[ADD4:%.*]], [[FOR_BODY]] ]
+; CHECK-AARCH64-NEXT:    [[MUL:%.*]] = fmul double [[X_0_LCSSA]], [[Y_0_LCSSA]]
+; CHECK-AARCH64-NEXT:    ret double [[MUL]]
+; CHECK-AARCH64:       for.body:
+; CHECK-AARCH64-NEXT:    [[I_018:%.*]] = phi i32 [ [[ADD5:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
+; CHECK-AARCH64-NEXT:    [[Y_017:%.*]] = phi double [ [[ADD4]], [[FOR_BODY]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-AARCH64-NEXT:    [[X_016:%.*]] = phi double [ [[ADD]], [[FOR_BODY]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-AARCH64-NEXT:    [[IDXPROM:%.*]] = zext i32 [[I_018]] to i64
+; CHECK-AARCH64-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 [[IDXPROM]]
+; CHECK-AARCH64-NEXT:    [[TMP0:%.*]] = load double, ptr [[ARRAYIDX]], align 8
+; CHECK-AARCH64-NEXT:    [[ADD]] = fadd double [[X_016]], [[TMP0]]
+; CHECK-AARCH64-NEXT:    [[ADD1:%.*]] = or disjoint i32 [[I_018]], 1
+; CHECK-AARCH64-NEXT:    [[IDXPROM2:%.*]] = zext i32 [[ADD1]] to i64
+; CHECK-AARCH64-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[IDXPROM2]]
+; CHECK-AARCH64-NEXT:    [[TMP1:%.*]] = load double, ptr [[ARRAYIDX3]], align 8
+; CHECK-AARCH64-NEXT:    [[ADD4]] = fadd double [[Y_017]], [[TMP1]]
+; CHECK-AARCH64-NEXT:    [[ADD5]] = add i32 [[I_018]], 2
+; CHECK-AARCH64-NEXT:    [[CMP:%.*]] = icmp ult i32 [[ADD5]], [[N]]
+; CHECK-AARCH64-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP]]
+;
+
 entry:
   %cmp15 = icmp eq i32 %n, 0
   br i1 %cmp15, label %for.cond.cleanup, label %for.body
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extract-subvector-long-input.ll b/llvm/test/Transforms/SLPVectorizer/extract-subvector-long-input.ll
similarity index 89%
rename from llvm/test/Transforms/SLPVectorizer/X86/extract-subvector-long-input.ll
rename to llvm/test/Transforms/SLPVectorizer/extract-subvector-long-input.ll
index f90456297d7cb..a1f4590a56919 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extract-subvector-long-input.ll
+++ b/llvm/test/Transforms/SLPVectorizer/extract-subvector-long-input.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -passes=slp-vectorizer -S -slp-threshold=-99999 -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: %if x86-registered-target %{ opt -passes=slp-vectorizer -S -slp-threshold=-99999 -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt -passes=slp-vectorizer -S -slp-threshold=-99999 -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s %}
 
 define void @test() {
 ; CHECK-LABEL: define void @test() {
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extract-vectorized-operand.ll b/llvm/test/Transforms/SLPVectorizer/extract-vectorized-operand.ll
similarity index 86%
rename from llvm/test/Transforms/SLPVectorizer/X86/extract-vectorized-operand.ll
rename to llvm/test/Transforms/SLPVectorizer/extract-vectorized-operand.ll
index 7a860719505f0..910754f990199 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extract-vectorized-operand.ll
+++ b/llvm/test/Transforms/SLPVectorizer/extract-vectorized-operand.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -S --passes=slp-vectorizer -slp-threshold=-99999 < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -slp-threshold=-99999 < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -slp-threshold=-99999 < %s -mtriple=aarch64-unknown-linux-gnu | FileCheck %s %}
 
 define void @test() {
 ; CHECK-LABEL: define void @test() {
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extract_with_non_const_index.ll b/llvm/test/Transforms/SLPVectorizer/extract_with_non_const_index.ll
similarity index 90%
rename from llvm/test/Transforms/SLPVectorizer/X86/extract_with_non_const_index.ll
rename to llvm/test/Transforms/SLPVectorizer/extract_with_non_const_index.ll
index 8277ca193540c..55378991d8a8e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extract_with_non_const_index.ll
+++ b/llvm/test/Transforms/SLPVectorizer/extract_with_non_const_index.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s
+; RUN: %if x86-registered-target %{ opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt < %s -mtriple=aarch64-unknown -passes=slp-vectorizer -S | FileCheck %s %}
 
 ; Reproducer for an issue discussed here:
 ; https://reviews.llvm.org/D108703#2974289
diff --git a/llvm/test/Transforms/SLPVectorizer/gathered-consecutive-loads-different-types.ll b/llvm/test/Transforms/SLPVectorizer/gathered-consecutive-loads-different-types.ll
new file mode 100644
index 0000000000000..a854c61db6d28
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/gathered-consecutive-loads-different-types.ll
@@ -0,0 +1,205 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer < %s | FileCheck %s
+
+define i32 @test(i8 %0) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: i8 [[TMP0:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP13_NOT_5:%.*]] = icmp eq i64 0, 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr addrspace(21) getelementptr inbounds (i8, ptr addrspace(21) null, i64 7), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i8> <i8 0, i8 poison>, i8 [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <2 x i8> zeroinitializer, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load volatile i8, ptr null, align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i8>, ptr addrspace(21) getelementptr inbounds (i8, ptr addrspace(21) null, i64 8), align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i8> [[TMP5]], <2 x i8> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 poison, i32 poison, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i8> [[TMP6]], <8 x i8> <i8 0, i8 0, i8 poison, i8 0, i8 0, i8 poison, i8 0, i8 0>, <8 x i32> <i32 8, i32 9, i32 0, i32 11, i32 12, i32 1, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <8 x i8> zeroinitializer, [[TMP7]]
+; CHECK-NEXT:    [[TEST_STRUCTCOPY_14_S14_CM_COERCE_SROA_2_0_COPYLOAD:%.*]] = load i48, ptr addrspace(21) getelementptr inbounds (i8, ptr addrspace(21) null, i64 8), align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x i48> <i48 poison, i48 0, i48 0, i48 0>, i48 [[TEST_STRUCTCOPY_14_S14_CM_COERCE_SROA_2_0_COPYLOAD]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = trunc <4 x i48> [[TMP9]] to <4 x i8>
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq <4 x i8> zeroinitializer, [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr addrspace(21) null, align 2
+; CHECK-NEXT:    [[TMP13:%.*]] = load volatile i8, ptr null, align 2
+; CHECK-NEXT:    [[TMP14:%.*]] = load <2 x i8>, ptr addrspace(21) getelementptr inbounds (i8, ptr addrspace(21) null, i64 8), align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <2 x i8> [[TMP14]], <2 x i8> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <8 x i8> [[TMP15]], <8 x i8> <i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 0, i8 0, i8 0>, <8 x i32> <i32 8, i32 poison, i32 10, i32 3, i32 4, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <8 x i8> [[TMP16]], i8 [[TMP12]], i32 1
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <8 x i8> <i8 0, i8 poison, i8 0, i8 poison, i8 0, i8 0, i8 0, i8 0>, i8 [[TMP0]], i32 3
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <8 x i8> [[TMP18]], i8 [[TMP13]], i32 1
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq <8 x i8> [[TMP17]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <64 x i1> <i1 poison, i1 false, i1 false, i1 false, i1 false, i1 false, i1 poison, i1 poison, i1 poison, i1 poison, i1 poison, i1 poison, i1 poison, i1 poison, i1 poison, i1 poison, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 poison, i1 poison, i1 poison, i1 poison, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 poison, i1 poison, i1 poison, i1 poison, i1 poison, i1 poison, i1 poison, i1 poison>, i1 [[CMP13_NOT_5]], i32 0
+; CHECK-NEXT:    [[TMP22:%.*]] = call <64 x i1> @llvm.vector.insert.v64i1.v8i1(<64 x i1> [[TMP21]], <8 x i1> [[TMP8]], i64 8)
+; CHECK-NEXT:    [[TMP23:%.*]] = call <64 x i1> @llvm.vector.insert.v64i1.v8i1(<64 x i1> [[TMP22]], <8 x i1> [[TMP20]], i64 56)
+; CHECK-NEXT:    [[TMP24:%.*]] = call <64 x i1> @llvm.vector.insert.v64i1.v4i1(<64 x i1> [[TMP23]], <4 x i1> [[TMP11]], i64 32)
+; CHECK-NEXT:    [[TMP25:%.*]] = call <64 x i1> @llvm.vector.insert.v64i1.v2i1(<64 x i1> [[TMP24]], <2 x i1> [[TMP3]], i64 6)
+; CHECK-NEXT:    [[TMP26:%.*]] = select <64 x i1> [[TMP25]], <64 x i32> zeroinitializer, <64 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = call i32 @llvm.vector.reduce.or.v64i32(<64 x i32> [[TMP26]])
+; CHECK-NEXT:    ret i32 [[TMP27]]
+;
+entry:
+  %cmp13.not.5 = icmp eq i64 0, 0
+  %add15.5 = select i1 %cmp13.not.5, i32 0, i32 0
+  %add26.5 = select i1 false, i32 0, i32 0
+  %add27.5 = or i32 %add15.5, %add26.5
+  %add37.5 = select i1 false, i32 0, i32 0
+  %add38.5 = or i32 %add27.5, %add37.5
+  %add15.6 = select i1 false, i32 0, i32 0
+  %add16.6 = or i32 %add15.6, %add38.5
+  %add37.6 = select i1 false, i32 0, i32 0
+  %add38.6 = or i32 %add16.6, %add37.6
+  %add15.7 = select i1 false, i32 0, i32 0
+  %add16.7 = or i32 %add15.7, %add38.6
+  %cmp23.not.7 = icmp eq i8 0, 0
+  %add26.7 = select i1 %cmp23.not.7, i32 0, i32 0
+  %add27.7 = or i32 %add16.7, %add26.7
+  %1 = load i8, ptr addrspace(21) getelementptr inbounds (i8, ptr addrspace(21) null, i64 7), align 1
+  %cmp34.not.7 = icmp eq i8 %1, 0
+  %add37.7 = select i1 %cmp34.not.7, i32 0, i32 0
+  %add38.7 = or i32 %add27.7, %add37.7
+  %cmp13.not.8 = icmp eq i8 0, 0
+  %add15.8 = select i1 %cmp13.not.8, i32 0, i32 0
+  %add16.8 = or i32 %add15.8, %add38.7
+  %2 = load volatile i8, ptr null, align 8
+  %cmp23.not.8 = icmp eq i8 0, 0
+  %add26.8 = select i1 %cmp23.not.8, i32 0, i32 0
+  %add27.8 = or i32 %add16.8, %add26.8
+  %3 = load i8, ptr addrspace(21) getelementptr inbounds (i8, ptr addrspace(21) null, i64 8), align 8
+  %cmp34.not.8 = icmp eq i8 %3, 0
+  %add37.8 = select i1 %cmp34.not.8, i32 0, i32 0
+  %add38.8 = or i32 %add27.8, %add37.8
+  %cmp13.not.9 = icmp eq i8 0, 0
+  %add15.9 = select i1 %cmp13.not.9, i32 0, i32 0
+  %add16.9 = or i32 %add15.9, %add38.8
+  %cmp23.not.9 = icmp eq i8 0, 0
+  %add26.9 = select i1 %cmp23.not.9, i32 0, i32 0
+  %add27.9 = or i32 %add16.9, %add26.9
+  %4 = load i8, ptr addrspace(21) getelementptr inbounds (i8, ptr addrspace(21) null, i64 9), align 1
+  %cmp34.not.9 = icmp eq i8 %4, 0
+  %add37.9 = select i1 %cmp34.not.9, i32 0, i32 0
+  %add38.9 = or i32 %add27.9, %add37.9
+  %cmp13.not.10 = icmp eq i8 0, 0
+  %add15.10 = select i1 %cmp13.not.10, i32 0, i32 0
+  %add16.10 = or i32 %add15.10, %add38.9
+  %cmp23.not.10 = icmp eq i8 0, 0
+  %add26.10 = select i1 %cmp23.not.10, i32 0, i32 0
+  %add27.10 = or i32 %add16.10, %add26.10
+  %add37.10 = select i1 false, i32 0, i32 0
+  %add38.10 = or i32 %add27.10, %add37.10
+  %add15.11 = select i1 false, i32 0, i32 0
+  %add16.11 = or i32 %add15.11, %add38.10
+  %add26.11 = select i1 false, i32 0, i32 0
+  %add27.11 = or i32 %add16.11, %add26.11
+  %add37.11 = select i1 false, i32 0, i32 0
+  %add38.11 = or i32 %add27.11, %add37.11
+  %add15.12 = select i1 false, i32 0, i32 0
+  %add16.12 = or i32 %add15.12, %add38.11
+  %add26.12 = select i1 false, i32 0, i32 0
+  %add27.12 = or i32 %add16.12, %add26.12
+  %add37.12 = select i1 false, i32 0, i32 0
+  %add38.12 = or i32 %add27.12, %add37.12
+  %add15.13 = select i1 false, i32 0, i32 0
+  %add16.13 = or i32 %add15.13, %add38.12
+  %add26.13 = select i1 false, i32 0, i32 0
+  %add27.13 = or i32 %add16.13, %add26.13
+  %add37.13 = select i1 false, i32 0, i32 0
+  %add38.13 = or i32 %add27.13, %add37.13
+  %test_structcopy_14.s14_cm.coerce.sroa.2.0.copyload = load i48, ptr addrspace(21) getelementptr inbounds (i8, ptr addrspace(21) null, i64 8), align 8
+  %s.sroa.9.8.extract.trunc.i198 = trunc i48 %test_structcopy_14.s14_cm.coerce.sroa.2.0.copyload to i8
+  %s.sroa.11.8.extract.trunc.i200 = trunc i48 0 to i8
+  %s.sroa.12.8.extract.trunc.i202 = trunc i48 0 to i8
+  %s.sroa.13.8.extract.trunc.i204 = trunc i48 0 to i8
+  %add.i210 = select i1 false, i32 0, i32 0
+  %add6.i211 = or i32 %add.i210, %add38.13
+  %add.1.i213 = select i1 false, i32 0, i32 0
+  %add6.1.i214 = or i32 %add6.i211, %add.1.i213
+  %add.2.i216 = select i1 false, i32 0, i32 0
+  %add6.2.i217 = or i32 %add6.1.i214, %add.2.i216
+  %add.4.i222 = select i1 false, i32 0, i32 0
+  %add6.4.i223 = or i32 %add6.2.i217, %add.4.i222
+  %add.5.i225 = select i1 false, i32 0, i32 0
+  %add6.5.i226 = or i32 %add6.4.i223, %add.5.i225
+  %add.7.i231 = select i1 false, i32 0, i32 0
+  %add6.7.i232 = or i32 %add6.5.i226, %add.7.i231
+  %cmp4.not.8.i233 = icmp eq i8 0, %s.sroa.9.8.extract.trunc.i198
+  %add.8.i234 = select i1 %cmp4.not.8.i233, i32 0, i32 0
+  %add6.8.i235 = or i32 %add6.7.i232, %add.8.i234
+  %cmp4.not.9.i236 = icmp eq i8 0, %s.sroa.11.8.extract.trunc.i200
+  %add.9.i237 = select i1 %cmp4.not.9.i236, i32 0, i32 0
+  %add6.9.i238 = or i32 %add6.8.i235, %add.9.i237
+  %cmp4.not.10.i239 = icmp eq i8 0, %s.sroa.12.8.extract.trunc.i202
+  %add.10.i240 = select i1 %cmp4.not.10.i239, i32 0, i32 0
+  %add6.10.i241 = or i32 %add6.9.i238, %add.10.i240
+  %cmp4.not.11.i242 = icmp eq i8 0, %s.sroa.13.8.extract.trunc.i204
+  %add.11.i243 = select i1 %cmp4.not.11.i242, i32 0, i32 0
+  %add6.11.i244 = or i32 %add6.10.i241, %add.11.i243
+  %add.12.i246 = select i1 false, i32 0, i32 0
+  %add6.12.i247 = or i32 %add6.11.i244, %add.12.i246
+  %add.13.i249 = select i1 false, i32 0, i32 0
+  %add6.13.i250 = or i32 %add6.12.i247, %add.13.i249
+  %add65 = select i1 false, i32 0, i32 0
+  %add66 = or i32 %add65, %add6.13.i250
+  %add65.1 = select i1 false, i32 0, i32 0
+  %add66.1 = or i32 %add65.1, %add66
+  %add65.2 = select i1 false, i32 0, i32 0
+  %add66.2 = or i32 %add65.2, %add66.1
+  %add65.3 = select i1 false, i32 0, i32 0
+  %add66.3 = or i32 %add65.3, %add66.2
+  %add65.4 = select i1 false, i32 0, i32 0
+  %add66.4 = or i32 %add65.4, %add66.3
+  %add65.5 = select i1 false, i32 0, i32 0
+  %add66.5 = or i32 %add65.5, %add66.4
+  %add65.6 = select i1 false, i32 0, i32 0
+  %add66.6 = or i32 %add65.6, %add66.5
+  %add65.7 = select i1 false, i32 0, i32 0
+  %add66.7 = or i32 %add65.7, %add66.6
+  %add65.8 = select i1 false, i32 0, i32 0
+  %add66.8 = or i32 %add65.8, %add66.7
+  %add65.9 = select i1 false, i32 0, i32 0
+  %add66.9 = or i32 %add65.9, %add66.8
+  %add65.10 = select i1 false, i32 0, i32 0
+  %add66.10 = or i32 %add65.10, %add66.9
+  %add65.11 = select i1 false, i32 0, i32 0
+  %add66.11 = or i32 %add65.11, %add66.10
+  %add65.12 = select i1 false, i32 0, i32 0
+  %add66.12 = or i32 %add65.12, %add66.11
+  %add65.13 = select i1 false, i32 0, i32 0
+  %add66.13 = or i32 %add65.13, %add66.12
+  %add.i254 = select i1 false, i32 0, i32 0
+  %add6.i255 = or i32 %add.i254, %add66.13
+  %add.1.i257 = select i1 false, i32 0, i32 0
+  %add6.1.i258 = or i32 %add6.i255, %add.1.i257
+  %add.2.i260 = select i1 false, i32 0, i32 0
+  %add6.2.i261 = or i32 %add6.1.i258, %add.2.i260
+  %add.3.i263 = select i1 false, i32 0, i32 0
+  %add6.3.i264 = or i32 %add6.2.i261, %add.3.i263
+  %cmp4.not.4.i265 = icmp eq i8 0, 0
+  %add.4.i266 = select i1 %cmp4.not.4.i265, i32 0, i32 0
+  %add6.4.i267 = or i32 %add6.3.i264, %add.4.i266
+  %5 = load i8, ptr addrspace(21) null, align 2
+  %6 = load volatile i8, ptr null, align 2
+  %cmp4.not.6.i271 = icmp eq i8 %5, %6
+  %add.6.i272 = select i1 %cmp4.not.6.i271, i32 0, i32 0
+  %add6.6.i273 = or i32 %add6.4.i267, %add.6.i272
+  %cmp4.not.7.i274 = icmp eq i8 0, 0
+  %add.7.i275 = select i1 %cmp4.not.7.i274, i32 0, i32 0
+  %add6.7.i276 = or i32 %add6.6.i273, %add.7.i275
+  %7 = load i8, ptr addrspace(21) getelementptr inbounds (i8, ptr addrspace(21) null, i64 8), align 8
+  %cmp4.not.8.i277 = icmp eq i8 %7, %0
+  %add.8.i278 = select i1 %cmp4.not.8.i277, i32 0, i32 0
+  %add6.8.i279 = or i32 %add6.7.i276, %add.8.i278
+  %8 = load i8, ptr addrspace(21) getelementptr inbounds (i8, ptr addrspace(21) null, i64 9), align 1
+  %cmp4.not.9.i280 = icmp eq i8 %8, 0
+  %add.9.i281 = select i1 %cmp4.not.9.i280, i32 0, i32 0
+  %add6.9.i282 = or i32 %add6.8.i279, %add.9.i281
+  %cmp4.not.10.i283 = icmp eq i8 0, 0
+  %add.10.i284 = select i1 %cmp4.not.10.i283, i32 0, i32 0
+  %add6.10.i285 = or i32 %add6.9.i282, %add.10.i284
+  %cmp4.not.12.i289 = icmp eq i8 0, 0
+  %add.12.i290 = select i1 %cmp4.not.12.i289, i32 0, i32 0
+  %add6.12.i291 = or i32 %add6.10.i285, %add.12.i290
+  %cmp4.not.7.i318 = icmp eq i8 0, 0
+  %add.7.i319 = select i1 %cmp4.not.7.i318, i32 0, i32 0
+  %add6.7.i320 = or i32 %add6.12.i291, %add.7.i319
+  ret i32 %add6.7.i320
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/icmp-altopcode-after-reordering.ll b/llvm/test/Transforms/SLPVectorizer/icmp-altopcode-after-reordering.ll
similarity index 90%
rename from llvm/test/Transforms/SLPVectorizer/X86/icmp-altopcode-after-reordering.ll
rename to llvm/test/Transforms/SLPVectorizer/icmp-altopcode-after-reordering.ll
index 17fe501391e1a..61e3c6cdb8861 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/icmp-altopcode-after-reordering.ll
+++ b/llvm/test/Transforms/SLPVectorizer/icmp-altopcode-after-reordering.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s %}
 
 define i32 @test(ptr %sptr, i64 %0) {
 ; CHECK-LABEL: define i32 @test(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/insert-shuffle.ll b/llvm/test/Transforms/SLPVectorizer/insert-shuffle.ll
similarity index 91%
rename from llvm/test/Transforms/SLPVectorizer/X86/insert-shuffle.ll
rename to llvm/test/Transforms/SLPVectorizer/insert-shuffle.ll
index 99df7c7130ffa..2d6e83b4add41 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/insert-shuffle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/insert-shuffle.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=slp-vectorizer -S -o - -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: %if x86-registered-target %{ opt -passes=slp-vectorizer -S -o - -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt -passes=slp-vectorizer -S -o - -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s %}
 
 %struct.sw = type { float, float, float, float }
 
diff --git a/llvm/test/Transforms/SLPVectorizer/insertelement-uses-vectorized-index.ll b/llvm/test/Transforms/SLPVectorizer/insertelement-uses-vectorized-index.ll
index 94f973e606436..0734fd80709f7 100644
--- a/llvm/test/Transforms/SLPVectorizer/insertelement-uses-vectorized-index.ll
+++ b/llvm/test/Transforms/SLPVectorizer/insertelement-uses-vectorized-index.ll
@@ -8,6 +8,8 @@ define void @test(ptr %0) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x ptr> <ptr null, ptr poison>, ptr [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint <2 x ptr> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[TMP8]] to i32
 ; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr null to i64
 ; CHECK-NEXT:    [[TMP3:%.*]] = trunc <2 x i64> [[TMP2]] to <2 x i32>
 ; CHECK-NEXT:    switch i32 0, label %[[NEWFUNCROOT994:.*]] [
@@ -18,7 +20,6 @@ define void @test(ptr %0) {
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[NEWFUNCROOT994]]:
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i32> [[TMP3]], i32 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[TMP5]], i64 [[TMP6]]
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/limit.ll b/llvm/test/Transforms/SLPVectorizer/limit.ll
similarity index 93%
rename from llvm/test/Transforms/SLPVectorizer/X86/limit.ll
rename to llvm/test/Transforms/SLPVectorizer/limit.ll
index 78ccd17608daf..b2c9ccb55d87b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/limit.ll
+++ b/llvm/test/Transforms/SLPVectorizer/limit.ll
@@ -1,8 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer -S | FileCheck %s
-
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
+; RUN: %if x86-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=aarch64-unknown-linux-gnu | FileCheck %s %}
 
 @b = common global [4 x i32] zeroinitializer, align 16
 @c = common global [4 x i32] zeroinitializer, align 16
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/operand-is-reduced-val.ll b/llvm/test/Transforms/SLPVectorizer/operand-is-reduced-val.ll
similarity index 88%
rename from llvm/test/Transforms/SLPVectorizer/X86/operand-is-reduced-val.ll
rename to llvm/test/Transforms/SLPVectorizer/operand-is-reduced-val.ll
index 5fcac3fbf3baf..408dda2211b8d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/operand-is-reduced-val.ll
+++ b/llvm/test/Transforms/SLPVectorizer/operand-is-reduced-val.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux < %s -slp-threshold=-10 | FileCheck %s
+; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux < %s -slp-threshold=-10 | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux < %s -slp-threshold=-10 | FileCheck %s %}
 
 define i64 @src(i32 %a) {
 ; CHECK-LABEL: define i64 @src(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduced-gathered-vectorized.ll b/llvm/test/Transforms/SLPVectorizer/reduced-gathered-vectorized.ll
similarity index 90%
rename from llvm/test/Transforms/SLPVectorizer/X86/reduced-gathered-vectorized.ll
rename to llvm/test/Transforms/SLPVectorizer/reduced-gathered-vectorized.ll
index 31f0e065cf77d..02ec2b1836501 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduced-gathered-vectorized.ll
+++ b/llvm/test/Transforms/SLPVectorizer/reduced-gathered-vectorized.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: %if x86-registered-target %{ opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt -passes=slp-vectorizer -S -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s %}
 
 define i16 @test() {
 ; CHECK-LABEL: @test(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-min-select.ll b/llvm/test/Transforms/SLPVectorizer/reduction-min-select.ll
similarity index 80%
rename from llvm/test/Transforms/SLPVectorizer/X86/reduction-min-select.ll
rename to llvm/test/Transforms/SLPVectorizer/reduction-min-select.ll
index cb62f2a45baac..669e05e10886d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-min-select.ll
+++ b/llvm/test/Transforms/SLPVectorizer/reduction-min-select.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: %if x86-registered-target %{ opt -S -passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt -S -passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s %}
 
 define void @test() {
 ; CHECK-LABEL: @test(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction_loads.ll b/llvm/test/Transforms/SLPVectorizer/reduction_loads.ll
similarity index 97%
rename from llvm/test/Transforms/SLPVectorizer/X86/reduction_loads.ll
rename to llvm/test/Transforms/SLPVectorizer/reduction_loads.ll
index 8b2b15283601a..1b37c0a77e94e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction_loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/reduction_loads.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-apple-macosx10.10.0 -mattr=+sse4.2 | FileCheck %s
+; RUN: %if x86-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-apple-macosx10.10.0 -mattr=+sse4.2 | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=aarch64-unknown-linux-gnu -mattr=+neon | FileCheck %s %}
 
 ; PR28474
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reorder-node.ll b/llvm/test/Transforms/SLPVectorizer/reorder-node.ll
similarity index 92%
rename from llvm/test/Transforms/SLPVectorizer/X86/reorder-node.ll
rename to llvm/test/Transforms/SLPVectorizer/reorder-node.ll
index 3a456798d7818..dd583ab3c84ad 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reorder-node.ll
+++ b/llvm/test/Transforms/SLPVectorizer/reorder-node.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-linux-gnu < %s | FileCheck %s
+; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-linux-gnu < %s | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-linux-gnu < %s | FileCheck %s %}
 
 define void @test(ptr noalias %arg, ptr noalias %arg1, ptr %arg2) {
 ; CHECK-LABEL: define void @test(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/return.ll b/llvm/test/Transforms/SLPVectorizer/return.ll
similarity index 90%
rename from llvm/test/Transforms/SLPVectorizer/X86/return.ll
rename to llvm/test/Transforms/SLPVectorizer/return.ll
index 31b328ca2f513..1ea4112eb6064 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/return.ll
+++ b/llvm/test/Transforms/SLPVectorizer/return.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer -S | FileCheck %s
-target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"
-target triple = "x86_64--linux-gnu"
+; RUN: %if x86-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=x86_64--linux-gnu | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=aarch64-unknown-linux-gnu | FileCheck %s %}
 
 @a = common global [4 x double] zeroinitializer, align 8
 @b = common global [4 x double] zeroinitializer, align 8
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reused-scalar-in-multi-node.ll b/llvm/test/Transforms/SLPVectorizer/reused-scalar-in-multi-node.ll
similarity index 90%
rename from llvm/test/Transforms/SLPVectorizer/X86/reused-scalar-in-multi-node.ll
rename to llvm/test/Transforms/SLPVectorizer/reused-scalar-in-multi-node.ll
index 13015a495a918..229979f444d8b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reused-scalar-in-multi-node.ll
+++ b/llvm/test/Transforms/SLPVectorizer/reused-scalar-in-multi-node.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
-; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s %}
 
 %struct.UFP = type { i32, i32, i32, [4 x i32] }
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/revec-reduction-logical.ll b/llvm/test/Transforms/SLPVectorizer/revec-reduction-logical.ll
similarity index 91%
rename from llvm/test/Transforms/SLPVectorizer/X86/revec-reduction-logical.ll
rename to llvm/test/Transforms/SLPVectorizer/revec-reduction-logical.ll
index dfe9799b2a7de..25f161e9f1276 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/revec-reduction-logical.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec-reduction-logical.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer -slp-revec -mtriple=x86_64 -S | FileCheck %s
+; RUN: %if x86-registered-target %{ opt < %s -passes=slp-vectorizer -slp-revec -mtriple=x86_64 -S | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt < %s -passes=slp-vectorizer -slp-revec -mtriple=aarch64-unknown-linux-gnu -S | FileCheck %s %}
 
 define i1 @logical_and_icmp_diff_preds(<4 x i32> %x) {
 ; CHECK-LABEL: @logical_and_icmp_diff_preds(
diff --git a/llvm/test/Transforms/SLPVectorizer/root-trunc-extract-reuse.ll b/llvm/test/Transforms/SLPVectorizer/root-trunc-extract-reuse.ll
index 34c068478c5f5..d4b737a6bc421 100644
--- a/llvm/test/Transforms/SLPVectorizer/root-trunc-extract-reuse.ll
+++ b/llvm/test/Transforms/SLPVectorizer/root-trunc-extract-reuse.ll
@@ -10,9 +10,9 @@ define i1 @test() {
 ; CHECK-NEXT:    br label [[ELSE]]
 ; CHECK:       else:
 ; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x i32> [ zeroinitializer, [[THEN]] ], [ zeroinitializer, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i32> [[TMP0]], i32 1
 ; CHECK-NEXT:    [[BF_CAST162:%.*]] = and i32 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> zeroinitializer, <2 x i32> [[TMP0]], <2 x i32> <i32 3, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> zeroinitializer, <2 x i32> [[TMP0]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[T13:%.*]] = and <2 x i32> [[TMP2]], zeroinitializer
 ; CHECK-NEXT:    br label [[ELSE1:%.*]]
 ; CHECK:       else1:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/same-scalars-reordered-in-reduction.ll b/llvm/test/Transforms/SLPVectorizer/same-scalars-reordered-in-reduction.ll
similarity index 83%
rename from llvm/test/Transforms/SLPVectorizer/X86/same-scalars-reordered-in-reduction.ll
rename to llvm/test/Transforms/SLPVectorizer/same-scalars-reordered-in-reduction.ll
index bb00f753ddb1f..e36aba4fd3a50 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/same-scalars-reordered-in-reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/same-scalars-reordered-in-reduction.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
-; RUN: opt -S -passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: %if x86-registered-target %{ opt -S -passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt -S -passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s %}
 
 define i32 @test() {
 ; CHECK-LABEL: define i32 @test() {
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/several_store_chains.ll b/llvm/test/Transforms/SLPVectorizer/several_store_chains.ll
similarity index 87%
rename from llvm/test/Transforms/SLPVectorizer/X86/several_store_chains.ll
rename to llvm/test/Transforms/SLPVectorizer/several_store_chains.ll
index 32b322a8ae0fb..3869075d35771 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/several_store_chains.ll
+++ b/llvm/test/Transforms/SLPVectorizer/several_store_chains.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -S -mtriple=x86_64-unknown -passes=slp-vectorizer | FileCheck %s
+; RUN: %if x86-registered-target %{ opt < %s -S -mtriple=x86_64-unknown -passes=slp-vectorizer | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt < %s -S -mtriple=aarch64-unknown -passes=slp-vectorizer | FileCheck %s %}
 
 ; Test for PR49898.
 define void @fusion_1506(ptr %temp_buf1) local_unnamed_addr {
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/shuffle-mask-resized.ll b/llvm/test/Transforms/SLPVectorizer/shuffle-mask-resized.ll
similarity index 85%
rename from llvm/test/Transforms/SLPVectorizer/X86/shuffle-mask-resized.ll
rename to llvm/test/Transforms/SLPVectorizer/shuffle-mask-resized.ll
index 7d493dac015a9..732b50396a460 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/shuffle-mask-resized.ll
+++ b/llvm/test/Transforms/SLPVectorizer/shuffle-mask-resized.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -slp-threshold=-10 < %s | FileCheck %s
+; RUN: %if x86-registered-target %{ opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -slp-threshold=-10 < %s | FileCheck %s %}
+; RUN: %if aaarch64-registered-target %{ opt -passes=slp-vectorizer -S -mtriple=aarch64-unknown-linux -slp-threshold=-10 < %s | FileCheck %s %}
 
 define i32 @test() {
 ; CHECK-LABEL: define i32 @test() {
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/splat-buildvector.ll b/llvm/test/Transforms/SLPVectorizer/splat-buildvector.ll
similarity index 80%
rename from llvm/test/Transforms/SLPVectorizer/X86/splat-buildvector.ll
rename to llvm/test/Transforms/SLPVectorizer/splat-buildvector.ll
index 5e5981bdaaa80..7532cd728225d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/splat-buildvector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/splat-buildvector.ll
@@ -1,5 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt --passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu %s -o - -slp-threshold=-100 | FileCheck %s
+; RUN: %if x86-registered-target %{ opt --passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu %s -o - -slp-threshold=-100 | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt --passes=slp-vectorizer -S -mtriple=aarch64-unknown-linux-gnu %s -o - -slp-threshold=-100 | FileCheck %s %}
+
 declare i64 @llvm.smax.i64(i64, i64)
 
 define i8 @foo(i64 %val_i64_57) {
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vectorize-reorder-alt-shuffle.ll b/llvm/test/Transforms/SLPVectorizer/vectorize-reorder-alt-shuffle.ll
similarity index 91%
rename from llvm/test/Transforms/SLPVectorizer/X86/vectorize-reorder-alt-shuffle.ll
rename to llvm/test/Transforms/SLPVectorizer/vectorize-reorder-alt-shuffle.ll
index 69cdbbdbd098b..631adf1fa81d5 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vectorize-reorder-alt-shuffle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/vectorize-reorder-alt-shuffle.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: %if x86-registered-target %{ opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt -passes=slp-vectorizer -S -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s %}
 
 define void @foo(ptr %c, ptr %d) {
 ; CHECK-LABEL: @foo(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vectorize-reordered-list.ll b/llvm/test/Transforms/SLPVectorizer/vectorize-reordered-list.ll
similarity index 84%
rename from llvm/test/Transforms/SLPVectorizer/X86/vectorize-reordered-list.ll
rename to llvm/test/Transforms/SLPVectorizer/vectorize-reordered-list.ll
index 4c3c712419a2e..a251ec06d7472 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vectorize-reordered-list.ll
+++ b/llvm/test/Transforms/SLPVectorizer/vectorize-reordered-list.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown %s | FileCheck %s
+; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown %s | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown %s | FileCheck %s %}
 
 define void @test(ptr %isec) {
 ; CHECK-LABEL: @test(
diff --git a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
index bbcdcb6f58674..6cb94e8f561bc 100644
--- a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
+++ b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt %s -passes='function(scalarizer,dce)' -scalarize-load-store -S | FileCheck %s
+; RUN: opt %s -passes='function(scalarizer<load-store>,dce)' -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 declare <4 x float> @ext(<4 x float>)
diff --git a/llvm/test/Transforms/Scalarizer/basic.ll b/llvm/test/Transforms/Scalarizer/basic.ll
index 28d5933ab9b87..190e8a089a5f6 100644
--- a/llvm/test/Transforms/Scalarizer/basic.ll
+++ b/llvm/test/Transforms/Scalarizer/basic.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt %s -passes='function(scalarizer,dce)' -scalarize-load-store -S | FileCheck %s
+; RUN: opt %s -passes='function(scalarizer<load-store>,dce)' -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 declare <4 x float> @ext(<4 x float>)
diff --git a/llvm/test/Transforms/Scalarizer/constant-extractelement.ll b/llvm/test/Transforms/Scalarizer/constant-extractelement.ll
index 50b7c3904999c..ad1561e8bae83 100644
--- a/llvm/test/Transforms/Scalarizer/constant-extractelement.ll
+++ b/llvm/test/Transforms/Scalarizer/constant-extractelement.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt %s -passes='function(scalarizer,dce)' -scalarize-load-store -S | FileCheck --check-prefixes=ALL %s
+; RUN: opt %s -passes='function(scalarizer<load-store>,dce)' -S | FileCheck --check-prefixes=ALL %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/Scalarizer/constant-insertelement.ll b/llvm/test/Transforms/Scalarizer/constant-insertelement.ll
index afb6d6a49296b..c1871ac92d255 100644
--- a/llvm/test/Transforms/Scalarizer/constant-insertelement.ll
+++ b/llvm/test/Transforms/Scalarizer/constant-insertelement.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt %s -passes='function(scalarizer,dce)' -scalarize-load-store -S | FileCheck --check-prefixes=ALL %s
+; RUN: opt %s -passes='function(scalarizer<load-store>,dce)' -S | FileCheck --check-prefixes=ALL %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/Scalarizer/dbginfo.ll b/llvm/test/Transforms/Scalarizer/dbginfo.ll
index 310b5aae02cf4..c388805fa6a2e 100644
--- a/llvm/test/Transforms/Scalarizer/dbginfo.ll
+++ b/llvm/test/Transforms/Scalarizer/dbginfo.ll
@@ -1,4 +1,4 @@
-; RUN: opt %s -passes='function(scalarizer)' -scalarize-load-store -S --experimental-debuginfo-iterators=false | FileCheck %s
+; RUN: opt %s -passes='function(scalarizer<load-store>)' -S --experimental-debuginfo-iterators=false | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 ; FIXME: the test output here changes if we use the RemoveDIs non-intrinsic
 ; debug-info format for the test. Specifically, the intrinsics no longer
diff --git a/llvm/test/Transforms/Scalarizer/min-bits.ll b/llvm/test/Transforms/Scalarizer/min-bits.ll
index be901d4990592..97cc71626e208 100644
--- a/llvm/test/Transforms/Scalarizer/min-bits.ll
+++ b/llvm/test/Transforms/Scalarizer/min-bits.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt %s -passes='function(scalarizer,dce)' -scalarize-load-store -scalarize-min-bits=16 -S | FileCheck %s --check-prefixes=CHECK,MIN16
-; RUN: opt %s -passes='function(scalarizer,dce)' -scalarize-load-store -scalarize-min-bits=32 -S | FileCheck %s --check-prefixes=CHECK,MIN32
+; RUN: opt %s -passes='function(scalarizer<load-store;min-bits=16>,dce)' -S | FileCheck %s --check-prefixes=CHECK,MIN16
+; RUN: opt %s -passes='function(scalarizer<load-store;min-bits=32>,dce)' -S | FileCheck %s --check-prefixes=CHECK,MIN32
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 define void @load_add_store_v2i16(ptr %pa, ptr %pb) {
diff --git a/llvm/test/Transforms/Scalarizer/opaque-ptr-bug.ll b/llvm/test/Transforms/Scalarizer/opaque-ptr-bug.ll
index 7d0ba0aa45f05..7d187dc33db88 100644
--- a/llvm/test/Transforms/Scalarizer/opaque-ptr-bug.ll
+++ b/llvm/test/Transforms/Scalarizer/opaque-ptr-bug.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt %s -passes='scalarizer,dce' -S -scalarize-load-store -o - | FileCheck %s
+; RUN: opt %s -passes='scalarizer<load-store>,dce' -S -o - | FileCheck %s
 
 ; This used to crash because the same (pointer) value was scattered by
 ; different amounts.
diff --git a/llvm/test/Transforms/Scalarizer/pass-param-parse-errors.ll b/llvm/test/Transforms/Scalarizer/pass-param-parse-errors.ll
new file mode 100644
index 0000000000000..7a241dac6ada1
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/pass-param-parse-errors.ll
@@ -0,0 +1,9 @@
+; RUN: not opt -passes='scalarizer<unknown>' -disable-output %s 2>&1 | FileCheck -check-prefix=UNKNOWNERR %s
+; RUN: not opt -passes='scalarizer<;>' -disable-output %s 2>&1 | FileCheck -check-prefix=UNKNOWNERR %s
+; RUN: not opt -passes='scalarizer<min-bits=>' -disable-output %s 2>&1 | FileCheck -check-prefix=MINBITS-EMPTY-ERR %s
+; RUN: not opt -passes='scalarizer<min-bits=x>' -disable-output %s 2>&1 | FileCheck -check-prefix=MINBITS-NOTINT-ERR %s
+; RUN: not opt -passes='scalarizer<no-min-bits=10>' -disable-output %s 2>&1 | FileCheck -check-prefix=UNKNOWNERR %s
+
+; UNKNOWNERR: invalid Scalarizer pass parameter '{{.*}}'
+; MINBITS-EMPTY-ERR: invalid argument to Scalarizer pass min-bits parameter: ''
+; MINBITS-NOTINT-ERR: invalid argument to Scalarizer pass min-bits parameter: 'x'
diff --git a/llvm/test/Transforms/Scalarizer/scatter-order.ll b/llvm/test/Transforms/Scalarizer/scatter-order.ll
index 1861b0f38446e..a98a2198b1ed2 100644
--- a/llvm/test/Transforms/Scalarizer/scatter-order.ll
+++ b/llvm/test/Transforms/Scalarizer/scatter-order.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt %s -passes='function(scalarizer)' -scalarize-load-store -S | FileCheck %s
+; RUN: opt %s -passes='function(scalarizer<load-store>)' -S | FileCheck %s
 
 ; This verifies that the order of extract element instructions is
 ; deterministic. In the past we could end up with different results depending
diff --git a/llvm/test/Transforms/Scalarizer/store-bug.ll b/llvm/test/Transforms/Scalarizer/store-bug.ll
index 1fccdc624e493..6d9cd46db5390 100644
--- a/llvm/test/Transforms/Scalarizer/store-bug.ll
+++ b/llvm/test/Transforms/Scalarizer/store-bug.ll
@@ -1,4 +1,4 @@
-; RUN: opt -passes='function(scalarizer)' -scalarize-load-store -S < %s | FileCheck %s
+; RUN: opt -passes='function(scalarizer<load-store>)' -S < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; This input caused the scalarizer not to clear cached results
diff --git a/llvm/test/Transforms/Scalarizer/variable-extractelement.ll b/llvm/test/Transforms/Scalarizer/variable-extractelement.ll
index 85361cdd6942d..9211ff91809c9 100644
--- a/llvm/test/Transforms/Scalarizer/variable-extractelement.ll
+++ b/llvm/test/Transforms/Scalarizer/variable-extractelement.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt %s -passes='function(scalarizer,dce)' -S | FileCheck --check-prefix=DEFAULT %s
-; RUN: opt %s -passes='function(scalarizer,dce)' -scalarize-variable-insert-extract=false -S | FileCheck --check-prefix=OFF %s
-; RUN: opt %s -passes='function(scalarizer,dce)' -scalarize-variable-insert-extract=true -S | FileCheck --check-prefix=DEFAULT %s
+; RUN: opt %s -passes='function(scalarizer<no-variable-insert-extract>,dce)' -S | FileCheck --check-prefix=OFF %s
+; RUN: opt %s -passes='function(scalarizer<variable-insert-extract>,dce)' -S | FileCheck --check-prefix=DEFAULT %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/Scalarizer/variable-insertelement.ll b/llvm/test/Transforms/Scalarizer/variable-insertelement.ll
index 452a94f301758..13122657932a0 100644
--- a/llvm/test/Transforms/Scalarizer/variable-insertelement.ll
+++ b/llvm/test/Transforms/Scalarizer/variable-insertelement.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt %s -passes='function(scalarizer,dce)' -S | FileCheck --check-prefix=DEFAULT %s
-; RUN: opt %s -passes='function(scalarizer,dce)' -scalarize-variable-insert-extract=false -S | FileCheck --check-prefix=OFF %s
-; RUN: opt %s -passes='function(scalarizer,dce)' -scalarize-variable-insert-extract=true -S | FileCheck --check-prefix=DEFAULT %s
+; RUN: opt %s -passes='function(scalarizer<no-variable-insert-extract>,dce)' -S | FileCheck --check-prefix=OFF %s
+; RUN: opt %s -passes='function(scalarizer<variable-insert-extract>,dce)' -S | FileCheck --check-prefix=DEFAULT %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/Scalarizer/vector-of-pointer-to-vector.ll b/llvm/test/Transforms/Scalarizer/vector-of-pointer-to-vector.ll
index 89ae34193257a..b1b71ecc801f4 100644
--- a/llvm/test/Transforms/Scalarizer/vector-of-pointer-to-vector.ll
+++ b/llvm/test/Transforms/Scalarizer/vector-of-pointer-to-vector.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt %s -passes='function(scalarizer,dce)' -scalarize-load-store -S | FileCheck %s
+; RUN: opt %s -passes='function(scalarizer<load-store>,dce)' -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 define <1 x i32> @f1(<1 x ptr> %src, i32 %index) {
diff --git a/llvm/test/Transforms/SimplifyCFG/hoist-cb-diff-attrs.ll b/llvm/test/Transforms/SimplifyCFG/hoist-cb-diff-attrs.ll
new file mode 100644
index 0000000000000..ee6c0cfd43b25
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/hoist-cb-diff-attrs.ll
@@ -0,0 +1,204 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-attributes --check-globals
+; RUN: opt < %s -passes='simplifycfg<hoist-common-insts>' -simplifycfg-require-and-preserve-domtree=1 -S | FileCheck %s
+
+declare ptr @foo(ptr %p, i64 %x)
+declare ptr @foo2(ptr %p, ptr %p2, i64 %x)
+declare void @side.effect()
+
+define ptr @test_hoist_int_attrs(i1 %c, ptr %p, ptr %p2, i64 %x) {
+; CHECK-LABEL: define {{[^@]+}}@test_hoist_int_attrs
+; CHECK-SAME: (i1 [[C:%.*]], ptr [[P:%.*]], ptr [[P2:%.*]], i64 [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call ptr @foo2(ptr align 32 dereferenceable_or_null(100) [[P]], ptr align 32 dereferenceable(50) [[P2]], i64 range(i64 10, 100000) [[X]]) #[[ATTR0:[0-9]+]]
+; CHECK-NEXT:    br i1 [[C]], label [[COMMON_RET:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    ret ptr [[R]]
+; CHECK:       else:
+; CHECK-NEXT:    call void @side.effect()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+if:
+  %r = call ptr @foo2(ptr align 64 dereferenceable_or_null(100) %p, ptr dereferenceable(50) align 64 %p2, i64 range(i64 10, 1000) %x) memory(read)
+  ret ptr %r
+
+else:
+  %r2 = call ptr @foo2(ptr align 32 dereferenceable_or_null(200) %p, ptr dereferenceable(100) align 32 %p2, i64 range(i64 10000, 100000) %x) memory(write)
+  call void @side.effect()
+  ret ptr %r2
+}
+
+define ptr @test_hoist_int_attrs2(i1 %c, ptr %p, i64 %x) {
+; CHECK-LABEL: define {{[^@]+}}@test_hoist_int_attrs2
+; CHECK-SAME: (i1 [[C:%.*]], ptr [[P:%.*]], i64 [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call ptr @foo(ptr dereferenceable(50) [[P]], i64 range(i64 10, 1000) [[X]]) #[[ATTR1:[0-9]+]]
+; CHECK-NEXT:    br i1 [[C]], label [[COMMON_RET:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    ret ptr [[R]]
+; CHECK:       else:
+; CHECK-NEXT:    call void @side.effect()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+if:
+  %r = call ptr @foo(ptr dereferenceable(50) %p, i64 range(i64 10, 1000) %x) memory(read)
+  ret ptr %r
+
+else:
+  %r2 = call ptr @foo(ptr dereferenceable(100) align 32 dereferenceable_or_null(200) %p, i64 range(i64 11, 100) %x) memory(none)
+  call void @side.effect()
+  ret ptr %r2
+}
+
+define ptr @test_hoist_bool_attrs2(i1 %c, ptr %p, i64 %x) {
+; CHECK-LABEL: define {{[^@]+}}@test_hoist_bool_attrs2
+; CHECK-SAME: (i1 [[C:%.*]], ptr [[P:%.*]], i64 [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call noundef ptr @foo(ptr nonnull [[P]], i64 noundef [[X]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    br i1 [[C]], label [[COMMON_RET:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    ret ptr [[R]]
+; CHECK:       else:
+; CHECK-NEXT:    call void @side.effect()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+if:
+  %r = call noundef ptr @foo(ptr readnone nonnull noundef %p, i64 noundef %x) cold mustprogress nocallback nofree nosync willreturn
+  ret ptr %r
+
+else:
+  %r2 = call noundef nonnull ptr @foo(ptr readonly nonnull %p, i64 noundef %x) mustprogress nocallback nofree willreturn
+  call void @side.effect()
+  ret ptr %r2
+}
+
+define ptr @test_hoist_bool_attrs3(i1 %c, ptr %p, i64 %x) {
+; CHECK-LABEL: define {{[^@]+}}@test_hoist_bool_attrs3
+; CHECK-SAME: (i1 [[C:%.*]], ptr [[P:%.*]], i64 [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call nonnull ptr @foo(ptr [[P]], i64 noundef [[X]]) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    br i1 [[C]], label [[COMMON_RET:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    ret ptr [[R]]
+; CHECK:       else:
+; CHECK-NEXT:    call void @side.effect()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+if:
+  %r = call nonnull ptr @foo(ptr readonly noundef %p, i64 noundef %x) cold nocallback nofree nosync willreturn alwaysinline
+  ret ptr %r
+
+else:
+  %r2 = call noundef nonnull ptr @foo(ptr writeonly nonnull %p, i64 noundef %x) nosync willreturn alwaysinline
+  call void @side.effect()
+  ret ptr %r2
+}
+
+define ptr @test_hoist_bool_attrs_fail_non_droppable(i1 %c, ptr %p, i64 %x) {
+; CHECK-LABEL: define {{[^@]+}}@test_hoist_bool_attrs_fail_non_droppable
+; CHECK-SAME: (i1 [[C:%.*]], ptr [[P:%.*]], i64 [[X:%.*]]) {
+; CHECK-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi ptr [ [[R:%.*]], [[IF]] ], [ [[R2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret ptr [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[R]] = call nonnull ptr @foo(ptr noundef readonly [[P]], i64 noundef [[X]]) #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[R2]] = call noundef nonnull ptr @foo(ptr nonnull writeonly [[P]], i64 noundef [[X]]) #[[ATTR5:[0-9]+]]
+; CHECK-NEXT:    call void @side.effect()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+if:
+  %r = call nonnull ptr @foo(ptr readonly noundef %p, i64 noundef %x) cold nocallback nofree nosync willreturn alwaysinline
+  ret ptr %r
+
+else:
+  %r2 = call noundef nonnull ptr @foo(ptr writeonly nonnull %p, i64 noundef %x) nosync willreturn
+  call void @side.effect()
+  ret ptr %r2
+}
+
+define ptr @test_hoist_bool_attrs_fail_non_droppable2(i1 %c, ptr %p, i64 %x) {
+; CHECK-LABEL: define {{[^@]+}}@test_hoist_bool_attrs_fail_non_droppable2
+; CHECK-SAME: (i1 [[C:%.*]], ptr [[P:%.*]], i64 [[X:%.*]]) {
+; CHECK-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi ptr [ [[R:%.*]], [[IF]] ], [ [[R2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret ptr [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[R]] = call nonnull ptr @foo(ptr noundef readonly [[P]], i64 noundef [[X]]) #[[ATTR6:[0-9]+]]
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[R2]] = call noundef nonnull ptr @foo(ptr nonnull writeonly byval(i64) [[P]], i64 noundef [[X]]) #[[ATTR5]]
+; CHECK-NEXT:    call void @side.effect()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+if:
+  %r = call nonnull ptr @foo(ptr readonly noundef %p, i64 noundef %x) cold nocallback nofree nosync willreturn
+  ret ptr %r
+
+else:
+  %r2 = call noundef nonnull ptr @foo(ptr byval(i64) writeonly nonnull %p, i64 noundef %x) nosync willreturn
+  call void @side.effect()
+  ret ptr %r2
+}
+
+define ptr @test_hoist_bool_attrs_fail_non_droppable3(i1 %c, ptr %p, i64 %x) {
+; CHECK-LABEL: define {{[^@]+}}@test_hoist_bool_attrs_fail_non_droppable3
+; CHECK-SAME: (i1 [[C:%.*]], ptr [[P:%.*]], i64 [[X:%.*]]) {
+; CHECK-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi ptr [ [[R:%.*]], [[IF]] ], [ [[R2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret ptr [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[R]] = call nonnull ptr @foo(ptr noundef readonly byval(i32) [[P]], i64 noundef [[X]]) #[[ATTR6]]
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[R2]] = call noundef nonnull ptr @foo(ptr nonnull writeonly byval(i64) [[P]], i64 noundef [[X]]) #[[ATTR5]]
+; CHECK-NEXT:    call void @side.effect()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+if:
+  %r = call nonnull ptr @foo(ptr byval(i32) readonly noundef %p, i64 noundef %x) cold nocallback nofree nosync willreturn
+  ret ptr %r
+
+else:
+  %r2 = call noundef nonnull ptr @foo(ptr byval(i64) writeonly nonnull %p, i64 noundef %x) nosync willreturn
+  call void @side.effect()
+  ret ptr %r2
+}
+
+define ptr @test_hoist_bool_attrs4(i1 %c, ptr %p, i64 %x) {
+; CHECK-LABEL: define {{[^@]+}}@test_hoist_bool_attrs4
+; CHECK-SAME: (i1 [[C:%.*]], ptr [[P:%.*]], i64 [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call nonnull ptr @foo(ptr byval(i64) [[P]], i64 noundef [[X]]) #[[ATTR5]]
+; CHECK-NEXT:    br i1 [[C]], label [[COMMON_RET:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    ret ptr [[R]]
+; CHECK:       else:
+; CHECK-NEXT:    call void @side.effect()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+if:
+  %r = call nonnull ptr @foo(ptr byval(i64) readonly noundef %p, i64 noundef %x) cold nocallback nofree nosync willreturn
+  ret ptr %r
+
+else:
+  %r2 = call noundef nonnull ptr @foo(ptr byval(i64) writeonly nonnull %p, i64 noundef %x) nosync willreturn
+  call void @side.effect()
+  ret ptr %r2
+}
+
+;.
+; CHECK: attributes #[[ATTR0]] = { memory(readwrite) }
+; CHECK: attributes #[[ATTR1]] = { memory(read) }
+; CHECK: attributes #[[ATTR2]] = { mustprogress nocallback nofree willreturn }
+; CHECK: attributes #[[ATTR3]] = { alwaysinline nosync willreturn }
+; CHECK: attributes #[[ATTR4]] = { alwaysinline cold nocallback nofree nosync willreturn }
+; CHECK: attributes #[[ATTR5]] = { nosync willreturn }
+; CHECK: attributes #[[ATTR6]] = { cold nocallback nofree nosync willreturn }
+;.
diff --git a/llvm/test/Transforms/SimplifyCFG/sink-cb-diff-attrs.ll b/llvm/test/Transforms/SimplifyCFG/sink-cb-diff-attrs.ll
new file mode 100644
index 0000000000000..fe9d87080dd11
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/sink-cb-diff-attrs.ll
@@ -0,0 +1,285 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-attributes --check-globals
+; RUN: opt < %s -passes='simplifycfg<sink-common-insts>' -simplifycfg-require-and-preserve-domtree=1 -S | FileCheck %s
+
+declare ptr @foo0()
+declare ptr @foo(ptr %p, i64 %x)
+declare ptr @foo2(ptr %p, ptr %p2, i64 %x)
+declare void @side.effect()
+
+define ptr @test_sink_no_args_oneside(i1 %c) {
+; CHECK-LABEL: define {{[^@]+}}@test_sink_no_args_oneside
+; CHECK-SAME: (i1 [[C:%.*]]) {
+; CHECK-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[END:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    call void @side.effect()
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[R2:%.*]] = call ptr @foo0()
+; CHECK-NEXT:    ret ptr [[R2]]
+;
+  br i1 %c, label %if, label %else
+if:
+  call void @side.effect()
+  %r = call ptr @foo0()
+  br label %end
+
+else:
+  %r2 = call ptr @foo0() readonly
+  br label %end
+end:
+  %pr = phi ptr [ %r, %if], [%r2, %else]
+  ret ptr %pr
+}
+
+define ptr @test_sink_no_args_oneside_fail(i1 %c) {
+; CHECK-LABEL: define {{[^@]+}}@test_sink_no_args_oneside_fail
+; CHECK-SAME: (i1 [[C:%.*]]) {
+; CHECK-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    call void @side.effect()
+; CHECK-NEXT:    [[R:%.*]] = call ptr @foo0()
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[R2:%.*]] = call ptr @foo0() #[[ATTR0:[0-9]+]]
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[PR:%.*]] = phi ptr [ [[R]], [[IF]] ], [ [[R2]], [[ELSE]] ]
+; CHECK-NEXT:    ret ptr [[PR]]
+;
+  br i1 %c, label %if, label %else
+if:
+  call void @side.effect()
+  %r = call ptr @foo0()
+  br label %end
+
+else:
+  %r2 = call ptr @foo0() readonly alwaysinline
+  br label %end
+end:
+  %pr = phi ptr [ %r, %if], [%r2, %else]
+  ret ptr %pr
+}
+
+define ptr @test_sink_int_attrs(i1 %c, ptr %p, ptr %p2, i64 %x) {
+; CHECK-LABEL: define {{[^@]+}}@test_sink_int_attrs
+; CHECK-SAME: (i1 [[C:%.*]], ptr [[P:%.*]], ptr [[P2:%.*]], i64 [[X:%.*]]) {
+; CHECK-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[END:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    call void @side.effect()
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[R2:%.*]] = call ptr @foo2(ptr align 32 dereferenceable_or_null(100) [[P]], ptr align 32 dereferenceable(50) [[P2]], i64 range(i64 10, 100000) [[X]]) #[[ATTR1:[0-9]+]]
+; CHECK-NEXT:    ret ptr [[R2]]
+;
+  br i1 %c, label %if, label %else
+if:
+  call void @side.effect()
+  %r = call ptr @foo2(ptr align 64 dereferenceable_or_null(100) %p, ptr dereferenceable(50) align 64 %p2, i64 range(i64 10, 1000) %x) memory(read)
+  br label %end
+
+else:
+  %r2 = call ptr @foo2(ptr align 32 dereferenceable_or_null(200) %p, ptr dereferenceable(100) align 32 %p2, i64 range(i64 10000, 100000) %x) memory(write)
+  br label %end
+end:
+  %pr = phi ptr [ %r, %if], [%r2, %else]
+  ret ptr %pr
+}
+
+define ptr @test_sink_int_attrs2(i1 %c, ptr %p, i64 %x) {
+; CHECK-LABEL: define {{[^@]+}}@test_sink_int_attrs2
+; CHECK-SAME: (i1 [[C:%.*]], ptr [[P:%.*]], i64 [[X:%.*]]) {
+; CHECK-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[END:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    call void @side.effect()
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[R2:%.*]] = call ptr @foo(ptr dereferenceable(50) [[P]], i64 range(i64 10, 1000) [[X]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    ret ptr [[R2]]
+;
+  br i1 %c, label %if, label %else
+if:
+  call void @side.effect()
+  %r = call ptr @foo(ptr dereferenceable(50) %p, i64 range(i64 10, 1000) %x) memory(read)
+  br label %end
+
+else:
+  %r2 = call ptr @foo(ptr dereferenceable(100) align 32 dereferenceable_or_null(200) %p, i64 range(i64 11, 100) %x) memory(none)
+  br label %end
+end:
+  %pr = phi ptr [ %r, %if], [%r2, %else]
+  ret ptr %pr
+}
+
+define ptr @test_sink_bool_attrs2(i1 %c, ptr %p, i64 %x) {
+; CHECK-LABEL: define {{[^@]+}}@test_sink_bool_attrs2
+; CHECK-SAME: (i1 [[C:%.*]], ptr [[P:%.*]], i64 [[X:%.*]]) {
+; CHECK-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[END:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    call void @side.effect()
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[R2:%.*]] = call noundef ptr @foo(ptr nonnull [[P]], i64 noundef [[X]]) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    ret ptr [[R2]]
+;
+  br i1 %c, label %if, label %else
+if:
+  call void @side.effect()
+  %r = call noundef ptr @foo(ptr readnone nonnull noundef %p, i64 noundef %x) cold mustprogress nocallback nofree nosync willreturn
+  br label %end
+
+else:
+  %r2 = call noundef nonnull ptr @foo(ptr readonly nonnull %p, i64 noundef %x) mustprogress nocallback nofree willreturn
+  br label %end
+end:
+  %pr = phi ptr [ %r, %if], [%r2, %else]
+  ret ptr %pr
+}
+
+define ptr @test_sink_bool_attrs3(i1 %c, ptr %p, i64 %x) {
+; CHECK-LABEL: define {{[^@]+}}@test_sink_bool_attrs3
+; CHECK-SAME: (i1 [[C:%.*]], ptr [[P:%.*]], i64 [[X:%.*]]) {
+; CHECK-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[END:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    call void @side.effect()
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[R2:%.*]] = call nonnull ptr @foo(ptr [[P]], i64 noundef [[X]]) #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    ret ptr [[R2]]
+;
+  br i1 %c, label %if, label %else
+if:
+  call void @side.effect()
+  %r = call nonnull ptr @foo(ptr readonly noundef %p, i64 noundef %x) cold nocallback nofree nosync willreturn alwaysinline
+  br label %end
+
+else:
+  %r2 = call noundef nonnull ptr @foo(ptr writeonly nonnull %p, i64 noundef %x) nosync willreturn alwaysinline
+  br label %end
+end:
+  %pr = phi ptr [ %r, %if], [%r2, %else]
+  ret ptr %pr
+}
+
+define ptr @test_sink_bool_attrs_fail_non_droppable(i1 %c, ptr %p, i64 %x) {
+; CHECK-LABEL: define {{[^@]+}}@test_sink_bool_attrs_fail_non_droppable
+; CHECK-SAME: (i1 [[C:%.*]], ptr [[P:%.*]], i64 [[X:%.*]]) {
+; CHECK-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    call void @side.effect()
+; CHECK-NEXT:    [[R:%.*]] = call nonnull ptr @foo(ptr noundef readonly [[P]], i64 noundef [[X]]) #[[ATTR5:[0-9]+]]
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[R2:%.*]] = call noundef nonnull ptr @foo(ptr nonnull writeonly [[P]], i64 noundef [[X]]) #[[ATTR6:[0-9]+]]
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[PR:%.*]] = phi ptr [ [[R]], [[IF]] ], [ [[R2]], [[ELSE]] ]
+; CHECK-NEXT:    ret ptr [[PR]]
+;
+  br i1 %c, label %if, label %else
+if:
+  call void @side.effect()
+  %r = call nonnull ptr @foo(ptr readonly noundef %p, i64 noundef %x) cold nocallback nofree nosync willreturn alwaysinline
+  br label %end
+
+else:
+  %r2 = call noundef nonnull ptr @foo(ptr writeonly nonnull %p, i64 noundef %x) nosync willreturn
+  br label %end
+end:
+  %pr = phi ptr [ %r, %if], [%r2, %else]
+  ret ptr %pr
+}
+
+define ptr @test_sink_bool_attrs_fail_non_droppable2(i1 %c, ptr %p, i64 %x) {
+; CHECK-LABEL: define {{[^@]+}}@test_sink_bool_attrs_fail_non_droppable2
+; CHECK-SAME: (i1 [[C:%.*]], ptr [[P:%.*]], i64 [[X:%.*]]) {
+; CHECK-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    call void @side.effect()
+; CHECK-NEXT:    [[R:%.*]] = call nonnull ptr @foo(ptr noundef readonly [[P]], i64 noundef [[X]]) #[[ATTR7:[0-9]+]]
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[R2:%.*]] = call noundef nonnull ptr @foo(ptr nonnull writeonly byval(i64) [[P]], i64 noundef [[X]]) #[[ATTR6]]
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[PR:%.*]] = phi ptr [ [[R]], [[IF]] ], [ [[R2]], [[ELSE]] ]
+; CHECK-NEXT:    ret ptr [[PR]]
+;
+  br i1 %c, label %if, label %else
+if:
+  call void @side.effect()
+  %r = call nonnull ptr @foo(ptr readonly noundef %p, i64 noundef %x) cold nocallback nofree nosync willreturn
+  br label %end
+
+else:
+  %r2 = call noundef nonnull ptr @foo(ptr byval(i64) writeonly nonnull %p, i64 noundef %x) nosync willreturn
+  br label %end
+end:
+  %pr = phi ptr [ %r, %if], [%r2, %else]
+  ret ptr %pr
+}
+
+define ptr @test_sink_bool_attrs_fail_non_droppable3(i1 %c, ptr %p, i64 %x) {
+; CHECK-LABEL: define {{[^@]+}}@test_sink_bool_attrs_fail_non_droppable3
+; CHECK-SAME: (i1 [[C:%.*]], ptr [[P:%.*]], i64 [[X:%.*]]) {
+; CHECK-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    call void @side.effect()
+; CHECK-NEXT:    [[R:%.*]] = call nonnull ptr @foo(ptr noundef readonly byval(i32) [[P]], i64 noundef [[X]]) #[[ATTR7]]
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[R2:%.*]] = call noundef nonnull ptr @foo(ptr nonnull writeonly byval(i64) [[P]], i64 noundef [[X]]) #[[ATTR8:[0-9]+]]
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[PR:%.*]] = phi ptr [ [[R]], [[IF]] ], [ [[R2]], [[ELSE]] ]
+; CHECK-NEXT:    ret ptr [[PR]]
+;
+  br i1 %c, label %if, label %else
+if:
+  call void @side.effect()
+  %r = call nonnull ptr @foo(ptr byval(i32) readonly noundef %p, i64 noundef %x) cold nocallback nofree nosync willreturn
+  br label %end
+
+else:
+  %r2 = call noundef nonnull ptr @foo(ptr byval(i64) writeonly nonnull %p, i64 noundef %x) nosync
+  br label %end
+end:
+  %pr = phi ptr [ %r, %if], [%r2, %else]
+  ret ptr %pr
+}
+
+define ptr @test_sink_bool_attrs4(i1 %c, ptr %p, i64 %x) {
+; CHECK-LABEL: define {{[^@]+}}@test_sink_bool_attrs4
+; CHECK-SAME: (i1 [[C:%.*]], ptr [[P:%.*]], i64 [[X:%.*]]) {
+; CHECK-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[END:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    call void @side.effect()
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[R2:%.*]] = call nonnull ptr @foo(ptr byval(i64) [[P]], i64 noundef [[X]]) #[[ATTR8]]
+; CHECK-NEXT:    ret ptr [[R2]]
+;
+  br i1 %c, label %if, label %else
+if:
+  call void @side.effect()
+  %r = call nonnull ptr @foo(ptr byval(i64) readonly noundef %p, i64 noundef %x) cold nocallback nofree nosync willreturn
+  br label %end
+
+else:
+  %r2 = call noundef nonnull ptr @foo(ptr byval(i64) writeonly nonnull %p, i64 noundef %x) nosync
+  br label %end
+end:
+  %pr = phi ptr [ %r, %if], [%r2, %else]
+  ret ptr %pr
+}
+
+;.
+; CHECK: attributes #[[ATTR0]] = { alwaysinline memory(read) }
+; CHECK: attributes #[[ATTR1]] = { memory(readwrite) }
+; CHECK: attributes #[[ATTR2]] = { memory(read) }
+; CHECK: attributes #[[ATTR3]] = { mustprogress nocallback nofree willreturn }
+; CHECK: attributes #[[ATTR4]] = { alwaysinline nosync willreturn }
+; CHECK: attributes #[[ATTR5]] = { alwaysinline cold nocallback nofree nosync willreturn }
+; CHECK: attributes #[[ATTR6]] = { nosync willreturn }
+; CHECK: attributes #[[ATTR7]] = { cold nocallback nofree nosync willreturn }
+; CHECK: attributes #[[ATTR8]] = { nosync }
+;.
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll
index d05c26241f87c..c6ccaa4f45076 100644
--- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll
@@ -1,10 +1,9 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -S | FileCheck %s
 
 define i32 @func(i32 %x) {
-; CHECK-LABEL: define i32 @func(
-; CHECK-SAME: i32 [[X:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-LABEL: @func(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[X:.*]], 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @foo(i1 [[TMP1]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[X]], 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @foo(i1 [[TMP3]])
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll.expected
index 6311a55a1f9de..d5d2eb71ac565 100644
--- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll.expected
@@ -1,10 +1,9 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -S | FileCheck %s
 
 define i32 @func(i32 %x) {
-; CHECK-LABEL: define i32 @func(
-; CHECK-SAME: i32 [[X:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-LABEL: @func(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @foo(i1 [[TMP1]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[X]], 2
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @foo(i1 [[TMP7]])
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values5.ll b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values5.ll
new file mode 100644
index 0000000000000..4d503e36350fe
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values5.ll
@@ -0,0 +1,305 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt < %s -S | FileCheck %s
+
+; Test a very long run of similar (but different) instructions in which the optimal
+; diff is sensitive to how names are mapped. This is already slightly reduced
+; from the original, but attempting to reduce it further ended up hiding another
+; bug.
+
+%structA = type { <3 x i32> }
+%structC = type { [50 x i32] }
+
+define dso_local void @main.resume.0(i64 %0, %structA %arg01, [23 x i32] %arg12, [30 x i32] %arg23) {
+; CHECK-LABEL: define dso_local void @main.resume.0(
+; CHECK-SAME: i64 [[TMP0:%.*]], { [[STRUCTA:%.*]], [23 x i32], [30 x i32] } [[TMP1:%.*]]) {
+; CHECK-NEXT:  entryresume.0:
+; CHECK-NEXT:    [[FOO:%.*]] = call ptr @getter(i32 108)
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { [[STRUCTA]], [23 x i32], [30 x i32] } [[TMP1]], 2
+; CHECK-NEXT:    [[DOTFCA_0_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 0
+; CHECK-NEXT:    [[DOTFCA_1_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 1
+; CHECK-NEXT:    [[DOTFCA_2_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 2
+; CHECK-NEXT:    [[DOTFCA_3_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 3
+; CHECK-NEXT:    [[DOTFCA_4_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 4
+; CHECK-NEXT:    [[DOTFCA_5_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 5
+; CHECK-NEXT:    [[DOTFCA_6_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 6
+; CHECK-NEXT:    [[DOTFCA_7_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 7
+; CHECK-NEXT:    [[DOTFCA_8_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 8
+; CHECK-NEXT:    [[DOTFCA_9_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 9
+; CHECK-NEXT:    [[DOTFCA_10_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 10
+; CHECK-NEXT:    [[DOTFCA_11_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 11
+; CHECK-NEXT:    [[DOTFCA_12_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 12
+; CHECK-NEXT:    [[DOTFCA_13_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 13
+; CHECK-NEXT:    [[DOTFCA_14_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 14
+; CHECK-NEXT:    [[DOTFCA_15_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 15
+; CHECK-NEXT:    [[DOTFCA_16_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 16
+; CHECK-NEXT:    [[DOTFCA_17_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 17
+; CHECK-NEXT:    [[DOTFCA_18_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 18
+; CHECK-NEXT:    [[DOTFCA_19_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 19
+; CHECK-NEXT:    [[DOTFCA_20_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 20
+; CHECK-NEXT:    [[DOTFCA_21_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 21
+; CHECK-NEXT:    [[DOTFCA_22_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 22
+; CHECK-NEXT:    [[DOTFCA_23_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 23
+; CHECK-NEXT:    [[DOTFCA_24_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 24
+; CHECK-NEXT:    [[DOTFCA_25_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 25
+; CHECK-NEXT:    [[DOTFCA_26_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 26
+; CHECK-NEXT:    [[DOTFCA_27_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 27
+; CHECK-NEXT:    [[TMP23:%.*]] = extractvalue [30 x i32] [[TMP3]], 28
+; CHECK-NEXT:    [[DOTFCA_29_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 29
+; CHECK-NEXT:    [[TMP4:%.*]] = freeze [[STRUCTC:%.*]] poison
+; CHECK-NEXT:    [[DOTFCA_0_0_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 0
+; CHECK-NEXT:    [[DOTFCA_0_1_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 1
+; CHECK-NEXT:    [[DOTFCA_0_2_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 2
+; CHECK-NEXT:    [[DOTFCA_0_3_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 3
+; CHECK-NEXT:    [[DOTFCA_0_4_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 4
+; CHECK-NEXT:    [[DOTFCA_0_5_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 5
+; CHECK-NEXT:    [[DOTFCA_0_6_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 6
+; CHECK-NEXT:    [[DOTFCA_0_7_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 7
+; CHECK-NEXT:    [[DOTFCA_0_8_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 8
+; CHECK-NEXT:    [[DOTFCA_0_9_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 9
+; CHECK-NEXT:    [[DOTFCA_0_10_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 10
+; CHECK-NEXT:    [[DOTFCA_0_11_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 11
+; CHECK-NEXT:    [[DOTFCA_0_12_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 12
+; CHECK-NEXT:    [[DOTFCA_0_13_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 13
+; CHECK-NEXT:    [[DOTFCA_0_14_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 14
+; CHECK-NEXT:    [[DOTFCA_0_15_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 15
+; CHECK-NEXT:    [[DOTFCA_0_16_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 16
+; CHECK-NEXT:    [[DOTFCA_0_17_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 17
+; CHECK-NEXT:    [[DOTFCA_0_18_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 18
+; CHECK-NEXT:    [[DOTFCA_0_19_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 19
+; CHECK-NEXT:    [[DOTFCA_0_20_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 20
+; CHECK-NEXT:    [[DOTFCA_0_21_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 21
+; CHECK-NEXT:    [[DOTFCA_0_22_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 22
+; CHECK-NEXT:    [[DOTFCA_0_23_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 23
+; CHECK-NEXT:    [[DOTFCA_0_24_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 24
+; CHECK-NEXT:    [[DOTFCA_0_25_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 25
+; CHECK-NEXT:    [[DOTFCA_0_26_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 26
+; CHECK-NEXT:    [[DOTFCA_0_27_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 27
+; CHECK-NEXT:    [[DOTFCA_0_28_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 28
+; CHECK-NEXT:    [[DOTFCA_0_29_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 29
+; CHECK-NEXT:    [[DOTFCA_0_30_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 30
+; CHECK-NEXT:    [[DOTFCA_0_31_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 31
+; CHECK-NEXT:    [[DOTFCA_0_32_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 32
+; CHECK-NEXT:    [[DOTFCA_0_33_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 33
+; CHECK-NEXT:    [[DOTFCA_0_34_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 34
+; CHECK-NEXT:    [[DOTFCA_0_35_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 35
+; CHECK-NEXT:    [[DOTFCA_0_36_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 36
+; CHECK-NEXT:    [[DOTFCA_0_37_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 37
+; CHECK-NEXT:    [[DOTFCA_0_38_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 38
+; CHECK-NEXT:    [[DOTFCA_0_39_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 39
+; CHECK-NEXT:    [[DOTFCA_0_40_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 40
+; CHECK-NEXT:    [[DOTFCA_0_41_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 41
+; CHECK-NEXT:    [[DOTFCA_0_42_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 42
+; CHECK-NEXT:    [[DOTFCA_0_43_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 43
+; CHECK-NEXT:    [[DOTFCA_0_44_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 44
+; CHECK-NEXT:    [[DOTFCA_0_45_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 45
+; CHECK-NEXT:    [[DOTFCA_0_46_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 46
+; CHECK-NEXT:    [[DOTFCA_0_47_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 47
+; CHECK-NEXT:    [[DOTFCA_0_48_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 48
+; CHECK-NEXT:    [[DOTFCA_0_49_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 49
+; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr i32 [[DOTFCA_0_EXTRACT]] to ptr
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP27]], align 4
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 2
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP29]], align 4
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 3
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP31]], align 4
+; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 4
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP33]], align 4
+; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 5
+; CHECK-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP35]], align 4
+; CHECK-NEXT:    [[TMP37:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 6
+; CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP37]], align 4
+; CHECK-NEXT:    [[TMP39:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 7
+; CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[TMP39]], align 4
+; CHECK-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 8
+; CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[TMP41]], align 4
+; CHECK-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 9
+; CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[TMP43]], align 4
+; CHECK-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 10
+; CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[TMP45]], align 4
+; CHECK-NEXT:    [[TMP47:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 11
+; CHECK-NEXT:    [[TMP28:%.*]] = load i32, ptr [[TMP47]], align 4
+; CHECK-NEXT:    [[TMP49:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 12
+; CHECK-NEXT:    [[TMP30:%.*]] = load i32, ptr [[TMP49]], align 4
+; CHECK-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 13
+; CHECK-NEXT:    [[TMP32:%.*]] = load i32, ptr [[TMP51]], align 4
+; CHECK-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 14
+; CHECK-NEXT:    [[TMP34:%.*]] = load i32, ptr [[TMP53]], align 4
+; CHECK-NEXT:    [[TMP55:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 15
+; CHECK-NEXT:    [[TMP36:%.*]] = load i32, ptr [[TMP55]], align 4
+; CHECK-NEXT:    [[TMP57:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 16
+; CHECK-NEXT:    [[TMP38:%.*]] = load i32, ptr [[TMP57]], align 4
+; CHECK-NEXT:    [[TMP59:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 17
+; CHECK-NEXT:    [[TMP40:%.*]] = load i32, ptr [[TMP59]], align 4
+; CHECK-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 18
+; CHECK-NEXT:    [[TMP42:%.*]] = load i32, ptr [[TMP61]], align 4
+; CHECK-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 19
+; CHECK-NEXT:    [[TMP44:%.*]] = load i32, ptr [[TMP63]], align 4
+; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 20
+; CHECK-NEXT:    [[TMP46:%.*]] = load i32, ptr [[TMP65]], align 4
+; CHECK-NEXT:    [[TMP67:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 21
+; CHECK-NEXT:    [[TMP48:%.*]] = load i32, ptr [[TMP67]], align 4
+; CHECK-NEXT:    [[TMP69:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 22
+; CHECK-NEXT:    [[TMP50:%.*]] = load i32, ptr [[TMP69]], align 4
+; CHECK-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 23
+; CHECK-NEXT:    [[TMP52:%.*]] = load i32, ptr [[TMP71]], align 4
+; CHECK-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 24
+; CHECK-NEXT:    [[TMP54:%.*]] = load i32, ptr [[TMP73]], align 4
+; CHECK-NEXT:    [[TMP75:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 25
+; CHECK-NEXT:    [[TMP56:%.*]] = load i32, ptr [[TMP75]], align 4
+; CHECK-NEXT:    [[TMP77:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 26
+; CHECK-NEXT:    [[TMP62:%.*]] = load i32, ptr [[TMP77]], align 4
+; CHECK-NEXT:    [[TMP60:%.*]] = inttoptr i32 [[DOTFCA_0_EXTRACT]] to ptr
+; CHECK-NEXT:    [[TMP58:%.*]] = extractvalue { [[STRUCTA]], [23 x i32], [30 x i32] } [[TMP1]], 0
+; CHECK-NEXT:    [[DOTFCA_0_EXTRACT57:%.*]] = extractvalue [[STRUCTA]] [[TMP58]], 0
+; CHECK-NEXT:    ret void
+;
+entryresume.0:
+  %1 = call ptr @getter(i32 108)
+  %2 = insertvalue { %structA, [23 x i32], [30 x i32] } poison, %structA %arg01, 0
+  %3 = insertvalue { %structA, [23 x i32], [30 x i32] } %2, [23 x i32] %arg12, 1
+  %4 = insertvalue { %structA, [23 x i32], [30 x i32] } %3, [30 x i32] %arg23, 2
+  %5 = extractvalue { %structA, [23 x i32], [30 x i32] } %4, 2
+  %.fca.0.extract = extractvalue [30 x i32] %5, 0
+  %.fca.1.extract = extractvalue [30 x i32] %5, 1
+  %.fca.2.extract = extractvalue [30 x i32] %5, 2
+  %.fca.3.extract = extractvalue [30 x i32] %5, 3
+  %.fca.4.extract = extractvalue [30 x i32] %5, 4
+  %.fca.5.extract = extractvalue [30 x i32] %5, 5
+  %.fca.6.extract = extractvalue [30 x i32] %5, 6
+  %.fca.7.extract = extractvalue [30 x i32] %5, 7
+  %.fca.8.extract = extractvalue [30 x i32] %5, 8
+  %.fca.9.extract = extractvalue [30 x i32] %5, 9
+  %.fca.10.extract = extractvalue [30 x i32] %5, 10
+  %.fca.11.extract = extractvalue [30 x i32] %5, 11
+  %.fca.12.extract = extractvalue [30 x i32] %5, 12
+  %.fca.13.extract = extractvalue [30 x i32] %5, 13
+  %.fca.14.extract = extractvalue [30 x i32] %5, 14
+  %.fca.15.extract = extractvalue [30 x i32] %5, 15
+  %.fca.16.extract = extractvalue [30 x i32] %5, 16
+  %.fca.17.extract = extractvalue [30 x i32] %5, 17
+  %.fca.18.extract = extractvalue [30 x i32] %5, 18
+  %.fca.19.extract = extractvalue [30 x i32] %5, 19
+  %.fca.20.extract = extractvalue [30 x i32] %5, 20
+  %.fca.21.extract = extractvalue [30 x i32] %5, 21
+  %.fca.22.extract = extractvalue [30 x i32] %5, 22
+  %.fca.23.extract = extractvalue [30 x i32] %5, 23
+  %.fca.24.extract = extractvalue [30 x i32] %5, 24
+  %.fca.25.extract = extractvalue [30 x i32] %5, 25
+  %.fca.26.extract = extractvalue [30 x i32] %5, 26
+  %.fca.27.extract = extractvalue [30 x i32] %5, 27
+  %.fca.28.extract = extractvalue [30 x i32] %5, 28
+  %.fca.29.extract = extractvalue [30 x i32] %5, 29
+  %6 = freeze %structC poison
+  %.fca.0.0.extract = extractvalue %structC %6, 0, 0
+  %.fca.0.1.extract = extractvalue %structC %6, 0, 1
+  %.fca.0.2.extract = extractvalue %structC %6, 0, 2
+  %.fca.0.3.extract = extractvalue %structC %6, 0, 3
+  %.fca.0.4.extract = extractvalue %structC %6, 0, 4
+  %.fca.0.5.extract = extractvalue %structC %6, 0, 5
+  %.fca.0.6.extract = extractvalue %structC %6, 0, 6
+  %.fca.0.7.extract = extractvalue %structC %6, 0, 7
+  %.fca.0.8.extract = extractvalue %structC %6, 0, 8
+  %.fca.0.9.extract = extractvalue %structC %6, 0, 9
+  %.fca.0.10.extract = extractvalue %structC %6, 0, 10
+  %.fca.0.11.extract = extractvalue %structC %6, 0, 11
+  %.fca.0.12.extract = extractvalue %structC %6, 0, 12
+  %.fca.0.13.extract = extractvalue %structC %6, 0, 13
+  %.fca.0.14.extract = extractvalue %structC %6, 0, 14
+  %.fca.0.15.extract = extractvalue %structC %6, 0, 15
+  %.fca.0.16.extract = extractvalue %structC %6, 0, 16
+  %.fca.0.17.extract = extractvalue %structC %6, 0, 17
+  %.fca.0.18.extract = extractvalue %structC %6, 0, 18
+  %.fca.0.19.extract = extractvalue %structC %6, 0, 19
+  %.fca.0.20.extract = extractvalue %structC %6, 0, 20
+  %.fca.0.21.extract = extractvalue %structC %6, 0, 21
+  %.fca.0.22.extract = extractvalue %structC %6, 0, 22
+  %.fca.0.23.extract = extractvalue %structC %6, 0, 23
+  %.fca.0.24.extract = extractvalue %structC %6, 0, 24
+  %.fca.0.25.extract = extractvalue %structC %6, 0, 25
+  %.fca.0.26.extract = extractvalue %structC %6, 0, 26
+  %.fca.0.27.extract = extractvalue %structC %6, 0, 27
+  %.fca.0.28.extract = extractvalue %structC %6, 0, 28
+  %.fca.0.29.extract = extractvalue %structC %6, 0, 29
+  %.fca.0.30.extract = extractvalue %structC %6, 0, 30
+  %.fca.0.31.extract = extractvalue %structC %6, 0, 31
+  %.fca.0.32.extract = extractvalue %structC %6, 0, 32
+  %.fca.0.33.extract = extractvalue %structC %6, 0, 33
+  %.fca.0.34.extract = extractvalue %structC %6, 0, 34
+  %.fca.0.35.extract = extractvalue %structC %6, 0, 35
+  %.fca.0.36.extract = extractvalue %structC %6, 0, 36
+  %.fca.0.37.extract = extractvalue %structC %6, 0, 37
+  %.fca.0.38.extract = extractvalue %structC %6, 0, 38
+  %.fca.0.39.extract = extractvalue %structC %6, 0, 39
+  %.fca.0.40.extract = extractvalue %structC %6, 0, 40
+  %.fca.0.41.extract = extractvalue %structC %6, 0, 41
+  %.fca.0.42.extract = extractvalue %structC %6, 0, 42
+  %.fca.0.43.extract = extractvalue %structC %6, 0, 43
+  %.fca.0.44.extract = extractvalue %structC %6, 0, 44
+  %.fca.0.45.extract = extractvalue %structC %6, 0, 45
+  %.fca.0.46.extract = extractvalue %structC %6, 0, 46
+  %.fca.0.47.extract = extractvalue %structC %6, 0, 47
+  %.fca.0.48.extract = extractvalue %structC %6, 0, 48
+  %.fca.0.49.extract = extractvalue %structC %6, 0, 49
+  %7 = inttoptr i32 %.fca.0.extract to ptr
+  %8 = load i32, ptr %7, align 4
+  %9 = getelementptr inbounds i32, ptr %7, i32 1
+  %10 = load i32, ptr %9, align 4
+  %11 = getelementptr inbounds i32, ptr %7, i32 2
+  %12 = load i32, ptr %11, align 4
+  %13 = getelementptr inbounds i32, ptr %7, i32 3
+  %14 = load i32, ptr %13, align 4
+  %15 = getelementptr inbounds i32, ptr %7, i32 4
+  %16 = load i32, ptr %15, align 4
+  %17 = getelementptr inbounds i32, ptr %7, i32 5
+  %18 = load i32, ptr %17, align 4
+  %19 = getelementptr inbounds i32, ptr %7, i32 6
+  %20 = load i32, ptr %19, align 4
+  %21 = getelementptr inbounds i32, ptr %7, i32 7
+  %22 = load i32, ptr %21, align 4
+  %23 = getelementptr inbounds i32, ptr %7, i32 8
+  %24 = load i32, ptr %23, align 4
+  %25 = getelementptr inbounds i32, ptr %7, i32 9
+  %26 = load i32, ptr %25, align 4
+  %27 = getelementptr inbounds i32, ptr %7, i32 10
+  %28 = load i32, ptr %27, align 4
+  %29 = getelementptr inbounds i32, ptr %7, i32 11
+  %30 = load i32, ptr %29, align 4
+  %31 = getelementptr inbounds i32, ptr %7, i32 12
+  %32 = load i32, ptr %31, align 4
+  %33 = getelementptr inbounds i32, ptr %7, i32 13
+  %34 = load i32, ptr %33, align 4
+  %35 = getelementptr inbounds i32, ptr %7, i32 14
+  %36 = load i32, ptr %35, align 4
+  %37 = getelementptr inbounds i32, ptr %7, i32 15
+  %38 = load i32, ptr %37, align 4
+  %39 = getelementptr inbounds i32, ptr %7, i32 16
+  %40 = load i32, ptr %39, align 4
+  %41 = getelementptr inbounds i32, ptr %7, i32 17
+  %42 = load i32, ptr %41, align 4
+  %43 = getelementptr inbounds i32, ptr %7, i32 18
+  %44 = load i32, ptr %43, align 4
+  %45 = getelementptr inbounds i32, ptr %7, i32 19
+  %46 = load i32, ptr %45, align 4
+  %47 = getelementptr inbounds i32, ptr %7, i32 20
+  %48 = load i32, ptr %47, align 4
+  %49 = getelementptr inbounds i32, ptr %7, i32 21
+  %50 = load i32, ptr %49, align 4
+  %51 = getelementptr inbounds i32, ptr %7, i32 22
+  %52 = load i32, ptr %51, align 4
+  %53 = getelementptr inbounds i32, ptr %7, i32 23
+  %54 = load i32, ptr %53, align 4
+  %55 = getelementptr inbounds i32, ptr %7, i32 24
+  %56 = load i32, ptr %55, align 4
+  %57 = getelementptr inbounds i32, ptr %7, i32 25
+  %58 = load i32, ptr %57, align 4
+  %59 = getelementptr inbounds i32, ptr %7, i32 26
+  %60 = load i32, ptr %59, align 4
+  %61 = inttoptr i32 %.fca.0.extract to ptr
+  %62 = extractvalue { %structA, [23 x i32], [30 x i32] } %4, 0
+  %.fca.0.extract57 = extractvalue %structA %62, 0
+  ret void
+}
+
+declare ptr @getter(i32)
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values5.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values5.ll.expected
new file mode 100644
index 0000000000000..40bf55e70a958
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values5.ll.expected
@@ -0,0 +1,308 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt < %s -S | FileCheck %s
+
+; Test a very long run of similar (but different) instructions in which the optimal
+; diff is sensitive to how names are mapped. This is already slightly reduced
+; from the original, but attempting to reduce it further ended up hiding another
+; bug.
+
+%structA = type { <3 x i32> }
+%structC = type { [50 x i32] }
+
+define dso_local void @main.resume.0(i64 %0, %structA %arg01, [23 x i32] %arg12, [30 x i32] %arg23) {
+; CHECK-LABEL: define dso_local void @main.resume.0(
+; CHECK-SAME: i64 [[TMP0:%.*]], [[STRUCTA:%.*]] [[ARG01:%.*]], [23 x i32] [[ARG12:%.*]], [30 x i32] [[ARG23:%.*]]) {
+; CHECK-NEXT:  entryresume.0:
+; CHECK-NEXT:    [[FOO:%.*]] = call ptr @getter(i32 108)
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { [[STRUCTA]], [23 x i32], [30 x i32] } poison, [[STRUCTA]] [[ARG01]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = insertvalue { [[STRUCTA]], [23 x i32], [30 x i32] } [[TMP6]], [23 x i32] [[ARG12]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { [[STRUCTA]], [23 x i32], [30 x i32] } [[TMP8]], [30 x i32] [[ARG23]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { [[STRUCTA]], [23 x i32], [30 x i32] } [[TMP1]], 2
+; CHECK-NEXT:    [[DOTFCA_0_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 0
+; CHECK-NEXT:    [[DOTFCA_1_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 1
+; CHECK-NEXT:    [[DOTFCA_2_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 2
+; CHECK-NEXT:    [[DOTFCA_3_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 3
+; CHECK-NEXT:    [[DOTFCA_4_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 4
+; CHECK-NEXT:    [[DOTFCA_5_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 5
+; CHECK-NEXT:    [[DOTFCA_6_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 6
+; CHECK-NEXT:    [[DOTFCA_7_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 7
+; CHECK-NEXT:    [[DOTFCA_8_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 8
+; CHECK-NEXT:    [[DOTFCA_9_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 9
+; CHECK-NEXT:    [[DOTFCA_10_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 10
+; CHECK-NEXT:    [[DOTFCA_11_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 11
+; CHECK-NEXT:    [[DOTFCA_12_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 12
+; CHECK-NEXT:    [[DOTFCA_13_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 13
+; CHECK-NEXT:    [[DOTFCA_14_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 14
+; CHECK-NEXT:    [[DOTFCA_15_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 15
+; CHECK-NEXT:    [[DOTFCA_16_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 16
+; CHECK-NEXT:    [[DOTFCA_17_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 17
+; CHECK-NEXT:    [[DOTFCA_18_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 18
+; CHECK-NEXT:    [[DOTFCA_19_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 19
+; CHECK-NEXT:    [[DOTFCA_20_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 20
+; CHECK-NEXT:    [[DOTFCA_21_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 21
+; CHECK-NEXT:    [[DOTFCA_22_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 22
+; CHECK-NEXT:    [[DOTFCA_23_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 23
+; CHECK-NEXT:    [[DOTFCA_24_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 24
+; CHECK-NEXT:    [[DOTFCA_25_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 25
+; CHECK-NEXT:    [[DOTFCA_26_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 26
+; CHECK-NEXT:    [[DOTFCA_27_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 27
+; CHECK-NEXT:    [[TMP23:%.*]] = extractvalue [30 x i32] [[TMP3]], 28
+; CHECK-NEXT:    [[DOTFCA_29_EXTRACT:%.*]] = extractvalue [30 x i32] [[TMP3]], 29
+; CHECK-NEXT:    [[TMP4:%.*]] = freeze [[STRUCTC:%.*]] poison
+; CHECK-NEXT:    [[DOTFCA_0_0_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 0
+; CHECK-NEXT:    [[DOTFCA_0_1_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 1
+; CHECK-NEXT:    [[DOTFCA_0_2_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 2
+; CHECK-NEXT:    [[DOTFCA_0_3_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 3
+; CHECK-NEXT:    [[DOTFCA_0_4_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 4
+; CHECK-NEXT:    [[DOTFCA_0_5_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 5
+; CHECK-NEXT:    [[DOTFCA_0_6_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 6
+; CHECK-NEXT:    [[DOTFCA_0_7_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 7
+; CHECK-NEXT:    [[DOTFCA_0_8_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 8
+; CHECK-NEXT:    [[DOTFCA_0_9_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 9
+; CHECK-NEXT:    [[DOTFCA_0_10_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 10
+; CHECK-NEXT:    [[DOTFCA_0_11_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 11
+; CHECK-NEXT:    [[DOTFCA_0_12_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 12
+; CHECK-NEXT:    [[DOTFCA_0_13_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 13
+; CHECK-NEXT:    [[DOTFCA_0_14_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 14
+; CHECK-NEXT:    [[DOTFCA_0_15_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 15
+; CHECK-NEXT:    [[DOTFCA_0_16_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 16
+; CHECK-NEXT:    [[DOTFCA_0_17_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 17
+; CHECK-NEXT:    [[DOTFCA_0_18_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 18
+; CHECK-NEXT:    [[DOTFCA_0_19_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 19
+; CHECK-NEXT:    [[DOTFCA_0_20_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 20
+; CHECK-NEXT:    [[DOTFCA_0_21_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 21
+; CHECK-NEXT:    [[DOTFCA_0_22_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 22
+; CHECK-NEXT:    [[DOTFCA_0_23_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 23
+; CHECK-NEXT:    [[DOTFCA_0_24_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 24
+; CHECK-NEXT:    [[DOTFCA_0_25_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 25
+; CHECK-NEXT:    [[DOTFCA_0_26_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 26
+; CHECK-NEXT:    [[DOTFCA_0_27_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 27
+; CHECK-NEXT:    [[DOTFCA_0_28_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 28
+; CHECK-NEXT:    [[DOTFCA_0_29_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 29
+; CHECK-NEXT:    [[DOTFCA_0_30_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 30
+; CHECK-NEXT:    [[DOTFCA_0_31_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 31
+; CHECK-NEXT:    [[DOTFCA_0_32_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 32
+; CHECK-NEXT:    [[DOTFCA_0_33_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 33
+; CHECK-NEXT:    [[DOTFCA_0_34_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 34
+; CHECK-NEXT:    [[DOTFCA_0_35_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 35
+; CHECK-NEXT:    [[DOTFCA_0_36_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 36
+; CHECK-NEXT:    [[DOTFCA_0_37_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 37
+; CHECK-NEXT:    [[DOTFCA_0_38_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 38
+; CHECK-NEXT:    [[DOTFCA_0_39_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 39
+; CHECK-NEXT:    [[DOTFCA_0_40_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 40
+; CHECK-NEXT:    [[DOTFCA_0_41_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 41
+; CHECK-NEXT:    [[DOTFCA_0_42_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 42
+; CHECK-NEXT:    [[DOTFCA_0_43_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 43
+; CHECK-NEXT:    [[DOTFCA_0_44_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 44
+; CHECK-NEXT:    [[DOTFCA_0_45_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 45
+; CHECK-NEXT:    [[DOTFCA_0_46_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 46
+; CHECK-NEXT:    [[DOTFCA_0_47_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 47
+; CHECK-NEXT:    [[DOTFCA_0_48_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 48
+; CHECK-NEXT:    [[DOTFCA_0_49_EXTRACT:%.*]] = extractvalue [[STRUCTC]] [[TMP4]], 0, 49
+; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr i32 [[DOTFCA_0_EXTRACT]] to ptr
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP27]], align 4
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 2
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP29]], align 4
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 3
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP31]], align 4
+; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 4
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP33]], align 4
+; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 5
+; CHECK-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP35]], align 4
+; CHECK-NEXT:    [[TMP37:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 6
+; CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP37]], align 4
+; CHECK-NEXT:    [[TMP39:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 7
+; CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[TMP39]], align 4
+; CHECK-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 8
+; CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[TMP41]], align 4
+; CHECK-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 9
+; CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[TMP43]], align 4
+; CHECK-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 10
+; CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[TMP45]], align 4
+; CHECK-NEXT:    [[TMP47:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 11
+; CHECK-NEXT:    [[TMP28:%.*]] = load i32, ptr [[TMP47]], align 4
+; CHECK-NEXT:    [[TMP49:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 12
+; CHECK-NEXT:    [[TMP30:%.*]] = load i32, ptr [[TMP49]], align 4
+; CHECK-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 13
+; CHECK-NEXT:    [[TMP32:%.*]] = load i32, ptr [[TMP51]], align 4
+; CHECK-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 14
+; CHECK-NEXT:    [[TMP34:%.*]] = load i32, ptr [[TMP53]], align 4
+; CHECK-NEXT:    [[TMP55:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 15
+; CHECK-NEXT:    [[TMP36:%.*]] = load i32, ptr [[TMP55]], align 4
+; CHECK-NEXT:    [[TMP57:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 16
+; CHECK-NEXT:    [[TMP38:%.*]] = load i32, ptr [[TMP57]], align 4
+; CHECK-NEXT:    [[TMP59:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 17
+; CHECK-NEXT:    [[TMP40:%.*]] = load i32, ptr [[TMP59]], align 4
+; CHECK-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 18
+; CHECK-NEXT:    [[TMP42:%.*]] = load i32, ptr [[TMP61]], align 4
+; CHECK-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 19
+; CHECK-NEXT:    [[TMP44:%.*]] = load i32, ptr [[TMP63]], align 4
+; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 20
+; CHECK-NEXT:    [[TMP46:%.*]] = load i32, ptr [[TMP65]], align 4
+; CHECK-NEXT:    [[TMP67:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 21
+; CHECK-NEXT:    [[TMP48:%.*]] = load i32, ptr [[TMP67]], align 4
+; CHECK-NEXT:    [[TMP69:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 22
+; CHECK-NEXT:    [[TMP50:%.*]] = load i32, ptr [[TMP69]], align 4
+; CHECK-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 23
+; CHECK-NEXT:    [[TMP52:%.*]] = load i32, ptr [[TMP71]], align 4
+; CHECK-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 24
+; CHECK-NEXT:    [[TMP54:%.*]] = load i32, ptr [[TMP73]], align 4
+; CHECK-NEXT:    [[TMP75:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 25
+; CHECK-NEXT:    [[TMP56:%.*]] = load i32, ptr [[TMP75]], align 4
+; CHECK-NEXT:    [[TMP77:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 26
+; CHECK-NEXT:    [[TMP62:%.*]] = load i32, ptr [[TMP77]], align 4
+; CHECK-NEXT:    [[TMP60:%.*]] = inttoptr i32 [[DOTFCA_0_EXTRACT]] to ptr
+; CHECK-NEXT:    [[TMP58:%.*]] = extractvalue { [[STRUCTA]], [23 x i32], [30 x i32] } [[TMP1]], 0
+; CHECK-NEXT:    [[DOTFCA_0_EXTRACT57:%.*]] = extractvalue [[STRUCTA]] [[TMP58]], 0
+; CHECK-NEXT:    ret void
+;
+entryresume.0:
+  %1 = call ptr @getter(i32 108)
+  %2 = insertvalue { %structA, [23 x i32], [30 x i32] } poison, %structA %arg01, 0
+  %3 = insertvalue { %structA, [23 x i32], [30 x i32] } %2, [23 x i32] %arg12, 1
+  %4 = insertvalue { %structA, [23 x i32], [30 x i32] } %3, [30 x i32] %arg23, 2
+  %5 = extractvalue { %structA, [23 x i32], [30 x i32] } %4, 2
+  %.fca.0.extract = extractvalue [30 x i32] %5, 0
+  %.fca.1.extract = extractvalue [30 x i32] %5, 1
+  %.fca.2.extract = extractvalue [30 x i32] %5, 2
+  %.fca.3.extract = extractvalue [30 x i32] %5, 3
+  %.fca.4.extract = extractvalue [30 x i32] %5, 4
+  %.fca.5.extract = extractvalue [30 x i32] %5, 5
+  %.fca.6.extract = extractvalue [30 x i32] %5, 6
+  %.fca.7.extract = extractvalue [30 x i32] %5, 7
+  %.fca.8.extract = extractvalue [30 x i32] %5, 8
+  %.fca.9.extract = extractvalue [30 x i32] %5, 9
+  %.fca.10.extract = extractvalue [30 x i32] %5, 10
+  %.fca.11.extract = extractvalue [30 x i32] %5, 11
+  %.fca.12.extract = extractvalue [30 x i32] %5, 12
+  %.fca.13.extract = extractvalue [30 x i32] %5, 13
+  %.fca.14.extract = extractvalue [30 x i32] %5, 14
+  %.fca.15.extract = extractvalue [30 x i32] %5, 15
+  %.fca.16.extract = extractvalue [30 x i32] %5, 16
+  %.fca.17.extract = extractvalue [30 x i32] %5, 17
+  %.fca.18.extract = extractvalue [30 x i32] %5, 18
+  %.fca.19.extract = extractvalue [30 x i32] %5, 19
+  %.fca.20.extract = extractvalue [30 x i32] %5, 20
+  %.fca.21.extract = extractvalue [30 x i32] %5, 21
+  %.fca.22.extract = extractvalue [30 x i32] %5, 22
+  %.fca.23.extract = extractvalue [30 x i32] %5, 23
+  %.fca.24.extract = extractvalue [30 x i32] %5, 24
+  %.fca.25.extract = extractvalue [30 x i32] %5, 25
+  %.fca.26.extract = extractvalue [30 x i32] %5, 26
+  %.fca.27.extract = extractvalue [30 x i32] %5, 27
+  %.fca.28.extract = extractvalue [30 x i32] %5, 28
+  %.fca.29.extract = extractvalue [30 x i32] %5, 29
+  %6 = freeze %structC poison
+  %.fca.0.0.extract = extractvalue %structC %6, 0, 0
+  %.fca.0.1.extract = extractvalue %structC %6, 0, 1
+  %.fca.0.2.extract = extractvalue %structC %6, 0, 2
+  %.fca.0.3.extract = extractvalue %structC %6, 0, 3
+  %.fca.0.4.extract = extractvalue %structC %6, 0, 4
+  %.fca.0.5.extract = extractvalue %structC %6, 0, 5
+  %.fca.0.6.extract = extractvalue %structC %6, 0, 6
+  %.fca.0.7.extract = extractvalue %structC %6, 0, 7
+  %.fca.0.8.extract = extractvalue %structC %6, 0, 8
+  %.fca.0.9.extract = extractvalue %structC %6, 0, 9
+  %.fca.0.10.extract = extractvalue %structC %6, 0, 10
+  %.fca.0.11.extract = extractvalue %structC %6, 0, 11
+  %.fca.0.12.extract = extractvalue %structC %6, 0, 12
+  %.fca.0.13.extract = extractvalue %structC %6, 0, 13
+  %.fca.0.14.extract = extractvalue %structC %6, 0, 14
+  %.fca.0.15.extract = extractvalue %structC %6, 0, 15
+  %.fca.0.16.extract = extractvalue %structC %6, 0, 16
+  %.fca.0.17.extract = extractvalue %structC %6, 0, 17
+  %.fca.0.18.extract = extractvalue %structC %6, 0, 18
+  %.fca.0.19.extract = extractvalue %structC %6, 0, 19
+  %.fca.0.20.extract = extractvalue %structC %6, 0, 20
+  %.fca.0.21.extract = extractvalue %structC %6, 0, 21
+  %.fca.0.22.extract = extractvalue %structC %6, 0, 22
+  %.fca.0.23.extract = extractvalue %structC %6, 0, 23
+  %.fca.0.24.extract = extractvalue %structC %6, 0, 24
+  %.fca.0.25.extract = extractvalue %structC %6, 0, 25
+  %.fca.0.26.extract = extractvalue %structC %6, 0, 26
+  %.fca.0.27.extract = extractvalue %structC %6, 0, 27
+  %.fca.0.28.extract = extractvalue %structC %6, 0, 28
+  %.fca.0.29.extract = extractvalue %structC %6, 0, 29
+  %.fca.0.30.extract = extractvalue %structC %6, 0, 30
+  %.fca.0.31.extract = extractvalue %structC %6, 0, 31
+  %.fca.0.32.extract = extractvalue %structC %6, 0, 32
+  %.fca.0.33.extract = extractvalue %structC %6, 0, 33
+  %.fca.0.34.extract = extractvalue %structC %6, 0, 34
+  %.fca.0.35.extract = extractvalue %structC %6, 0, 35
+  %.fca.0.36.extract = extractvalue %structC %6, 0, 36
+  %.fca.0.37.extract = extractvalue %structC %6, 0, 37
+  %.fca.0.38.extract = extractvalue %structC %6, 0, 38
+  %.fca.0.39.extract = extractvalue %structC %6, 0, 39
+  %.fca.0.40.extract = extractvalue %structC %6, 0, 40
+  %.fca.0.41.extract = extractvalue %structC %6, 0, 41
+  %.fca.0.42.extract = extractvalue %structC %6, 0, 42
+  %.fca.0.43.extract = extractvalue %structC %6, 0, 43
+  %.fca.0.44.extract = extractvalue %structC %6, 0, 44
+  %.fca.0.45.extract = extractvalue %structC %6, 0, 45
+  %.fca.0.46.extract = extractvalue %structC %6, 0, 46
+  %.fca.0.47.extract = extractvalue %structC %6, 0, 47
+  %.fca.0.48.extract = extractvalue %structC %6, 0, 48
+  %.fca.0.49.extract = extractvalue %structC %6, 0, 49
+  %7 = inttoptr i32 %.fca.0.extract to ptr
+  %8 = load i32, ptr %7, align 4
+  %9 = getelementptr inbounds i32, ptr %7, i32 1
+  %10 = load i32, ptr %9, align 4
+  %11 = getelementptr inbounds i32, ptr %7, i32 2
+  %12 = load i32, ptr %11, align 4
+  %13 = getelementptr inbounds i32, ptr %7, i32 3
+  %14 = load i32, ptr %13, align 4
+  %15 = getelementptr inbounds i32, ptr %7, i32 4
+  %16 = load i32, ptr %15, align 4
+  %17 = getelementptr inbounds i32, ptr %7, i32 5
+  %18 = load i32, ptr %17, align 4
+  %19 = getelementptr inbounds i32, ptr %7, i32 6
+  %20 = load i32, ptr %19, align 4
+  %21 = getelementptr inbounds i32, ptr %7, i32 7
+  %22 = load i32, ptr %21, align 4
+  %23 = getelementptr inbounds i32, ptr %7, i32 8
+  %24 = load i32, ptr %23, align 4
+  %25 = getelementptr inbounds i32, ptr %7, i32 9
+  %26 = load i32, ptr %25, align 4
+  %27 = getelementptr inbounds i32, ptr %7, i32 10
+  %28 = load i32, ptr %27, align 4
+  %29 = getelementptr inbounds i32, ptr %7, i32 11
+  %30 = load i32, ptr %29, align 4
+  %31 = getelementptr inbounds i32, ptr %7, i32 12
+  %32 = load i32, ptr %31, align 4
+  %33 = getelementptr inbounds i32, ptr %7, i32 13
+  %34 = load i32, ptr %33, align 4
+  %35 = getelementptr inbounds i32, ptr %7, i32 14
+  %36 = load i32, ptr %35, align 4
+  %37 = getelementptr inbounds i32, ptr %7, i32 15
+  %38 = load i32, ptr %37, align 4
+  %39 = getelementptr inbounds i32, ptr %7, i32 16
+  %40 = load i32, ptr %39, align 4
+  %41 = getelementptr inbounds i32, ptr %7, i32 17
+  %42 = load i32, ptr %41, align 4
+  %43 = getelementptr inbounds i32, ptr %7, i32 18
+  %44 = load i32, ptr %43, align 4
+  %45 = getelementptr inbounds i32, ptr %7, i32 19
+  %46 = load i32, ptr %45, align 4
+  %47 = getelementptr inbounds i32, ptr %7, i32 20
+  %48 = load i32, ptr %47, align 4
+  %49 = getelementptr inbounds i32, ptr %7, i32 21
+  %50 = load i32, ptr %49, align 4
+  %51 = getelementptr inbounds i32, ptr %7, i32 22
+  %52 = load i32, ptr %51, align 4
+  %53 = getelementptr inbounds i32, ptr %7, i32 23
+  %54 = load i32, ptr %53, align 4
+  %55 = getelementptr inbounds i32, ptr %7, i32 24
+  %56 = load i32, ptr %55, align 4
+  %57 = getelementptr inbounds i32, ptr %7, i32 25
+  %58 = load i32, ptr %57, align 4
+  %59 = getelementptr inbounds i32, ptr %7, i32 26
+  %60 = load i32, ptr %59, align 4
+  %61 = inttoptr i32 %.fca.0.extract to ptr
+  %62 = extractvalue { %structA, [23 x i32], [30 x i32] } %4, 0
+  %.fca.0.extract57 = extractvalue %structA %62, 0
+  ret void
+}
+
+declare ptr @getter(i32)
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values5.test b/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values5.test
new file mode 100644
index 0000000000000..a04a9fa3e9078
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values5.test
@@ -0,0 +1,2 @@
+# RUN: cp -f %S/Inputs/stable_ir_values5.ll %t.ll && %update_test_checks %t.ll
+# RUN: diff -u %t.ll %S/Inputs/stable_ir_values5.ll.expected
diff --git a/llvm/test/tools/llvm-readtapi/compare-ignore-archs.test b/llvm/test/tools/llvm-readtapi/compare-ignore-archs.test
new file mode 100644
index 0000000000000..a9a79128caadc
--- /dev/null
+++ b/llvm/test/tools/llvm-readtapi/compare-ignore-archs.test
@@ -0,0 +1,68 @@
+; RUN: rm -rf %t
+; RUN: split-file %s %t  
+
+; RUN: llvm-readtapi --compare %t/all_archs.tbd  %t/missing_archs.tbd --ignore-arch armv7 --ignore-arch armv7s 2>&1 | FileCheck %s --allow-empty --implicit-check-not warning: --implicit-check-not error:
+; RUN: not llvm-readtapi --compare %t/all_archs.tbd  %t/missing_archs.tbd --ignore-arch armv7s 2>&1 | FileCheck %s --check-prefix ARMV7
+
+all_archs.tbd
+; ARMV7: < {{.*}}all_archs.tbd
+; ARMV7: > {{.*}}missing_archs.tbd
+
+; ARMV7:      Reexported Libraries
+; ARMV7-NEXT: 	    armv7-apple-ios
+; ARMV7:      Symbols
+; ARMV7-NEXT: 	    armv7-apple-ios
+; ARMV7:      Inlined Reexported Frameworks/Libraries
+; ARMV7-NEXT: 	/System/Library/Frameworks/FooCore.framework/FooCore
+; ARMV7: 		  Symbols
+; ARMV7-NEXT: 			armv7-apple-ios
+
+;--- all_archs.tbd
+--- !tapi-tbd
+tbd-version:     4
+targets:         [ armv7-ios, armv7s-ios, arm64-ios, arm64e-ios ]
+install-name:    '/System/Library/Frameworks/Foo.framework/Foo'
+current-version: 1986.34.9
+reexported-libraries:
+  - targets:         [ armv7-ios, armv7s-ios, arm64-ios, arm64e-ios ]
+    libraries:       [ '/System/Library/Frameworks/FooCore.framework/FooCore' ]
+exports:
+  - targets:         [ armv7-ios, armv7s-ios, arm64-ios, arm64e-ios ]
+    symbols:         [ _AllRequestsKeyPathFragment, _AnalyticsLoggingSubsystem, _AnyRequestKeyPathFragment, 
+                       _bar_getBarPointSize_ints, _bar_newBarMessage, _bar_serialize ]
+  - targets:         [ arm64-ios, arm64e-ios ]
+    symbols:         [ __ZN3lingo11MapEdgeRoad6lengthEv, 
+                       __ZTVN3lingo11MapEdgeRoadE, __ZTVN3lingo7MapNodeE, __ZTVN5bar19GeometryPathElementE ]
+--- !tapi-tbd
+tbd-version:     4
+targets:         [ armv7-ios, armv7s-ios, arm64-ios, arm64e-ios ]
+install-name:    '/System/Library/Frameworks/FooCore.framework/FooCore'
+current-version: 1986.34.9
+exports:
+  - targets:         [ armv7-ios, armv7s-ios, arm64-ios, arm64e-ios ]
+    symbols:         [ _sym, _workgroupsym, _taskgroup_sim, meta_sim ]
+...
+
+;--- missing_archs.tbd
+--- !tapi-tbd
+tbd-version:     4
+targets:         [ arm64-ios, arm64e-ios ]
+install-name:    '/System/Library/Frameworks/Foo.framework/Foo'
+current-version: 1986.34.9
+reexported-libraries:
+  - targets:         [ arm64-ios, arm64e-ios ]
+    libraries:       [ '/System/Library/Frameworks/FooCore.framework/FooCore' ]
+exports:
+  - targets:         [ arm64-ios, arm64e-ios ]
+    symbols:         [ _AllRequestsKeyPathFragment, _AnalyticsLoggingSubsystem, _AnyRequestKeyPathFragment, 
+                       _bar_getBarPointSize_ints, _bar_newBarMessage, _bar_serialize, __ZN3lingo11MapEdgeRoad6lengthEv, 
+                       __ZTVN3lingo11MapEdgeRoadE, __ZTVN3lingo7MapNodeE, __ZTVN5bar19GeometryPathElementE ]
+--- !tapi-tbd
+tbd-version:     4
+targets:         [ arm64-ios, arm64e-ios ]
+install-name:    '/System/Library/Frameworks/FooCore.framework/FooCore'
+current-version: 1986.34.9
+exports:
+  - targets:         [ arm64-ios, arm64e-ios ]
+    symbols:         [ _sym, _workgroupsym, _taskgroup_sim, meta_sim ]
+...
diff --git a/llvm/test/tools/llvm-reduce/mir/preserve-func-info.mir b/llvm/test/tools/llvm-reduce/mir/preserve-func-info.mir
index f735dfd5cbbf0..cbddf89f3b47b 100644
--- a/llvm/test/tools/llvm-reduce/mir/preserve-func-info.mir
+++ b/llvm/test/tools/llvm-reduce/mir/preserve-func-info.mir
@@ -17,6 +17,7 @@
 # RESULT-NEXT: noPhis:          false
 # RESULT-NEXT: isSSA:           false
 # RESULT-NEXT: noVRegs:         false
+# RESULT-NEXT: hasFakeUses: true
 # RESULT-NEXT: callsEHReturn: true
 # RESULT-NEXT: callsUnwindInit: true
 # RESULT-NEXT: hasEHCatchret: true
@@ -47,6 +48,7 @@ hasWinCFI:       true
 noPhis:          false
 isSSA:           false
 noVRegs:         false
+hasFakeUses: true
 failsVerification: true
 tracksDebugUserValues: true
 callsEHReturn: true
diff --git a/llvm/tools/llvm-debuginfod-find/llvm-debuginfod-find.cpp b/llvm/tools/llvm-debuginfod-find/llvm-debuginfod-find.cpp
index 699fcf8a7dbcd..77862737bccd2 100644
--- a/llvm/tools/llvm-debuginfod-find/llvm-debuginfod-find.cpp
+++ b/llvm/tools/llvm-debuginfod-find/llvm-debuginfod-find.cpp
@@ -67,7 +67,6 @@ static std::vector<std::string> DebugFileDirectory;
 
 static void parseArgs(int argc, char **argv) {
   DebuginfodFindOptTable Tbl;
-  llvm::StringRef ToolName = argv[0];
   llvm::BumpPtrAllocator A;
   llvm::StringSaver Saver{A};
   opt::InputArgList Args =
@@ -77,9 +76,15 @@ static void parseArgs(int argc, char **argv) {
       });
 
   if (Args.hasArg(OPT_help)) {
-    Tbl.printHelp(llvm::outs(),
-                  "llvm-debuginfod-find [options] <input build_id>",
-                  ToolName.str().c_str());
+    Tbl.printHelp(
+        llvm::outs(), "llvm-debuginfod-find [options] <input build_id>",
+        "llvm-debuginfod-find: Fetch debuginfod artifacts\n\n"
+        "This program is a frontend to the debuginfod client library. The "
+        "cache directory, request timeout (in seconds), and debuginfod server "
+        "urls are set by these environment variables:\n"
+        "DEBUGINFOD_CACHE_PATH (default set by sys::path::cache_directory)\n"
+        "DEBUGINFOD_TIMEOUT (defaults to 90s)\n"
+        "DEBUGINFOD_URLS=[comma separated URLs] (defaults to empty)");
     std::exit(0);
   }
 
diff --git a/llvm/tools/llvm-readtapi/TapiOpts.td b/llvm/tools/llvm-readtapi/TapiOpts.td
index 3b8f26579bd4a..3d13dc99fc963 100644
--- a/llvm/tools/llvm-readtapi/TapiOpts.td
+++ b/llvm/tools/llvm-readtapi/TapiOpts.td
@@ -35,3 +35,8 @@ def delete_input : FF<"delete-input", "delete and replace input file on success"
 def delete_private_libraries : FF<"delete-private-libraries", "delete private system dynamic libraries and frameworks">;
 def t: FF<"t", "logs each library loaded, useful for debugging problems with search paths where the wrong library is loaded">;
 
+
+//
+// Compare options
+//
+defm ignore_arch : JS<"ignore-arch", "<architecture> slice to ignore for comparison", "<architecture>">;
diff --git a/llvm/tools/llvm-readtapi/llvm-readtapi.cpp b/llvm/tools/llvm-readtapi/llvm-readtapi.cpp
index c287dac4cd239..6e42ed76949ff 100644
--- a/llvm/tools/llvm-readtapi/llvm-readtapi.cpp
+++ b/llvm/tools/llvm-readtapi/llvm-readtapi.cpp
@@ -71,9 +71,14 @@ struct StubOptions {
   bool TraceLibs = false;
 };
 
+struct CompareOptions {
+  ArchitectureSet ArchsToIgnore;
+};
+
 struct Context {
   std::vector<std::string> Inputs;
   StubOptions StubOpt;
+  CompareOptions CmpOpt;
   std::unique_ptr<llvm::raw_fd_stream> OutStream;
   FileType WriteFT = FileType::TBD_V5;
   bool Compact = false;
@@ -160,6 +165,28 @@ static bool handleCompareAction(const Context &Ctx) {
   auto LeftIF = getInterfaceFile(Ctx.Inputs.front());
   auto RightIF = getInterfaceFile(Ctx.Inputs.at(1));
 
+  // Remove all architectures to ignore before running comparison.
+  auto removeArchFromIF = [](auto &IF, const ArchitectureSet &ArchSet,
+                             const Architecture ArchToRemove) {
+    if (!ArchSet.has(ArchToRemove))
+      return;
+    if (ArchSet.count() == 1)
+      return;
+    auto OutIF = IF->remove(ArchToRemove);
+    if (!OutIF)
+      ExitOnErr(OutIF.takeError());
+    IF = std::move(*OutIF);
+  };
+
+  if (!Ctx.CmpOpt.ArchsToIgnore.empty()) {
+    const ArchitectureSet LeftArchs = LeftIF->getArchitectures();
+    const ArchitectureSet RightArchs = RightIF->getArchitectures();
+    for (const auto Arch : Ctx.CmpOpt.ArchsToIgnore) {
+      removeArchFromIF(LeftIF, LeftArchs, Arch);
+      removeArchFromIF(RightIF, RightArchs, Arch);
+    }
+  }
+
   raw_ostream &OS = Ctx.OutStream ? *Ctx.OutStream : outs();
   return DiffEngine(LeftIF.get(), RightIF.get()).compareFiles(OS);
 }
@@ -497,12 +524,20 @@ int main(int Argc, char **Argv) {
       reportError("unsupported filetype '" + FT + "'");
   }
 
-  if (opt::Arg *A = Args.getLastArg(OPT_arch_EQ)) {
-    StringRef Arch = A->getValue();
-    Ctx.Arch = getArchitectureFromName(Arch);
-    if (Ctx.Arch == AK_unknown)
-      reportError("unsupported architecture '" + Arch);
-  }
+  auto SanitizeArch = [&](opt::Arg *A) {
+    StringRef ArchStr = A->getValue();
+    auto Arch = getArchitectureFromName(ArchStr);
+    if (Arch == AK_unknown)
+      reportError("unsupported architecture '" + ArchStr);
+    return Arch;
+  };
+
+  if (opt::Arg *A = Args.getLastArg(OPT_arch_EQ))
+    Ctx.Arch = SanitizeArch(A);
+
+  for (opt::Arg *A : Args.filtered(OPT_ignore_arch_EQ))
+    Ctx.CmpOpt.ArchsToIgnore.set(SanitizeArch(A));
+
   // Handle top level and exclusive operation.
   SmallVector<opt::Arg *, 1> ActionArgs(Args.filtered(OPT_action_group));
 
diff --git a/llvm/tools/llvm-reduce/ReducerWorkItem.cpp b/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
index 5409b6dc7459d..5f6a22cb3a974 100644
--- a/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
+++ b/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
@@ -403,6 +403,7 @@ static std::unique_ptr<MachineFunction> cloneMF(MachineFunction *SrcMF,
   DstMF->setHasEHCatchret(SrcMF->hasEHCatchret());
   DstMF->setHasEHScopes(SrcMF->hasEHScopes());
   DstMF->setHasEHFunclets(SrcMF->hasEHFunclets());
+  DstMF->setHasFakeUses(SrcMF->hasFakeUses());
   DstMF->setIsOutlined(SrcMF->isOutlined());
 
   if (!SrcMF->getLandingPads().empty() ||
diff --git a/llvm/unittests/ADT/APFloatTest.cpp b/llvm/unittests/ADT/APFloatTest.cpp
index cd8a00fb18d7f..6008f00e42a98 100644
--- a/llvm/unittests/ADT/APFloatTest.cpp
+++ b/llvm/unittests/ADT/APFloatTest.cpp
@@ -824,6 +824,11 @@ TEST(APFloatTest, IsSmallestNormalized) {
     const fltSemantics &Semantics =
         APFloat::EnumToSemantics(static_cast<APFloat::Semantics>(I));
 
+    // For Float8E8M0FNU format, the below cases are tested
+    // through Float8E8M0FNUSmallest and Float8E8M0FNUNext tests.
+    if (I == APFloat::S_Float8E8M0FNU)
+      continue;
+
     EXPECT_FALSE(APFloat::getZero(Semantics, false).isSmallestNormalized());
     EXPECT_FALSE(APFloat::getZero(Semantics, true).isSmallestNormalized());
 
@@ -1917,6 +1922,57 @@ TEST(DoubleAPFloatTest, isInteger) {
   EXPECT_FALSE(T3.isInteger());
 }
 
+// Test to check if the full range of Float8E8M0FNU
+// values are being represented correctly.
+TEST(APFloatTest, Float8E8M0FNUValues) {
+  // High end of the range
+  auto test = APFloat(APFloat::Float8E8M0FNU(), "0x1.0p127");
+  EXPECT_EQ(0x1.0p127, test.convertToDouble());
+
+  test = APFloat(APFloat::Float8E8M0FNU(), "0x1.0p126");
+  EXPECT_EQ(0x1.0p126, test.convertToDouble());
+
+  test = APFloat(APFloat::Float8E8M0FNU(), "0x1.0p125");
+  EXPECT_EQ(0x1.0p125, test.convertToDouble());
+
+  // tests the fix in makeLargest()
+  test = APFloat::getLargest(APFloat::Float8E8M0FNU());
+  EXPECT_EQ(0x1.0p127, test.convertToDouble());
+
+  // tests overflow to nan
+  APFloat nan = APFloat(APFloat::Float8E8M0FNU(), "nan");
+  test = APFloat(APFloat::Float8E8M0FNU(), "0x1.0p128");
+  EXPECT_TRUE(test.bitwiseIsEqual(nan));
+
+  // Mid of the range
+  test = APFloat(APFloat::Float8E8M0FNU(), "0x1.0p0");
+  EXPECT_EQ(1.0, test.convertToDouble());
+
+  test = APFloat(APFloat::Float8E8M0FNU(), "0x1.0p1");
+  EXPECT_EQ(2.0, test.convertToDouble());
+
+  test = APFloat(APFloat::Float8E8M0FNU(), "0x1.0p2");
+  EXPECT_EQ(4.0, test.convertToDouble());
+
+  // Low end of the range
+  test = APFloat(APFloat::Float8E8M0FNU(), "0x1.0p-125");
+  EXPECT_EQ(0x1.0p-125, test.convertToDouble());
+
+  test = APFloat(APFloat::Float8E8M0FNU(), "0x1.0p-126");
+  EXPECT_EQ(0x1.0p-126, test.convertToDouble());
+
+  test = APFloat(APFloat::Float8E8M0FNU(), "0x1.0p-127");
+  EXPECT_EQ(0x1.0p-127, test.convertToDouble());
+
+  // Smallest value
+  test = APFloat::getSmallest(APFloat::Float8E8M0FNU());
+  EXPECT_EQ(0x1.0p-127, test.convertToDouble());
+
+  // Value below the smallest, but clamped to the smallest
+  test = APFloat(APFloat::Float8E8M0FNU(), "0x1.0p-128");
+  EXPECT_EQ(0x1.0p-127, test.convertToDouble());
+}
+
 TEST(APFloatTest, getLargest) {
   EXPECT_EQ(3.402823466e+38f, APFloat::getLargest(APFloat::IEEEsingle()).convertToFloat());
   EXPECT_EQ(1.7976931348623158e+308, APFloat::getLargest(APFloat::IEEEdouble()).convertToDouble());
@@ -1929,6 +1985,8 @@ TEST(APFloatTest, getLargest) {
       30, APFloat::getLargest(APFloat::Float8E4M3B11FNUZ()).convertToDouble());
   EXPECT_EQ(3.40116213421e+38f,
             APFloat::getLargest(APFloat::FloatTF32()).convertToFloat());
+  EXPECT_EQ(1.701411834e+38f,
+            APFloat::getLargest(APFloat::Float8E8M0FNU()).convertToDouble());
   EXPECT_EQ(28, APFloat::getLargest(APFloat::Float6E3M2FN()).convertToDouble());
   EXPECT_EQ(7.5,
             APFloat::getLargest(APFloat::Float6E2M3FN()).convertToDouble());
@@ -2012,6 +2070,13 @@ TEST(APFloatTest, getSmallest) {
   EXPECT_TRUE(test.isFiniteNonZero());
   EXPECT_TRUE(test.isDenormal());
   EXPECT_TRUE(test.bitwiseIsEqual(expected));
+
+  test = APFloat::getSmallest(APFloat::Float8E8M0FNU());
+  expected = APFloat(APFloat::Float8E8M0FNU(), "0x1.0p-127");
+  EXPECT_FALSE(test.isNegative());
+  EXPECT_TRUE(test.isFiniteNonZero());
+  EXPECT_FALSE(test.isDenormal());
+  EXPECT_TRUE(test.bitwiseIsEqual(expected));
 }
 
 TEST(APFloatTest, getSmallestNormalized) {
@@ -2118,6 +2183,14 @@ TEST(APFloatTest, getSmallestNormalized) {
   EXPECT_FALSE(test.isDenormal());
   EXPECT_TRUE(test.bitwiseIsEqual(expected));
   EXPECT_TRUE(test.isSmallestNormalized());
+
+  test = APFloat::getSmallestNormalized(APFloat::Float8E8M0FNU(), false);
+  expected = APFloat(APFloat::Float8E8M0FNU(), "0x1.0p-127");
+  EXPECT_FALSE(test.isNegative());
+  EXPECT_TRUE(test.isFiniteNonZero());
+  EXPECT_FALSE(test.isDenormal());
+  EXPECT_TRUE(test.bitwiseIsEqual(expected));
+  EXPECT_TRUE(test.isSmallestNormalized());
 }
 
 TEST(APFloatTest, getZero) {
@@ -5326,6 +5399,104 @@ TEST(APFloatTest, Float8ExhaustivePair) {
   }
 }
 
+TEST(APFloatTest, Float8E8M0FNUExhaustivePair) {
+  // Test each pair of 8-bit values for Float8E8M0FNU format
+  APFloat::Semantics Sem = APFloat::S_Float8E8M0FNU;
+  const llvm::fltSemantics &S = APFloat::EnumToSemantics(Sem);
+  for (int i = 0; i < 256; i++) {
+    for (int j = 0; j < 256; j++) {
+      SCOPED_TRACE("sem=" + std::to_string(Sem) + ",i=" + std::to_string(i) +
+                   ",j=" + std::to_string(j));
+      APFloat x(S, APInt(8, i));
+      APFloat y(S, APInt(8, j));
+
+      bool losesInfo;
+      APFloat xd = x;
+      xd.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
+                 &losesInfo);
+      EXPECT_FALSE(losesInfo);
+      APFloat yd = y;
+      yd.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
+                 &losesInfo);
+      EXPECT_FALSE(losesInfo);
+
+      // Add
+      APFloat z = x;
+      z.add(y, APFloat::rmNearestTiesToEven);
+      APFloat zd = xd;
+      zd.add(yd, APFloat::rmNearestTiesToEven);
+      zd.convert(S, APFloat::rmNearestTiesToEven, &losesInfo);
+      EXPECT_TRUE(z.bitwiseIsEqual(zd))
+          << "sem=" << Sem << ", i=" << i << ", j=" << j;
+
+      // Subtract
+      if (i >= j) {
+        z = x;
+        z.subtract(y, APFloat::rmNearestTiesToEven);
+        zd = xd;
+        zd.subtract(yd, APFloat::rmNearestTiesToEven);
+        zd.convert(S, APFloat::rmNearestTiesToEven, &losesInfo);
+        EXPECT_TRUE(z.bitwiseIsEqual(zd))
+            << "sem=" << Sem << ", i=" << i << ", j=" << j;
+      }
+
+      // Multiply
+      z = x;
+      z.multiply(y, APFloat::rmNearestTiesToEven);
+      zd = xd;
+      zd.multiply(yd, APFloat::rmNearestTiesToEven);
+      zd.convert(S, APFloat::rmNearestTiesToEven, &losesInfo);
+      EXPECT_TRUE(z.bitwiseIsEqual(zd))
+          << "sem=" << Sem << ", i=" << i << ", j=" << j;
+
+      // Divide
+      z = x;
+      z.divide(y, APFloat::rmNearestTiesToEven);
+      zd = xd;
+      zd.divide(yd, APFloat::rmNearestTiesToEven);
+      zd.convert(S, APFloat::rmNearestTiesToEven, &losesInfo);
+      EXPECT_TRUE(z.bitwiseIsEqual(zd))
+          << "sem=" << Sem << ", i=" << i << ", j=" << j;
+
+      // Mod
+      z = x;
+      z.mod(y);
+      zd = xd;
+      zd.mod(yd);
+      zd.convert(S, APFloat::rmNearestTiesToEven, &losesInfo);
+      EXPECT_TRUE(z.bitwiseIsEqual(zd))
+          << "sem=" << Sem << ", i=" << i << ", j=" << j;
+      APFloat mod_cached = z;
+      // When one of them is a NaN, the result is a NaN.
+      // When i < j, the mod is 'i' since it is the smaller
+      // number. Otherwise the mod is always zero since
+      // both x and y are powers-of-two in this format.
+      // Since this format does not support zero and it is
+      // represented as the smallest normalized value, we
+      // test for isSmallestNormalized().
+      if (i == 255 || j == 255)
+        EXPECT_TRUE(z.isNaN());
+      else if (i >= j)
+        EXPECT_TRUE(z.isSmallestNormalized());
+      else
+        EXPECT_TRUE(z.bitwiseIsEqual(x));
+
+      // Remainder
+      z = x;
+      z.remainder(y);
+      zd = xd;
+      zd.remainder(yd);
+      zd.convert(S, APFloat::rmNearestTiesToEven, &losesInfo);
+      EXPECT_TRUE(z.bitwiseIsEqual(zd))
+          << "sem=" << Sem << ", i=" << i << ", j=" << j;
+      // Since this format has only exponents (i.e. no precision)
+      // we expect the remainder and mod to provide the same results.
+      EXPECT_TRUE(z.bitwiseIsEqual(mod_cached))
+          << "sem=" << Sem << ", i=" << i << ", j=" << j;
+    }
+  }
+}
+
 TEST(APFloatTest, Float6ExhaustivePair) {
   // Test each pair of 6-bit floats with non-standard semantics
   for (APFloat::Semantics Sem :
@@ -5801,6 +5972,46 @@ TEST(APFloatTest, Float8E4M3FNExhaustive) {
   }
 }
 
+TEST(APFloatTest, Float8E8M0FNUExhaustive) {
+  // Test each of the 256 Float8E8M0FNU values.
+  for (int i = 0; i < 256; i++) {
+    APFloat test(APFloat::Float8E8M0FNU(), APInt(8, i));
+    SCOPED_TRACE("i=" + std::to_string(i));
+
+    // isLargest
+    if (i == 254) {
+      EXPECT_TRUE(test.isLargest());
+      EXPECT_EQ(abs(test).convertToDouble(), 0x1.0p127);
+    } else {
+      EXPECT_FALSE(test.isLargest());
+    }
+
+    // isSmallest
+    if (i == 0) {
+      EXPECT_TRUE(test.isSmallest());
+      EXPECT_EQ(abs(test).convertToDouble(), 0x1.0p-127);
+    } else {
+      EXPECT_FALSE(test.isSmallest());
+    }
+
+    // convert to Double
+    bool losesInfo;
+    std::string val = std::to_string(i - 127); // 127 is the bias
+    llvm::SmallString<16> str("0x1.0p");
+    str += val;
+    APFloat test2(APFloat::IEEEdouble(), str);
+
+    APFloat::opStatus status = test.convert(
+        APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven, &losesInfo);
+    EXPECT_EQ(status, APFloat::opOK);
+    EXPECT_FALSE(losesInfo);
+    if (i == 255)
+      EXPECT_TRUE(test.isNaN());
+    else
+      EXPECT_EQ(test.convertToDouble(), test2.convertToDouble());
+  }
+}
+
 TEST(APFloatTest, Float8E5M2FNUZNext) {
   APFloat test(APFloat::Float8E5M2FNUZ(), APFloat::uninitialized);
   APFloat expected(APFloat::Float8E5M2FNUZ(), APFloat::uninitialized);
@@ -7077,6 +7288,12 @@ TEST(APFloatTest, getExactLog2) {
     auto SemEnum = static_cast<APFloat::Semantics>(I);
     const fltSemantics &Semantics = APFloat::EnumToSemantics(SemEnum);
 
+    // For the Float8E8M0FNU format, the below cases along
+    // with some more corner cases are tested through
+    // Float8E8M0FNUGetExactLog2.
+    if (I == APFloat::S_Float8E8M0FNU)
+      continue;
+
     APFloat One(Semantics, "1.0");
 
     if (I == APFloat::S_PPCDoubleDouble) {
@@ -7146,6 +7363,250 @@ TEST(APFloatTest, getExactLog2) {
   }
 }
 
+TEST(APFloatTest, Float8E8M0FNUGetZero) {
+#ifdef GTEST_HAS_DEATH_TEST
+#ifndef NDEBUG
+  EXPECT_DEATH(APFloat::getZero(APFloat::Float8E8M0FNU(), false),
+               "This floating point format does not support Zero");
+  EXPECT_DEATH(APFloat::getZero(APFloat::Float8E8M0FNU(), true),
+               "This floating point format does not support Zero");
+#endif
+#endif
+}
+
+TEST(APFloatTest, Float8E8M0FNUGetSignedValues) {
+#ifdef GTEST_HAS_DEATH_TEST
+#ifndef NDEBUG
+  EXPECT_DEATH(APFloat(APFloat::Float8E8M0FNU(), "-64"),
+               "This floating point format does not support signed values");
+  EXPECT_DEATH(APFloat(APFloat::Float8E8M0FNU(), "-0x1.0p128"),
+               "This floating point format does not support signed values");
+  EXPECT_DEATH(APFloat(APFloat::Float8E8M0FNU(), "-inf"),
+               "This floating point format does not support signed values");
+  EXPECT_DEATH(APFloat::getNaN(APFloat::Float8E8M0FNU(), true),
+               "This floating point format does not support signed values");
+  EXPECT_DEATH(APFloat::getInf(APFloat::Float8E8M0FNU(), true),
+               "This floating point format does not support signed values");
+  EXPECT_DEATH(APFloat::getSmallest(APFloat::Float8E8M0FNU(), true),
+               "This floating point format does not support signed values");
+  EXPECT_DEATH(APFloat::getSmallestNormalized(APFloat::Float8E8M0FNU(), true),
+               "This floating point format does not support signed values");
+  EXPECT_DEATH(APFloat::getLargest(APFloat::Float8E8M0FNU(), true),
+               "This floating point format does not support signed values");
+  APFloat x = APFloat(APFloat::Float8E8M0FNU(), "4");
+  APFloat y = APFloat(APFloat::Float8E8M0FNU(), "8");
+  EXPECT_DEATH(x.subtract(y, APFloat::rmNearestTiesToEven),
+               "This floating point format does not support signed values");
+#endif
+#endif
+}
+
+TEST(APFloatTest, Float8E8M0FNUGetInf) {
+  // The E8M0 format does not support infinity and the
+  // all ones representation is treated as NaN.
+  APFloat t = APFloat::getInf(APFloat::Float8E8M0FNU());
+  EXPECT_TRUE(t.isNaN());
+  EXPECT_FALSE(t.isInfinity());
+}
+
+TEST(APFloatTest, Float8E8M0FNUFromString) {
+  // Exactly representable
+  EXPECT_EQ(64, APFloat(APFloat::Float8E8M0FNU(), "64").convertToDouble());
+  // Overflow to NaN
+  EXPECT_TRUE(APFloat(APFloat::Float8E8M0FNU(), "0x1.0p128").isNaN());
+  // Inf converted to NaN
+  EXPECT_TRUE(APFloat(APFloat::Float8E8M0FNU(), "inf").isNaN());
+  // NaN converted to NaN
+  EXPECT_TRUE(APFloat(APFloat::Float8E8M0FNU(), "nan").isNaN());
+}
+
+TEST(APFloatTest, Float8E8M0FNUDivideByZero) {
+  APFloat x(APFloat::Float8E8M0FNU(), "1");
+  APFloat zero(APFloat::Float8E8M0FNU(), "0");
+  x.divide(zero, APFloat::rmNearestTiesToEven);
+
+  // Zero is represented as the smallest normalized value
+  // in this format i.e 2^-127.
+  // This tests the fix in convertFromDecimalString() function.
+  EXPECT_EQ(0x1.0p-127, zero.convertToDouble());
+
+  // [1 / (2^-127)] = 2^127
+  EXPECT_EQ(0x1.0p127, x.convertToDouble());
+}
+
+TEST(APFloatTest, Float8E8M0FNUGetExactLog2) {
+  const fltSemantics &Semantics = APFloat::Float8E8M0FNU();
+  APFloat One(Semantics, "1.0");
+  EXPECT_EQ(0, One.getExactLog2());
+
+  // In the Float8E8M0FNU format, 3 is rounded-up to 4.
+  // So, we expect 2 as the result.
+  EXPECT_EQ(2, APFloat(Semantics, "3.0").getExactLog2());
+  EXPECT_EQ(2, APFloat(Semantics, "3.0").getExactLog2Abs());
+
+  // In the Float8E8M0FNU format, 5 is rounded-down to 4.
+  // So, we expect 2 as the result.
+  EXPECT_EQ(2, APFloat(Semantics, "5.0").getExactLog2());
+  EXPECT_EQ(2, APFloat(Semantics, "5.0").getExactLog2Abs());
+
+  // Exact power-of-two value.
+  EXPECT_EQ(3, APFloat(Semantics, "8.0").getExactLog2());
+  EXPECT_EQ(3, APFloat(Semantics, "8.0").getExactLog2Abs());
+
+  // Negative exponent value.
+  EXPECT_EQ(-2, APFloat(Semantics, "0.25").getExactLog2());
+  EXPECT_EQ(-2, APFloat(Semantics, "0.25").getExactLog2Abs());
+
+  int MinExp = APFloat::semanticsMinExponent(Semantics);
+  int MaxExp = APFloat::semanticsMaxExponent(Semantics);
+  int Precision = APFloat::semanticsPrecision(Semantics);
+
+  // Values below the minExp getting capped to minExp.
+  EXPECT_EQ(-127,
+            scalbn(One, MinExp - Precision - 1, APFloat::rmNearestTiesToEven)
+                .getExactLog2());
+  EXPECT_EQ(-127, scalbn(One, MinExp - Precision, APFloat::rmNearestTiesToEven)
+                      .getExactLog2());
+
+  // Values above the maxExp overflow to NaN, and getExactLog2() returns
+  // INT_MIN for these cases.
+  EXPECT_EQ(
+      INT_MIN,
+      scalbn(One, MaxExp + 1, APFloat::rmNearestTiesToEven).getExactLog2());
+
+  // This format can represent [minExp, maxExp].
+  // So, the result is the same as the 'Exp' of the scalbn.
+  for (int i = MinExp - Precision + 1; i <= MaxExp; ++i) {
+    EXPECT_EQ(i, scalbn(One, i, APFloat::rmNearestTiesToEven).getExactLog2());
+  }
+}
+
+TEST(APFloatTest, Float8E8M0FNUSmallest) {
+  APFloat test(APFloat::getSmallest(APFloat::Float8E8M0FNU()));
+  EXPECT_EQ(0x1.0p-127, test.convertToDouble());
+
+  // For E8M0 format, there are no denorms.
+  // So, getSmallest is equal to isSmallestNormalized().
+  EXPECT_TRUE(test.isSmallestNormalized());
+  EXPECT_EQ(fcPosNormal, test.classify());
+
+  test = APFloat::getAllOnesValue(APFloat::Float8E8M0FNU());
+  EXPECT_FALSE(test.isSmallestNormalized());
+  EXPECT_TRUE(test.isNaN());
+}
+
+TEST(APFloatTest, Float8E8M0FNUNext) {
+  APFloat test(APFloat::getSmallest(APFloat::Float8E8M0FNU()));
+  // Increment of 1 should reach 2^-126
+  EXPECT_EQ(APFloat::opOK, test.next(false));
+  EXPECT_FALSE(test.isSmallestNormalized());
+  EXPECT_EQ(0x1.0p-126, test.convertToDouble());
+
+  // Decrement of 1, again, should reach 2^-127
+  // i.e. smallest normalized
+  EXPECT_EQ(APFloat::opOK, test.next(true));
+  EXPECT_TRUE(test.isSmallestNormalized());
+
+  // Decrement again, but gets capped at the smallest normalized
+  EXPECT_EQ(APFloat::opOK, test.next(true));
+  EXPECT_TRUE(test.isSmallestNormalized());
+}
+
+TEST(APFloatTest, Float8E8M0FNUFMA) {
+  APFloat f1(APFloat::Float8E8M0FNU(), "4.0");
+  APFloat f2(APFloat::Float8E8M0FNU(), "2.0");
+  APFloat f3(APFloat::Float8E8M0FNU(), "8.0");
+
+  // Exact value: 4*2 + 8 = 16.
+  f1.fusedMultiplyAdd(f2, f3, APFloat::rmNearestTiesToEven);
+  EXPECT_EQ(16.0, f1.convertToDouble());
+
+  // 4*2 + 4 = 12 but it gets rounded-up to 16.
+  f1 = APFloat(APFloat::Float8E8M0FNU(), "4.0");
+  f1.fusedMultiplyAdd(f2, f1, APFloat::rmNearestTiesToEven);
+  EXPECT_EQ(16.0, f1.convertToDouble());
+
+  // 4*2 + 2 = 10 but it gets rounded-down to 8.
+  f1 = APFloat(APFloat::Float8E8M0FNU(), "4.0");
+  f1.fusedMultiplyAdd(f2, f2, APFloat::rmNearestTiesToEven);
+  EXPECT_EQ(8.0, f1.convertToDouble());
+
+  // All of them using the same value.
+  f1 = APFloat(APFloat::Float8E8M0FNU(), "1.0");
+  f1.fusedMultiplyAdd(f1, f1, APFloat::rmNearestTiesToEven);
+  EXPECT_EQ(2.0, f1.convertToDouble());
+}
+
+TEST(APFloatTest, ConvertDoubleToE8M0FNU) {
+  bool losesInfo;
+  APFloat test(APFloat::IEEEdouble(), "1.0");
+  APFloat::opStatus status = test.convert(
+      APFloat::Float8E8M0FNU(), APFloat::rmNearestTiesToEven, &losesInfo);
+  EXPECT_EQ(1.0, test.convertToDouble());
+  EXPECT_FALSE(losesInfo);
+  EXPECT_EQ(status, APFloat::opOK);
+
+  // For E8M0, zero encoding is represented as the smallest normalized value.
+  test = APFloat(APFloat::IEEEdouble(), "0.0");
+  status = test.convert(APFloat::Float8E8M0FNU(), APFloat::rmNearestTiesToEven,
+                        &losesInfo);
+  EXPECT_TRUE(test.isSmallestNormalized());
+  EXPECT_EQ(0x1.0p-127, test.convertToDouble());
+  EXPECT_FALSE(losesInfo);
+  EXPECT_EQ(status, APFloat::opOK);
+
+  // Test that the conversion of a power-of-two value is precise.
+  test = APFloat(APFloat::IEEEdouble(), "8.0");
+  status = test.convert(APFloat::Float8E8M0FNU(), APFloat::rmNearestTiesToEven,
+                        &losesInfo);
+  EXPECT_EQ(8.0f, test.convertToDouble());
+  EXPECT_FALSE(losesInfo);
+  EXPECT_EQ(status, APFloat::opOK);
+
+  // Test to check round-down conversion to power-of-two.
+  // The fractional part of 9 is "001" (i.e. 1.125x2^3=9).
+  test = APFloat(APFloat::IEEEdouble(), "9.0");
+  status = test.convert(APFloat::Float8E8M0FNU(), APFloat::rmNearestTiesToEven,
+                        &losesInfo);
+  EXPECT_EQ(8.0f, test.convertToDouble());
+  EXPECT_TRUE(losesInfo);
+  EXPECT_EQ(status, APFloat::opInexact);
+
+  // Test to check round-up conversion to power-of-two.
+  // The fractional part of 13 is "101" (i.e. 1.625x2^3=13).
+  test = APFloat(APFloat::IEEEdouble(), "13.0");
+  status = test.convert(APFloat::Float8E8M0FNU(), APFloat::rmNearestTiesToEven,
+                        &losesInfo);
+  EXPECT_EQ(16.0f, test.convertToDouble());
+  EXPECT_TRUE(losesInfo);
+  EXPECT_EQ(status, APFloat::opInexact);
+
+  // Test to check round-up conversion to power-of-two.
+  // The fractional part of 12 is "100" (i.e. 1.5x2^3=12).
+  test = APFloat(APFloat::IEEEdouble(), "12.0");
+  status = test.convert(APFloat::Float8E8M0FNU(), APFloat::rmNearestTiesToEven,
+                        &losesInfo);
+  EXPECT_EQ(16.0f, test.convertToDouble());
+  EXPECT_TRUE(losesInfo);
+  EXPECT_EQ(status, APFloat::opInexact);
+
+  // Overflow to NaN.
+  test = APFloat(APFloat::IEEEdouble(), "0x1.0p128");
+  status = test.convert(APFloat::Float8E8M0FNU(), APFloat::rmNearestTiesToEven,
+                        &losesInfo);
+  EXPECT_TRUE(test.isNaN());
+  EXPECT_TRUE(losesInfo);
+  EXPECT_EQ(status, APFloat::opOverflow | APFloat::opInexact);
+
+  // Underflow to smallest normalized value.
+  test = APFloat(APFloat::IEEEdouble(), "0x1.0p-128");
+  status = test.convert(APFloat::Float8E8M0FNU(), APFloat::rmNearestTiesToEven,
+                        &losesInfo);
+  EXPECT_TRUE(test.isSmallestNormalized());
+  EXPECT_TRUE(losesInfo);
+  EXPECT_EQ(status, APFloat::opUnderflow | APFloat::opInexact);
+}
+
 TEST(APFloatTest, Float6E3M2FNFromString) {
   // Exactly representable
   EXPECT_EQ(28, APFloat(APFloat::Float6E3M2FN(), "28").convertToDouble());
diff --git a/llvm/unittests/IR/AttributesTest.cpp b/llvm/unittests/IR/AttributesTest.cpp
index b713cd38d2f51..f73f2b20e9fea 100644
--- a/llvm/unittests/IR/AttributesTest.cpp
+++ b/llvm/unittests/IR/AttributesTest.cpp
@@ -447,8 +447,8 @@ TEST(Attributes, SetIntersect) {
 
     ConstantRange CR0(APInt(32, 0), APInt(32, 10));
     ConstantRange CR1(APInt(32, 15), APInt(32, 20));
-    ArrayRef<ConstantRange> CRL0 = {CR0};
-    ArrayRef<ConstantRange> CRL1 = {CR0, CR1};
+    ConstantRange CRL0[] = {CR0};
+    ConstantRange CRL1[] = {CR0, CR1};
     Type *T0 = Type::getInt32Ty(C0);
     Type *T1 = Type::getInt64Ty(C0);
     Attribute Attr0 = BuildAttr(C0, Kind, V0, T0, CR0, CRL0);
@@ -596,6 +596,38 @@ TEST(Attributes, SetIntersectByValAlign) {
   }
 }
 
+TEST(Attributes, ListIntersectDifferingMustPreserve) {
+  LLVMContext C;
+  std::optional<AttributeList> Res;
+  {
+    AttributeList AL0;
+    AttributeList AL1;
+    AL1 = AL1.addFnAttribute(C, Attribute::ReadOnly);
+    AL0 = AL0.addParamAttribute(C, 0, Attribute::SExt);
+    Res = AL0.intersectWith(C, AL1);
+    ASSERT_FALSE(Res.has_value());
+    Res = AL1.intersectWith(C, AL0);
+    ASSERT_FALSE(Res.has_value());
+  }
+  {
+    AttributeList AL0;
+    AttributeList AL1;
+    AL1 = AL1.addFnAttribute(C, Attribute::AlwaysInline);
+    AL0 = AL0.addParamAttribute(C, 0, Attribute::ReadOnly);
+    Res = AL0.intersectWith(C, AL1);
+    ASSERT_FALSE(Res.has_value());
+    Res = AL1.intersectWith(C, AL0);
+    ASSERT_FALSE(Res.has_value());
+
+    AL0 = AL0.addFnAttribute(C, Attribute::AlwaysInline);
+    AL1 = AL1.addParamAttribute(C, 1, Attribute::SExt);
+    Res = AL0.intersectWith(C, AL1);
+    ASSERT_FALSE(Res.has_value());
+    Res = AL1.intersectWith(C, AL0);
+    ASSERT_FALSE(Res.has_value());
+  }
+}
+
 TEST(Attributes, ListIntersect) {
   LLVMContext C;
   AttributeList AL0;
@@ -610,11 +642,16 @@ TEST(Attributes, ListIntersect) {
 
   AL0 = AL0.addParamAttribute(C, 1, Attribute::NoUndef);
   Res = AL0.intersectWith(C, AL1);
-  ASSERT_FALSE(Res.has_value());
+  ASSERT_TRUE(Res.has_value());
+  ASSERT_TRUE(Res->hasRetAttr(Attribute::NoUndef));
+  ASSERT_FALSE(Res->hasParamAttr(1, Attribute::NoUndef));
 
   AL1 = AL1.addParamAttribute(C, 2, Attribute::NoUndef);
   Res = AL0.intersectWith(C, AL1);
-  ASSERT_FALSE(Res.has_value());
+  ASSERT_TRUE(Res.has_value());
+  ASSERT_TRUE(Res->hasRetAttr(Attribute::NoUndef));
+  ASSERT_FALSE(Res->hasParamAttr(1, Attribute::NoUndef));
+  ASSERT_FALSE(Res->hasParamAttr(2, Attribute::NoUndef));
 
   AL0 = AL0.addParamAttribute(C, 2, Attribute::NoUndef);
   AL1 = AL1.addParamAttribute(C, 1, Attribute::NoUndef);
@@ -692,7 +729,17 @@ TEST(Attributes, ListIntersect) {
 
   AL1 = AL1.addParamAttribute(C, 3, Attribute::ReadNone);
   Res = AL0.intersectWith(C, AL1);
-  ASSERT_FALSE(Res.has_value());
+  ASSERT_TRUE(Res.has_value());
+  ASSERT_TRUE(Res.has_value());
+  ASSERT_TRUE(Res->hasFnAttr(Attribute::AlwaysInline));
+  ASSERT_TRUE(Res->hasFnAttr(Attribute::ReadOnly));
+  ASSERT_TRUE(Res->hasRetAttr(Attribute::NoUndef));
+  ASSERT_FALSE(Res->hasRetAttr(Attribute::NonNull));
+  ASSERT_TRUE(Res->hasParamAttr(1, Attribute::NoUndef));
+  ASSERT_TRUE(Res->hasParamAttr(2, Attribute::NoUndef));
+  ASSERT_FALSE(Res->hasParamAttr(2, Attribute::NonNull));
+  ASSERT_FALSE(Res->hasParamAttr(2, Attribute::ReadNone));
+  ASSERT_FALSE(Res->hasParamAttr(3, Attribute::ReadNone));
 
   AL0 = AL0.addParamAttribute(C, 3, Attribute::ReadNone);
   Res = AL0.intersectWith(C, AL1);
diff --git a/llvm/unittests/IR/ConstantFPRangeTest.cpp b/llvm/unittests/IR/ConstantFPRangeTest.cpp
index 722e6566730da..158d08f9b77a0 100644
--- a/llvm/unittests/IR/ConstantFPRangeTest.cpp
+++ b/llvm/unittests/IR/ConstantFPRangeTest.cpp
@@ -150,17 +150,84 @@ static void EnumerateTwoInterestingConstantFPRanges(Fn TestFn,
 
 template <typename Fn>
 static void EnumerateValuesInConstantFPRange(const ConstantFPRange &CR,
-                                             Fn TestFn) {
+                                             Fn TestFn, bool IgnoreNaNPayload) {
   const fltSemantics &Sem = CR.getSemantics();
-  unsigned Bits = APFloat::semanticsSizeInBits(Sem);
-  assert(Bits < 32 && "Too many bits");
-  for (unsigned I = 0, E = (1U << Bits) - 1; I != E; ++I) {
-    APFloat V(Sem, APInt(Bits, I));
-    if (CR.contains(V))
-      TestFn(V);
+  if (IgnoreNaNPayload) {
+    if (CR.containsSNaN()) {
+      TestFn(APFloat::getSNaN(Sem, false));
+      TestFn(APFloat::getSNaN(Sem, true));
+    }
+    if (CR.containsQNaN()) {
+      TestFn(APFloat::getQNaN(Sem, false));
+      TestFn(APFloat::getQNaN(Sem, true));
+    }
+    if (CR.isNaNOnly())
+      return;
+    APFloat Lower = CR.getLower();
+    const APFloat &Upper = CR.getUpper();
+    auto Next = [&](APFloat &V) {
+      if (V.bitwiseIsEqual(Upper))
+        return false;
+      strictNext(V);
+      return true;
+    };
+    do
+      TestFn(Lower);
+    while (Next(Lower));
+  } else {
+    unsigned Bits = APFloat::semanticsSizeInBits(Sem);
+    assert(Bits < 32 && "Too many bits");
+    for (unsigned I = 0, E = (1U << Bits) - 1; I != E; ++I) {
+      APFloat V(Sem, APInt(Bits, I));
+      if (CR.contains(V))
+        TestFn(V);
+    }
   }
 }
 
+template <typename Fn>
+static bool AnyOfValueInConstantFPRange(const ConstantFPRange &CR, Fn TestFn,
+                                        bool IgnoreNaNPayload) {
+  const fltSemantics &Sem = CR.getSemantics();
+  if (IgnoreNaNPayload) {
+    if (CR.containsSNaN()) {
+      if (TestFn(APFloat::getSNaN(Sem, false)))
+        return true;
+      if (TestFn(APFloat::getSNaN(Sem, true)))
+        return true;
+    }
+    if (CR.containsQNaN()) {
+      if (TestFn(APFloat::getQNaN(Sem, false)))
+        return true;
+      if (TestFn(APFloat::getQNaN(Sem, true)))
+        return true;
+    }
+    if (CR.isNaNOnly())
+      return false;
+    APFloat Lower = CR.getLower();
+    const APFloat &Upper = CR.getUpper();
+    auto Next = [&](APFloat &V) {
+      if (V.bitwiseIsEqual(Upper))
+        return false;
+      strictNext(V);
+      return true;
+    };
+    do {
+      if (TestFn(Lower))
+        return true;
+    } while (Next(Lower));
+  } else {
+    unsigned Bits = APFloat::semanticsSizeInBits(Sem);
+    assert(Bits < 32 && "Too many bits");
+    for (unsigned I = 0, E = (1U << Bits) - 1; I != E; ++I) {
+      APFloat V(Sem, APInt(Bits, I));
+      if (CR.contains(V) && TestFn(V))
+        return true;
+    }
+  }
+  return false;
+}
+
 TEST_F(ConstantFPRangeTest, Basics) {
   EXPECT_TRUE(Full.isFullSet());
   EXPECT_FALSE(Full.isEmptySet());
@@ -263,12 +330,16 @@ TEST_F(ConstantFPRangeTest, SingleElement) {
   EXPECT_EQ(*One.getSingleElement(), APFloat(1.0));
   EXPECT_EQ(*PosZero.getSingleElement(), APFloat::getZero(Sem));
   EXPECT_EQ(*PosInf.getSingleElement(), APFloat::getInf(Sem));
+  ConstantFPRange PosZeroOrNaN = PosZero.unionWith(NaN);
+  EXPECT_EQ(*PosZeroOrNaN.getSingleElement(/*ExcludesNaN=*/true),
+            APFloat::getZero(Sem));
 
   EXPECT_FALSE(Full.isSingleElement());
   EXPECT_FALSE(Empty.isSingleElement());
   EXPECT_TRUE(One.isSingleElement());
   EXPECT_FALSE(Some.isSingleElement());
   EXPECT_FALSE(Zero.isSingleElement());
+  EXPECT_TRUE(PosZeroOrNaN.isSingleElement(/*ExcludesNaN=*/true));
 }
 
 TEST_F(ConstantFPRangeTest, ExhaustivelyEnumerate) {
@@ -368,13 +439,16 @@ TEST_F(ConstantFPRangeTest, FPClassify) {
       [](const ConstantFPRange &CR) {
         unsigned Mask = fcNone;
         bool HasPos = false, HasNeg = false;
-        EnumerateValuesInConstantFPRange(CR, [&](const APFloat &V) {
-          Mask |= V.classify();
-          if (V.isNegative())
-            HasNeg = true;
-          else
-            HasPos = true;
-        });
+        EnumerateValuesInConstantFPRange(
+            CR,
+            [&](const APFloat &V) {
+              Mask |= V.classify();
+              if (V.isNegative())
+                HasNeg = true;
+              else
+                HasPos = true;
+            },
+            /*IgnoreNaNPayload=*/true);
 
         std::optional<bool> SignBit = std::nullopt;
         if (HasPos != HasNeg)
@@ -425,4 +499,36 @@ TEST_F(ConstantFPRangeTest, MismatchedSemantics) {
 #endif
 #endif
 
+TEST_F(ConstantFPRangeTest, makeAllowedFCmpRegion) {
+  for (auto Pred : FCmpInst::predicates()) {
+    EnumerateConstantFPRanges(
+        [Pred](const ConstantFPRange &CR) {
+          ConstantFPRange Res =
+              ConstantFPRange::makeAllowedFCmpRegion(Pred, CR);
+          ConstantFPRange Optimal =
+              ConstantFPRange::getEmpty(CR.getSemantics());
+          EnumerateValuesInConstantFPRange(
+              ConstantFPRange::getFull(CR.getSemantics()),
+              [&](const APFloat &V) {
+                if (AnyOfValueInConstantFPRange(
+                        CR,
+                        [&](const APFloat &U) {
+                          return FCmpInst::compare(V, U, Pred);
+                        },
+                        /*IgnoreNaNPayload=*/true))
+                  Optimal = Optimal.unionWith(ConstantFPRange(V));
+              },
+              /*IgnoreNaNPayload=*/true);
+
+          EXPECT_TRUE(Res.contains(Optimal))
+              << "Wrong result for makeAllowedFCmpRegion(" << Pred << ", " << CR
+              << "). Expected " << Optimal << ", but got " << Res;
+          EXPECT_EQ(Res, Optimal)
+              << "Suboptimal result for makeAllowedFCmpRegion(" << Pred << ", "
+              << CR << ")";
+        },
+        /*Exhaustive=*/false);
+  }
+}
+
 } // anonymous namespace
diff --git a/llvm/unittests/IR/InstructionsTest.cpp b/llvm/unittests/IR/InstructionsTest.cpp
index 481fe96607e48..0af812564c026 100644
--- a/llvm/unittests/IR/InstructionsTest.cpp
+++ b/llvm/unittests/IR/InstructionsTest.cpp
@@ -1559,12 +1559,61 @@ TEST(InstructionsTest, FPCallIsFPMathOperator) {
       CallInst::Create(AVFFnTy, AVFCallee, {}, ""));
   EXPECT_TRUE(isa<FPMathOperator>(AVFCall));
 
-  Type *AAVFTy = ArrayType::get(AVFTy, 2);
-  FunctionType *AAVFFnTy = FunctionType::get(AAVFTy, {});
-  Value *AAVFCallee = Constant::getNullValue(PtrTy);
-  std::unique_ptr<CallInst> AAVFCall(
-      CallInst::Create(AAVFFnTy, AAVFCallee, {}, ""));
-  EXPECT_TRUE(isa<FPMathOperator>(AAVFCall));
+  Type *StructITy = StructType::get(ITy, ITy);
+  FunctionType *StructIFnTy = FunctionType::get(StructITy, {});
+  Value *StructICallee = Constant::getNullValue(PtrTy);
+  std::unique_ptr<CallInst> StructICall(
+      CallInst::Create(StructIFnTy, StructICallee, {}, ""));
+  EXPECT_FALSE(isa<FPMathOperator>(StructICall));
+
+  Type *EmptyStructTy = StructType::get(C);
+  FunctionType *EmptyStructFnTy = FunctionType::get(EmptyStructTy, {});
+  Value *EmptyStructCallee = Constant::getNullValue(PtrTy);
+  std::unique_ptr<CallInst> EmptyStructCall(
+      CallInst::Create(EmptyStructFnTy, EmptyStructCallee, {}, ""));
+  EXPECT_FALSE(isa<FPMathOperator>(EmptyStructCall));
+
+  Type *NamedStructFTy = StructType::create({FTy, FTy}, "AStruct");
+  FunctionType *NamedStructFFnTy = FunctionType::get(NamedStructFTy, {});
+  Value *NamedStructFCallee = Constant::getNullValue(PtrTy);
+  std::unique_ptr<CallInst> NamedStructFCall(
+      CallInst::Create(NamedStructFFnTy, NamedStructFCallee, {}, ""));
+  EXPECT_FALSE(isa<FPMathOperator>(NamedStructFCall));
+
+  Type *MixedStructTy = StructType::get(FTy, ITy);
+  FunctionType *MixedStructFnTy = FunctionType::get(MixedStructTy, {});
+  Value *MixedStructCallee = Constant::getNullValue(PtrTy);
+  std::unique_ptr<CallInst> MixedStructCall(
+      CallInst::Create(MixedStructFnTy, MixedStructCallee, {}, ""));
+  EXPECT_FALSE(isa<FPMathOperator>(MixedStructCall));
+
+  Type *StructFTy = StructType::get(FTy, FTy, FTy);
+  FunctionType *StructFFnTy = FunctionType::get(StructFTy, {});
+  Value *StructFCallee = Constant::getNullValue(PtrTy);
+  std::unique_ptr<CallInst> StructFCall(
+      CallInst::Create(StructFFnTy, StructFCallee, {}, ""));
+  EXPECT_TRUE(isa<FPMathOperator>(StructFCall));
+
+  Type *StructVFTy = StructType::get(VFTy, VFTy, VFTy, VFTy);
+  FunctionType *StructVFFnTy = FunctionType::get(StructVFTy, {});
+  Value *StructVFCallee = Constant::getNullValue(PtrTy);
+  std::unique_ptr<CallInst> StructVFCall(
+      CallInst::Create(StructVFFnTy, StructVFCallee, {}, ""));
+  EXPECT_TRUE(isa<FPMathOperator>(StructVFCall));
+
+  Type *NestedStructFTy = StructType::get(StructFTy, StructFTy, StructFTy);
+  FunctionType *NestedStructFFnTy = FunctionType::get(NestedStructFTy, {});
+  Value *NestedStructFCallee = Constant::getNullValue(PtrTy);
+  std::unique_ptr<CallInst> NestedStructFCall(
+      CallInst::Create(NestedStructFFnTy, NestedStructFCallee, {}, ""));
+  EXPECT_FALSE(isa<FPMathOperator>(NestedStructFCall));
+
+  Type *AStructFTy = ArrayType::get(StructFTy, 5);
+  FunctionType *AStructFFnTy = FunctionType::get(AStructFTy, {});
+  Value *AStructFCallee = Constant::getNullValue(PtrTy);
+  std::unique_ptr<CallInst> AStructFCall(
+      CallInst::Create(AStructFFnTy, AStructFCallee, {}, ""));
+  EXPECT_FALSE(isa<FPMathOperator>(AStructFCall));
 }
 
 TEST(InstructionsTest, FNegInstruction) {
diff --git a/llvm/unittests/SandboxIR/CMakeLists.txt b/llvm/unittests/SandboxIR/CMakeLists.txt
index 622496ada567f..1e83bda7a1f6d 100644
--- a/llvm/unittests/SandboxIR/CMakeLists.txt
+++ b/llvm/unittests/SandboxIR/CMakeLists.txt
@@ -6,6 +6,7 @@ set(LLVM_LINK_COMPONENTS
   )
 
 add_llvm_unittest(SandboxIRTests
+  IntrinsicInstTest.cpp
   PassTest.cpp
   RegionTest.cpp
   SandboxIRTest.cpp
diff --git a/llvm/unittests/SandboxIR/IntrinsicInstTest.cpp b/llvm/unittests/SandboxIR/IntrinsicInstTest.cpp
new file mode 100644
index 0000000000000..2272cfc114616
--- /dev/null
+++ b/llvm/unittests/SandboxIR/IntrinsicInstTest.cpp
@@ -0,0 +1,88 @@
+//===- IntrinsicInstTest.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/SandboxIR/IntrinsicInst.h"
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Value.h"
+#include "llvm/SandboxIR/Context.h"
+#include "llvm/SandboxIR/Function.h"
+#include "llvm/Support/SourceMgr.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+struct IntrinsicInstTest : public testing::Test {
+  LLVMContext C;
+  std::unique_ptr<Module> M;
+
+  void parseIR(LLVMContext &C, const char *IR) {
+    SMDiagnostic Err;
+    M = parseAssemblyString(IR, Err, C);
+    if (!M)
+      Err.print("SandboxIRTest", errs());
+  }
+  BasicBlock *getBasicBlockByName(Function &F, StringRef Name) {
+    for (BasicBlock &BB : F)
+      if (BB.getName() == Name)
+        return &BB;
+    llvm_unreachable("Expected to find basic block!");
+  }
+};
+
+TEST_F(IntrinsicInstTest, Basic) {
+  parseIR(C, R"IR(
+declare void @llvm.sideeffect()
+declare void @llvm.assume(i1)
+declare i8 @llvm.uadd.sat.i8(i8, i8)
+declare i8 @llvm.smax.i8(i8, i8)
+
+define void @foo(i8 %v1, i1 %cond) {
+  call void @llvm.sideeffect()
+  call void @llvm.assume(i1 %cond)
+  call i8 @llvm.uadd.sat.i8(i8 %v1, i8 %v1)
+  call i8 @llvm.smax.i8(i8 %v1, i8 %v1)
+  ret void
+}
+)IR");
+
+  llvm::Function *LLVMF = &*M->getFunction("foo");
+  auto *LLVMBB = &*LLVMF->begin();
+  auto LLVMIt = LLVMBB->begin();
+
+  sandboxir::Context Ctx(C);
+  sandboxir::Function *F = Ctx.createFunction(LLVMF);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  auto ItE = BB->getTerminator()->getIterator();
+  for (; It != ItE; ++It, ++LLVMIt) {
+    auto *I = &*It;
+    auto *LLVMI = &*LLVMIt;
+    // Check classof().
+    EXPECT_TRUE(isa<sandboxir::IntrinsicInst>(I));
+    // Check getIntrinsicID().
+    EXPECT_EQ(cast<sandboxir::IntrinsicInst>(I)->getIntrinsicID(),
+              cast<llvm::IntrinsicInst>(LLVMI)->getIntrinsicID());
+    // Check isAssociative().
+    EXPECT_EQ(cast<sandboxir::IntrinsicInst>(I)->isAssociative(),
+              cast<llvm::IntrinsicInst>(LLVMI)->isAssociative());
+    // Check isCommutative().
+    EXPECT_EQ(cast<sandboxir::IntrinsicInst>(I)->isCommutative(),
+              cast<llvm::IntrinsicInst>(LLVMI)->isCommutative());
+    // Check isAssumeLikeIntrinsic().
+    EXPECT_EQ(cast<sandboxir::IntrinsicInst>(I)->isAssumeLikeIntrinsic(),
+              cast<llvm::IntrinsicInst>(LLVMI)->isAssumeLikeIntrinsic());
+    // Check mayLowerToFunctionCall().
+    auto ID = cast<sandboxir::IntrinsicInst>(I)->getIntrinsicID();
+    EXPECT_EQ(sandboxir::IntrinsicInst::mayLowerToFunctionCall(ID),
+              llvm::IntrinsicInst::mayLowerToFunctionCall(ID));
+  }
+}
diff --git a/llvm/unittests/SandboxIR/PassTest.cpp b/llvm/unittests/SandboxIR/PassTest.cpp
index 10fe59b654a2e..b380ae9fd475a 100644
--- a/llvm/unittests/SandboxIR/PassTest.cpp
+++ b/llvm/unittests/SandboxIR/PassTest.cpp
@@ -13,6 +13,7 @@
 #include "llvm/SandboxIR/Context.h"
 #include "llvm/SandboxIR/Function.h"
 #include "llvm/SandboxIR/PassManager.h"
+#include "llvm/SandboxIR/Region.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gtest/gtest.h"
 
@@ -86,6 +87,68 @@ define void @foo() {
 #endif
 }
 
+TEST_F(PassTest, RegionPass) {
+  auto *F = parseFunction(R"IR(
+define i8 @foo(i8 %v0, i8 %v1) {
+  %t0 = add i8 %v0, 1
+  %t1 = add i8 %t0, %v1, !sandboxvec !0
+  %t2 = add i8 %t1, %v1, !sandboxvec !0
+  ret i8 %t1
+}
+
+!0 = distinct !{!"sandboxregion"}
+)IR",
+                          "foo");
+
+  class TestPass final : public RegionPass {
+    unsigned &InstCount;
+
+  public:
+    TestPass(unsigned &InstCount)
+        : RegionPass("test-pass"), InstCount(InstCount) {}
+    bool runOnRegion(Region &R) final {
+      for ([[maybe_unused]] auto &Inst : R) {
+        ++InstCount;
+      }
+      return false;
+    }
+  };
+  unsigned InstCount = 0;
+  TestPass TPass(InstCount);
+  // Check getName(),
+  EXPECT_EQ(TPass.getName(), "test-pass");
+  // Check runOnRegion();
+  llvm::SmallVector<std::unique_ptr<Region>> Regions =
+      Region::createRegionsFromMD(*F);
+  ASSERT_EQ(Regions.size(), 1u);
+  TPass.runOnRegion(*Regions[0]);
+  EXPECT_EQ(InstCount, 2u);
+#ifndef NDEBUG
+  {
+    // Check print().
+    std::string Buff;
+    llvm::raw_string_ostream SS(Buff);
+    TPass.print(SS);
+    EXPECT_EQ(Buff, "test-pass");
+  }
+  {
+    // Check operator<<().
+    std::string Buff;
+    llvm::raw_string_ostream SS(Buff);
+    SS << TPass;
+    EXPECT_EQ(Buff, "test-pass");
+  }
+  // Check pass name assertions.
+  class TestNamePass final : public RegionPass {
+  public:
+    TestNamePass(llvm::StringRef Name) : RegionPass(Name) {}
+    bool runOnRegion(Region &F) { return false; }
+  };
+  EXPECT_DEATH(TestNamePass("white space"), ".*whitespace.*");
+  EXPECT_DEATH(TestNamePass("-dash"), ".*start with.*");
+#endif
+}
+
 TEST_F(PassTest, FunctionPassManager) {
   auto *F = parseFunction(R"IR(
 define void @foo() {
@@ -136,6 +199,67 @@ define void @foo() {
 #endif // NDEBUG
 }
 
+TEST_F(PassTest, RegionPassManager) {
+  auto *F = parseFunction(R"IR(
+define i8 @foo(i8 %v0, i8 %v1) {
+  %t0 = add i8 %v0, 1
+  %t1 = add i8 %t0, %v1, !sandboxvec !0
+  %t2 = add i8 %t1, %v1, !sandboxvec !0
+  ret i8 %t1
+}
+
+!0 = distinct !{!"sandboxregion"}
+)IR",
+                          "foo");
+
+  class TestPass1 final : public RegionPass {
+    unsigned &InstCount;
+
+  public:
+    TestPass1(unsigned &InstCount)
+        : RegionPass("test-pass1"), InstCount(InstCount) {}
+    bool runOnRegion(Region &R) final {
+      for ([[maybe_unused]] auto &Inst : R)
+        ++InstCount;
+      return false;
+    }
+  };
+  class TestPass2 final : public RegionPass {
+    unsigned &InstCount;
+
+  public:
+    TestPass2(unsigned &InstCount)
+        : RegionPass("test-pass2"), InstCount(InstCount) {}
+    bool runOnRegion(Region &R) final {
+      for ([[maybe_unused]] auto &Inst : R)
+        ++InstCount;
+      return false;
+    }
+  };
+  unsigned InstCount1 = 0;
+  unsigned InstCount2 = 0;
+  TestPass1 TPass1(InstCount1);
+  TestPass2 TPass2(InstCount2);
+
+  RegionPassManager RPM("test-rpm");
+  RPM.addPass(&TPass1);
+  RPM.addPass(&TPass2);
+  // Check runOnRegion().
+  llvm::SmallVector<std::unique_ptr<Region>> Regions =
+      Region::createRegionsFromMD(*F);
+  ASSERT_EQ(Regions.size(), 1u);
+  RPM.runOnRegion(*Regions[0]);
+  EXPECT_EQ(InstCount1, 2u);
+  EXPECT_EQ(InstCount2, 2u);
+#ifndef NDEBUG
+  // Check dump().
+  std::string Buff;
+  llvm::raw_string_ostream SS(Buff);
+  RPM.print(SS);
+  EXPECT_EQ(Buff, "test-rpm(test-pass1,test-pass2)");
+#endif // NDEBUG
+}
+
 TEST_F(PassTest, PassRegistry) {
   class TestPass1 final : public FunctionPass {
   public:
diff --git a/llvm/unittests/SandboxIR/SandboxIRTest.cpp b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
index 7206ee34d36e3..5f39fdc0b9133 100644
--- a/llvm/unittests/SandboxIR/SandboxIRTest.cpp
+++ b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
@@ -1470,68 +1470,6 @@ define i32 @foo(i32 %arg0, i32 %arg1) {
   EXPECT_EQ(Glob0->getOperand(0), Glob1);
 }
 
-TEST_F(SandboxIRTest, GetExpected) {
-  parseIR(C, R"IR(
-define float @foo(float %v, ptr %ptr) {
-  %add = fadd float %v, %v
-  store float %v, ptr %ptr
-  ret float %v
-}
-define void @bar(float %v, ptr %ptr) {
-  ret void
-}
-)IR");
-  llvm::Function &Foo = *M->getFunction("foo");
-  sandboxir::Context Ctx(C);
-
-  Ctx.createFunction(&Foo);
-  auto *FooBB = cast<sandboxir::BasicBlock>(Ctx.getValue(&*Foo.begin()));
-  auto FooIt = FooBB->begin();
-  auto Add = cast<sandboxir::Instruction>(&*FooIt++);
-  auto *S0 = cast<sandboxir::Instruction>(&*FooIt++);
-  auto *RetF = cast<sandboxir::Instruction>(&*FooIt++);
-  // getExpectedValue
-  EXPECT_EQ(sandboxir::Utils::getExpectedValue(Add), Add);
-  EXPECT_EQ(sandboxir::Utils::getExpectedValue(S0),
-            cast<sandboxir::StoreInst>(S0)->getValueOperand());
-  EXPECT_EQ(sandboxir::Utils::getExpectedValue(RetF),
-            cast<sandboxir::ReturnInst>(RetF)->getReturnValue());
-  // getExpectedType
-  EXPECT_EQ(sandboxir::Utils::getExpectedType(Add), Add->getType());
-  EXPECT_EQ(sandboxir::Utils::getExpectedType(S0),
-            cast<sandboxir::StoreInst>(S0)->getValueOperand()->getType());
-  EXPECT_EQ(sandboxir::Utils::getExpectedType(RetF),
-            cast<sandboxir::ReturnInst>(RetF)->getReturnValue()->getType());
-
-  // getExpectedValue for void returns
-  llvm::Function &Bar = *M->getFunction("bar");
-  Ctx.createFunction(&Bar);
-  auto *BarBB = cast<sandboxir::BasicBlock>(Ctx.getValue(&*Bar.begin()));
-  auto BarIt = BarBB->begin();
-  auto *RetV = cast<sandboxir::Instruction>(&*BarIt++);
-  EXPECT_EQ(sandboxir::Utils::getExpectedValue(RetV), nullptr);
-}
-
-TEST_F(SandboxIRTest, GetNumBits) {
-  parseIR(C, R"IR(
-define void @foo(float %arg0, double %arg1, i8 %arg2, i64 %arg3) {
-bb0:
-  ret void
-}
-)IR");
-  llvm::Function &Foo = *M->getFunction("foo");
-  sandboxir::Context Ctx(C);
-  sandboxir::Function *F = Ctx.createFunction(&Foo);
-  const DataLayout &DL = M->getDataLayout();
-  // getNumBits for scalars
-  EXPECT_EQ(sandboxir::Utils::getNumBits(F->getArg(0), DL),
-            DL.getTypeSizeInBits(Type::getFloatTy(C)));
-  EXPECT_EQ(sandboxir::Utils::getNumBits(F->getArg(1), DL),
-            DL.getTypeSizeInBits(Type::getDoubleTy(C)));
-  EXPECT_EQ(sandboxir::Utils::getNumBits(F->getArg(2), DL), 8u);
-  EXPECT_EQ(sandboxir::Utils::getNumBits(F->getArg(3), DL), 64u);
-}
-
 TEST_F(SandboxIRTest, RAUW_RUWIf) {
   parseIR(C, R"IR(
 define void @foo(ptr %ptr) {
@@ -1997,76 +1935,6 @@ define void @foo(i8 %v1, ptr %ptr) {
   }
 }
 
-TEST_F(SandboxIRTest, Instruction_isStackSaveOrRestoreIntrinsic) {
-  parseIR(C, R"IR(
-declare void @llvm.sideeffect()
-define void @foo(i8 %v1, ptr %ptr) {
-  %add = add i8 %v1, %v1
-  %stacksave = call ptr @llvm.stacksave()
-  call void @llvm.stackrestore(ptr %stacksave)
-  call void @llvm.sideeffect()
-  ret void
-}
-)IR");
-  llvm::Function *LLVMF = &*M->getFunction("foo");
-  sandboxir::Context Ctx(C);
-  sandboxir::Function *F = Ctx.createFunction(LLVMF);
-  auto *BB = &*F->begin();
-  auto It = BB->begin();
-  auto *Add = cast<sandboxir::BinaryOperator>(&*It++);
-  auto *StackSave = cast<sandboxir::CallInst>(&*It++);
-  auto *StackRestore = cast<sandboxir::CallInst>(&*It++);
-  auto *Other = cast<sandboxir::CallInst>(&*It++);
-  auto *Ret = cast<sandboxir::ReturnInst>(&*It++);
-
-  EXPECT_FALSE(Add->isStackSaveOrRestoreIntrinsic());
-  EXPECT_TRUE(StackSave->isStackSaveOrRestoreIntrinsic());
-  EXPECT_TRUE(StackRestore->isStackSaveOrRestoreIntrinsic());
-  EXPECT_FALSE(Other->isStackSaveOrRestoreIntrinsic());
-  EXPECT_FALSE(Ret->isStackSaveOrRestoreIntrinsic());
-}
-
-TEST_F(SandboxIRTest, Instruction_isMemDepCandidate) {
-  parseIR(C, R"IR(
-declare void @llvm.fake.use(...)
-declare void @llvm.sideeffect()
-declare void @llvm.pseudoprobe(i64, i64, i32, i64)
-declare void @bar()
-define void @foo(i8 %v1, ptr %ptr) {
-  %add0 = add i8 %v1, %v1
-  %ld0 = load i8, ptr %ptr
-  store i8 %v1, ptr %ptr
-  call void @llvm.sideeffect()
-  call void @llvm.pseudoprobe(i64 42, i64 1, i32 0, i64 -1)
-  call void @llvm.fake.use(ptr %ptr)
-  call void @bar()
-  ret void
-}
-)IR");
-  llvm::Function *LLVMF = &*M->getFunction("foo");
-  sandboxir::Context Ctx(C);
-  sandboxir::Function *F = Ctx.createFunction(LLVMF);
-  auto *BB = &*F->begin();
-  auto It = BB->begin();
-  auto *Add0 = cast<sandboxir::BinaryOperator>(&*It++);
-  auto *Ld0 = cast<sandboxir::LoadInst>(&*It++);
-  auto *St0 = cast<sandboxir::StoreInst>(&*It++);
-  auto *SideEffect0 = cast<sandboxir::CallInst>(&*It++);
-  auto *PseudoProbe0 = cast<sandboxir::CallInst>(&*It++);
-  auto *OtherIntrinsic0 = cast<sandboxir::CallInst>(&*It++);
-  auto *CallBar = cast<sandboxir::CallInst>(&*It++);
-  auto *Ret = cast<sandboxir::ReturnInst>(&*It++);
-
-  EXPECT_FALSE(Add0->isMemDepCandidate());
-  EXPECT_TRUE(Ld0->isMemDepCandidate());
-  EXPECT_TRUE(St0->isMemDepCandidate());
-  EXPECT_FALSE(SideEffect0->isMemDepCandidate());
-  EXPECT_FALSE(PseudoProbe0->isMemDepCandidate());
-  EXPECT_TRUE(OtherIntrinsic0->isMemDepCandidate());
-  EXPECT_TRUE(CallBar->isMemDepCandidate());
-  EXPECT_FALSE(Ret->isMemDepCandidate());
-}
-
 TEST_F(SandboxIRTest, VAArgInst) {
   parseIR(C, R"IR(
 define void @foo(ptr %va) {
@@ -2092,7 +1960,7 @@ define void @foo(ptr %va) {
   // Check create().
   auto *NewVATy = sandboxir::Type::getInt8Ty(Ctx);
   auto *NewVA = sandboxir::VAArgInst::create(Arg, NewVATy, Ret->getIterator(),
-                                             Ret->getParent(), Ctx, "NewVA");
+                                             Ctx, "NewVA");
   EXPECT_EQ(NewVA->getNextNode(), Ret);
   EXPECT_EQ(NewVA->getType(), NewVATy);
 #ifndef NDEBUG
@@ -2121,8 +1989,8 @@ define void @foo(i8 %arg) {
   EXPECT_EQ(Freeze->getOperand(0), Arg);
 
   // Check create().
-  auto *NewFreeze = sandboxir::FreezeInst::create(
-      Arg, Ret->getIterator(), Ret->getParent(), Ctx, "NewFreeze");
+  auto *NewFreeze =
+      sandboxir::FreezeInst::create(Arg, Ret->getIterator(), Ctx, "NewFreeze");
   EXPECT_EQ(NewFreeze->getNextNode(), Ret);
 #ifndef NDEBUG
   EXPECT_EQ(NewFreeze->getName(), "NewFreeze");
@@ -2169,7 +2037,7 @@ define void @foo() {
   // Check create().
   auto *NewFence =
       sandboxir::FenceInst::create(AtomicOrdering::Release, Ret->getIterator(),
-                                   BB, Ctx, SyncScope::SingleThread);
+                                   Ctx, SyncScope::SingleThread);
   EXPECT_EQ(NewFence->getNextNode(), Ret);
   EXPECT_EQ(NewFence->getOrdering(), AtomicOrdering::Release);
   EXPECT_EQ(NewFence->getSyncScopeID(), SyncScope::SingleThread);
@@ -2224,7 +2092,7 @@ define void @foo(i1 %c0, i8 %v0, i8 %v1, i1 %c1) {
   {
     // Check SelectInst::create() InsertBefore.
     auto *NewSel = cast<sandboxir::SelectInst>(sandboxir::SelectInst::create(
-        Cond0, V0, V1, /*InsertBefore=*/Ret, Ctx));
+        Cond0, V0, V1, /*InsertBefore=*/Ret->getIterator(), Ctx));
     EXPECT_EQ(NewSel->getCondition(), Cond0);
     EXPECT_EQ(NewSel->getTrueValue(), V0);
     EXPECT_EQ(NewSel->getFalseValue(), V1);
@@ -2246,8 +2114,8 @@ define void @foo(i1 %c0, i8 %v0, i8 %v1, i1 %c1) {
     auto *FortyTwo =
         sandboxir::ConstantInt::get(sandboxir::Type::getInt1Ty(Ctx), 42,
                                     /*IsSigned=*/false);
-    auto *NewSel =
-        sandboxir::SelectInst::create(False, FortyTwo, FortyTwo, Ret, Ctx);
+    auto *NewSel = sandboxir::SelectInst::create(False, FortyTwo, FortyTwo,
+                                                 Ret->getIterator(), Ctx);
     EXPECT_TRUE(isa<sandboxir::Constant>(NewSel));
     EXPECT_EQ(NewSel, FortyTwo);
   }
@@ -2279,7 +2147,7 @@ define void @foo(<2 x i8> %vec, i32 %idx) {
 
   auto *NewI1 =
       cast<sandboxir::ExtractElementInst>(sandboxir::ExtractElementInst::create(
-          ArgVec, ArgIdx, Ret, Ctx, "NewExtrBeforeRet"));
+          ArgVec, ArgIdx, Ret->getIterator(), Ctx, "NewExtrBeforeRet"));
   EXPECT_EQ(NewI1->getOperand(0), ArgVec);
   EXPECT_EQ(NewI1->getOperand(1), ArgIdx);
   EXPECT_EQ(NewI1->getNextNode(), Ret);
@@ -2325,7 +2193,7 @@ define void @foo(i8 %v0, i8 %v1, <2 x i8> %vec) {
   auto *Idx = Ins0->getOperand(2);
   auto *NewI1 =
       cast<sandboxir::InsertElementInst>(sandboxir::InsertElementInst::create(
-          Poison, Arg0, Idx, Ret, Ctx, "NewIns1"));
+          Poison, Arg0, Idx, Ret->getIterator(), Ctx, "NewIns1"));
   EXPECT_EQ(NewI1->getOperand(0), Poison);
   EXPECT_EQ(NewI1->getNextNode(), Ret);
 
@@ -2375,13 +2243,14 @@ define void @foo(<2 x i8> %v1, <2 x i8> %v2) {
   auto CreateShuffleWithMask = [&](auto &&...Indices) {
     SmallVector<int, 4> Mask = {Indices...};
     return cast<sandboxir::ShuffleVectorInst>(
-        sandboxir::ShuffleVectorInst::create(ArgV1, ArgV2, Mask, Ret, Ctx));
+        sandboxir::ShuffleVectorInst::create(ArgV1, ArgV2, Mask,
+                                             Ret->getIterator(), Ctx));
   };
 
   // create (InsertBefore)
   auto *NewI1 =
       cast<sandboxir::ShuffleVectorInst>(sandboxir::ShuffleVectorInst::create(
-          ArgV1, ArgV2, ArrayRef<int>({0, 2, 1, 3}), Ret, Ctx,
+          ArgV1, ArgV2, ArrayRef<int>({0, 2, 1, 3}), Ret->getIterator(), Ctx,
           "NewShuffleBeforeRet"));
   EXPECT_EQ(NewI1->getOperand(0), ArgV1);
   EXPECT_EQ(NewI1->getOperand(1), ArgV2);
@@ -2790,8 +2659,8 @@ define void @foo({i32, float} %agg) {
   // create before instruction
   auto *NewExtBeforeRet =
       cast<sandboxir::ExtractValueInst>(sandboxir::ExtractValueInst::create(
-          ArgAgg, ArrayRef<unsigned>({0}), Ret->getIterator(), Ret->getParent(),
-          Ctx, "NewExtBeforeRet"));
+          ArgAgg, ArrayRef<unsigned>({0}), Ret->getIterator(), Ctx,
+          "NewExtBeforeRet"));
   EXPECT_EQ(NewExtBeforeRet->getNextNode(), Ret);
 #ifndef NDEBUG
   EXPECT_EQ(NewExtBeforeRet->getName(), "NewExtBeforeRet");
@@ -2800,7 +2669,7 @@ define void @foo({i32, float} %agg) {
   // create at end of BB
   auto *NewExtAtEnd =
       cast<sandboxir::ExtractValueInst>(sandboxir::ExtractValueInst::create(
-          ArgAgg, ArrayRef<unsigned>({0}), BB->end(), BB, Ctx, "NewExtAtEnd"));
+          ArgAgg, ArrayRef<unsigned>({0}), BB->end(), Ctx, "NewExtAtEnd"));
   EXPECT_EQ(NewExtAtEnd->getPrevNode(), Ret);
 #ifndef NDEBUG
   EXPECT_EQ(NewExtAtEnd->getName(), "NewExtAtEnd");
@@ -2808,7 +2677,7 @@ define void @foo({i32, float} %agg) {
 
   // Test the path that creates a folded constant.
   auto *ShouldBeConstant = sandboxir::ExtractValueInst::create(
-      Const1->getOperand(0), ArrayRef<unsigned>({0}), BB->end(), BB, Ctx);
+      Const1->getOperand(0), ArrayRef<unsigned>({0}), BB->end(), Ctx);
   EXPECT_TRUE(isa<sandboxir::Constant>(ShouldBeConstant));
 
   auto *Zero = sandboxir::ConstantInt::get(sandboxir::Type::getInt32Ty(Ctx), 0);
@@ -2897,8 +2766,8 @@ define void @foo({i32, float} %agg, i32 %i) {
   // create before instruction
   auto *NewInsBeforeRet =
       cast<sandboxir::InsertValueInst>(sandboxir::InsertValueInst::create(
-          ArgAgg, ArgInt, ArrayRef<unsigned>({0}), Ret->getIterator(),
-          Ret->getParent(), Ctx, "NewInsBeforeRet"));
+          ArgAgg, ArgInt, ArrayRef<unsigned>({0}), Ret->getIterator(), Ctx,
+          "NewInsBeforeRet"));
   EXPECT_EQ(NewInsBeforeRet->getNextNode(), Ret);
 #ifndef NDEBUG
   EXPECT_EQ(NewInsBeforeRet->getName(), "NewInsBeforeRet");
@@ -2907,8 +2776,7 @@ define void @foo({i32, float} %agg, i32 %i) {
   // create at end of BB
   auto *NewInsAtEnd =
       cast<sandboxir::InsertValueInst>(sandboxir::InsertValueInst::create(
-          ArgAgg, ArgInt, ArrayRef<unsigned>({0}), BB->end(), BB, Ctx,
-          "NewInsAtEnd"));
+          ArgAgg, ArgInt, ArrayRef<unsigned>({0}), BB, Ctx, "NewInsAtEnd"));
   EXPECT_EQ(NewInsAtEnd->getPrevNode(), Ret);
 #ifndef NDEBUG
   EXPECT_EQ(NewInsAtEnd->getName(), "NewInsAtEnd");
@@ -2917,7 +2785,7 @@ define void @foo({i32, float} %agg, i32 %i) {
   // Test the path that creates a folded constant.
   auto *Zero = sandboxir::ConstantInt::get(sandboxir::Type::getInt32Ty(Ctx), 0);
   auto *ShouldBeConstant = sandboxir::InsertValueInst::create(
-      Const1->getOperand(0), Zero, ArrayRef<unsigned>({0}), BB->end(), BB, Ctx);
+      Const1->getOperand(0), Zero, ArrayRef<unsigned>({0}), BB, Ctx);
   auto *ExpectedConstant = Const2->getOperand(0);
   EXPECT_TRUE(isa<sandboxir::Constant>(ShouldBeConstant));
   EXPECT_EQ(ShouldBeConstant, ExpectedConstant);
@@ -3023,7 +2891,7 @@ define void @foo(i1 %cond0, i1 %cond2) {
 
   {
     // Check unconditional BranchInst::create() InsertBefore.
-    auto *Br = sandboxir::BranchInst::create(BB1, /*InsertBefore=*/Ret1, Ctx);
+    auto *Br = sandboxir::BranchInst::create(BB1, Ret1->getIterator(), Ctx);
     EXPECT_FALSE(Br->isConditional());
     EXPECT_TRUE(Br->isUnconditional());
 #ifndef NDEBUG
@@ -3052,7 +2920,7 @@ define void @foo(i1 %cond0, i1 %cond2) {
   {
     // Check conditional BranchInst::create() InsertBefore.
     auto *Br = sandboxir::BranchInst::create(BB1, BB2, Cond0,
-                                             /*InsertBefore=*/Ret1, Ctx);
+                                             Ret1->getIterator(), Ctx);
     EXPECT_TRUE(Br->isConditional());
     EXPECT_EQ(Br->getCondition(), Cond0);
     unsigned SuccIdx = 0;
@@ -3105,9 +2973,8 @@ define void @foo(ptr %arg0, ptr %arg1) {
   // Check getAlign()
   EXPECT_EQ(Ld->getAlign(), 64);
   // Check create(InsertBefore)
-  sandboxir::LoadInst *NewLd =
-      sandboxir::LoadInst::create(Ld->getType(), Arg1, Align(8),
-                                  /*InsertBefore=*/Ret, Ctx, "NewLd");
+  sandboxir::LoadInst *NewLd = sandboxir::LoadInst::create(
+      Ld->getType(), Arg1, Align(8), Ret->getIterator(), Ctx, "NewLd");
   EXPECT_FALSE(NewLd->isVolatile());
   OrigVolatileValue = NewLd->isVolatile();
   NewLd->setVolatile(true);
@@ -3119,10 +2986,9 @@ define void @foo(ptr %arg0, ptr %arg1) {
   EXPECT_EQ(NewLd->getAlign(), 8);
   EXPECT_EQ(NewLd->getName(), "NewLd");
   // Check create(InsertBefore, IsVolatile=true)
-  sandboxir::LoadInst *NewVLd =
-      sandboxir::LoadInst::create(VLd->getType(), Arg1, Align(8),
-                                  /*InsertBefore=*/Ret,
-                                  /*IsVolatile=*/true, Ctx, "NewVLd");
+  sandboxir::LoadInst *NewVLd = sandboxir::LoadInst::create(
+      VLd->getType(), Arg1, Align(8), Ret->getIterator(),
+      /*IsVolatile=*/true, Ctx, "NewVLd");
 
   EXPECT_TRUE(NewVLd->isVolatile());
   OrigVolatileValue = NewVLd->isVolatile();
@@ -3186,8 +3052,7 @@ define void @foo(i8 %val, ptr %ptr) {
   EXPECT_EQ(St->getAlign(), 64);
   // Check create(InsertBefore)
   sandboxir::StoreInst *NewSt =
-      sandboxir::StoreInst::create(Val, Ptr, Align(8),
-                                   /*InsertBefore=*/Ret, Ctx);
+      sandboxir::StoreInst::create(Val, Ptr, Align(8), Ret->getIterator(), Ctx);
   EXPECT_FALSE(NewSt->isVolatile());
   OrigVolatileValue = NewSt->isVolatile();
   NewSt->setVolatile(true);
@@ -3201,8 +3066,7 @@ define void @foo(i8 %val, ptr %ptr) {
   EXPECT_EQ(NewSt->getNextNode(), Ret);
   // Check create(InsertBefore, IsVolatile=true)
   sandboxir::StoreInst *NewVSt =
-      sandboxir::StoreInst::create(Val, Ptr, Align(8),
-                                   /*InsertBefore=*/Ret,
+      sandboxir::StoreInst::create(Val, Ptr, Align(8), Ret->getIterator(),
                                    /*IsVolatile=*/true, Ctx);
   EXPECT_TRUE(NewVSt->isVolatile());
   OrigVolatileValue = NewVSt->isVolatile();
@@ -3262,11 +3126,11 @@ define i8 @foo(i8 %val) {
 
   // Check create(InsertBefore) a void ReturnInst.
   auto *NewRet1 = cast<sandboxir::ReturnInst>(
-      sandboxir::ReturnInst::create(nullptr, /*InsertBefore=*/Ret, Ctx));
+      sandboxir::ReturnInst::create(nullptr, Ret->getIterator(), Ctx));
   EXPECT_EQ(NewRet1->getReturnValue(), nullptr);
   // Check create(InsertBefore) a non-void ReturnInst.
   auto *NewRet2 = cast<sandboxir::ReturnInst>(
-      sandboxir::ReturnInst::create(Val, /*InsertBefore=*/Ret, Ctx));
+      sandboxir::ReturnInst::create(Val, Ret->getIterator(), Ctx));
   EXPECT_EQ(NewRet2->getReturnValue(), Val);
 
   // Check create(InsertAtEnd) a void ReturnInst.
@@ -3454,7 +3318,7 @@ define i8 @foo(i8 %arg) {
   {
     // Check create() WhereIt.
     auto *Call = cast<sandboxir::CallInst>(sandboxir::CallInst::create(
-        FTy, &F, Args, /*WhereIt=*/Ret->getIterator(), BB, Ctx));
+        FTy, &F, Args, /*WhereIt=*/Ret->getIterator(), Ctx));
     EXPECT_EQ(Call->getNextNode(), Ret);
     EXPECT_EQ(Call->getCalledFunction(), &F);
     EXPECT_EQ(range_size(Call->args()), 1u);
@@ -3463,7 +3327,7 @@ define i8 @foo(i8 %arg) {
   {
     // Check create() InsertBefore.
     auto *Call = cast<sandboxir::CallInst>(
-        sandboxir::CallInst::create(FTy, &F, Args, /*InsertBefore=*/Ret, Ctx));
+        sandboxir::CallInst::create(FTy, &F, Args, Ret->getIterator(), Ctx));
     EXPECT_EQ(Call->getNextNode(), Ret);
     EXPECT_EQ(Call->getCalledFunction(), &F);
     EXPECT_EQ(range_size(Call->args()), 1u);
@@ -3542,7 +3406,7 @@ define void @foo(i8 %arg) {
     auto *InsertBefore = &*BB0->begin();
     auto *NewInvoke = cast<sandboxir::InvokeInst>(sandboxir::InvokeInst::create(
         F.getFunctionType(), &F, NormalBB, ExceptionBB, Args,
-        /*WhereIt=*/InsertBefore->getIterator(), /*WhereBB=*/BB0, Ctx));
+        InsertBefore->getIterator(), Ctx));
     EXPECT_EQ(NewInvoke->getNormalDest(), NormalBB);
     EXPECT_EQ(NewInvoke->getUnwindDest(), ExceptionBB);
     EXPECT_EQ(NewInvoke->getNextNode(), InsertBefore);
@@ -3551,9 +3415,9 @@ define void @foo(i8 %arg) {
     // Check create() InsertBefore.
     SmallVector<sandboxir::Value *> Args({Arg});
     auto *InsertBefore = &*BB0->begin();
-    auto *NewInvoke = cast<sandboxir::InvokeInst>(
-        sandboxir::InvokeInst::create(F.getFunctionType(), &F, NormalBB,
-                                      ExceptionBB, Args, InsertBefore, Ctx));
+    auto *NewInvoke = cast<sandboxir::InvokeInst>(sandboxir::InvokeInst::create(
+        F.getFunctionType(), &F, NormalBB, ExceptionBB, Args,
+        InsertBefore->getIterator(), Ctx));
     EXPECT_EQ(NewInvoke->getNormalDest(), NormalBB);
     EXPECT_EQ(NewInvoke->getUnwindDest(), ExceptionBB);
     EXPECT_EQ(NewInvoke->getNextNode(), InsertBefore);
@@ -3562,8 +3426,7 @@ define void @foo(i8 %arg) {
     // Check create() InsertAtEnd.
     SmallVector<sandboxir::Value *> Args({Arg});
     auto *NewInvoke = cast<sandboxir::InvokeInst>(sandboxir::InvokeInst::create(
-        F.getFunctionType(), &F, NormalBB, ExceptionBB, Args,
-        /*InsertAtEnd=*/BB0, Ctx));
+        F.getFunctionType(), &F, NormalBB, ExceptionBB, Args, BB0, Ctx));
     EXPECT_EQ(NewInvoke->getNormalDest(), NormalBB);
     EXPECT_EQ(NewInvoke->getUnwindDest(), ExceptionBB);
     EXPECT_EQ(NewInvoke->getParent(), BB0);
@@ -3652,8 +3515,7 @@ define void @foo(i8 %arg) {
     // Check create() WhereIt, WhereBB.
     SmallVector<sandboxir::Value *> Args({Arg});
     auto *NewCallBr = cast<sandboxir::CallBrInst>(sandboxir::CallBrInst::create(
-        F.getFunctionType(), &F, BB1, {BB2}, Args, /*WhereIt=*/BB0->end(),
-        /*WhereBB=*/BB0, Ctx));
+        F.getFunctionType(), &F, BB1, {BB2}, Args, BB0->end(), Ctx));
     EXPECT_EQ(NewCallBr->getDefaultDest(), BB1);
     EXPECT_EQ(NewCallBr->getIndirectDests().size(), 1u);
     EXPECT_EQ(NewCallBr->getIndirectDests()[0], BB2);
@@ -3664,8 +3526,9 @@ define void @foo(i8 %arg) {
     // Check create() InsertBefore
     SmallVector<sandboxir::Value *> Args({Arg});
     auto *InsertBefore = &*BB0->rbegin();
-    auto *NewCallBr = cast<sandboxir::CallBrInst>(sandboxir::CallBrInst::create(
-        F.getFunctionType(), &F, BB1, {BB2}, Args, InsertBefore, Ctx));
+    auto *NewCallBr = cast<sandboxir::CallBrInst>(
+        sandboxir::CallBrInst::create(F.getFunctionType(), &F, BB1, {BB2}, Args,
+                                      InsertBefore->getIterator(), Ctx));
     EXPECT_EQ(NewCallBr->getDefaultDest(), BB1);
     EXPECT_EQ(NewCallBr->getIndirectDests().size(), 1u);
     EXPECT_EQ(NewCallBr->getIndirectDests()[0], BB2);
@@ -3674,9 +3537,8 @@ define void @foo(i8 %arg) {
   {
     // Check create() InsertAtEnd.
     SmallVector<sandboxir::Value *> Args({Arg});
-    auto *NewCallBr = cast<sandboxir::CallBrInst>(
-        sandboxir::CallBrInst::create(F.getFunctionType(), &F, BB1, {BB2}, Args,
-                                      /*InsertAtEnd=*/BB0, Ctx));
+    auto *NewCallBr = cast<sandboxir::CallBrInst>(sandboxir::CallBrInst::create(
+        F.getFunctionType(), &F, BB1, {BB2}, Args, BB0, Ctx));
     EXPECT_EQ(NewCallBr->getDefaultDest(), BB1);
     EXPECT_EQ(NewCallBr->getIndirectDests().size(), 1u);
     EXPECT_EQ(NewCallBr->getIndirectDests()[0], BB2);
@@ -3735,10 +3597,9 @@ define void @foo() {
     EXPECT_EQ(LPad->isFilter(Idx), LLVMLPad->isFilter(Idx));
   // Check create().
   auto *BBRet = &*BB->begin();
-  auto *NewLPad =
-      cast<sandboxir::LandingPadInst>(sandboxir::LandingPadInst::create(
-          sandboxir::Type::getInt8Ty(Ctx), 0, BBRet->getIterator(),
-          BBRet->getParent(), Ctx, "NewLPad"));
+  auto *NewLPad = cast<sandboxir::LandingPadInst>(
+      sandboxir::LandingPadInst::create(sandboxir::Type::getInt8Ty(Ctx), 0,
+                                        BBRet->getIterator(), Ctx, "NewLPad"));
   EXPECT_EQ(NewLPad->getNextNode(), BBRet);
   EXPECT_FALSE(NewLPad->isCleanup());
 #ifndef NDEBUG
@@ -3817,7 +3678,7 @@ define void @foo() {
   }
   // Check CatchPadInst::create().
   auto *NewCPI = cast<sandboxir::CatchPadInst>(sandboxir::CatchPadInst::create(
-      CS, {}, BBRet->getIterator(), BB, Ctx, "NewCPI"));
+      CS, {}, BBRet->getIterator(), Ctx, "NewCPI"));
   EXPECT_EQ(NewCPI->getCatchSwitch(), CS);
   EXPECT_EQ(NewCPI->arg_size(), 0u);
   EXPECT_EQ(NewCPI->getNextNode(), BBRet);
@@ -3827,7 +3688,7 @@ define void @foo() {
   // Check CleanupPadInst::create().
   auto *NewCLPI =
       cast<sandboxir::CleanupPadInst>(sandboxir::CleanupPadInst::create(
-          CS, {}, BBRet->getIterator(), BB, Ctx, "NewCLPI"));
+          CS, {}, BBRet->getIterator(), Ctx, "NewCLPI"));
   EXPECT_EQ(NewCLPI->getParentPad(), CS);
   EXPECT_EQ(NewCLPI->arg_size(), 0u);
   EXPECT_EQ(NewCLPI->getNextNode(), BBRet);
@@ -6101,7 +5962,7 @@ define void @foo() {
   EXPECT_EQ(UI->getNumOfIRInstrs(), 1u);
   // Check create(InsertBefore)
   sandboxir::UnreachableInst *NewUI =
-      sandboxir::UnreachableInst::create(/*InsertBefore=*/UI, Ctx);
+      sandboxir::UnreachableInst::create(UI->getIterator(), Ctx);
   EXPECT_EQ(NewUI->getNextNode(), UI);
   // Check create(InsertAtEnd)
   sandboxir::UnreachableInst *NewUIEnd =
diff --git a/llvm/unittests/SandboxIR/TrackerTest.cpp b/llvm/unittests/SandboxIR/TrackerTest.cpp
index 9c7710a6a9297..6d060854949e1 100644
--- a/llvm/unittests/SandboxIR/TrackerTest.cpp
+++ b/llvm/unittests/SandboxIR/TrackerTest.cpp
@@ -527,9 +527,9 @@ define void @foo(ptr %ptr) {
 
   Ctx.save();
   // Check create(InsertBefore) with tracking enabled.
-  sandboxir::LoadInst *NewLd =
-      sandboxir::LoadInst::create(Ld->getType(), Ptr, Align(8),
-                                  /*InsertBefore=*/Ld, Ctx, "NewLd");
+  sandboxir::LoadInst *NewLd = sandboxir::LoadInst::create(
+      Ld->getType(), Ptr, Align(8),
+      /*InsertBefore=*/Ld->getIterator(), Ctx, "NewLd");
   It = BB->begin();
   EXPECT_EQ(&*It++, NewLd);
   EXPECT_EQ(&*It++, Ld);
diff --git a/llvm/unittests/SandboxIR/UtilsTest.cpp b/llvm/unittests/SandboxIR/UtilsTest.cpp
index 7ce8f40f8985b..a803c2a1cf977 100644
--- a/llvm/unittests/SandboxIR/UtilsTest.cpp
+++ b/llvm/unittests/SandboxIR/UtilsTest.cpp
@@ -135,3 +135,83 @@ define void @foo(ptr %ptr) {
   EXPECT_FALSE(sandboxir::Utils::atLowerAddress(L1, L0, SE, DL));
   EXPECT_FALSE(sandboxir::Utils::atLowerAddress(L3, V3L3, SE, DL));
 }
+
+TEST_F(UtilsTest, GetExpected) {
+  parseIR(C, R"IR(
+define float @foo(float %v, ptr %ptr) {
+  %add = fadd float %v, %v
+  store float %v, ptr %ptr
+  ret float %v
+}
+define void @bar(float %v, ptr %ptr) {
+  ret void
+}
+)IR");
+  llvm::Function &Foo = *M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+
+  Ctx.createFunction(&Foo);
+  auto *FooBB = cast<sandboxir::BasicBlock>(Ctx.getValue(&*Foo.begin()));
+  auto FooIt = FooBB->begin();
+  auto Add = cast<sandboxir::Instruction>(&*FooIt++);
+  auto *S0 = cast<sandboxir::Instruction>(&*FooIt++);
+  auto *RetF = cast<sandboxir::Instruction>(&*FooIt++);
+  // getExpectedValue
+  EXPECT_EQ(sandboxir::Utils::getExpectedValue(Add), Add);
+  EXPECT_EQ(sandboxir::Utils::getExpectedValue(S0),
+            cast<sandboxir::StoreInst>(S0)->getValueOperand());
+  EXPECT_EQ(sandboxir::Utils::getExpectedValue(RetF),
+            cast<sandboxir::ReturnInst>(RetF)->getReturnValue());
+  // getExpectedType
+  EXPECT_EQ(sandboxir::Utils::getExpectedType(Add), Add->getType());
+  EXPECT_EQ(sandboxir::Utils::getExpectedType(S0),
+            cast<sandboxir::StoreInst>(S0)->getValueOperand()->getType());
+  EXPECT_EQ(sandboxir::Utils::getExpectedType(RetF),
+            cast<sandboxir::ReturnInst>(RetF)->getReturnValue()->getType());
+
+  // getExpectedValue for void returns
+  llvm::Function &Bar = *M->getFunction("bar");
+  Ctx.createFunction(&Bar);
+  auto *BarBB = cast<sandboxir::BasicBlock>(Ctx.getValue(&*Bar.begin()));
+  auto BarIt = BarBB->begin();
+  auto *RetV = cast<sandboxir::Instruction>(&*BarIt++);
+  EXPECT_EQ(sandboxir::Utils::getExpectedValue(RetV), nullptr);
+}
+
+TEST_F(UtilsTest, GetNumBits) {
+  parseIR(C, R"IR(
+define void @foo(float %arg0, double %arg1, i8 %arg2, i64 %arg3, ptr %arg4) {
+bb0:
+  %ld0 = load float, ptr %arg4
+  %ld1 = load double, ptr %arg4
+  %ld2 = load i8, ptr %arg4
+  %ld3 = load i64, ptr %arg4
+  ret void
+}
+)IR");
+  llvm::Function &Foo = *M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  sandboxir::Function *F = Ctx.createFunction(&Foo);
+  const DataLayout &DL = M->getDataLayout();
+  // getNumBits for scalars via the Value overload
+  EXPECT_EQ(sandboxir::Utils::getNumBits(F->getArg(0), DL),
+            DL.getTypeSizeInBits(Type::getFloatTy(C)));
+  EXPECT_EQ(sandboxir::Utils::getNumBits(F->getArg(1), DL),
+            DL.getTypeSizeInBits(Type::getDoubleTy(C)));
+  EXPECT_EQ(sandboxir::Utils::getNumBits(F->getArg(2), DL), 8u);
+  EXPECT_EQ(sandboxir::Utils::getNumBits(F->getArg(3), DL), 64u);
+
+  auto &BB = *F->begin();
+  auto It = BB.begin();
+  auto *L0 = cast<sandboxir::LoadInst>(&*It++);
+  auto *L1 = cast<sandboxir::LoadInst>(&*It++);
+  auto *L2 = cast<sandboxir::LoadInst>(&*It++);
+  auto *L3 = cast<sandboxir::LoadInst>(&*It++);
+  // getNumBits for scalars via the Instruction overload
+  EXPECT_EQ(sandboxir::Utils::getNumBits(L0),
+            DL.getTypeSizeInBits(Type::getFloatTy(C)));
+  EXPECT_EQ(sandboxir::Utils::getNumBits(L1),
+            DL.getTypeSizeInBits(Type::getDoubleTy(C)));
+  EXPECT_EQ(sandboxir::Utils::getNumBits(L2), 8u);
+  EXPECT_EQ(sandboxir::Utils::getNumBits(L3), 64u);
+}
diff --git a/llvm/unittests/Support/KnownBitsTest.cpp b/llvm/unittests/Support/KnownBitsTest.cpp
index b701757aed5eb..551c1a8107494 100644
--- a/llvm/unittests/Support/KnownBitsTest.cpp
+++ b/llvm/unittests/Support/KnownBitsTest.cpp
@@ -521,16 +521,15 @@ TEST(KnownBitsTest, BinaryExhaustive) {
       [](const APInt &N1, const APInt &N2) { return APIntOps::mulhu(N1, N2); },
       /*CheckOptimality=*/false);
 
-  testBinaryOpExhaustive("avgFloorS", KnownBits::avgFloorS, APIntOps::avgFloorS,
-                         /*CheckOptimality=*/false);
+  testBinaryOpExhaustive("avgFloorS", KnownBits::avgFloorS,
+                         APIntOps::avgFloorS);
 
   testBinaryOpExhaustive("avgFloorU", KnownBits::avgFloorU,
                          APIntOps::avgFloorU);
 
   testBinaryOpExhaustive("avgCeilU", KnownBits::avgCeilU, APIntOps::avgCeilU);
 
-  testBinaryOpExhaustive("avgCeilS", KnownBits::avgCeilS, APIntOps::avgCeilS,
-                         /*CheckOptimality=*/false);
+  testBinaryOpExhaustive("avgCeilS", KnownBits::avgCeilS, APIntOps::avgCeilS);
 }
 
 TEST(KnownBitsTest, UnaryExhaustive) {
diff --git a/llvm/unittests/TableGen/ParserEntryPointTest.cpp b/llvm/unittests/TableGen/ParserEntryPointTest.cpp
index 28290e8ecdf68..920e06fb0c3fa 100644
--- a/llvm/unittests/TableGen/ParserEntryPointTest.cpp
+++ b/llvm/unittests/TableGen/ParserEntryPointTest.cpp
@@ -33,7 +33,7 @@ TEST(Parser, SanityTest) {
   bool ProcessResult = TableGenParseFile(SrcMgr, Records);
   EXPECT_FALSE(ProcessResult);
 
-  Record *Foo = Records.getDef("Foo");
+  const Record *Foo = Records.getDef("Foo");
   std::optional<StringRef> Field = Foo->getValueAsOptionalString("strField");
   EXPECT_TRUE(Field.has_value());
   EXPECT_EQ(*Field, "value");
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
index 13db80ab5c68e..5b5d45f6c574b 100644
--- a/llvm/unittests/TargetParser/TargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -45,7 +45,7 @@ const char *ARMArch[] = {
     "armv8m.main", "iwmmxt",    "iwmmxt2",      "xscale",      "armv8.1-m.main",
     "armv9-a",     "armv9",     "armv9a",       "armv9.1-a",   "armv9.1a",
     "armv9.2-a",   "armv9.2a",  "armv9.3-a",    "armv9.3a",    "armv9.4-a",
-    "armv9.4a",    "armv9.5-a", "armv9.5a",
+    "armv9.4a",    "armv9.5-a", "armv9.5a",     "armv9.6a",    "armv9.6-a",
 };
 
 std::string FormatExtensionFlags(int64_t Flags) {
@@ -611,6 +611,8 @@ TEST(TargetParserTest, testARMArch) {
                           ARMBuildAttrs::CPUArch::v9_A));
   EXPECT_TRUE(testARMArch("armv9.5-a", "generic", "v9.5a",
                           ARMBuildAttrs::CPUArch::v9_A));
+  EXPECT_TRUE(testARMArch("armv9.6-a", "generic", "v9.6a",
+                          ARMBuildAttrs::CPUArch::v9_A));
   EXPECT_TRUE(
       testARMArch("armv8-r", "generic", "v8r", ARMBuildAttrs::CPUArch::v8_R));
   EXPECT_TRUE(testARMArch("armv8-m.base", "generic", "v8m.base",
@@ -925,6 +927,7 @@ TEST(TargetParserTest, ARMparseArchProfile) {
     case ARM::ArchKind::ARMV9_3A:
     case ARM::ArchKind::ARMV9_4A:
     case ARM::ArchKind::ARMV9_5A:
+    case ARM::ArchKind::ARMV9_6A:
       EXPECT_EQ(ARM::ProfileKind::A, ARM::parseArchProfile(ARMArch[i]));
       break;
     default:
@@ -1186,6 +1189,7 @@ TEST(TargetParserTest, testAArch64Arch) {
   EXPECT_TRUE(testAArch64Arch("armv9.3-a"));
   EXPECT_TRUE(testAArch64Arch("armv9.4-a"));
   EXPECT_TRUE(testAArch64Arch("armv9.5-a"));
+  EXPECT_TRUE(testAArch64Arch("armv9.6-a"));
 }
 
 bool testAArch64Extension(StringRef CPUName, StringRef ArchExt) {
@@ -1434,6 +1438,7 @@ TEST(TargetParserTest, AArch64ArchFeatures) {
   EXPECT_EQ(AArch64::ARMV9_3A.ArchFeature, "+v9.3a");
   EXPECT_EQ(AArch64::ARMV9_4A.ArchFeature, "+v9.4a");
   EXPECT_EQ(AArch64::ARMV9_5A.ArchFeature, "+v9.5a");
+  EXPECT_EQ(AArch64::ARMV9_6A.ArchFeature, "+v9.6a");
   EXPECT_EQ(AArch64::ARMV8R.ArchFeature, "+v8r");
 }
 
@@ -1463,7 +1468,7 @@ TEST(TargetParserTest, AArch64ArchPartialOrder) {
 
   for (const auto *A :
        {&AArch64::ARMV9_1A, &AArch64::ARMV9_2A, &AArch64::ARMV9_3A,
-        &AArch64::ARMV9_4A, &AArch64::ARMV9_5A})
+        &AArch64::ARMV9_4A, &AArch64::ARMV9_5A, &AArch64::ARMV9_6A})
     EXPECT_TRUE(A->implies(AArch64::ARMV9A));
 
   EXPECT_TRUE(AArch64::ARMV8_1A.implies(AArch64::ARMV8A));
@@ -1481,6 +1486,7 @@ TEST(TargetParserTest, AArch64ArchPartialOrder) {
   EXPECT_TRUE(AArch64::ARMV9_3A.implies(AArch64::ARMV9_2A));
   EXPECT_TRUE(AArch64::ARMV9_4A.implies(AArch64::ARMV9_3A));
   EXPECT_TRUE(AArch64::ARMV9_5A.implies(AArch64::ARMV9_4A));
+  EXPECT_TRUE(AArch64::ARMV9_6A.implies(AArch64::ARMV9_5A));
 
   EXPECT_TRUE(AArch64::ARMV9A.implies(AArch64::ARMV8_5A));
   EXPECT_TRUE(AArch64::ARMV9_1A.implies(AArch64::ARMV8_6A));
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
index d8b6f519982eb..fb8d3780684f8 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
@@ -29,7 +29,108 @@ struct DependencyGraphTest : public testing::Test {
   }
 };
 
-TEST_F(DependencyGraphTest, DGNode_IsMem) {
+TEST_F(DependencyGraphTest, isStackSaveOrRestoreIntrinsic) {
+  parseIR(C, R"IR(
+declare void @llvm.sideeffect()
+define void @foo(i8 %v1, ptr %ptr) {
+  %add = add i8 %v1, %v1
+  %stacksave = call ptr @llvm.stacksave()
+  call void @llvm.stackrestore(ptr %stacksave)
+  call void @llvm.sideeffect()
+  ret void
+}
+)IR");
+  llvm::Function *LLVMF = &*M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  sandboxir::Function *F = Ctx.createFunction(LLVMF);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  auto *Add = cast<sandboxir::BinaryOperator>(&*It++);
+  auto *StackSave = cast<sandboxir::CallInst>(&*It++);
+  auto *StackRestore = cast<sandboxir::CallInst>(&*It++);
+  auto *Other = cast<sandboxir::CallInst>(&*It++);
+  auto *Ret = cast<sandboxir::ReturnInst>(&*It++);
+
+  using DGNode = sandboxir::DGNode;
+  EXPECT_FALSE(DGNode::isStackSaveOrRestoreIntrinsic(Add));
+  EXPECT_TRUE(DGNode::isStackSaveOrRestoreIntrinsic(StackSave));
+  EXPECT_TRUE(DGNode::isStackSaveOrRestoreIntrinsic(StackRestore));
+  EXPECT_FALSE(DGNode::isStackSaveOrRestoreIntrinsic(Other));
+  EXPECT_FALSE(DGNode::isStackSaveOrRestoreIntrinsic(Ret));
+}
+
+TEST_F(DependencyGraphTest, Instruction_isMemDepCandidate) {
+  parseIR(C, R"IR(
+declare void @llvm.fake.use(...)
+declare void @llvm.sideeffect()
+declare void @llvm.pseudoprobe(i64, i64, i32, i64)
+declare void @bar()
+define void @foo(i8 %v1, ptr %ptr) {
+  %add0 = add i8 %v1, %v1
+  %ld0 = load i8, ptr %ptr
+  store i8 %v1, ptr %ptr
+  call void @llvm.sideeffect()
+  call void @llvm.pseudoprobe(i64 42, i64 1, i32 0, i64 -1)
+  call void @llvm.fake.use(ptr %ptr)
+  call void @bar()
+  ret void
+}
+)IR");
+  llvm::Function *LLVMF = &*M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  sandboxir::Function *F = Ctx.createFunction(LLVMF);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  auto *Add0 = cast<sandboxir::BinaryOperator>(&*It++);
+  auto *Ld0 = cast<sandboxir::LoadInst>(&*It++);
+  auto *St0 = cast<sandboxir::StoreInst>(&*It++);
+  auto *SideEffect0 = cast<sandboxir::CallInst>(&*It++);
+  auto *PseudoProbe0 = cast<sandboxir::CallInst>(&*It++);
+  auto *OtherIntrinsic0 = cast<sandboxir::CallInst>(&*It++);
+  auto *CallBar = cast<sandboxir::CallInst>(&*It++);
+  auto *Ret = cast<sandboxir::ReturnInst>(&*It++);
+
+  using DGNode = sandboxir::DGNode;
+
+  EXPECT_FALSE(DGNode::isMemDepCandidate(Add0));
+  EXPECT_TRUE(DGNode::isMemDepCandidate(Ld0));
+  EXPECT_TRUE(DGNode::isMemDepCandidate(St0));
+  EXPECT_FALSE(DGNode::isMemDepCandidate(SideEffect0));
+  EXPECT_FALSE(DGNode::isMemDepCandidate(PseudoProbe0));
+  EXPECT_TRUE(DGNode::isMemDepCandidate(OtherIntrinsic0));
+  EXPECT_TRUE(DGNode::isMemDepCandidate(CallBar));
+  EXPECT_FALSE(DGNode::isMemDepCandidate(Ret));
+}
+
+TEST_F(DependencyGraphTest, Instruction_isMemIntrinsic) {
+  parseIR(C, R"IR(
+declare void @llvm.sideeffect()
+declare void @llvm.pseudoprobe(i64)
+declare void @llvm.assume(i1)
+
+define void @foo(ptr %ptr, i1 %cond) {
+  call void @llvm.sideeffect()
+  call void @llvm.pseudoprobe(i64 42)
+  call void @llvm.assume(i1 %cond)
+  ret void
+}
+)IR");
+  llvm::Function *LLVMF = &*M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  sandboxir::Function *F = Ctx.createFunction(LLVMF);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  auto *SideEffect = cast<sandboxir::IntrinsicInst>(&*It++);
+  auto *PseudoProbe = cast<sandboxir::IntrinsicInst>(&*It++);
+  auto *OtherIntrinsic = cast<sandboxir::IntrinsicInst>(&*It++);
+
+  using DGNode = sandboxir::DGNode;
+  EXPECT_FALSE(DGNode::isMemIntrinsic(SideEffect));
+  EXPECT_FALSE(DGNode::isMemIntrinsic(PseudoProbe));
+  EXPECT_TRUE(DGNode::isMemIntrinsic(OtherIntrinsic));
+}
+
+TEST_F(DependencyGraphTest, MemDGNode) {
   parseIR(C, R"IR(
 declare void @llvm.sideeffect()
 declare void @llvm.pseudoprobe(i64, i64, i32, i64)
@@ -66,16 +167,16 @@ define void @foo(i8 %v1, ptr %ptr) {
 
   sandboxir::DependencyGraph DAG;
   DAG.extend({&*BB->begin(), BB->getTerminator()});
-  EXPECT_TRUE(DAG.getNode(Store)->isMem());
-  EXPECT_TRUE(DAG.getNode(Load)->isMem());
-  EXPECT_FALSE(DAG.getNode(Add)->isMem());
-  EXPECT_TRUE(DAG.getNode(StackSave)->isMem());
-  EXPECT_TRUE(DAG.getNode(StackRestore)->isMem());
-  EXPECT_FALSE(DAG.getNode(SideEffect)->isMem());
-  EXPECT_FALSE(DAG.getNode(PseudoProbe)->isMem());
-  EXPECT_TRUE(DAG.getNode(FakeUse)->isMem());
-  EXPECT_TRUE(DAG.getNode(Call)->isMem());
-  EXPECT_FALSE(DAG.getNode(Ret)->isMem());
+  EXPECT_TRUE(isa<llvm::sandboxir::MemDGNode>(DAG.getNode(Store)));
+  EXPECT_TRUE(isa<llvm::sandboxir::MemDGNode>(DAG.getNode(Load)));
+  EXPECT_FALSE(isa<llvm::sandboxir::MemDGNode>(DAG.getNode(Add)));
+  EXPECT_TRUE(isa<llvm::sandboxir::MemDGNode>(DAG.getNode(StackSave)));
+  EXPECT_TRUE(isa<llvm::sandboxir::MemDGNode>(DAG.getNode(StackRestore)));
+  EXPECT_FALSE(isa<llvm::sandboxir::MemDGNode>(DAG.getNode(SideEffect)));
+  EXPECT_FALSE(isa<llvm::sandboxir::MemDGNode>(DAG.getNode(PseudoProbe)));
+  EXPECT_TRUE(isa<llvm::sandboxir::MemDGNode>(DAG.getNode(FakeUse)));
+  EXPECT_TRUE(isa<llvm::sandboxir::MemDGNode>(DAG.getNode(Call)));
+  EXPECT_FALSE(isa<llvm::sandboxir::MemDGNode>(DAG.getNode(Ret)));
 }
 
 TEST_F(DependencyGraphTest, Basic) {
@@ -115,3 +216,100 @@ define void @foo(ptr %ptr, i8 %v0, i8 %v1) {
   EXPECT_THAT(N1->memPreds(), testing::ElementsAre(N0));
   EXPECT_THAT(N2->memPreds(), testing::ElementsAre(N1));
 }
+
+TEST_F(DependencyGraphTest, MemDGNode_getPrevNode_getNextNode) {
+  parseIR(C, R"IR(
+define void @foo(ptr %ptr, i8 %v0, i8 %v1) {
+  store i8 %v0, ptr %ptr
+  add i8 %v0, %v0
+  store i8 %v1, ptr %ptr
+  ret void
+}
+)IR");
+  llvm::Function *LLVMF = &*M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  auto *F = Ctx.createFunction(LLVMF);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  auto *S0 = cast<sandboxir::StoreInst>(&*It++);
+  [[maybe_unused]] auto *Add = cast<sandboxir::BinaryOperator>(&*It++);
+  auto *S1 = cast<sandboxir::StoreInst>(&*It++);
+  [[maybe_unused]] auto *Ret = cast<sandboxir::ReturnInst>(&*It++);
+
+  sandboxir::DependencyGraph DAG;
+  DAG.extend({&*BB->begin(), BB->getTerminator()});
+
+  auto *S0N = cast<sandboxir::MemDGNode>(DAG.getNode(S0));
+  auto *S1N = cast<sandboxir::MemDGNode>(DAG.getNode(S1));
+
+  EXPECT_EQ(S0N->getPrevNode(), nullptr);
+  EXPECT_EQ(S0N->getNextNode(), S1N);
+
+  EXPECT_EQ(S1N->getPrevNode(), S0N);
+  EXPECT_EQ(S1N->getNextNode(), nullptr);
+}
+
+TEST_F(DependencyGraphTest, DGNodeRange) {
+  parseIR(C, R"IR(
+define void @foo(ptr %ptr, i8 %v0, i8 %v1) {
+  add i8 %v0, %v0
+  store i8 %v0, ptr %ptr
+  add i8 %v0, %v0
+  store i8 %v1, ptr %ptr
+  ret void
+}
+)IR");
+  llvm::Function *LLVMF = &*M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  auto *F = Ctx.createFunction(LLVMF);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  auto *Add0 = cast<sandboxir::BinaryOperator>(&*It++);
+  auto *S0 = cast<sandboxir::StoreInst>(&*It++);
+  auto *Add1 = cast<sandboxir::BinaryOperator>(&*It++);
+  auto *S1 = cast<sandboxir::StoreInst>(&*It++);
+  auto *Ret = cast<sandboxir::ReturnInst>(&*It++);
+
+  sandboxir::DependencyGraph DAG;
+  DAG.extend({&*BB->begin(), BB->getTerminator()});
+
+  auto *S0N = cast<sandboxir::MemDGNode>(DAG.getNode(S0));
+  auto *S1N = cast<sandboxir::MemDGNode>(DAG.getNode(S1));
+
+  // Check empty range.
+  EXPECT_THAT(sandboxir::MemDGNodeIntervalBuilder::makeEmpty(),
+              testing::ElementsAre());
+
+  // Returns the pointers in Range.
+  auto getPtrVec = [](const auto &Range) {
+    SmallVector<const sandboxir::DGNode *> Vec;
+    for (const sandboxir::DGNode &N : Range)
+      Vec.push_back(&N);
+    return Vec;
+  };
+  // Both TopN and BotN are memory.
+  EXPECT_THAT(
+      getPtrVec(sandboxir::MemDGNodeIntervalBuilder::make({S0, S1}, DAG)),
+      testing::ElementsAre(S0N, S1N));
+  // Only TopN is memory.
+  EXPECT_THAT(
+      getPtrVec(sandboxir::MemDGNodeIntervalBuilder::make({S0, Ret}, DAG)),
+      testing::ElementsAre(S0N, S1N));
+  EXPECT_THAT(
+      getPtrVec(sandboxir::MemDGNodeIntervalBuilder::make({S0, Add1}, DAG)),
+      testing::ElementsAre(S0N));
+  // Only BotN is memory.
+  EXPECT_THAT(
+      getPtrVec(sandboxir::MemDGNodeIntervalBuilder::make({Add0, S1}, DAG)),
+      testing::ElementsAre(S0N, S1N));
+  EXPECT_THAT(
+      getPtrVec(sandboxir::MemDGNodeIntervalBuilder::make({Add0, S0}, DAG)),
+      testing::ElementsAre(S0N));
+  // Neither TopN or BotN is memory.
+  EXPECT_THAT(
+      getPtrVec(sandboxir::MemDGNodeIntervalBuilder::make({Add0, Ret}, DAG)),
+      testing::ElementsAre(S0N, S1N));
+  EXPECT_THAT(
+      getPtrVec(sandboxir::MemDGNodeIntervalBuilder::make({Add0, Add0}, DAG)),
+      testing::ElementsAre());
+}
diff --git a/llvm/utils/TableGen/AsmWriterEmitter.cpp b/llvm/utils/TableGen/AsmWriterEmitter.cpp
index 3f1f937e5fd8a..83205b50f6e28 100644
--- a/llvm/utils/TableGen/AsmWriterEmitter.cpp
+++ b/llvm/utils/TableGen/AsmWriterEmitter.cpp
@@ -597,7 +597,7 @@ emitRegisterNameString(raw_ostream &O, StringRef AltName,
     } else {
       // Make sure the register has an alternate name for this index.
       std::vector<const Record *> AltNameList =
-          Reg.TheDef->getValueAsListOfConstDefs("RegAltNameIndices");
+          Reg.TheDef->getValueAsListOfDefs("RegAltNameIndices");
       unsigned Idx = 0, e;
       for (e = AltNameList.size();
            Idx < e && (AltNameList[Idx]->getName() != AltName); ++Idx)
@@ -1012,7 +1012,7 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) {
       if (PassSubtarget) {
         // We only consider ReqFeatures predicates if PassSubtarget
         std::vector<const Record *> RF =
-            CGA.TheDef->getValueAsListOfConstDefs("Predicates");
+            CGA.TheDef->getValueAsListOfDefs("Predicates");
         copy_if(RF, std::back_inserter(ReqFeatures), [](const Record *R) {
           return R->getValueAsBit("AssemblerMatcherPredicate");
         });
diff --git a/llvm/utils/TableGen/CallingConvEmitter.cpp b/llvm/utils/TableGen/CallingConvEmitter.cpp
index bed5aa86846bf..c8f263e15d96b 100644
--- a/llvm/utils/TableGen/CallingConvEmitter.cpp
+++ b/llvm/utils/TableGen/CallingConvEmitter.cpp
@@ -14,6 +14,7 @@
 #include "Common/CodeGenTarget.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TGTimer.h"
 #include "llvm/TableGen/TableGenBackend.h"
 #include <deque>
 #include <set>
@@ -51,7 +52,7 @@ void CallingConvEmitter::run(raw_ostream &O) {
 
   // Emit prototypes for all of the non-custom CC's so that they can forward ref
   // each other.
-  Records.startTimer("Emit prototypes");
+  Records.getTimer().startTimer("Emit prototypes");
   O << "#ifndef GET_CC_REGISTER_LISTS\n\n";
   for (const Record *CC : CCs) {
     if (!CC->getValueAsBit("Custom")) {
@@ -71,7 +72,7 @@ void CallingConvEmitter::run(raw_ostream &O) {
   }
 
   // Emit each non-custom calling convention description in full.
-  Records.startTimer("Emit full descriptions");
+  Records.getTimer().startTimer("Emit full descriptions");
   for (const Record *CC : CCs) {
     if (!CC->getValueAsBit("Custom")) {
       EmitCallingConv(CC, O);
@@ -110,7 +111,7 @@ void CallingConvEmitter::EmitCallingConv(const Record *CC, raw_ostream &O) {
     const Record *Action = CCActions->getElementAsRecord(i);
     SwiftAction =
         llvm::any_of(Action->getSuperClasses(),
-                     [](const std::pair<Record *, SMRange> &Class) {
+                     [](const std::pair<const Record *, SMRange> &Class) {
                        std::string Name = Class.first->getNameInitAsString();
                        return StringRef(Name).starts_with("CCIfSwift");
                      });
diff --git a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
index dd728da8076a7..751ac3dd0af10 100644
--- a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
@@ -3698,7 +3698,7 @@ static bool hasNullFragReference(DagInit *DI) {
   DefInit *OpDef = dyn_cast<DefInit>(DI->getOperator());
   if (!OpDef)
     return false;
-  Record *Operator = OpDef->getDef();
+  const Record *Operator = OpDef->getDef();
 
   // If this is the null fragment, return true.
   if (Operator->getName() == "null_frag")
diff --git a/llvm/utils/TableGen/Common/CodeGenHwModes.cpp b/llvm/utils/TableGen/Common/CodeGenHwModes.cpp
index fda13b3d8a440..f5b5d3feed7c3 100644
--- a/llvm/utils/TableGen/Common/CodeGenHwModes.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenHwModes.cpp
@@ -39,8 +39,8 @@ LLVM_DUMP_METHOD
 void HwMode::dump() const { dbgs() << Name << ": " << Features << '\n'; }
 
 HwModeSelect::HwModeSelect(const Record *R, CodeGenHwModes &CGH) {
-  std::vector<const Record *> Modes = R->getValueAsListOfConstDefs("Modes");
-  std::vector<const Record *> Objects = R->getValueAsListOfConstDefs("Objects");
+  std::vector<const Record *> Modes = R->getValueAsListOfDefs("Modes");
+  std::vector<const Record *> Objects = R->getValueAsListOfDefs("Objects");
   if (Modes.size() != Objects.size()) {
     PrintError(
         R->getLoc(),
diff --git a/llvm/utils/TableGen/Common/CodeGenInstruction.cpp b/llvm/utils/TableGen/Common/CodeGenInstruction.cpp
index 452b084aa6f7d..7fedc17701c47 100644
--- a/llvm/utils/TableGen/Common/CodeGenInstruction.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenInstruction.cpp
@@ -485,8 +485,8 @@ CodeGenInstruction::CodeGenInstruction(const Record *R)
   isCodeGenOnly = R->getValueAsBit("isCodeGenOnly");
   isPseudo = R->getValueAsBit("isPseudo");
   isMeta = R->getValueAsBit("isMeta");
-  ImplicitDefs = R->getValueAsListOfConstDefs("Defs");
-  ImplicitUses = R->getValueAsListOfConstDefs("Uses");
+  ImplicitDefs = R->getValueAsListOfDefs("Defs");
+  ImplicitUses = R->getValueAsListOfDefs("Uses");
 
   // This flag is only inferred from the pattern.
   hasChain = false;
diff --git a/llvm/utils/TableGen/Common/CodeGenRegisters.cpp b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
index 81a15334d63dc..b53492dafb25b 100644
--- a/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
@@ -80,7 +80,7 @@ void CodeGenSubRegIndex::updateComponents(CodeGenRegBank &RegBank) {
     return;
 
   std::vector<const Record *> Comps =
-      TheDef->getValueAsListOfConstDefs("ComposedOf");
+      TheDef->getValueAsListOfDefs("ComposedOf");
   if (!Comps.empty()) {
     if (Comps.size() != 2)
       PrintFatalError(TheDef->getLoc(),
@@ -93,7 +93,7 @@ void CodeGenSubRegIndex::updateComponents(CodeGenRegBank &RegBank) {
   }
 
   std::vector<const Record *> Parts =
-      TheDef->getValueAsListOfConstDefs("CoveringSubRegIndices");
+      TheDef->getValueAsListOfDefs("CoveringSubRegIndices");
   if (!Parts.empty()) {
     if (Parts.size() < 2)
       PrintFatalError(TheDef->getLoc(),
@@ -169,9 +169,8 @@ CodeGenRegister::CodeGenRegister(const Record *R, unsigned Enum)
 
 void CodeGenRegister::buildObjectGraph(CodeGenRegBank &RegBank) {
   std::vector<const Record *> SRIs =
-      TheDef->getValueAsListOfConstDefs("SubRegIndices");
-  std::vector<const Record *> SRs =
-      TheDef->getValueAsListOfConstDefs("SubRegs");
+      TheDef->getValueAsListOfDefs("SubRegIndices");
+  std::vector<const Record *> SRs = TheDef->getValueAsListOfDefs("SubRegs");
 
   if (SRIs.size() != SRs.size())
     PrintFatalError(TheDef->getLoc(),
@@ -629,7 +628,7 @@ struct TupleExpander : SetTheory::Expander {
   void expand(SetTheory &ST, const Record *Def,
               SetTheory::RecSet &Elts) override {
     std::vector<const Record *> Indices =
-        Def->getValueAsListOfConstDefs("SubRegIndices");
+        Def->getValueAsListOfDefs("SubRegIndices");
     unsigned Dim = Indices.size();
     ListInit *SubRegs = Def->getValueAsListInit("SubRegs");
     if (Dim != SubRegs->size())
@@ -698,9 +697,8 @@ struct TupleExpander : SetTheory::Expander {
                         "Register tuple redefines register '" + Name + "'.");
 
       // Copy Proto super-classes.
-      ArrayRef<std::pair<Record *, SMRange>> Supers = Proto->getSuperClasses();
-      for (const auto &SuperPair : Supers)
-        NewReg->addSuperClass(SuperPair.first, SuperPair.second);
+      for (const auto &[Super, Loc] : Proto->getSuperClasses())
+        NewReg->addSuperClass(Super, Loc);
 
       // Copy Proto fields.
       for (unsigned i = 0, e = Proto->getValues().size(); i != e; ++i) {
@@ -764,8 +762,7 @@ CodeGenRegisterClass::CodeGenRegisterClass(CodeGenRegBank &RegBank,
     : TheDef(R), Name(std::string(R->getName())),
       TopoSigs(RegBank.getNumTopoSigs()), EnumValue(-1), TSFlags(0) {
   GeneratePressureSet = R->getValueAsBit("GeneratePressureSet");
-  std::vector<const Record *> TypeList =
-      R->getValueAsListOfConstDefs("RegTypes");
+  std::vector<const Record *> TypeList = R->getValueAsListOfDefs("RegTypes");
   if (TypeList.empty())
     PrintFatalError(R->getLoc(), "RegTypes list must not be empty!");
   for (unsigned i = 0, e = TypeList.size(); i != e; ++i) {
diff --git a/llvm/utils/TableGen/Common/CodeGenSchedule.cpp b/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
index de2cb67b1f1d5..9c37fbe9c4b42 100644
--- a/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
@@ -310,13 +310,13 @@ static void processSTIPredicate(STIPredicateFunction &Fn,
   // definitions. Each unique opcode will be associated with an OpcodeInfo
   // object.
   for (const Record *Def : Fn.getDefinitions()) {
-    ConstRecVec Classes = Def->getValueAsListOfConstDefs("Classes");
+    ConstRecVec Classes = Def->getValueAsListOfDefs("Classes");
     for (const Record *EC : Classes) {
       const Record *Pred = EC->getValueAsDef("Predicate");
       if (!Predicate2Index.contains(Pred))
         Predicate2Index[Pred] = NumUniquePredicates++;
 
-      ConstRecVec Opcodes = EC->getValueAsListOfConstDefs("Opcodes");
+      ConstRecVec Opcodes = EC->getValueAsListOfDefs("Opcodes");
       for (const Record *Opcode : Opcodes) {
         if (!Opcode2Index.contains(Opcode)) {
           Opcode2Index[Opcode] = OpcodeMappings.size();
@@ -341,14 +341,14 @@ static void processSTIPredicate(STIPredicateFunction &Fn,
   // Construct a OpcodeInfo object for every unique opcode declared by an
   // InstructionEquivalenceClass definition.
   for (const Record *Def : Fn.getDefinitions()) {
-    ConstRecVec Classes = Def->getValueAsListOfConstDefs("Classes");
+    ConstRecVec Classes = Def->getValueAsListOfDefs("Classes");
     const Record *SchedModel = Def->getValueAsDef("SchedModel");
     unsigned ProcIndex = ProcModelMap.find(SchedModel)->second;
     APInt ProcMask(ProcModelMap.size(), 0);
     ProcMask.setBit(ProcIndex);
 
     for (const Record *EC : Classes) {
-      ConstRecVec Opcodes = EC->getValueAsListOfConstDefs("Opcodes");
+      ConstRecVec Opcodes = EC->getValueAsListOfDefs("Opcodes");
 
       std::vector<int64_t> OpIndices =
           EC->getValueAsListOfInts("OperandIndices");
@@ -532,8 +532,8 @@ void CodeGenSchedModels::collectProcModels() {
   ProcModels.reserve(ProcRecords.size() + 1);
 
   // Use idx=0 for NoModel/NoItineraries.
-  Record *NoModelDef = Records.getDef("NoSchedModel");
-  Record *NoItinsDef = Records.getDef("NoItineraries");
+  const Record *NoModelDef = Records.getDef("NoSchedModel");
+  const Record *NoItinsDef = Records.getDef("NoItineraries");
   ProcModels.emplace_back(0, "NoSchedModel", NoModelDef, NoItinsDef);
   ProcModelMap[NoModelDef] = 0;
 
@@ -552,7 +552,7 @@ void CodeGenSchedModels::addProcModel(const Record *ProcDef) {
 
   std::string Name = std::string(ModelKey->getName());
   if (ModelKey->isSubClassOf("SchedMachineModel")) {
-    Record *ItinsDef = ModelKey->getValueAsDef("Itineraries");
+    const Record *ItinsDef = ModelKey->getValueAsDef("Itineraries");
     ProcModels.emplace_back(ProcModels.size(), Name, ModelKey, ItinsDef);
   } else {
     // An itinerary is defined without a machine model. Infer a new model.
@@ -669,14 +669,14 @@ void CodeGenSchedModels::collectSchedRW() {
   for (CodeGenSchedRW &CGRW : SchedWrites) {
     if (!CGRW.IsSequence)
       continue;
-    findRWs(CGRW.TheDef->getValueAsListOfConstDefs("Writes"), CGRW.Sequence,
+    findRWs(CGRW.TheDef->getValueAsListOfDefs("Writes"), CGRW.Sequence,
             /*IsRead=*/false);
   }
   // Initialize Aliases vectors.
   for (const Record *ADef : AliasDefs) {
-    Record *AliasDef = ADef->getValueAsDef("AliasRW");
+    const Record *AliasDef = ADef->getValueAsDef("AliasRW");
     getSchedRW(AliasDef).IsAlias = true;
-    Record *MatchDef = ADef->getValueAsDef("MatchRW");
+    const Record *MatchDef = ADef->getValueAsDef("MatchRW");
     CodeGenSchedRW &RW = getSchedRW(MatchDef);
     if (RW.IsAlias)
       PrintFatalError(ADef->getLoc(), "Cannot Alias an Alias");
@@ -781,7 +781,7 @@ void CodeGenSchedModels::expandRWSeqForProc(
   for (const Record *Rec : SchedWrite.Aliases) {
     const CodeGenSchedRW &AliasRW = getSchedRW(Rec->getValueAsDef("AliasRW"));
     if (Rec->getValueInit("SchedModel")->isComplete()) {
-      Record *ModelDef = Rec->getValueAsDef("SchedModel");
+      const Record *ModelDef = Rec->getValueAsDef("SchedModel");
       if (&getProcModel(ModelDef) != &ProcModel)
         continue;
     }
@@ -854,11 +854,10 @@ void CodeGenSchedModels::collectSchedClasses() {
   // Create a SchedClass for each unique combination of itinerary class and
   // SchedRW list.
   for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
-    Record *ItinDef = Inst->TheDef->getValueAsDef("Itinerary");
+    const Record *ItinDef = Inst->TheDef->getValueAsDef("Itinerary");
     IdxVec Writes, Reads;
     if (!Inst->TheDef->isValueUnset("SchedRW"))
-      findRWs(Inst->TheDef->getValueAsListOfConstDefs("SchedRW"), Writes,
-              Reads);
+      findRWs(Inst->TheDef->getValueAsListOfDefs("SchedRW"), Writes, Reads);
 
     // ProcIdx == 0 indicates the class applies to all processors.
     unsigned SCIdx = addSchedClass(ItinDef, Writes, Reads, /*ProcIndices*/ {0});
@@ -920,8 +919,7 @@ void CodeGenSchedModels::collectSchedClasses() {
                         << InstName);
       IdxVec Writes;
       IdxVec Reads;
-      findRWs(RWDef->getValueAsListOfConstDefs("OperandReadWrites"), Writes,
-              Reads);
+      findRWs(RWDef->getValueAsListOfDefs("OperandReadWrites"), Writes, Reads);
       LLVM_DEBUG({
         for (unsigned WIdx : Writes)
           dbgs() << " " << SchedWrites[WIdx].Name;
@@ -1052,7 +1050,7 @@ void CodeGenSchedModels::createInstRWClass(const Record *InstRWDef) {
         if (OrigNumInstrs == InstDefs.size()) {
           assert(SchedClasses[OldSCIdx].ProcIndices[0] == 0 &&
                  "expected a generic SchedClass");
-          Record *RWModelDef = InstRWDef->getValueAsDef("SchedModel");
+          const Record *RWModelDef = InstRWDef->getValueAsDef("SchedModel");
           // Make sure we didn't already have a InstRW containing this
           // instruction on this model.
           for (const Record *RWD : RWDefs) {
@@ -1130,8 +1128,7 @@ void CodeGenSchedModels::collectProcItins() {
     if (!ProcModel.hasItineraries())
       continue;
 
-    ConstRecVec ItinRecords =
-        ProcModel.ItinsDef->getValueAsListOfConstDefs("IID");
+    ConstRecVec ItinRecords = ProcModel.ItinsDef->getValueAsListOfDefs("IID");
     assert(!ItinRecords.empty() && "ProcModel.hasItineraries is incorrect");
 
     // Populate ItinDefList with Itinerary records.
@@ -1224,8 +1221,7 @@ void CodeGenSchedModels::inferFromItinClass(const Record *ItinClassDef,
     // For all ItinRW entries.
     bool HasMatch = false;
     for (const Record *Rec : PM.ItinRWDefs) {
-      ConstRecVec Matched =
-          Rec->getValueAsListOfConstDefs("MatchedItinClasses");
+      ConstRecVec Matched = Rec->getValueAsListOfDefs("MatchedItinClasses");
       if (!llvm::is_contained(Matched, ItinClassDef))
         continue;
       if (HasMatch)
@@ -1234,8 +1230,7 @@ void CodeGenSchedModels::inferFromItinClass(const Record *ItinClassDef,
                             " in ItinResources for " + PM.ModelName);
       HasMatch = true;
       IdxVec Writes, Reads;
-      findRWs(Rec->getValueAsListOfConstDefs("OperandReadWrites"), Writes,
-              Reads);
+      findRWs(Rec->getValueAsListOfDefs("OperandReadWrites"), Writes, Reads);
       inferFromRW(Writes, Reads, FromClassIdx, PIdx);
     }
   }
@@ -1257,7 +1252,7 @@ void CodeGenSchedModels::inferFromInstRWs(unsigned SCIdx) {
     if (II == IE)
       continue;
     IdxVec Writes, Reads;
-    findRWs(Rec->getValueAsListOfConstDefs("OperandReadWrites"), Writes, Reads);
+    findRWs(Rec->getValueAsListOfDefs("OperandReadWrites"), Writes, Reads);
     unsigned PIdx = getProcModel(Rec->getValueAsDef("SchedModel")).Index;
     inferFromRW(Writes, Reads, SCIdx, PIdx); // May mutate SchedClasses.
     SchedClasses[SCIdx].InstRWProcIndices.insert(PIdx);
@@ -1284,7 +1279,7 @@ struct PredCheck {
   unsigned RWIdx;
   const Record *Predicate;
 
-  PredCheck(bool r, unsigned w, Record *p)
+  PredCheck(bool r, unsigned w, const Record *p)
       : IsRead(r), RWIdx(w), Predicate(p) {}
 };
 
@@ -1323,7 +1318,7 @@ class PredTransitions {
 #endif
 
 private:
-  bool mutuallyExclusive(Record *PredDef, ArrayRef<Record *> Preds,
+  bool mutuallyExclusive(const Record *PredDef, ArrayRef<const Record *> Preds,
                          ArrayRef<PredCheck> Term);
   void getIntersectingVariants(const CodeGenSchedRW &SchedRW, unsigned TransIdx,
                                std::vector<TransVariant> &IntersectingVariants);
@@ -1341,8 +1336,8 @@ class PredTransitions {
 // predicates are not exclusive because the predicates for a given SchedWrite
 // are always checked in the order they are defined in the .td file. Later
 // conditions implicitly negate any prior condition.
-bool PredTransitions::mutuallyExclusive(Record *PredDef,
-                                        ArrayRef<Record *> Preds,
+bool PredTransitions::mutuallyExclusive(const Record *PredDef,
+                                        ArrayRef<const Record *> Preds,
                                         ArrayRef<PredCheck> Term) {
   for (const PredCheck &PC : Term) {
     if (PC.Predicate == PredDef)
@@ -1350,8 +1345,7 @@ bool PredTransitions::mutuallyExclusive(Record *PredDef,
 
     const CodeGenSchedRW &SchedRW = SchedModels.getSchedRW(PC.RWIdx, PC.IsRead);
     assert(SchedRW.HasVariants && "PredCheck must refer to a SchedVariant");
-    ConstRecVec Variants =
-        SchedRW.TheDef->getValueAsListOfConstDefs("Variants");
+    ConstRecVec Variants = SchedRW.TheDef->getValueAsListOfDefs("Variants");
     if (any_of(Variants, [PredDef](const Record *R) {
           return R->getValueAsDef("Predicate") == PredDef;
         })) {
@@ -1388,9 +1382,9 @@ bool PredTransitions::mutuallyExclusive(Record *PredDef,
   return false;
 }
 
-static std::vector<Record *> getAllPredicates(ArrayRef<TransVariant> Variants,
-                                              unsigned ProcId) {
-  std::vector<Record *> Preds;
+static std::vector<const Record *>
+getAllPredicates(ArrayRef<TransVariant> Variants, unsigned ProcId) {
+  std::vector<const Record *> Preds;
   for (auto &Variant : Variants) {
     if (!Variant.VarOrSeqDef->isSubClassOf("SchedVar"))
       continue;
@@ -1412,7 +1406,7 @@ void PredTransitions::getIntersectingVariants(
   if (SchedRW.HasVariants) {
     unsigned VarProcIdx = 0;
     if (SchedRW.TheDef->getValueInit("SchedModel")->isComplete()) {
-      Record *ModelDef = SchedRW.TheDef->getValueAsDef("SchedModel");
+      const Record *ModelDef = SchedRW.TheDef->getValueAsDef("SchedModel");
       VarProcIdx = SchedModels.getProcModel(ModelDef).Index;
     }
     if (VarProcIdx == 0 || VarProcIdx == TransVec[TransIdx].ProcIndex) {
@@ -1431,7 +1425,7 @@ void PredTransitions::getIntersectingVariants(
     // that processor.
     unsigned AliasProcIdx = 0;
     if ((*AI)->getValueInit("SchedModel")->isComplete()) {
-      Record *ModelDef = (*AI)->getValueAsDef("SchedModel");
+      const Record *ModelDef = (*AI)->getValueAsDef("SchedModel");
       AliasProcIdx = SchedModels.getProcModel(ModelDef).Index;
     }
     if (AliasProcIdx && AliasProcIdx != TransVec[TransIdx].ProcIndex)
@@ -1457,13 +1451,13 @@ void PredTransitions::getIntersectingVariants(
     if (AliasProcIdx == 0)
       GenericRW = true;
   }
-  std::vector<Record *> AllPreds =
+  std::vector<const Record *> AllPreds =
       getAllPredicates(Variants, TransVec[TransIdx].ProcIndex);
   for (TransVariant &Variant : Variants) {
     // Don't expand variants if the processor models don't intersect.
     // A zero processor index means any processor.
     if (Variant.VarOrSeqDef->isSubClassOf("SchedVar")) {
-      Record *PredDef = Variant.VarOrSeqDef->getValueAsDef("Predicate");
+      const Record *PredDef = Variant.VarOrSeqDef->getValueAsDef("Predicate");
       if (mutuallyExclusive(PredDef, AllPreds, TransVec[TransIdx].PredTerm))
         continue;
     }
@@ -1495,10 +1489,10 @@ void PredTransitions::pushVariant(const TransVariant &VInfo, bool IsRead) {
   // then the whole transition is specific to this processor.
   IdxVec SelectedRWs;
   if (VInfo.VarOrSeqDef->isSubClassOf("SchedVar")) {
-    Record *PredDef = VInfo.VarOrSeqDef->getValueAsDef("Predicate");
+    const Record *PredDef = VInfo.VarOrSeqDef->getValueAsDef("Predicate");
     Trans.PredTerm.emplace_back(IsRead, VInfo.RWIdx, PredDef);
     ConstRecVec SelectedDefs =
-        VInfo.VarOrSeqDef->getValueAsListOfConstDefs("Selected");
+        VInfo.VarOrSeqDef->getValueAsListOfDefs("Selected");
     SchedModels.findRWs(SelectedDefs, SelectedRWs, IsRead);
   } else {
     assert(VInfo.VarOrSeqDef->isSubClassOf("WriteSequence") &&
@@ -1769,8 +1763,7 @@ bool CodeGenSchedModels::hasSuperGroup(ConstRecVec &SubUnits,
   for (const Record *ProcResourceDef : PM.ProcResourceDefs) {
     if (!ProcResourceDef->isSubClassOf("ProcResGroup"))
       continue;
-    ConstRecVec SuperUnits =
-        ProcResourceDef->getValueAsListOfConstDefs("Resources");
+    ConstRecVec SuperUnits = ProcResourceDef->getValueAsListOfDefs("Resources");
     auto RI = SubUnits.begin(), RE = SubUnits.end();
     for (; RI != RE; ++RI) {
       if (!is_contained(SuperUnits, *RI)) {
@@ -1789,12 +1782,12 @@ void CodeGenSchedModels::verifyProcResourceGroups(CodeGenProcModel &PM) {
     if (!PM.ProcResourceDefs[i]->isSubClassOf("ProcResGroup"))
       continue;
     ConstRecVec CheckUnits =
-        PM.ProcResourceDefs[i]->getValueAsListOfConstDefs("Resources");
+        PM.ProcResourceDefs[i]->getValueAsListOfDefs("Resources");
     for (unsigned j = i + 1; j < e; ++j) {
       if (!PM.ProcResourceDefs[j]->isSubClassOf("ProcResGroup"))
         continue;
       ConstRecVec OtherUnits =
-          PM.ProcResourceDefs[j]->getValueAsListOfConstDefs("Resources");
+          PM.ProcResourceDefs[j]->getValueAsListOfDefs("Resources");
       if (std::find_first_of(CheckUnits.begin(), CheckUnits.end(),
                              OtherUnits.begin(),
                              OtherUnits.end()) != CheckUnits.end()) {
@@ -1833,7 +1826,7 @@ void CodeGenSchedModels::collectRegisterFiles() {
                       "Invalid RegisterFile with zero physical registers");
     }
 
-    ConstRecVec RegisterClasses = RF->getValueAsListOfConstDefs("RegClasses");
+    ConstRecVec RegisterClasses = RF->getValueAsListOfDefs("RegClasses");
     std::vector<int64_t> RegisterCosts = RF->getValueAsListOfInts("RegCosts");
     ListInit *MoveElimInfo = RF->getValueAsListInit("AllowMoveElimination");
     for (unsigned I = 0, E = RegisterClasses.size(); I < E; ++I) {
@@ -1868,11 +1861,10 @@ void CodeGenSchedModels::collectProcResources() {
     // This class may have a default ReadWrite list which can be overriden by
     // InstRW definitions.
     for (const Record *RW : SC.InstRWs) {
-      Record *RWModelDef = RW->getValueAsDef("SchedModel");
+      const Record *RWModelDef = RW->getValueAsDef("SchedModel");
       unsigned PIdx = getProcModel(RWModelDef).Index;
       IdxVec Writes, Reads;
-      findRWs(RW->getValueAsListOfConstDefs("OperandReadWrites"), Writes,
-              Reads);
+      findRWs(RW->getValueAsListOfDefs("OperandReadWrites"), Writes, Reads);
       collectRWResources(Writes, Reads, PIdx);
     }
 
@@ -2016,7 +2008,7 @@ void CodeGenSchedModels::collectItinProcResources(const Record *ItinClassDef) {
     // For all ItinRW entries.
     bool HasMatch = false;
     for (const Record *R : PM.ItinRWDefs) {
-      ConstRecVec Matched = R->getValueAsListOfConstDefs("MatchedItinClasses");
+      ConstRecVec Matched = R->getValueAsListOfDefs("MatchedItinClasses");
       if (!llvm::is_contained(Matched, ItinClassDef))
         continue;
       if (HasMatch)
@@ -2025,7 +2017,7 @@ void CodeGenSchedModels::collectItinProcResources(const Record *ItinClassDef) {
                             " in ItinResources for " + PM.ModelName);
       HasMatch = true;
       IdxVec Writes, Reads;
-      findRWs(R->getValueAsListOfConstDefs("OperandReadWrites"), Writes, Reads);
+      findRWs(R->getValueAsListOfDefs("OperandReadWrites"), Writes, Reads);
       collectRWResources(Writes, Reads, PIdx);
     }
   }
@@ -2192,7 +2184,7 @@ bool CodeGenProcModel::isUnsupported(const CodeGenInstruction &Inst) const {
 
 bool CodeGenProcModel::hasReadOfWrite(const Record *WriteDef) const {
   for (auto &RADef : ReadAdvanceDefs) {
-    ConstRecVec ValidWrites = RADef->getValueAsListOfConstDefs("ValidWrites");
+    ConstRecVec ValidWrites = RADef->getValueAsListOfDefs("ValidWrites");
     if (is_contained(ValidWrites, WriteDef))
       return true;
   }
diff --git a/llvm/utils/TableGen/Common/CodeGenTarget.cpp b/llvm/utils/TableGen/Common/CodeGenTarget.cpp
index 7aa945a3aae06..9883cf5cf35fb 100644
--- a/llvm/utils/TableGen/Common/CodeGenTarget.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenTarget.cpp
@@ -133,7 +133,7 @@ bool CodeGenTarget::getAllowRegisterRenaming() const {
 ///
 const Record *CodeGenTarget::getAsmParser() const {
   std::vector<const Record *> LI =
-      TargetRec->getValueAsListOfConstDefs("AssemblyParsers");
+      TargetRec->getValueAsListOfDefs("AssemblyParsers");
   if (AsmParserNum >= LI.size())
     PrintFatalError("Target does not have an AsmParser #" +
                     Twine(AsmParserNum) + "!");
@@ -145,7 +145,7 @@ const Record *CodeGenTarget::getAsmParser() const {
 ///
 const Record *CodeGenTarget::getAsmParserVariant(unsigned Idx) const {
   std::vector<const Record *> LI =
-      TargetRec->getValueAsListOfConstDefs("AssemblyParserVariants");
+      TargetRec->getValueAsListOfDefs("AssemblyParserVariants");
   if (Idx >= LI.size())
     PrintFatalError("Target does not have an AsmParserVariant #" + Twine(Idx) +
                     "!");
@@ -163,7 +163,7 @@ unsigned CodeGenTarget::getAsmParserVariantCount() const {
 ///
 const Record *CodeGenTarget::getAsmWriter() const {
   std::vector<const Record *> LI =
-      TargetRec->getValueAsListOfConstDefs("AssemblyWriters");
+      TargetRec->getValueAsListOfDefs("AssemblyWriters");
   if (AsmWriterNum >= LI.size())
     PrintFatalError("Target does not have an AsmWriter #" +
                     Twine(AsmWriterNum) + "!");
@@ -407,7 +407,7 @@ ComplexPattern::ComplexPattern(const Record *R) {
   Ty = R->getValueAsDef("Ty");
   NumOperands = R->getValueAsInt("NumOperands");
   SelectFunc = std::string(R->getValueAsString("SelectFunc"));
-  RootNodes = R->getValueAsListOfConstDefs("RootNodes");
+  RootNodes = R->getValueAsListOfDefs("RootNodes");
 
   // FIXME: This is a hack to statically increase the priority of patterns which
   // maps a sub-dag to a complex pattern. e.g. favors LEA over ADD. To get best
diff --git a/llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.h b/llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.h
index 4e519f9bf1846..de6ccd4c023e2 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.h
+++ b/llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.h
@@ -32,7 +32,7 @@ inline bool isSpecificDef(const Init &N, StringRef Def) {
 /// subclass of the given class and coerce it to a def if it is. This is
 /// primarily useful for testing for subclasses of GIDefKind and similar in
 /// DagInit's since DagInit's support any type inside them.
-inline Record *getDefOfSubClass(const Init &N, StringRef Cls) {
+inline const Record *getDefOfSubClass(const Init &N, StringRef Cls) {
   if (const DefInit *OpI = dyn_cast<DefInit>(&N))
     if (OpI->getDef()->isSubClassOf(Cls))
       return OpI->getDef();
diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h
index 315606417fc9e..00fe073057c5c 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h
+++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h
@@ -55,7 +55,7 @@ enum {
   GISF_IgnoreCopies = 0x1,
 };
 
-using GISelFlags = std::uint16_t;
+using GISelFlags = std::uint32_t;
 
 //===- Helper functions ---------------------------------------------------===//
 
@@ -1998,7 +1998,7 @@ class CopyOrAddZeroRegRenderer : public OperandRenderer {
 
 public:
   CopyOrAddZeroRegRenderer(unsigned NewInsnID, StringRef SymbolicName,
-                           Record *ZeroRegisterDef)
+                           const Record *ZeroRegisterDef)
       : OperandRenderer(OR_CopyOrAddZeroReg), NewInsnID(NewInsnID),
         SymbolicName(SymbolicName), ZeroRegisterDef(ZeroRegisterDef) {
     assert(!SymbolicName.empty() && "Cannot copy from an unspecified source");
diff --git a/llvm/utils/TableGen/Common/PredicateExpander.cpp b/llvm/utils/TableGen/Common/PredicateExpander.cpp
index 314e563ba90bb..e54df89937c4a 100644
--- a/llvm/utils/TableGen/Common/PredicateExpander.cpp
+++ b/llvm/utils/TableGen/Common/PredicateExpander.cpp
@@ -297,7 +297,7 @@ void PredicateExpander::expandOpcodeSwitchStatement(
 void PredicateExpander::expandStatement(raw_ostream &OS, const Record *Rec) {
   // Assume that padding has been added by the caller.
   if (Rec->isSubClassOf("MCOpcodeSwitchStatement")) {
-    expandOpcodeSwitchStatement(OS, Rec->getValueAsListOfConstDefs("Cases"),
+    expandOpcodeSwitchStatement(OS, Rec->getValueAsListOfDefs("Cases"),
                                 Rec->getValueAsDef("DefaultCase"));
     return;
   }
diff --git a/llvm/utils/TableGen/CompressInstEmitter.cpp b/llvm/utils/TableGen/CompressInstEmitter.cpp
index f46ceb5174229..5ee02f4fbf499 100644
--- a/llvm/utils/TableGen/CompressInstEmitter.cpp
+++ b/llvm/utils/TableGen/CompressInstEmitter.cpp
@@ -479,7 +479,7 @@ void CompressInstEmitter::evaluateCompressPat(const Record *Rec) {
 
   // Get the target features for the CompressPat.
   std::vector<const Record *> PatReqFeatures;
-  std::vector<const Record *> RF = Rec->getValueAsListOfConstDefs("Predicates");
+  std::vector<const Record *> RF = Rec->getValueAsListOfDefs("Predicates");
   copy_if(RF, std::back_inserter(PatReqFeatures), [](const Record *R) {
     return R->getValueAsBit("AssemblerMatcherPredicate");
   });
@@ -688,7 +688,7 @@ void CompressInstEmitter::emitCompressInstEmitter(raw_ostream &OS,
     // Add Dest instruction required features.
     std::vector<const Record *> ReqFeatures;
     std::vector<const Record *> RF =
-        Dest.TheDef->getValueAsListOfConstDefs("Predicates");
+        Dest.TheDef->getValueAsListOfDefs("Predicates");
     copy_if(RF, std::back_inserter(ReqFeatures), [](const Record *R) {
       return R->getValueAsBit("AssemblerMatcherPredicate");
     });
diff --git a/llvm/utils/TableGen/DAGISelEmitter.cpp b/llvm/utils/TableGen/DAGISelEmitter.cpp
index 2cceb22afdb99..d3b653b0fba27 100644
--- a/llvm/utils/TableGen/DAGISelEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelEmitter.cpp
@@ -16,6 +16,7 @@
 #include "Common/DAGISelMatcher.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TGTimer.h"
 #include "llvm/TableGen/TableGenBackend.h"
 using namespace llvm;
 
@@ -132,7 +133,8 @@ struct PatternSortingPredicate {
 } // End anonymous namespace
 
 void DAGISelEmitter::run(raw_ostream &OS) {
-  Records.startTimer("Parse patterns");
+  TGTimer &Timer = Records.getTimer();
+  Timer.startTimer("Parse patterns");
   emitSourceFileHeader("DAG Instruction Selector for the " +
                            CGP.getTargetInfo().getName().str() + " target",
                        OS);
@@ -163,7 +165,7 @@ void DAGISelEmitter::run(raw_ostream &OS) {
              });
 
   // Add all the patterns to a temporary list so we can sort them.
-  Records.startTimer("Sort patterns");
+  Timer.startTimer("Sort patterns");
   std::vector<const PatternToMatch *> Patterns;
   for (const PatternToMatch &PTM : CGP.ptms())
     Patterns.push_back(&PTM);
@@ -173,7 +175,7 @@ void DAGISelEmitter::run(raw_ostream &OS) {
   llvm::stable_sort(Patterns, PatternSortingPredicate(CGP));
 
   // Convert each variant of each pattern into a Matcher.
-  Records.startTimer("Convert to matchers");
+  Timer.startTimer("Convert to matchers");
   SmallVector<Matcher *, 0> PatternMatchers;
   for (const PatternToMatch *PTM : Patterns) {
     for (unsigned Variant = 0;; ++Variant) {
@@ -187,12 +189,12 @@ void DAGISelEmitter::run(raw_ostream &OS) {
   std::unique_ptr<Matcher> TheMatcher =
       std::make_unique<ScopeMatcher>(std::move(PatternMatchers));
 
-  Records.startTimer("Optimize matchers");
+  Timer.startTimer("Optimize matchers");
   OptimizeMatcher(TheMatcher, CGP);
 
   // Matcher->dump();
 
-  Records.startTimer("Emit matcher table");
+  Timer.startTimer("Emit matcher table");
   EmitMatcherTable(TheMatcher.get(), CGP, OS);
 }
 
diff --git a/llvm/utils/TableGen/DFAPacketizerEmitter.cpp b/llvm/utils/TableGen/DFAPacketizerEmitter.cpp
index 42155e78d0a26..537bee55978bd 100644
--- a/llvm/utils/TableGen/DFAPacketizerEmitter.cpp
+++ b/llvm/utils/TableGen/DFAPacketizerEmitter.cpp
@@ -108,7 +108,7 @@ int DFAPacketizerEmitter::collectAllFuncUnits(
   int totalFUs = 0;
   // Parse functional units for all the itineraries.
   for (const Record *Proc : ProcItinList) {
-    std::vector<const Record *> FUs = Proc->getValueAsListOfConstDefs("FU");
+    std::vector<const Record *> FUs = Proc->getValueAsListOfDefs("FU");
 
     LLVM_DEBUG(dbgs() << "    FU:"
                       << " (" << FUs.size() << " FUs) " << Proc->getName());
@@ -139,7 +139,7 @@ int DFAPacketizerEmitter::collectAllComboFuncs(
   int numCombos = 0;
   for (unsigned i = 0, N = ComboFuncList.size(); i < N; ++i) {
     const Record *Func = ComboFuncList[i];
-    std::vector<const Record *> FUs = Func->getValueAsListOfConstDefs("CFD");
+    std::vector<const Record *> FUs = Func->getValueAsListOfDefs("CFD");
 
     LLVM_DEBUG(dbgs() << "    CFD:" << i << " (" << FUs.size() << " combo FUs) "
                       << Func->getName() << "\n");
@@ -151,7 +151,7 @@ int DFAPacketizerEmitter::collectAllComboFuncs(
       const Record *FuncData = FUs[j];
       const Record *ComboFunc = FuncData->getValueAsDef("TheComboFunc");
       const std::vector<const Record *> FuncList =
-          FuncData->getValueAsListOfConstDefs("FuncList");
+          FuncData->getValueAsListOfDefs("FuncList");
       const std::string &ComboFuncName = std::string(ComboFunc->getName());
       uint64_t ComboBit = FUNameToBitsMap[ComboFuncName];
       uint64_t ComboResources = ComboBit;
diff --git a/llvm/utils/TableGen/DXILEmitter.cpp b/llvm/utils/TableGen/DXILEmitter.cpp
index bab53adbaefe3..06bf7a0c0a837 100644
--- a/llvm/utils/TableGen/DXILEmitter.cpp
+++ b/llvm/utils/TableGen/DXILEmitter.cpp
@@ -116,7 +116,7 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
   }
 
   // Get overload records
-  std::vector<const Record *> Recs = R->getValueAsListOfConstDefs("overloads");
+  std::vector<const Record *> Recs = R->getValueAsListOfDefs("overloads");
 
   // Sort records in ascending order of DXIL version
   AscendingSortByVersion(Recs);
@@ -126,7 +126,7 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
   }
 
   // Get stage records
-  Recs = R->getValueAsListOfConstDefs("stages");
+  Recs = R->getValueAsListOfDefs("stages");
 
   if (Recs.empty()) {
     PrintFatalError(R, Twine("Atleast one specification of valid stage for ") +
@@ -141,7 +141,7 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
   }
 
   // Get attribute records
-  Recs = R->getValueAsListOfConstDefs("attributes");
+  Recs = R->getValueAsListOfDefs("attributes");
 
   // Sort records in ascending order of DXIL version
   AscendingSortByVersion(Recs);
diff --git a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
index cca805ea4ae0c..2524a443f3454 100644
--- a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
@@ -53,6 +53,7 @@
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/StringMatcher.h"
+#include "llvm/TableGen/TGTimer.h"
 #include "llvm/TableGen/TableGenBackend.h"
 #include <cstdint>
 
@@ -2681,7 +2682,7 @@ void GICombinerEmitter::gatherRules(std::vector<RuleMatcher> &ActiveRules,
                                     ArrayRef<const Record *> RulesAndGroups) {
   for (const Record *Rec : RulesAndGroups) {
     if (!Rec->isValueUnset("Rules")) {
-      gatherRules(ActiveRules, Rec->getValueAsListOfConstDefs("Rules"));
+      gatherRules(ActiveRules, Rec->getValueAsListOfDefs("Rules"));
       continue;
     }
 
@@ -2718,16 +2719,17 @@ void GICombinerEmitter::run(raw_ostream &OS) {
   InstructionOpcodeMatcher::initOpcodeValuesMap(Target);
   LLTOperandMatcher::initTypeIDValuesMap();
 
-  Records.startTimer("Gather rules");
+  TGTimer &Timer = Records.getTimer();
+  Timer.startTimer("Gather rules");
   std::vector<RuleMatcher> Rules;
-  gatherRules(Rules, Combiner->getValueAsListOfConstDefs("Rules"));
+  gatherRules(Rules, Combiner->getValueAsListOfDefs("Rules"));
   if (ErrorsPrinted)
     PrintFatalError(Combiner->getLoc(), "Failed to parse one or more rules");
 
   if (StopAfterParse)
     return;
 
-  Records.startTimer("Creating Match Table");
+  Timer.startTimer("Creating Match Table");
   unsigned MaxTemporaries = 0;
   for (const auto &Rule : Rules)
     MaxTemporaries = std::max(MaxTemporaries, Rule.countRendererFns());
@@ -2744,7 +2746,7 @@ void GICombinerEmitter::run(raw_ostream &OS) {
 
   const MatchTable Table = buildMatchTable(Rules);
 
-  Records.startTimer("Emit combiner");
+  Timer.startTimer("Emit combiner");
 
   emitSourceFileHeader(getClassName().str() + " Combiner Match Table", OS);
 
diff --git a/llvm/utils/TableGen/InstrDocsEmitter.cpp b/llvm/utils/TableGen/InstrDocsEmitter.cpp
index d32cfa2354545..5f22fb417198b 100644
--- a/llvm/utils/TableGen/InstrDocsEmitter.cpp
+++ b/llvm/utils/TableGen/InstrDocsEmitter.cpp
@@ -214,7 +214,7 @@ static void EmitInstrDocs(const RecordKeeper &RK, raw_ostream &OS) {
 
     // Predicates.
     std::vector<const Record *> Predicates =
-        II->TheDef->getValueAsListOfConstDefs("Predicates");
+        II->TheDef->getValueAsListOfDefs("Predicates");
     if (!Predicates.empty()) {
       OS << "Predicates: ";
       ListSeparator LS;
diff --git a/llvm/utils/TableGen/InstrInfoEmitter.cpp b/llvm/utils/TableGen/InstrInfoEmitter.cpp
index 5653434ddd682..a7039ff7e31e9 100644
--- a/llvm/utils/TableGen/InstrInfoEmitter.cpp
+++ b/llvm/utils/TableGen/InstrInfoEmitter.cpp
@@ -28,6 +28,7 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TGTimer.h"
 #include "llvm/TableGen/TableGenBackend.h"
 #include <cassert>
 #include <cstdint>
@@ -934,14 +935,15 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
   const Record *InstrInfo = Target.getInstructionSet();
 
   // Collect all of the operand info records.
-  Records.startTimer("Collect operand info");
+  TGTimer &Timer = Records.getTimer();
+  Timer.startTimer("Collect operand info");
   OperandInfoListTy OperandInfoList;
   OperandInfoMapTy OperandInfoMap;
   unsigned OperandInfoSize =
       CollectOperandInfo(OperandInfoList, OperandInfoMap);
 
   // Collect all of the instruction's implicit uses and defs.
-  Records.startTimer("Collect uses/defs");
+  Timer.startTimer("Collect uses/defs");
   std::map<std::vector<const Record *>, unsigned> EmittedLists;
   std::vector<std::vector<const Record *>> ImplicitLists;
   unsigned ImplicitListSize = 0;
@@ -979,7 +981,7 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
   OS << "namespace llvm {\n\n";
 
   // Emit all of the MCInstrDesc records in reverse ENUM ordering.
-  Records.startTimer("Emit InstrDesc records");
+  Timer.startTimer("Emit InstrDesc records");
   OS << "static_assert(sizeof(MCOperandInfo) % sizeof(MCPhysReg) == 0);\n";
   OS << "static constexpr unsigned " << TargetName << "ImpOpBase = sizeof "
      << TargetName << "InstrTable::OperandInfo / (sizeof(MCPhysReg));\n\n";
@@ -998,13 +1000,13 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
   OS << "  }, {\n";
 
   // Emit all of the operand info records.
-  Records.startTimer("Emit operand info");
+  Timer.startTimer("Emit operand info");
   EmitOperandInfo(OS, OperandInfoList);
 
   OS << "  }, {\n";
 
   // Emit all of the instruction's implicit uses and defs.
-  Records.startTimer("Emit uses/defs");
+  Timer.startTimer("Emit uses/defs");
   for (auto &List : ImplicitLists) {
     OS << "    /* " << EmittedLists[List] << " */";
     for (auto &Reg : List)
@@ -1015,7 +1017,7 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
   OS << "  }\n};\n\n";
 
   // Emit the array of instruction names.
-  Records.startTimer("Emit instruction names");
+  Timer.startTimer("Emit instruction names");
   InstrNames.layout();
   InstrNames.emitStringLiteralDef(OS, Twine("extern const char ") + TargetName +
                                           "InstrNameData[]");
@@ -1076,7 +1078,7 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
   }
 
   // MCInstrInfo initialization routine.
-  Records.startTimer("Emit initialization routine");
+  Timer.startTimer("Emit initialization routine");
   OS << "static inline void Init" << TargetName
      << "MCInstrInfo(MCInstrInfo *II) {\n";
   OS << "  II->InitMCInstrInfo(" << TargetName << "Descs.Insts, " << TargetName
@@ -1156,22 +1158,22 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
 
   OS << "#endif // GET_INSTRINFO_CTOR_DTOR\n\n";
 
-  Records.startTimer("Emit operand name mappings");
+  Timer.startTimer("Emit operand name mappings");
   emitOperandNameMappings(OS, Target, NumberedInstructions);
 
-  Records.startTimer("Emit operand type mappings");
+  Timer.startTimer("Emit operand type mappings");
   emitOperandTypeMappings(OS, Target, NumberedInstructions);
 
-  Records.startTimer("Emit logical operand size mappings");
+  Timer.startTimer("Emit logical operand size mappings");
   emitLogicalOperandSizeMappings(OS, TargetName, NumberedInstructions);
 
-  Records.startTimer("Emit logical operand type mappings");
+  Timer.startTimer("Emit logical operand type mappings");
   emitLogicalOperandTypeMappings(OS, TargetName, NumberedInstructions);
 
-  Records.startTimer("Emit helper methods");
+  Timer.startTimer("Emit helper methods");
   emitMCIIHelperMethods(OS, TargetName);
 
-  Records.startTimer("Emit verifier methods");
+  Timer.startTimer("Emit verifier methods");
   emitFeatureVerifier(OS, Target);
 }
 
@@ -1357,11 +1359,12 @@ void InstrInfoEmitter::emitEnums(raw_ostream &OS) {
   OS << "#endif // GET_INSTRINFO_SCHED_ENUM\n\n";
 }
 
-static void EmitInstrInfo(const RecordKeeper &RK, raw_ostream &OS) {
-  RK.startTimer("Analyze DAG patterns");
-  InstrInfoEmitter(RK).run(OS);
-  RK.startTimer("Emit map table");
-  EmitMapTable(RK, OS);
+static void EmitInstrInfo(const RecordKeeper &Records, raw_ostream &OS) {
+  TGTimer &Timer = Records.getTimer();
+  Timer.startTimer("Analyze DAG patterns");
+  InstrInfoEmitter(Records).run(OS);
+  Timer.startTimer("Emit map table");
+  EmitMapTable(Records, OS);
 }
 
 static TableGen::Emitter::Opt X("gen-instr-info", EmitInstrInfo,
diff --git a/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp b/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp
index 6ca2fea41230b..ce509c7ef1aba 100644
--- a/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp
+++ b/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp
@@ -101,7 +101,7 @@ void MacroFusionPredicatorEmitter::emitMacroFusionImpl(
 
   for (const Record *Fusion : Fusions) {
     std::vector<const Record *> Predicates =
-        Fusion->getValueAsListOfConstDefs("Predicates");
+        Fusion->getValueAsListOfDefs("Predicates");
     bool IsCommutable = Fusion->getValueAsBit("IsCommutable");
 
     OS << "bool is" << Fusion->getName() << "(\n";
diff --git a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
index 9e09bdae76fd4..d2d2bd91445a1 100644
--- a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
+++ b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
@@ -15,6 +15,7 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TGTimer.h"
 #include "llvm/TableGen/TableGenBackend.h"
 #include <vector>
 using namespace llvm;
@@ -298,13 +299,14 @@ void PseudoLoweringEmitter::run(raw_ostream &OS) {
   StringRef Classes[] = {"PseudoInstExpansion", "Instruction"};
 
   // Process the pseudo expansion definitions, validating them as we do so.
-  Records.startTimer("Process definitions");
+  TGTimer &Timer = Records.getTimer();
+  Timer.startTimer("Process definitions");
   for (const Record *Inst : Records.getAllDerivedDefinitions(Classes))
     evaluateExpansion(Inst);
 
   // Generate expansion code to lower the pseudo to an MCInst of the real
   // instruction.
-  Records.startTimer("Emit expansion code");
+  Timer.startTimer("Emit expansion code");
   emitLoweringEmitter(OS);
 }
 
diff --git a/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp b/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
index 1fbcba59f964d..23496a37d5ea1 100644
--- a/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
+++ b/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
@@ -172,7 +172,7 @@ static void emitRISCVProcs(const RecordKeeper &RK, raw_ostream &OS) {
   // Iterate on all definition records.
   for (const Record *Rec :
        RK.getAllDerivedDefinitionsIfDefined("RISCVProcessorModel")) {
-    const std::vector<Record *> &Features =
+    const std::vector<const Record *> &Features =
         Rec->getValueAsListOfDefs("Features");
     bool FastScalarUnalignedAccess = any_of(Features, [&](auto &Feature) {
       return Feature->getValueAsString("Name") == "unaligned-scalar-mem";
diff --git a/llvm/utils/TableGen/RegisterBankEmitter.cpp b/llvm/utils/TableGen/RegisterBankEmitter.cpp
index 460f286543b17..a2fcf55e85132 100644
--- a/llvm/utils/TableGen/RegisterBankEmitter.cpp
+++ b/llvm/utils/TableGen/RegisterBankEmitter.cpp
@@ -19,6 +19,7 @@
 #include "llvm/Support/MathExtras.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TGTimer.h"
 #include "llvm/TableGen/TableGenBackend.h"
 
 #define DEBUG_TYPE "register-bank-emitter"
@@ -385,7 +386,8 @@ void RegisterBankEmitter::run(raw_ostream &OS) {
   const CodeGenRegBank &RegisterClassHierarchy = Target.getRegBank();
   const CodeGenHwModes &CGH = Target.getHwModes();
 
-  Records.startTimer("Analyze records");
+  TGTimer &Timer = Records.getTimer();
+  Timer.startTimer("Analyze records");
   std::vector<RegisterBank> Banks;
   for (const auto &V : Records.getAllDerivedDefinitions("RegisterBank")) {
     SmallPtrSet<const CodeGenRegisterClass *, 8> VisitedRCs;
@@ -407,7 +409,7 @@ void RegisterBankEmitter::run(raw_ostream &OS) {
   }
 
   // Warn about ambiguous MIR caused by register bank/class name clashes.
-  Records.startTimer("Warn ambiguous");
+  Timer.startTimer("Warn ambiguous");
   for (const auto &Class : RegisterClassHierarchy.getRegClasses()) {
     for (const auto &Bank : Banks) {
       if (Bank.getName().lower() == StringRef(Class.getName()).lower()) {
@@ -420,7 +422,7 @@ void RegisterBankEmitter::run(raw_ostream &OS) {
     }
   }
 
-  Records.startTimer("Emit output");
+  Timer.startTimer("Emit output");
   emitSourceFileHeader("Register Bank Source Fragments", OS);
   OS << "#ifdef GET_REGBANK_DECLARATIONS\n"
      << "#undef GET_REGBANK_DECLARATIONS\n";
diff --git a/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/llvm/utils/TableGen/RegisterInfoEmitter.cpp
index 7d81a83ef2b0a..371ee75d1b49e 100644
--- a/llvm/utils/TableGen/RegisterInfoEmitter.cpp
+++ b/llvm/utils/TableGen/RegisterInfoEmitter.cpp
@@ -33,6 +33,7 @@
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/SetTheory.h"
+#include "llvm/TableGen/TGTimer.h"
 #include "llvm/TableGen/TableGenBackend.h"
 #include <algorithm>
 #include <cassert>
@@ -1802,16 +1803,17 @@ void RegisterInfoEmitter::runTargetDesc(raw_ostream &OS) {
 }
 
 void RegisterInfoEmitter::run(raw_ostream &OS) {
-  Records.startTimer("Print enums");
+  TGTimer &Timer = Records.getTimer();
+  Timer.startTimer("Print enums");
   runEnums(OS);
 
-  Records.startTimer("Print MC registers");
+  Timer.startTimer("Print MC registers");
   runMCDesc(OS);
 
-  Records.startTimer("Print header fragment");
+  Timer.startTimer("Print header fragment");
   runTargetHeader(OS);
 
-  Records.startTimer("Print target registers");
+  Timer.startTimer("Print target registers");
   runTargetDesc(OS);
 
   if (RegisterInfoDebug)
diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp
index d21e19c060afc..17b84d06fe857 100644
--- a/llvm/utils/TableGen/SubtargetEmitter.cpp
+++ b/llvm/utils/TableGen/SubtargetEmitter.cpp
@@ -284,7 +284,7 @@ unsigned SubtargetEmitter::FeatureKeyValues(raw_ostream &OS,
        << "\"" << CommandLineName << "\", "
        << "\"" << Desc << "\", " << Target << "::" << Name << ", ";
 
-    ConstRecVec ImpliesList = Feature->getValueAsListOfConstDefs("Implies");
+    ConstRecVec ImpliesList = Feature->getValueAsListOfDefs("Implies");
 
     printFeatureMask(OS, ImpliesList, FeatureMap);
 
@@ -320,9 +320,9 @@ unsigned SubtargetEmitter::CPUKeyValues(raw_ostream &OS,
 
   for (const Record *Processor : ProcessorList) {
     StringRef Name = Processor->getValueAsString("Name");
-    ConstRecVec FeatureList = Processor->getValueAsListOfConstDefs("Features");
+    ConstRecVec FeatureList = Processor->getValueAsListOfDefs("Features");
     ConstRecVec TuneFeatureList =
-        Processor->getValueAsListOfConstDefs("TuneFeatures");
+        Processor->getValueAsListOfDefs("TuneFeatures");
 
     // Emit as "{ "cpu", "description", 0, { f1 , f2 , ... fn } },".
     OS << " { "
@@ -354,7 +354,7 @@ void SubtargetEmitter::FormItineraryStageString(const std::string &Name,
                                                 std::string &ItinString,
                                                 unsigned &NStages) {
   // Get states list
-  ConstRecVec StageList = ItinData->getValueAsListOfConstDefs("Stages");
+  ConstRecVec StageList = ItinData->getValueAsListOfDefs("Stages");
 
   // For each stage
   unsigned N = NStages = StageList.size();
@@ -367,7 +367,7 @@ void SubtargetEmitter::FormItineraryStageString(const std::string &Name,
     ItinString += "  { " + itostr(Cycles) + ", ";
 
     // Get unit list
-    ConstRecVec UnitList = Stage->getValueAsListOfConstDefs("Units");
+    ConstRecVec UnitList = Stage->getValueAsListOfDefs("Units");
 
     // For each unit
     for (unsigned j = 0, M = UnitList.size(); j < M;) {
@@ -415,7 +415,7 @@ void SubtargetEmitter::FormItineraryBypassString(const std::string &Name,
                                                  const Record *ItinData,
                                                  std::string &ItinString,
                                                  unsigned NOperandCycles) {
-  ConstRecVec BypassList = ItinData->getValueAsListOfConstDefs("Bypasses");
+  ConstRecVec BypassList = ItinData->getValueAsListOfDefs("Bypasses");
   unsigned N = BypassList.size();
   unsigned i = 0;
   ListSeparator LS;
@@ -445,7 +445,7 @@ void SubtargetEmitter::EmitStageAndOperandCycleData(
     if (!ItinsDefSet.insert(ProcModel.ItinsDef).second)
       continue;
 
-    ConstRecVec FUs = ProcModel.ItinsDef->getValueAsListOfConstDefs("FU");
+    ConstRecVec FUs = ProcModel.ItinsDef->getValueAsListOfDefs("FU");
     if (FUs.empty())
       continue;
 
@@ -459,7 +459,7 @@ void SubtargetEmitter::EmitStageAndOperandCycleData(
 
     OS << "} // end namespace " << Name << "FU\n";
 
-    ConstRecVec BPs = ProcModel.ItinsDef->getValueAsListOfConstDefs("BP");
+    ConstRecVec BPs = ProcModel.ItinsDef->getValueAsListOfDefs("BP");
     if (!BPs.empty()) {
       OS << "\n// Pipeline forwarding paths for itineraries \"" << Name
          << "\"\n"
@@ -1005,7 +1005,7 @@ void SubtargetEmitter::ExpandProcResources(
     const Record *PRDef = PRVec[i];
     ConstRecVec SubResources;
     if (PRDef->isSubClassOf("ProcResGroup"))
-      SubResources = PRDef->getValueAsListOfConstDefs("Resources");
+      SubResources = PRDef->getValueAsListOfDefs("Resources");
     else {
       SubResources.push_back(PRDef);
       PRDef = SchedModels.findProcResUnits(PRDef, PM, PRDef->getLoc());
@@ -1027,7 +1027,7 @@ void SubtargetEmitter::ExpandProcResources(
     for (const Record *PR : PM.ProcResourceDefs) {
       if (PR == PRDef || !PR->isSubClassOf("ProcResGroup"))
         continue;
-      ConstRecVec SuperResources = PR->getValueAsListOfConstDefs("Resources");
+      ConstRecVec SuperResources = PR->getValueAsListOfDefs("Resources");
       ConstRecIter SubI = SubResources.begin(), SubE = SubResources.end();
       for (; SubI != SubE; ++SubI) {
         if (!is_contained(SuperResources, *SubI)) {
@@ -1104,18 +1104,16 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel,
       if (RWDef) {
         Writes.clear();
         Reads.clear();
-        SchedModels.findRWs(
-            RWDef->getValueAsListOfConstDefs("OperandReadWrites"), Writes,
-            Reads);
+        SchedModels.findRWs(RWDef->getValueAsListOfDefs("OperandReadWrites"),
+                            Writes, Reads);
       }
     }
     if (Writes.empty()) {
       // Check this processor's itinerary class resources.
       for (const Record *I : ProcModel.ItinRWDefs) {
-        ConstRecVec Matched =
-            I->getValueAsListOfConstDefs("MatchedItinClasses");
+        ConstRecVec Matched = I->getValueAsListOfDefs("MatchedItinClasses");
         if (is_contained(Matched, SC.ItinClassDef)) {
-          SchedModels.findRWs(I->getValueAsListOfConstDefs("OperandReadWrites"),
+          SchedModels.findRWs(I->getValueAsListOfDefs("OperandReadWrites"),
                               Writes, Reads);
           break;
         }
@@ -1165,8 +1163,7 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel,
         SCDesc.RetireOOO |= WriteRes->getValueAsBit("RetireOOO");
 
         // Create an entry for each ProcResource listed in WriteRes.
-        ConstRecVec PRVec =
-            WriteRes->getValueAsListOfConstDefs("ProcResources");
+        ConstRecVec PRVec = WriteRes->getValueAsListOfDefs("ProcResources");
         std::vector<int64_t> ReleaseAtCycles =
             WriteRes->getValueAsListOfInts("ReleaseAtCycles");
 
@@ -1276,7 +1273,7 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel,
         break;
       }
       ConstRecVec ValidWrites =
-          ReadAdvance->getValueAsListOfConstDefs("ValidWrites");
+          ReadAdvance->getValueAsListOfDefs("ValidWrites");
       IdxVec WriteIDs;
       if (ValidWrites.empty())
         WriteIDs.push_back(0);
diff --git a/llvm/utils/TableGen/X86RecognizableInstr.cpp b/llvm/utils/TableGen/X86RecognizableInstr.cpp
index 4386e8361712b..60fc1d1ecbfab 100644
--- a/llvm/utils/TableGen/X86RecognizableInstr.cpp
+++ b/llvm/utils/TableGen/X86RecognizableInstr.cpp
@@ -154,7 +154,7 @@ RecognizableInstr::RecognizableInstr(DisassemblerTables &tables,
       UID(uid), Spec(&tables.specForUID(uid)) {
   // Check for 64-bit inst which does not require REX
   // FIXME: Is there some better way to check for In64BitMode?
-  for (const Record *Predicate : Rec->getValueAsListOfConstDefs("Predicates")) {
+  for (const Record *Predicate : Rec->getValueAsListOfDefs("Predicates")) {
     if (Predicate->getName().contains("Not64Bit") ||
         Predicate->getName().contains("In32Bit")) {
       Is32Bit = true;
diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
index b861bd010e2b2..8ed600e5629e9 100644
--- a/llvm/utils/UpdateTestChecks/common.py
+++ b/llvm/utils/UpdateTestChecks/common.py
@@ -454,7 +454,7 @@ def collect_original_check_lines(ti: TestInfo, prefix_set: set):
                     continue
 
                 if check_kind == "LABEL":
-                    m = IR_FUNCTION_RE.match(line)
+                    m = IR_FUNCTION_LABEL_RE.match(line)
                     if m is not None:
                         func_name = m.group(1)
                         if (
@@ -572,6 +572,9 @@ def invoke_tool(exe, cmd_args, ir, preprocess_cmd=None, verbose=False):
 )
 
 IR_FUNCTION_RE = re.compile(r'^\s*define\s+(?:internal\s+)?[^@]*@"?([\w.$-]+)"?\s*\(')
+IR_FUNCTION_LABEL_RE = re.compile(
+    r'^\s*(?:define\s+(?:internal\s+)?[^@]*)?@"?([\w.$-]+)"?\s*\('
+)
 TRIPLE_IR_RE = re.compile(r'^\s*target\s+triple\s*=\s*"([^"]+)"$')
 TRIPLE_ARG_RE = re.compile(r"-m?triple[= ]([^ ]+)")
 MARCH_ARG_RE = re.compile(r"-march[= ]([^ ]+)")
@@ -1534,14 +1537,58 @@ def diffify_line(line, mapper):
 
         candidate_matches = find_diff_matching(lhs_lines, rhs_lines)
 
-        # Apply commits greedily on a match-by-match basis
-        matches = [(-1, -1)]
-        committed_anything = False
+        candidate_matches = [
+            (old_begin + lhs_idx, new_begin + rhs_idx)
+            for lhs_idx, rhs_idx in candidate_matches
+        ]
+
+        # Candidate matches may conflict if they require conflicting mappings of
+        # names. We want to determine a large set of compatible candidates,
+        # because that leads to a small diff.
+        #
+        # We think of the candidates as vertices in a conflict graph. The
+        # conflict graph has edges between incompatible candidates. We want to
+        # find a large independent set in this graph.
+        #
+        # Greedily selecting candidates and removing incompatible ones has the
+        # disadvantage that making few bad decisions early on can have huge
+        # consequences.
+        #
+        # Instead, we implicitly compute multiple independent sets by greedily
+        # assigning a *coloring* to the conflict graph. Then, we select the
+        # largest color class (which is the largest independent set we found),
+        # commit to all candidates in it, and recurse.
+        #
+        # Note that we don't actually materialize the conflict graph. Instead,
+        # each color class tracks the information needed to decide implicitly
+        # whether a vertex conflicts (has an edge to) any of the vertices added
+        # to the color class so far.
+        class Color:
+            def __init__(self):
+                # (lhs_idx, rhs_idx) of matches in this color
+                self.matches = []
+
+                # rhs_name -> lhs_name mappings required by this color
+                self.mapping = {}
+
+                # lhs_names committed for this color
+                self.committed = set()
+
+        colors = []
+
         for lhs_idx, rhs_idx in candidate_matches:
             lhs_line = old_line_infos[lhs_idx]
             rhs_line = new_line_infos[rhs_idx]
 
-            local_commits = {}
+            # We scan through the uncommitted names in the candidate line and
+            # filter out the color classes to which the candidate could be
+            # assigned.
+            #
+            # Simultaneously, we prepare a new color class in case the candidate
+            # conflicts with all colors that have been established so far.
+            compatible_colors = colors[:]
+            new_color = Color()
+            new_color.matches.append((lhs_idx, rhs_idx))
 
             for lhs_value, rhs_value in zip(lhs_line.values, rhs_line.values):
                 if new_mapping[rhs_value.name] in committed_names:
@@ -1554,39 +1601,71 @@ def diffify_line(line, mapper):
                     else:
                         break
 
-                if rhs_value.name in local_commits:
+                if rhs_value.name in new_color.mapping:
                     # Same, but for a possible commit happening on the same line
-                    if local_commits[rhs_value.name] == lhs_value.name:
+                    if new_color.color[rhs_value.name] == lhs_value.name:
                         continue
                     else:
                         break
 
-                if lhs_value.name in committed_names:
-                    # We can't map this value because the name we would map it to has already been
-                    # committed for something else. Give up on this line.
+                if (
+                    lhs_value.name in committed_names
+                    or lhs_value.name in new_color.committed
+                ):
+                    # We can't map this value because the name we would map it
+                    # to has already been committed for something else. Give up
+                    # on this line.
                     break
 
-                local_commits[rhs_value.name] = lhs_value.name
-            else:
-                # No reason not to add any commitments for this line
-                for rhs_var, lhs_var in local_commits.items():
-                    new_mapping[rhs_var] = lhs_var
-                    committed_names.add(lhs_var)
-                    committed_anything = True
-
-                    if (
-                        lhs_var != rhs_var
-                        and lhs_var in new_mapping
-                        and new_mapping[lhs_var] == lhs_var
-                    ):
-                        new_mapping[lhs_var] = "conflict_" + lhs_var
+                new_color.mapping[rhs_value.name] = lhs_value.name
+                new_color.committed.add(lhs_value.name)
 
-                matches.append((lhs_idx, rhs_idx))
+                color_idx = 0
+                while color_idx < len(compatible_colors):
+                    color = compatible_colors[color_idx]
+                    compatible = True
+                    if rhs_value.name in color.mapping:
+                        compatible = color.mapping[rhs_value.name] == lhs_value.name
+                    else:
+                        compatible = lhs_value.name not in color.committed
+                    if compatible:
+                        color_idx += 1
+                    else:
+                        del compatible_colors[color_idx]
+            else:
+                # We never broke out of the loop, which means that at a minimum,
+                # this line is viable standalone
+                if compatible_colors:
+                    color = max(compatible_colors, key=lambda color: len(color.matches))
+                    color.mapping.update(new_color.mapping)
+                    color.committed.update(new_color.committed)
+                    color.matches.append((lhs_idx, rhs_idx))
+                else:
+                    colors.append(new_color)
+
+        if colors:
+            # Pick the largest color class. This gives us a large independent
+            # (non-conflicting) set of candidate matches. Assign all names
+            # required by the independent set and recurse.
+            max_color = max(colors, key=lambda color: len(color.matches))
+
+            for rhs_var, lhs_var in max_color.mapping.items():
+                new_mapping[rhs_var] = lhs_var
+                committed_names.add(lhs_var)
+
+                if (
+                    lhs_var != rhs_var
+                    and lhs_var in new_mapping
+                    and new_mapping[lhs_var] == lhs_var
+                ):
+                    new_mapping[lhs_var] = "conflict_" + lhs_var
 
-        matches.append((old_end, new_end))
+            matches = (
+                [(old_begin - 1, new_begin - 1)]
+                + max_color.matches
+                + [(old_end, new_end)]
+            )
 
-        # Recursively handle sequences between matches
-        if committed_anything:
             for (lhs_prev, rhs_prev), (lhs_next, rhs_next) in zip(matches, matches[1:]):
                 recurse(lhs_prev + 1, lhs_next, rhs_prev + 1, rhs_next)
 
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clangd/refactor/tweaks/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clangd/refactor/tweaks/BUILD.gn
index 255dca3a6015a..8d19295b4d3d8 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clangd/refactor/tweaks/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clangd/refactor/tweaks/BUILD.gn
@@ -35,6 +35,7 @@ source_set("tweaks") {
     "RemoveUsingNamespace.cpp",
     "ScopifyEnum.cpp",
     "SpecialMembers.cpp",
+    "SwapBinaryOperands.cpp",
     "SwapIfBranches.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clangd/unittests/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clangd/unittests/BUILD.gn
index b46bac97b41fa..7deefe9dc0613 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clangd/unittests/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clangd/unittests/BUILD.gn
@@ -150,6 +150,7 @@ unittest("ClangdTests") {
     "tweaks/ScopifyEnumTests.cpp",
     "tweaks/ShowSelectionTreeTests.cpp",
     "tweaks/SpecialMembersTests.cpp",
+    "tweaks/SwapBinaryOperandsTests.cpp",
     "tweaks/SwapIfBranchesTests.cpp",
     "tweaks/TweakTesting.cpp",
     "tweaks/TweakTests.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
index 83a10d5b18bc8..bcd36779b85ef 100644
--- a/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
@@ -79,6 +79,7 @@ static_library("Sema") {
     "SemaExprMember.cpp",
     "SemaExprObjC.cpp",
     "SemaFixItUtils.cpp",
+    "SemaFunctionEffects.cpp",
     "SemaHLSL.cpp",
     "SemaHexagon.cpp",
     "SemaInit.cpp",
diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 3e165c3d6f634..d850a7f20952d 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -632,7 +632,6 @@ if (current_toolchain == default_toolchain) {
       "__memory/unique_temporary_buffer.h",
       "__memory/uses_allocator.h",
       "__memory/uses_allocator_construction.h",
-      "__memory/voidify.h",
       "__memory_resource/memory_resource.h",
       "__memory_resource/monotonic_buffer_resource.h",
       "__memory_resource/polymorphic_allocator.h",
diff --git a/llvm/utils/gn/secondary/lldb/source/Plugins/SymbolFile/DWARF/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Plugins/SymbolFile/DWARF/BUILD.gn
index ec99bf0e8ca26..b2f572ad3b668 100644
--- a/llvm/utils/gn/secondary/lldb/source/Plugins/SymbolFile/DWARF/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Plugins/SymbolFile/DWARF/BUILD.gn
@@ -50,7 +50,6 @@ static_library("DWARF") {
     "DWARFContext.cpp",
     "DWARFDIE.cpp",
     "DWARFDataExtractor.cpp",
-    "DWARFDebugArangeSet.cpp",
     "DWARFDebugAranges.cpp",
     "DWARFDebugInfo.cpp",
     "DWARFDebugInfoEntry.cpp",
diff --git a/llvm/utils/gn/secondary/lldb/tools/lldb-server/BUILD.gn b/llvm/utils/gn/secondary/lldb/tools/lldb-server/BUILD.gn
index bb1f3ad20b9d3..65517976ff63d 100644
--- a/llvm/utils/gn/secondary/lldb/tools/lldb-server/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/tools/lldb-server/BUILD.gn
@@ -62,7 +62,6 @@ executable("lldb-server") {
   # Reaches into Plugins/ObjectFile/Mach-O.
   include_dirs = [ "//lldb/source" ]
   sources = [
-    "Acceptor.cpp",
     "LLDBServerUtilities.cpp",
     "SystemInitializerLLGS.cpp",
     "lldb-gdbserver.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/TableGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/TableGen/BUILD.gn
index bd2402bf8d381..d90df7bc0e57a 100644
--- a/llvm/utils/gn/secondary/llvm/lib/TableGen/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/TableGen/BUILD.gn
@@ -12,6 +12,7 @@ static_library("TableGen") {
     "StringMatcher.cpp",
     "TGLexer.cpp",
     "TGParser.cpp",
+    "TGTimer.cpp",
     "TableGenBackend.cpp",
     "TableGenBackendSkeleton.cpp",
   ]
diff --git a/llvm/utils/gn/secondary/llvm/unittests/SandboxIR/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/SandboxIR/BUILD.gn
index 11045f4315877..578b2b18d6ee9 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/SandboxIR/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/SandboxIR/BUILD.gn
@@ -8,6 +8,7 @@ unittest("SandboxIRTests") {
     "//llvm/lib/SandboxIR",
   ]
   sources = [
+    "IntrinsicInstTest.cpp",
     "PassTest.cpp",
     "RegionTest.cpp",
     "SandboxIRTest.cpp",
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index a1785073547ad..00432b8d31778 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -1394,10 +1394,14 @@ def getDefaultSubstitutions(test, tmpDir, tmpBase, normalize_slashes=False):
     substitutions = []
     substitutions.extend(test.config.substitutions)
     tmpName = tmpBase + ".tmp"
-    baseName = os.path.basename(tmpBase)
+    tmpBaseName = os.path.basename(tmpBase)
+    sourceBaseName = os.path.basename(sourcepath)
 
     substitutions.append(("%{pathsep}", os.pathsep))
-    substitutions.append(("%basename_t", baseName))
+    substitutions.append(("%basename_t", tmpBaseName))
+
+    substitutions.append(("%{s:basename}", sourceBaseName))
+    substitutions.append(("%{t:stem}", tmpBaseName))
 
     substitutions.extend(
         [
diff --git a/llvm/utils/lit/tests/substitutions.py b/llvm/utils/lit/tests/substitutions.py
new file mode 100644
index 0000000000000..aa461531628e9
--- /dev/null
+++ b/llvm/utils/lit/tests/substitutions.py
@@ -0,0 +1,7 @@
+# Basic test for substitutions.
+#
+# RUN: echo %{s:basename} | FileCheck %s --check-prefix=BASENAME
+# RUN: echo %{t:stem} %basename_t | FileCheck %s --check-prefix=TMPBASENAME
+
+# BASENAME: substitutions.py
+# TMPBASENAME: [[FIRST:[^[:space:]]+]] [[FIRST]]
diff --git a/mlir/include/mlir-c/BuiltinTypes.h b/mlir/include/mlir-c/BuiltinTypes.h
index 6dc25a56b8e61..6875fab7bf796 100644
--- a/mlir/include/mlir-c/BuiltinTypes.h
+++ b/mlir/include/mlir-c/BuiltinTypes.h
@@ -179,6 +179,16 @@ MLIR_CAPI_EXPORTED bool mlirTypeIsAFloat8E3M4(MlirType type);
 /// context.
 MLIR_CAPI_EXPORTED MlirType mlirFloat8E3M4TypeGet(MlirContext ctx);
 
+/// Returns the typeID of an Float8E8M0FNU type.
+MLIR_CAPI_EXPORTED MlirTypeID mlirFloat8E8M0FNUTypeGetTypeID(void);
+
+/// Checks whether the given type is an f8E8M0FNU type.
+MLIR_CAPI_EXPORTED bool mlirTypeIsAFloat8E8M0FNU(MlirType type);
+
+/// Creates an f8E8M0FNU type in the given context. The type is owned by the
+/// context.
+MLIR_CAPI_EXPORTED MlirType mlirFloat8E8M0FNUTypeGet(MlirContext ctx);
+
 /// Returns the typeID of an BFloat16 type.
 MLIR_CAPI_EXPORTED MlirTypeID mlirBFloat16TypeGetTypeID(void);
 
diff --git a/mlir/include/mlir-c/Dialect/SparseTensor.h b/mlir/include/mlir-c/Dialect/SparseTensor.h
index 125469f57c5f5..c816c1b58690e 100644
--- a/mlir/include/mlir-c/Dialect/SparseTensor.h
+++ b/mlir/include/mlir-c/Dialect/SparseTensor.h
@@ -39,6 +39,7 @@ enum MlirSparseTensorLevelFormat {
 enum MlirSparseTensorLevelPropertyNondefault {
   MLIR_SPARSE_PROPERTY_NON_UNIQUE = 0x0001,
   MLIR_SPARSE_PROPERTY_NON_ORDERED = 0x0002,
+  MLIR_SPARSE_PROPERTY_SOA = 0x0004,
 };
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Analysis/Presburger/PresburgerSpace.h b/mlir/include/mlir/Analysis/Presburger/PresburgerSpace.h
index cff7957989871..97573b6e45301 100644
--- a/mlir/include/mlir/Analysis/Presburger/PresburgerSpace.h
+++ b/mlir/include/mlir/Analysis/Presburger/PresburgerSpace.h
@@ -75,7 +75,7 @@ class Identifier {
   template <typename T>
   explicit Identifier(T value)
       : value(llvm::PointerLikeTypeTraits<T>::getAsVoidPointer(value)) {
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
     idType = llvm::getTypeName<T>();
 #endif
   }
@@ -84,7 +84,7 @@ class Identifier {
   /// the type of the identifier used to create it.
   template <typename T>
   T getValue() const {
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
     assert(llvm::getTypeName<T>() == idType &&
            "Identifier was initialized with a different type than the one used "
            "to retrieve it.");
@@ -108,7 +108,7 @@ class Identifier {
   /// The value of the identifier.
   void *value = nullptr;
 
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
   /// TypeID of the identifiers in space. This should be used in asserts only.
   llvm::StringRef idType;
 #endif
diff --git a/mlir/include/mlir/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.h b/mlir/include/mlir/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.h
index 451c466fa0c95..642e99d963ef6 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.h
+++ b/mlir/include/mlir/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.h
@@ -10,6 +10,7 @@
 #define MLIR_DIALECT_AFFINE_IR_VALUEBOUNDSOPINTERFACEIMPL_H
 
 #include "mlir/Support/LLVM.h"
+#include <cstdint>
 
 namespace mlir {
 class DialectRegistry;
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index 030160821bd82..88e82ce48959b 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -1737,7 +1737,8 @@ def LLVM_ConstantOp
 // Atomic operations.
 //
 
-def LLVM_AtomicRMWType : AnyTypeOf<[LLVM_AnyFloat, LLVM_AnyPointer, AnySignlessInteger]>;
+def LLVM_AtomicRMWType
+    : AnyTypeOf<[LLVM_AnyFloat, LLVM_AnyPointer, AnySignlessInteger, LLVM_AnyFixedVector]>;
 
 def LLVM_AtomicRMWOp : LLVM_MemAccessOpBase<"atomicrmw", [
       TypesMatchWith<"result #0 and operand #1 have the same type",
diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMDialect.h b/mlir/include/mlir/Dialect/LLVMIR/NVVMDialect.h
index 08019e77ae6af..4fd00ff929bd7 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMDialect.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMDialect.h
@@ -35,7 +35,9 @@ enum NVVMMemorySpace {
   /// Global memory space identifier.
   kGlobalMemorySpace = 1,
   /// Shared memory space identifier.
-  kSharedMemorySpace = 3
+  kSharedMemorySpace = 3,
+  /// Constant memory space identifier.
+  kConstantMemorySpace = 4
 };
 
 /// Return the element type and number of elements associated with a wmma matrix
diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
index 2338f7d2da729..f9036cf96e9a1 100644
--- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
@@ -73,6 +73,17 @@ def ApplyTilingCanonicalizationPatternsOp : Op<Transform_Dialect,
   let assemblyFormat = "attr-dict";
 }
 
+def ApplyFoldAddIntoDestPatternsOp : Op<Transform_Dialect,
+    "apply_patterns.linalg.fold_add_into_dest",
+    [DeclareOpInterfaceMethods<PatternDescriptorOpInterface>]> {
+  let description = [{
+    Collects patterns to replace linalg.add when destination passing suffices
+    for achieving the sum.
+  }];
+
+  let assemblyFormat = "attr-dict";
+}
+
 //===----------------------------------------------------------------------===//
 // BufferizeToAllocationOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
index e2bb1c119c3c5..1afe7ff4ff912 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
@@ -762,6 +762,14 @@ LogicalResult copyToGPUPrivateMemory(OpBuilder &b, Value src, Value dst);
 /// memory is freed when going outside of the scope.
 LogicalResult deallocateGPUPrivateMemory(OpBuilder &, Value /*buffer*/);
 
+/// Return true if there's dedicated logic in the Linalg Vectorizer to
+/// vectorize this Op, false otherwise.
+///
+/// Note that this helper merely implements a very high level check and that the
+/// vectorizer also requires various additional pre-conditions to be met for it
+/// to work (these are checked by the vectorizer itself).
+bool hasVectorizationImpl(Operation *);
+
 /// Emit a suitable vector form for an operation. If provided,
 /// `inputVectorSizes` are used to vectorize this operation. `inputVectorSizes`
 /// must match the rank of the iteration space of the operation and the sizes
@@ -1748,6 +1756,10 @@ void populateFoldReshapeOpsByCollapsingPatterns(
 void populateConstantFoldLinalgOperations(RewritePatternSet &patterns,
                                           const ControlFusionFn &controlFn);
 
+/// Pattern to replace `linalg.add` when destination passing on a contraction op
+/// suffices for achieving the sum.
+void populateFoldAddIntoDestPatterns(RewritePatternSet &patterns);
+
 /// Pattern to fuse a `tensor.pad` operation with the producer of its source,
 /// if the producer is a `linalg` operation with all parallel iterator types.
 void populateFuseTensorPadWithProducerLinalgOpPatterns(
diff --git a/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h b/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
index 1c094c1c1328a..ca3326dbbef51 100644
--- a/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
+++ b/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
@@ -106,9 +106,9 @@ inline bool isSameViewOrTrivialAlias(MemrefValue a, MemrefValue b) {
   return skipFullyAliasingOperations(a) == skipFullyAliasingOperations(b);
 }
 
-/// Walk up the source chain until something an op other than a `memref.subview`
-/// or `memref.cast` is found.
-MemrefValue skipSubViewsAndCasts(MemrefValue source);
+/// Walk up the source chain until we find an operation that is not a view of
+/// the source memref (i.e. implements ViewLikeOpInterface).
+MemrefValue skipViewLikeOps(MemrefValue source);
 
 } // namespace memref
 } // namespace mlir
diff --git a/mlir/include/mlir/Dialect/Tensor/Utils/Utils.h b/mlir/include/mlir/Dialect/Tensor/Utils/Utils.h
index 84d06d456bb68..ed1ec1e871482 100644
--- a/mlir/include/mlir/Dialect/Tensor/Utils/Utils.h
+++ b/mlir/include/mlir/Dialect/Tensor/Utils/Utils.h
@@ -14,12 +14,22 @@
 namespace mlir {
 namespace tensor {
 
-// Return a PadOp that pads `source` to `type` size where the static
-// sizes are assumed to be greater than the dynamic sizes. If `type` has dynamic
-// dimensions the padding width is set to zero. The op performs "high" padding
-// (i.e. it adds trailing padding values until the desired size is met).
-PadOp createPadHighOp(RankedTensorType type, Value source, Value pad,
-                      bool nofold, Location loc, OpBuilder &builder);
+// Return a PadOp that pads `source` to `resType` size. The op performs "high"
+// padding, i.e. it adds trailing padding values until the desired size is met.
+// Output sizes are assumed to be greater than the input sizes. The padding
+// width is calculated as: resDim - sourceDim.
+//
+// Handling static sizes is trivial. Dynamic dimensions are trickier (*):
+//  1. dynamic input sizes are extracted from `source`
+//  2. for dynamic output dims, there are two options:
+//    2.1 all output dynamic dim sizes are specified in `dynOutDim`,
+//    2.2 `dynOutDim` is empty and the corresponding padding width is set to 0.
+//
+// (*) Note that `resType` is just a shape and it only encodes the actual sizes
+// for _static_ dimensions.
+PadOp createPadHighOp(RankedTensorType resType, Value source, Value pad,
+                      bool nofold, Location loc, OpBuilder &builder,
+                      SmallVector<Value> dynOutDim = {});
 
 // Creates dim ops for each dynamic dimension of the ranked tensor argument and
 // returns these as values.
diff --git a/mlir/include/mlir/Dialect/Transform/Interfaces/TransformInterfaces.h b/mlir/include/mlir/Dialect/Transform/Interfaces/TransformInterfaces.h
index 43193e4cd4cf6..e51aac02936b5 100644
--- a/mlir/include/mlir/Dialect/Transform/Interfaces/TransformInterfaces.h
+++ b/mlir/include/mlir/Dialect/Transform/Interfaces/TransformInterfaces.h
@@ -196,7 +196,7 @@ class TransformState {
   /// should be emitted when the value is used.
   using InvalidatedHandleMap = DenseMap<Value, std::function<void(Location)>>;
 
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
   /// Debug only: A timestamp is associated with each transform IR value, so
   /// that invalid iterator usage can be detected more reliably.
   using TransformIRTimestampMapping = DenseMap<Value, int64_t>;
@@ -211,7 +211,7 @@ class TransformState {
     ValueMapping values;
     ValueMapping reverseValues;
 
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
     TransformIRTimestampMapping timestamps;
     void incrementTimestamp(Value value) { ++timestamps[value]; }
 #endif // LLVM_ENABLE_ABI_BREAKING_CHECKS
@@ -248,7 +248,7 @@ class TransformState {
   auto getPayloadOps(Value value) const {
     ArrayRef<Operation *> view = getPayloadOpsView(value);
 
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
     // Memorize the current timestamp and make sure that it has not changed
     // when incrementing or dereferencing the iterator returned by this
     // function. The timestamp is incremented when the "direct" mapping is
@@ -259,7 +259,7 @@ class TransformState {
     // When ops are replaced/erased, they are replaced with nullptr (until
     // the data structure is compacted). Do not enumerate these ops.
     return llvm::make_filter_range(view, [=](Operation *op) {
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
       [[maybe_unused]] bool sameTimestamp =
           currentTimestamp == this->getMapping(value).timestamps.lookup(value);
       assert(sameTimestamp && "iterator was invalidated during iteration");
@@ -277,7 +277,7 @@ class TransformState {
   auto getPayloadValues(Value handleValue) const {
     ArrayRef<Value> view = getPayloadValuesView(handleValue);
 
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
     // Memorize the current timestamp and make sure that it has not changed
     // when incrementing or dereferencing the iterator returned by this
     // function. The timestamp is incremented when the "values" mapping is
diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index b6d2cc29cd1bf..45fd1c6e3f938 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -1290,12 +1290,12 @@ def Vector_TransferReadOp :
     specifies if the transfer is guaranteed to be within the source bounds. If
     set to "false", accesses (including the starting point) may run
     out-of-bounds along the respective vector dimension as the index increases.
-    Non-vector and broadcast dimensions *must* always be in-bounds. The
-    `in_bounds` array length has to be equal to the vector rank. This attribute
-    has a default value: `false` (i.e. "out-of-bounds"). When skipped in the
-    textual IR, the default value is assumed. Similarly, the OP printer will
-    omit this attribute when all dimensions are out-of-bounds (i.e. the default
-    value is used).
+    Non-vector dimensions *must* always be in-bounds. The `in_bounds` array
+    length has to be equal to the vector rank. This attribute has a default
+    value: `false` (i.e. "out-of-bounds"). When skipped in the textual IR, the
+    default value is assumed. Similarly, the OP printer will omit this
+    attribute when all dimensions are out-of-bounds (i.e. the default value is
+    used).
 
     A `vector.transfer_read` can be lowered to a simple load if all dimensions
     are specified to be within bounds and no `mask` was specified.
@@ -1535,12 +1535,12 @@ def Vector_TransferWriteOp :
     specifies if the transfer is guaranteed to be within the source bounds. If
     set to "false", accesses (including the starting point) may run
     out-of-bounds along the respective vector dimension as the index increases.
-    Non-vector and broadcast dimensions *must* always be in-bounds. The
-    `in_bounds` array length has to be equal to the vector rank. This attribute
-    has a default value: `false` (i.e. "out-of-bounds"). When skipped in the
-    textual IR, the default value is assumed. Similarly, the OP printer will
-    omit this attribute when all dimensions are out-of-bounds (i.e. the default
-    value is used).
+    Non-vector dimensions *must* always be in-bounds. The `in_bounds` array
+    length has to be equal to the vector rank. This attribute has a default
+    value: `false` (i.e. "out-of-bounds"). When skipped in the textual IR, the
+    default value is assumed. Similarly, the OP printer will omit this
+    attribute when all dimensions are out-of-bounds (i.e. the default value is
+    used).
 
      A `vector.transfer_write` can be lowered to a simple store if all
      dimensions are specified to be within bounds and no `mask` was specified.
diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h
index 7ac0cf77fe59b..d6c51d20571fd 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h
@@ -11,6 +11,7 @@
 
 #include "mlir/Bytecode/BytecodeOpInterface.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/TypeUtilities.h"
diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td
index 26eec0d4f2082..2aaa7fd4221ab 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td
@@ -142,4 +142,36 @@ def XeGPU_FenceScopeAttr:
     let assemblyFormat = "$value";
 }
 
+def XeGPU_SGMapAttr : XeGPUAttr<"SGMap", "sg_map"> {
+  let summary = [{
+    Describes the mapping between work item (WI) and the 2D tensor specified by the tensor descriptor.
+  }];
+  let description = [{
+    To distribute the XeGPU operation to work items, the tensor_desc must be specified with the sg_map
+    attribute at the tensor description creation time.
+    Within the `sg_map`, `wi_layout` specifies the layout of work items,
+    describing the mapping of work items to the tensor.
+    wi_layout[0] x wi_layout[1] must be equal to the total number of work items within a subgroup.
+    `wi_data` specifies the minimum number of data elements assigned to each work item for a single distribution.
+
+    E.g., #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>
+    In this example, the subgroup has 16 work items in wi_layout=[1, 16],
+    each accessing 1 element as specified by wi_data=[1, 1].
+
+    `wi_data[0] * wi_data[1]` can be greater than 1, meaning that each work item operates on multiple elements,
+    which is eventually lowered to "SIMT-flavor" vector, like SPIR-V vector or llvm vector, or packed to a storage data type.
+    The multiple elements indicated by `wi_data` can only be from one dimension and must be contiguous in the memory along either dimension.
+  }];
+  let parameters = (ins
+    ArrayRefParameter<"uint32_t">:$wi_layout,
+    ArrayRefParameter<"uint32_t">:$wi_data);
+
+  let builders = [
+    AttrBuilder<(ins)>
+  ];
+
+  let hasCustomAssemblyFormat = 1;
+  let genVerifyDecl = 1;
+}
+
 #endif // MLIR_DIALECT_XEGPU_IR_XEGPUATTRS_TD
diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
index e24a056de2caf..239ce0aa8e003 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
@@ -424,9 +424,9 @@ def XeGPU_CreateDescOp: XeGPU_Op<"create_tdesc", [Pure, ViewLikeOpInterface]> {
     It accepts the following parameters:
 
     * source: a 1D memref or pointer (uint64_t) represents the flattened memory object.
-    * offsets: a array containing offsets of each access point. Its size
+    * offsets: a vector containing offsets of each access point. Its size
       is fixed to the hardware supportted subgroup size, e.g., 16 on PVC,
-      implying each element in the array corresponds to a work-item (SIMT lane)
+      implying each element in the vector corresponds to a work-item (SIMT lane)
       in the subgroup.
 
     The first dimension of the result TensorDesc corresponds to work-items, so it should
@@ -436,56 +436,59 @@ def XeGPU_CreateDescOp: XeGPU_Op<"create_tdesc", [Pure, ViewLikeOpInterface]> {
     Example 1. It assumes subgroup size is 4, and accesses a[0], a[16], a[32], a[64]
     ```mlir
     %a = memref.alloc() : memref<1024xf32>
-    %1 = xegpu.create_tdesc %a[0, 16, 32, 64]: memref<1024xf32> -> TensorDesc<4xf32>
+    %0 = arith.constant dense<[0, 16, 32, 64]> : vector<4xindex>
+    %1 = xegpu.create_tdesc %a, %0: memref<1024xf32>, vector<4xindex> -> TensorDesc<4xf32>
     ```
 
     Example 2. It assumes subgroup size is 4, and each workitem access 8 elements.
                It will access totally 32 data elements: a[0:7], a[16:23], a[32:39], a[64:71]
     ```mlir
     %0 = memref.alloc() : memref<1024xf32>
-    %1 = xegpu.create_tdesc %0[0, 16, 32, 64] : memref<1024xf32> -> TensorDesc<4x8xf32, chunk_size = 8>
+    %off = arith.constant dense<[0, 16, 32, 64]> : vector<4xindex>
+    %1 = xegpu.create_tdesc %0, %off : memref<1024xf32>, vector<4xindex>
+          -> TensorDesc<4x8xf32, #xegpu.scattered_tdesc_attr<chunk_size = 8>>
     ```
 
     Example 3. It is similar to Example 2, but there is some overlaps among workitems.
                It accesses: a[0:7], a[4:11], a[8:15], a[12:19]
     ```mlir
     %0 = memref.alloc() : memref<1024xf32>
-    %1 = xegpu.create_tdesc %0[0, 4, 8, 12] : memref<1024xf32> -> TensorDesc<4x8xf32, chunk_size = 8>>
+    %off = arith.constant dense<[0, 4, 8, 12]> : vector<4xindex>
+    %1 = xegpu.create_tdesc %0, %off : memref<1024xf32>, vector<4xindex>
+          -> TensorDesc<4x8xf32, #xegpu.scattered_tdesc_attr<chunk_size = 8>>
     ```
   }];
 
   let arguments = (ins XeGPU_BaseAddrType: $source,
-                       Variadic<Index>: $offsets,
-                       DenseI64ArrayAttr: $const_offsets);
+                       XeGPU_OffsetType: $offsets);
   let results = (outs XeGPU_TensorDesc:$TensorDesc);
 
+  let builders = [
+    OpBuilder<(ins "xegpu::TensorDescType": $TensorDesc, "mlir::Value": $source,
+                   "llvm::ArrayRef<OpFoldResult>": $offsets)>,
+    OpBuilder<(ins "xegpu::TensorDescType": $TensorDesc, "mlir::Value": $source,
+                   "llvm::ArrayRef<int64_t>": $offsets)>,
+  ];
+
   let assemblyFormat = [{
-    $source
-    custom<DynamicIndexList>($offsets, $const_offsets)
-    attr-dict `:`  type($source) `->` qualified(type($TensorDesc))
+    $source `,` $offsets attr-dict `:`  type($source) `,` type($offsets) `->` qualified(type($TensorDesc))
   }];
 
-  let extraClassDeclaration = extraBaseClassDeclaration # [{
+  let extraClassDeclaration = [{
     xegpu::TensorDescType getTensorDescType() {
       return getTensorDesc().getType();
     }
 
-    SmallVector<OpFoldResult> getMixedOffsets() {
-      Builder b(getContext());
-      return getMixedValues(getConstOffsets(), getOffsets(), b);
+    mlir::VectorType getOffsetsType() {
+      return getOffsets().getType();
     }
 
     size_t getNumOffsets() {
-      return getMixedOffsets().size();
+      return getOffsetsType().getNumElements();
     }
 
     mlir::Value getViewSource() { return getSource(); }
 
-    OpFoldResult getOffset(unsigned idx) {
-      assert(idx < getNumOffsets() && "Invalid out of bound access.");
-      return getMixedOffsets()[idx];
-    }
-
     unsigned getSourceMemorySpace() {
       auto srcTy = getSource().getType();
       if (auto memrefTy = llvm::dyn_cast<mlir::MemRefType>(srcTy)) {
@@ -550,7 +553,7 @@ def XeGPU_LoadGatherOp : XeGPU_Op<"load", [AllRanksMatch<["value", "TensorDesc"]
     describes the data being loaded at the subgroup level, so its size is
     consistent with the number of work-items in a subgroup. When the chunk size
     is larger than 2, the output vector is a 2D vector, with dim-1 correspoding
-    to work-items, and dim-0 corresponding to the chunk_size loaded by each work-item.
+    to work-items, and dim-0 corresponding to the chunk size loaded by each work-item.
     Specially, there is a transpose effect on the result (as compared to the TensorDesc)
     due to the hardware implementation. Therefore, a transpose attribute is introduced
     on purpose, making sure users are aware of this implicit transformation.
@@ -558,16 +561,25 @@ def XeGPU_LoadGatherOp : XeGPU_Op<"load", [AllRanksMatch<["value", "TensorDesc"]
     The mask operand masks out memory access so that it is safe to pass out-of-boundary
     addresses/offsets as long as they are masked. It applies to slots of SIMD lanes.
 
-  Example:
+  Example 1:
   ```mlir
-    %2 = xegpu.load %1, %0 {transpose,
-                            l1_hint = #xegpu.cache_hint<cached>,
+    %2 = xegpu.load %1, %0 {l1_hint = #xegpu.cache_hint<cached>,
                             l2_hint = #xegpu.cache_hint<uncached>,
                             l3_hint = #xegpu.cache_hint<uncached>}
           : !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<memory_space=global>>,
             vector<16xi1> -> vector<16xf32>
   ```
 
+  Example 2:
+  ```mlir
+    %2 = xegpu.load %1, %0 {transpose,
+                            l1_hint = #xegpu.cache_hint<cached>,
+                            l2_hint = #xegpu.cache_hint<uncached>,
+                            l3_hint = #xegpu.cache_hint<uncached>}
+          : !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<memory_space=global, chunk_size=8>>,
+            vector<16xi1> -> vector<8x16xf32>
+  ```
+
   }];
 
   let arguments = (ins XeGPU_TensorDesc: $TensorDesc,
@@ -610,17 +622,27 @@ def XeGPU_StoreScatterOp : XeGPU_Op<"store", [AllElementCountsMatch<["value", "T
   let description = [{ It (aka. store) stores data to scattered memory locations. The value is
   typically a 1D vector. But when the chunk size of the TensorDesc is larger than 1, it will be
   a 2D vector instead. For the later case, dim-1 of the value correspods to the simd lanes
-  and the dim-0 of the value corresponds to the chunk_size stored per lane. So `store_scatter`
+  and the dim-0 of the value corresponds to the chunk size stored per lane. So `store_scatter`
   has transpose effect, which is similar to `load_gather`. Therefore, a transpose attribute is
   introduced on purpose, making sure users are aware of this implicit transformation.
 
-  Example:
+  Example 1:
   ```mlir
     %3 = xegpu.store %0, %1, %2 {l1_hint = #xegpu.cache_hint<uncached>,
                                  l2_hint = #xegpu.cache_hint<write_back>,
                                  l3_hint = #xegpu.cache_hint<write_through>}
-          : vector<16xf32>, !xegpu.tensor_desc<16xf32, #xegpu.tdesc_attr<scattered=true>>, vector<16xi1>
+          : vector<16xf32>, !xegpu.tensor_desc<16xf32, #xegpu.scattered_tdesc_attr<>>, vector<16xi1>
+  ```
+
+  Example 2:
+  ```mlir
+    %3 = xegpu.store %0, %1, %2 {transpose,
+                                 l1_hint = #xegpu.cache_hint<uncached>,
+                                 l2_hint = #xegpu.cache_hint<write_back>,
+                                 l3_hint = #xegpu.cache_hint<write_through>}
+          : vector<8x16xf32>, !xegpu.tensor_desc<16x8xf32, #xegpu.scattered_tdesc_attr<chunk_size=8>>, vector<16xi1>
   ```
+
   }];
 
   let arguments = (ins
@@ -666,40 +688,39 @@ def XeGPU_UpdateOffsetOp: XeGPU_Op<"update_offset",
 
     Example:
     ```mlir
-      %2 = xegpu.update_offset %1, [32, 32, 32, 32]
-            : !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>
+      %off = arith.constant dense<[32, 32, 32, 32]> : vector<4xindex>
+      %2 = xegpu.update_offset %1, %off :
+              !xegpu.tensor_desc<4x2xf32, #xegpu.scattered_tdesc_attr<>>, vector<4xindex>
     ```
   }];
 
   let arguments = (ins XeGPU_TensorDesc: $TensorDesc,
-                       Variadic<Index>: $offsets,
-                       DenseI64ArrayAttr: $const_offsets);
+                       XeGPU_OffsetType: $offsets);
   let results = (outs XeGPU_TensorDesc: $result);
 
-  let extraClassDeclaration = extraBaseClassDeclaration # [{
+  let builders = [
+    OpBuilder<(ins "mlir::Value": $TensorDesc,
+                   "llvm::ArrayRef<OpFoldResult>": $offsets)>,
+    OpBuilder<(ins "mlir::Value": $TensorDesc,
+                   "llvm::ArrayRef<int64_t>": $offsets)>
+  ];
+
+  let extraClassDeclaration = [{
     xegpu::TensorDescType getTensorDescType() {
       return getTensorDesc().getType();
     }
 
-    SmallVector<OpFoldResult> getMixedOffsets() {
-      Builder b(getContext());
-      return getMixedValues(getConstOffsets(), getOffsets(), b);
+    mlir::VectorType getOffsetsType() {
+      return getOffsets().getType();
     }
 
     size_t getNumOffsets() {
-      return getMixedOffsets().size();
-    }
-
-    OpFoldResult getOffset(unsigned idx) {
-      assert(idx < getNumOffsets() && "Invalid out of bound access.");
-      return getMixedOffsets()[idx];
+      return getOffsetsType().getNumElements();
     }
   }];
 
   let assemblyFormat = [{
-    $TensorDesc `,`
-    custom<DynamicIndexList>($offsets, $const_offsets)
-    attr-dict `:` qualified(type($TensorDesc))
+    $TensorDesc `,` $offsets attr-dict `:` qualified(type($TensorDesc)) `,` type($offsets)
   }];
 }
 
diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td
index 0ce1211664b5b..d09c5c1870d50 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td
@@ -63,7 +63,7 @@ def XeGPU_TensorDesc: XeGPUTypeDef<"TensorDesc", "tensor_desc",
     element-type ::= float-type | integer-type | index-type
     dim-list := (static-dim-list `x`)?
     static-dim-list ::= decimal-literal `x` decimal-literal
-    attr-list = (, memory_space = value)? (, arr_len = value)? (, boundary_check = value)? (, scattered = value)?
+    attr-list = (, memory_space = value)? (, arr_len = value)? (, boundary_check = value)? (, scattered = value)? (, sg_map `<` wi_layout = value, wi_data = value `>`)?
     ```
 
     Examples:
@@ -77,12 +77,16 @@ def XeGPU_TensorDesc: XeGPUTypeDef<"TensorDesc", "tensor_desc",
 
     // A TensorDesc with 8x16 f32 elements for a memory region in shared memory space.
     xegpu.tensor_desc<8x16xf32, #xegpu.tdesc_attr<memory_space = slm>>
+
+    // A TensorDesc with a sg_map
+    xegpu.tensor_desc<8x16xf32, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>>
     ```
   }];
 
   let parameters = (ins ArrayRefParameter<"int64_t">: $shape,
                         "mlir::Type": $elementType,
-                        OptionalParameter<"mlir::Attribute">: $encoding);
+                        OptionalParameter<"mlir::Attribute">: $encoding,
+                        OptionalParameter<"mlir::Attribute">: $sg_map);
 
   let builders = [
     TypeBuilderWithInferredContext<(ins
@@ -90,14 +94,16 @@ def XeGPU_TensorDesc: XeGPUTypeDef<"TensorDesc", "tensor_desc",
       "mlir::Type": $elementType,
       CArg<"int", "1">: $array_length,
       CArg<"bool", "true">: $boundary_check,
-      CArg<"xegpu::MemorySpace", "xegpu::MemorySpace::Global">:$memory_space)>,
+      CArg<"xegpu::MemorySpace", "xegpu::MemorySpace::Global">:$memory_space,
+      CArg<"mlir::Attribute", "mlir::Attribute()">:$sg_map)>,
     TypeBuilderWithInferredContext<(ins
       "llvm::ArrayRef<int64_t>": $shape,
       "mlir::Type": $elementType,
       CArg<"int", "1">: $chunk_size,
-      CArg<"xegpu::MemorySpace", "xegpu::MemorySpace::Global">:$memory_space)>
+      CArg<"xegpu::MemorySpace", "xegpu::MemorySpace::Global">:$memory_space,
+      CArg<"mlir::Attribute", "mlir::Attribute()">:$sg_map)>
   ];
-
+  
   let extraClassDeclaration = [{
     using TensorType::clone;
     using mlir::ShapedType::Trait<TensorDescType>::getElementTypeBitWidth;
@@ -121,6 +127,10 @@ def XeGPU_TensorDesc: XeGPUTypeDef<"TensorDesc", "tensor_desc",
       return llvm::dyn_cast_if_present<ScatterTensorDescAttr>(getEncoding());
     }
 
+    SGMapAttr getSGMapAttr() const {
+      return llvm::dyn_cast_if_present<SGMapAttr>(getSgMap());
+    }
+
     xegpu::MemorySpace getMemorySpace() const {
       auto block_attr = getEncodingAsBlockTensorDescAttr();
       if (block_attr && block_attr.getMemorySpace())
diff --git a/mlir/include/mlir/IR/AttrTypeBase.td b/mlir/include/mlir/IR/AttrTypeBase.td
index d176b36068f7a..cbe4f0d67574b 100644
--- a/mlir/include/mlir/IR/AttrTypeBase.td
+++ b/mlir/include/mlir/IR/AttrTypeBase.td
@@ -281,6 +281,12 @@ class AttrDef<Dialect dialect, string name, list<Trait> traits = [],
   let predicate = CPred<"::llvm::isa<" # cppType # ">($_self)">;
 }
 
+// Provide a LocationAttrDef for dialects to provide their own locations
+// that subclass LocationAttr.
+class LocationAttrDef<Dialect dialect, string name, list<Trait> traits = []>
+    : AttrDef<dialect, name, traits # [NativeAttrTrait<"IsLocation">],
+              "::mlir::LocationAttr">;
+
 // Define a new type, named `name`, belonging to `dialect` that inherits from
 // the given C++ base class.
 class TypeDef<Dialect dialect, string name, list<Trait> traits = [],
diff --git a/mlir/include/mlir/IR/Attributes.h b/mlir/include/mlir/IR/Attributes.h
index 8a077865b51b5..d347013295d5f 100644
--- a/mlir/include/mlir/IR/Attributes.h
+++ b/mlir/include/mlir/IR/Attributes.h
@@ -322,12 +322,19 @@ class AttributeInterface
 // Core AttributeTrait
 //===----------------------------------------------------------------------===//
 
-/// This trait is used to determine if an attribute is mutable or not. It is
-/// attached on an attribute if the corresponding ImplType defines a `mutate`
-/// function with proper signature.
 namespace AttributeTrait {
+/// This trait is used to determine if an attribute is mutable or not. It is
+/// attached on an attribute if the corresponding ConcreteType defines a
+/// `mutate` function with proper signature.
 template <typename ConcreteType>
 using IsMutable = detail::StorageUserTrait::IsMutable<ConcreteType>;
+
+/// This trait is used to determine if an attribute is a location or not. It is
+/// attached to an attribute by the user if they intend the attribute to be used
+/// as a location.
+template <typename ConcreteType>
+struct IsLocation : public AttributeTrait::TraitBase<ConcreteType, IsLocation> {
+};
 } // namespace AttributeTrait
 
 } // namespace mlir.
diff --git a/mlir/include/mlir/IR/Builders.h b/mlir/include/mlir/IR/Builders.h
index ee5d787962530..04a8bddc3cd59 100644
--- a/mlir/include/mlir/IR/Builders.h
+++ b/mlir/include/mlir/IR/Builders.h
@@ -70,6 +70,7 @@ class Builder {
   FloatType getFloat8E4M3FNUZType();
   FloatType getFloat8E4M3B11FNUZType();
   FloatType getFloat8E3M4Type();
+  FloatType getFloat8E8M0FNUType();
   FloatType getBF16Type();
   FloatType getF16Type();
   FloatType getTF32Type();
diff --git a/mlir/include/mlir/IR/BuiltinLocationAttributes.td b/mlir/include/mlir/IR/BuiltinLocationAttributes.td
index 5a72404dea15b..bbe566ce97777 100644
--- a/mlir/include/mlir/IR/BuiltinLocationAttributes.td
+++ b/mlir/include/mlir/IR/BuiltinLocationAttributes.td
@@ -18,7 +18,7 @@ include "mlir/IR/BuiltinDialect.td"
 
 // Base class for Builtin dialect location attributes.
 class Builtin_LocationAttr<string name, list<Trait> traits = []>
-    : AttrDef<Builtin_Dialect, name, traits, "::mlir::LocationAttr"> {
+    : LocationAttrDef<Builtin_Dialect, name, traits> {
   let cppClassName = name;
   let mnemonic = ?;
 }
diff --git a/mlir/include/mlir/IR/BuiltinTypes.h b/mlir/include/mlir/IR/BuiltinTypes.h
index 91e68b4066dd6..25535408f4528 100644
--- a/mlir/include/mlir/IR/BuiltinTypes.h
+++ b/mlir/include/mlir/IR/BuiltinTypes.h
@@ -70,6 +70,7 @@ class FloatType : public Type {
   static FloatType getFloat4E2M1FN(MLIRContext *ctx);
   static FloatType getFloat6E2M3FN(MLIRContext *ctx);
   static FloatType getFloat6E3M2FN(MLIRContext *ctx);
+  static FloatType getFloat8E8M0FNU(MLIRContext *ctx);
 
   /// Methods for support type inquiry through isa, cast, and dyn_cast.
   static bool classof(Type type);
@@ -416,11 +417,12 @@ inline bool BaseMemRefType::isValidElementType(Type type) {
 }
 
 inline bool FloatType::classof(Type type) {
-  return llvm::isa<
-      Float4E2M1FNType, Float6E2M3FNType, Float6E3M2FNType, Float8E5M2Type,
-      Float8E4M3Type, Float8E4M3FNType, Float8E5M2FNUZType, Float8E4M3FNUZType,
-      Float8E4M3B11FNUZType, Float8E3M4Type, BFloat16Type, Float16Type,
-      FloatTF32Type, Float32Type, Float64Type, Float80Type, Float128Type>(type);
+  return llvm::isa<Float4E2M1FNType, Float6E2M3FNType, Float6E3M2FNType,
+                   Float8E5M2Type, Float8E4M3Type, Float8E4M3FNType,
+                   Float8E5M2FNUZType, Float8E4M3FNUZType,
+                   Float8E4M3B11FNUZType, Float8E3M4Type, Float8E8M0FNUType,
+                   BFloat16Type, Float16Type, FloatTF32Type, Float32Type,
+                   Float64Type, Float80Type, Float128Type>(type);
 }
 
 inline FloatType FloatType::getFloat4E2M1FN(MLIRContext *ctx) {
@@ -463,6 +465,10 @@ inline FloatType FloatType::getFloat8E3M4(MLIRContext *ctx) {
   return Float8E3M4Type::get(ctx);
 }
 
+inline FloatType FloatType::getFloat8E8M0FNU(MLIRContext *ctx) {
+  return Float8E8M0FNUType::get(ctx);
+}
+
 inline FloatType FloatType::getBF16(MLIRContext *ctx) {
   return BFloat16Type::get(ctx);
 }
diff --git a/mlir/include/mlir/IR/BuiltinTypes.td b/mlir/include/mlir/IR/BuiltinTypes.td
index b2b41b16beec2..dca228097d782 100644
--- a/mlir/include/mlir/IR/BuiltinTypes.td
+++ b/mlir/include/mlir/IR/BuiltinTypes.td
@@ -296,6 +296,29 @@ def Builtin_Float6E3M2FN : Builtin_FloatType<"Float6E3M2FN", "f6E3M2FN"> {
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// Float8E8M0FNUType
+
+def Builtin_Float8E8M0FNU : Builtin_FloatType<"Float8E8M0FNU", "f8E8M0FNU"> {
+  let summary = "8-bit floating point with 8-bit exponent, no mantissa or sign";
+  let description = [{
+    An 8-bit floating point type with no sign bit, 8 bits exponent and no
+    mantissa. This is not a standard type as defined by IEEE-754; it is intended
+    to be used for representing scaling factors, so it cannot represent zeros
+    and negative numbers. The values it can represent are powers of two in the
+    range [-127,127] and NaN.
+
+      * bit encoding: S0E8M0
+      * exponent bias: 127
+      * infinities: Not supported
+      * NaNs: Supported with all bits set to 1
+      * denormals: Not supported
+
+    Open Compute Project (OCP) microscaling formats (MX) specification:
+    https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // BFloat16Type
 
diff --git a/mlir/include/mlir/IR/CommonTypeConstraints.td b/mlir/include/mlir/IR/CommonTypeConstraints.td
index 211385245555a..48e4c24f83865 100644
--- a/mlir/include/mlir/IR/CommonTypeConstraints.td
+++ b/mlir/include/mlir/IR/CommonTypeConstraints.td
@@ -353,6 +353,8 @@ def F6E2M3FN : Type<CPred<"$_self.isFloat6E2M3FN()">, "f6E2M3FN type">,
                BuildableType<"$_builder.getFloat6E2M3FNType()">;
 def F6E3M2FN : Type<CPred<"$_self.isFloat6E3M2FN()">, "f6E3M2FN type">,
                BuildableType<"$_builder.getFloat6E3M2FNType()">;
+def F8E8M0FNU : Type<CPred<"$_self.isFloat8E8M0FNU()">, "f8E8M0FNU type">,
+                BuildableType<"$_builder.getFloat8E8M0FNUType()">;
 
 def AnyComplex : Type<CPred<"::llvm::isa<::mlir::ComplexType>($_self)">,
                       "complex-type", "::mlir::ComplexType">;
diff --git a/mlir/include/mlir/IR/Location.h b/mlir/include/mlir/IR/Location.h
index 423b4d19b5b94..5eb1bfaf4afcd 100644
--- a/mlir/include/mlir/IR/Location.h
+++ b/mlir/include/mlir/IR/Location.h
@@ -32,7 +32,10 @@ class LocationAttr : public Attribute {
 public:
   using Attribute::Attribute;
 
-  /// Walk all of the locations nested under, and including, the current.
+  /// Walk all of the locations nested directly under, and including, the
+  /// current. This means that if a location is nested under a non-location
+  /// attribute, it will *not* be walked by this method. This walk is performed
+  /// in pre-order to get this behavior.
   WalkResult walk(function_ref<WalkResult(Location)> walkFn);
 
   /// Return an instance of the given location type if one is nested under the
diff --git a/mlir/include/mlir/IR/OpBase.td b/mlir/include/mlir/IR/OpBase.td
index 4481e56615b8b..5c82c041c62ee 100644
--- a/mlir/include/mlir/IR/OpBase.td
+++ b/mlir/include/mlir/IR/OpBase.td
@@ -206,7 +206,9 @@ def AnyRegion : Region<CPred<"true">, "any region">;
 // A region with the given number of blocks.
 class SizedRegion<int numBlocks> : Region<
   CPred<"::llvm::hasNItems($_self, " # numBlocks # ")">,
-  "region with " # numBlocks # " blocks">;
+  "region with " # numBlocks # " blocks"> {
+  int blocks = numBlocks;
+}
 
 // A region with at least the given number of blocks.
 class MinSizedRegion<int numBlocks> : Region<
diff --git a/mlir/include/mlir/IR/PDLPatternMatch.h.inc b/mlir/include/mlir/IR/PDLPatternMatch.h.inc
index 7f82bb1e98f78..3b561dda1a398 100644
--- a/mlir/include/mlir/IR/PDLPatternMatch.h.inc
+++ b/mlir/include/mlir/IR/PDLPatternMatch.h.inc
@@ -623,7 +623,7 @@ template <typename PDLFnT, std::size_t... I>
 void assertArgs(PatternRewriter &rewriter, ArrayRef<PDLValue> values,
                 std::index_sequence<I...>) {
   // We only want to do verification in debug builds, same as with `assert`.
-#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+#ifndef NDEBUG
   using FnTraitsT = llvm::function_traits<PDLFnT>;
   auto errorFn = [&](const Twine &msg) -> LogicalResult {
     llvm::report_fatal_error(msg);
diff --git a/mlir/include/mlir/IR/Types.h b/mlir/include/mlir/IR/Types.h
index 1b52b97f29b5f..acd0f894abbbe 100644
--- a/mlir/include/mlir/IR/Types.h
+++ b/mlir/include/mlir/IR/Types.h
@@ -135,6 +135,7 @@ class Type {
   bool isFloat8E4M3FNUZ() const;
   bool isFloat8E4M3B11FNUZ() const;
   bool isFloat8E3M4() const;
+  bool isFloat8E8M0FNU() const;
   bool isBF16() const;
   bool isF16() const;
   bool isTF32() const;
diff --git a/mlir/include/mlir/Interfaces/VectorInterfaces.td b/mlir/include/mlir/Interfaces/VectorInterfaces.td
index 7ea62c2ae2ab1..be939bad14b7b 100644
--- a/mlir/include/mlir/Interfaces/VectorInterfaces.td
+++ b/mlir/include/mlir/Interfaces/VectorInterfaces.td
@@ -234,12 +234,9 @@ def VectorTransferOpInterface : OpInterface<"VectorTransferOpInterface"> {
       return constExpr && constExpr.getValue() == 0;
     }
 
-    /// Return "true" if the vector transfer dimension `dim` is in-bounds. Also
-    /// return "true" if the dimension is a broadcast dimension. Return "false"
-    /// otherwise.
+    /// Return "true" if the vector transfer dimension `dim` is in-bounds.
+    /// Return "false" otherwise.
     bool isDimInBounds(unsigned dim) {
-      if ($_op.isBroadcastDim(dim))
-        return true;
       auto inBounds = $_op.getInBounds();
       return ::llvm::cast<::mlir::BoolAttr>(inBounds[dim]).getValue();
     }
diff --git a/mlir/include/mlir/TableGen/Attribute.h b/mlir/include/mlir/TableGen/Attribute.h
index d0a9430d4ed6c..62720e74849fc 100644
--- a/mlir/include/mlir/TableGen/Attribute.h
+++ b/mlir/include/mlir/TableGen/Attribute.h
@@ -204,7 +204,7 @@ class EnumAttr : public Attribute {
   std::vector<EnumAttrCase> getAllCases() const;
 
   bool genSpecializedAttr() const;
-  llvm::Record *getBaseAttrClass() const;
+  const llvm::Record *getBaseAttrClass() const;
   StringRef getSpecializedAttrClassName() const;
   bool printBitEnumPrimaryGroups() const;
 };
diff --git a/mlir/include/mlir/TableGen/GenInfo.h b/mlir/include/mlir/TableGen/GenInfo.h
index ef2e12f07df16..9dec0e4f1bd91 100644
--- a/mlir/include/mlir/TableGen/GenInfo.h
+++ b/mlir/include/mlir/TableGen/GenInfo.h
@@ -21,8 +21,8 @@ class RecordKeeper;
 namespace mlir {
 
 /// Generator function to invoke.
-using GenFunction = std::function<bool(const llvm::RecordKeeper &recordKeeper,
-                                       raw_ostream &os)>;
+using GenFunction =
+    std::function<bool(const llvm::RecordKeeper &records, raw_ostream &os)>;
 
 /// Structure to group information about a generator (argument to invoke via
 /// mlir-tblgen, description, and generator function).
@@ -34,9 +34,9 @@ class GenInfo {
       : arg(arg), description(description), generator(std::move(generator)) {}
 
   /// Invokes the generator and returns whether the generator failed.
-  bool invoke(const llvm::RecordKeeper &recordKeeper, raw_ostream &os) const {
+  bool invoke(const llvm::RecordKeeper &records, raw_ostream &os) const {
     assert(generator && "Cannot call generator with null generator");
-    return generator(recordKeeper, os);
+    return generator(records, os);
   }
 
   /// Returns the command line option that may be passed to 'mlir-tblgen' to
diff --git a/mlir/include/mlir/TableGen/Predicate.h b/mlir/include/mlir/TableGen/Predicate.h
index fd5a5a5dc99ae..2eeb253d56b26 100644
--- a/mlir/include/mlir/TableGen/Predicate.h
+++ b/mlir/include/mlir/TableGen/Predicate.h
@@ -104,7 +104,7 @@ class CombinedPred : public Pred {
   const llvm::Record *getCombinerDef() const;
 
   // Get the predicates that are combined by this predicate.
-  std::vector<llvm::Record *> getChildren() const;
+  std::vector<const llvm::Record *> getChildren() const;
 };
 
 // A combined predicate that requires all child predicates of 'CPred' type to
diff --git a/mlir/lib/Analysis/Presburger/PresburgerSpace.cpp b/mlir/lib/Analysis/Presburger/PresburgerSpace.cpp
index e3abbdfeb3f3b..532c464925b1a 100644
--- a/mlir/lib/Analysis/Presburger/PresburgerSpace.cpp
+++ b/mlir/lib/Analysis/Presburger/PresburgerSpace.cpp
@@ -18,9 +18,11 @@ using namespace presburger;
 bool Identifier::isEqual(const Identifier &other) const {
   if (value == nullptr || other.value == nullptr)
     return false;
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
   assert(value != other.value ||
          (value == other.value && idType == other.idType &&
           "Values of Identifiers are equal but their types do not match."));
+#endif
   return value == other.value;
 }
 
diff --git a/mlir/lib/AsmParser/Parser.cpp b/mlir/lib/AsmParser/Parser.cpp
index 83eec3244009d..8f19487d80fa3 100644
--- a/mlir/lib/AsmParser/Parser.cpp
+++ b/mlir/lib/AsmParser/Parser.cpp
@@ -631,7 +631,8 @@ class OperationParser : public Parser {
 
   /// Parse a location alias, that is a sequence looking like: #loc42
   /// The alias may have already be defined or may be defined later, in which
-  /// case an OpaqueLoc is used a placeholder.
+  /// case an OpaqueLoc is used a placeholder. The caller must ensure that the
+  /// token is actually an alias, which means it must not contain a dot.
   ParseResult parseLocationAlias(LocationAttr &loc);
 
   /// This is the structure of a result specifier in the assembly syntax,
@@ -1917,9 +1918,11 @@ class CustomOpAsmParser : public AsmParserImpl<OpAsmParser> {
 
     Token tok = parser.getToken();
 
-    // Check to see if we are parsing a location alias.
-    // Otherwise, we parse the location directly.
-    if (tok.is(Token::hash_identifier)) {
+    // Check to see if we are parsing a location alias. We are parsing a
+    // location alias if the token is a hash identifier *without* a dot in it -
+    // the dot signifies a dialect attribute. Otherwise, we parse the location
+    // directly.
+    if (tok.is(Token::hash_identifier) && !tok.getSpelling().contains('.')) {
       if (parser.parseLocationAlias(directLoc))
         return failure();
     } else if (parser.parseLocationInstance(directLoc)) {
@@ -2086,11 +2089,9 @@ ParseResult OperationParser::parseLocationAlias(LocationAttr &loc) {
   Token tok = getToken();
   consumeToken(Token::hash_identifier);
   StringRef identifier = tok.getSpelling().drop_front();
-  if (identifier.contains('.')) {
-    return emitError(tok.getLoc())
-           << "expected location, but found dialect attribute: '#" << identifier
-           << "'";
-  }
+  assert(!identifier.contains('.') &&
+         "unexpected dialect attribute token, expected alias");
+
   if (state.asmState)
     state.asmState->addAttrAliasUses(identifier, tok.getLocRange());
 
@@ -2120,10 +2121,11 @@ OperationParser::parseTrailingLocationSpecifier(OpOrArgument opOrArgument) {
     return failure();
   Token tok = getToken();
 
-  // Check to see if we are parsing a location alias.
-  // Otherwise, we parse the location directly.
+  // Check to see if we are parsing a location alias. We are parsing a location
+  // alias if the token is a hash identifier *without* a dot in it - the dot
+  // signifies a dialect attribute. Otherwise, we parse the location directly.
   LocationAttr directLoc;
-  if (tok.is(Token::hash_identifier)) {
+  if (tok.is(Token::hash_identifier) && !tok.getSpelling().contains('.')) {
     if (parseLocationAlias(directLoc))
       return failure();
   } else if (parseLocationInstance(directLoc)) {
diff --git a/mlir/lib/AsmParser/TokenKinds.def b/mlir/lib/AsmParser/TokenKinds.def
index 2b29177b7dff0..49da8c3dea5fa 100644
--- a/mlir/lib/AsmParser/TokenKinds.def
+++ b/mlir/lib/AsmParser/TokenKinds.def
@@ -104,6 +104,7 @@ TOK_KEYWORD(f8E3M4)
 TOK_KEYWORD(f4E2M1FN)
 TOK_KEYWORD(f6E2M3FN)
 TOK_KEYWORD(f6E3M2FN)
+TOK_KEYWORD(f8E8M0FNU)
 TOK_KEYWORD(f128)
 TOK_KEYWORD(false)
 TOK_KEYWORD(floordiv)
diff --git a/mlir/lib/AsmParser/TypeParser.cpp b/mlir/lib/AsmParser/TypeParser.cpp
index 60903a86ff8ce..c614eb39b364b 100644
--- a/mlir/lib/AsmParser/TypeParser.cpp
+++ b/mlir/lib/AsmParser/TypeParser.cpp
@@ -49,6 +49,7 @@ OptionalParseResult Parser::parseOptionalType(Type &type) {
   case Token::kw_f8E4M3FNUZ:
   case Token::kw_f8E4M3B11FNUZ:
   case Token::kw_f8E3M4:
+  case Token::kw_f8E8M0FNU:
   case Token::kw_bf16:
   case Token::kw_f16:
   case Token::kw_tf32:
@@ -336,6 +337,9 @@ Type Parser::parseNonFunctionType() {
   case Token::kw_f8E3M4:
     consumeToken(Token::kw_f8E3M4);
     return builder.getFloat8E3M4Type();
+  case Token::kw_f8E8M0FNU:
+    consumeToken(Token::kw_f8E8M0FNU);
+    return builder.getFloat8E8M0FNUType();
   case Token::kw_bf16:
     consumeToken(Token::kw_bf16);
     return builder.getBF16Type();
diff --git a/mlir/lib/Bindings/Python/DialectSparseTensor.cpp b/mlir/lib/Bindings/Python/DialectSparseTensor.cpp
index 584981cfe99bf..a730bf500be98 100644
--- a/mlir/lib/Bindings/Python/DialectSparseTensor.cpp
+++ b/mlir/lib/Bindings/Python/DialectSparseTensor.cpp
@@ -33,7 +33,8 @@ static void populateDialectSparseTensorSubmodule(const py::module &m) {
   py::enum_<MlirSparseTensorLevelPropertyNondefault>(m, "LevelProperty",
                                                      py::module_local())
       .value("non_ordered", MLIR_SPARSE_PROPERTY_NON_ORDERED)
-      .value("non_unique", MLIR_SPARSE_PROPERTY_NON_UNIQUE);
+      .value("non_unique", MLIR_SPARSE_PROPERTY_NON_UNIQUE)
+      .value("soa", MLIR_SPARSE_PROPERTY_SOA);
 
   mlir_attribute_subclass(m, "EncodingAttr",
                           mlirAttributeIsASparseTensorEncodingAttr)
diff --git a/mlir/lib/Bindings/Python/IRTypes.cpp b/mlir/lib/Bindings/Python/IRTypes.cpp
index 5a369b5d4938c..6f192bc4bffee 100644
--- a/mlir/lib/Bindings/Python/IRTypes.cpp
+++ b/mlir/lib/Bindings/Python/IRTypes.cpp
@@ -331,6 +331,27 @@ class PyFloat8E3M4Type : public PyConcreteType<PyFloat8E3M4Type, PyFloatType> {
   }
 };
 
+/// Floating Point Type subclass - Float8E8M0FNUType.
+class PyFloat8E8M0FNUType
+    : public PyConcreteType<PyFloat8E8M0FNUType, PyFloatType> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirTypeIsAFloat8E8M0FNU;
+  static constexpr GetTypeIDFunctionTy getTypeIdFunction =
+      mlirFloat8E8M0FNUTypeGetTypeID;
+  static constexpr const char *pyClassName = "Float8E8M0FNUType";
+  using PyConcreteType::PyConcreteType;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](DefaultingPyMlirContext context) {
+          MlirType t = mlirFloat8E8M0FNUTypeGet(context->get());
+          return PyFloat8E8M0FNUType(context->getRef(), t);
+        },
+        py::arg("context") = py::none(), "Create a float8_e8m0fnu type.");
+  }
+};
+
 /// Floating Point Type subclass - BF16Type.
 class PyBF16Type : public PyConcreteType<PyBF16Type, PyFloatType> {
 public:
@@ -953,6 +974,7 @@ void mlir::python::populateIRTypes(py::module &m) {
   PyFloat8E4M3B11FNUZType::bind(m);
   PyFloat8E5M2FNUZType::bind(m);
   PyFloat8E3M4Type::bind(m);
+  PyFloat8E8M0FNUType::bind(m);
   PyBF16Type::bind(m);
   PyF16Type::bind(m);
   PyTF32Type::bind(m);
diff --git a/mlir/lib/CAPI/Dialect/SparseTensor.cpp b/mlir/lib/CAPI/Dialect/SparseTensor.cpp
index f2a0ab33c0224..cf25b5263678f 100644
--- a/mlir/lib/CAPI/Dialect/SparseTensor.cpp
+++ b/mlir/lib/CAPI/Dialect/SparseTensor.cpp
@@ -36,7 +36,9 @@ static_assert(
 static_assert(static_cast<int>(MLIR_SPARSE_PROPERTY_NON_ORDERED) ==
                       static_cast<int>(LevelPropNonDefault::Nonordered) &&
                   static_cast<int>(MLIR_SPARSE_PROPERTY_NON_UNIQUE) ==
-                      static_cast<int>(LevelPropNonDefault::Nonunique),
+                      static_cast<int>(LevelPropNonDefault::Nonunique) &&
+                  static_cast<int>(MLIR_SPARSE_PROPERTY_SOA) ==
+                      static_cast<int>(LevelPropNonDefault::SoA),
               "MlirSparseTensorLevelProperty (C-API) and "
               "LevelPropertyNondefault (C++) mismatch");
 
diff --git a/mlir/lib/CAPI/IR/BuiltinTypes.cpp b/mlir/lib/CAPI/IR/BuiltinTypes.cpp
index efc1e857a39c7..252ff54afe0c5 100644
--- a/mlir/lib/CAPI/IR/BuiltinTypes.cpp
+++ b/mlir/lib/CAPI/IR/BuiltinTypes.cpp
@@ -205,6 +205,18 @@ MlirType mlirFloat8E3M4TypeGet(MlirContext ctx) {
   return wrap(FloatType::getFloat8E3M4(unwrap(ctx)));
 }
 
+MlirTypeID mlirFloat8E8M0FNUTypeGetTypeID() {
+  return wrap(Float8E8M0FNUType::getTypeID());
+}
+
+bool mlirTypeIsAFloat8E8M0FNU(MlirType type) {
+  return unwrap(type).isFloat8E8M0FNU();
+}
+
+MlirType mlirFloat8E8M0FNUTypeGet(MlirContext ctx) {
+  return wrap(FloatType::getFloat8E8M0FNU(unwrap(ctx)));
+}
+
 MlirTypeID mlirBFloat16TypeGetTypeID() {
   return wrap(BFloat16Type::getTypeID());
 }
diff --git a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
index 1886dfa870961..d551506485a45 100644
--- a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
@@ -139,8 +139,6 @@ void ConvertToLLVMPattern::getMemRefDescriptorSizes(
     strides[i] = runningStride;
 
     int64_t staticSize = memRefType.getShape()[i];
-    if (staticSize == 0)
-      continue;
     bool useSizeAsStride = stride == 1;
     if (staticSize == ShapedType::kDynamic)
       stride = ShapedType::kDynamic;
diff --git a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
index fd6369b5bb4ee..5a92fa839e984 100644
--- a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
@@ -250,7 +250,8 @@ Type LLVMTypeConverter::convertFloatType(FloatType type) const {
   if (type.isFloat8E5M2() || type.isFloat8E4M3() || type.isFloat8E4M3FN() ||
       type.isFloat8E5M2FNUZ() || type.isFloat8E4M3FNUZ() ||
       type.isFloat8E4M3B11FNUZ() || type.isFloat8E3M4() ||
-      type.isFloat4E2M1FN() || type.isFloat6E2M3FN() || type.isFloat6E3M2FN())
+      type.isFloat4E2M1FN() || type.isFloat6E2M3FN() || type.isFloat6E3M2FN() ||
+      type.isFloat8E8M0FNU())
     return IntegerType::get(&getContext(), type.getWidth());
   return type;
 }
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
index 5bbf24ffdbe14..638b99e7602d6 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
@@ -88,15 +88,14 @@ linalgIntBroadcastExtSIAdd(PatternRewriter &rewriter, Location loc, Value bias,
       .getResult(0);
 }
 
-// Broadcast the source value to all the outer dimensions of the result value.
-// If required, the element type is expanded using an arith.extsi operation.
-static mlir::Value linalgBroadcastAndMaybeExtSI(PatternRewriter &rewriter,
-                                                Location loc, Value source,
-                                                Value result) {
+// Construct the affine map that a linalg generic would use to broadcast the
+// source tensor into the shape of the result tensor.
+static AffineMap getBroadcastingMap(PatternRewriter &rewriter, Value source,
+                                    Value result) {
   ShapedType resultTy = cast<ShapedType>(result.getType());
   ShapedType sourceTy = cast<ShapedType>(source.getType());
-  int64_t resultRank = resultTy.getRank();
-  int64_t sourceRank = sourceTy.getRank();
+  const int64_t resultRank = resultTy.getRank();
+  const int64_t sourceRank = sourceTy.getRank();
 
   // The source tensor is broadcast to all the outer dimensions of the
   // result tensor.
@@ -115,14 +114,21 @@ static mlir::Value linalgBroadcastAndMaybeExtSI(PatternRewriter &rewriter,
     }
   }
 
-  // Creating maps for the input and output of the broacast-like generic op.
-  SmallVector<AffineMap, 2> indexingMaps = {
-      // Broadcast the last dimension of the bias to all output dimensions.
-      AffineMap::get(/*dimCount=*/resultRank,
-                     /*symbolCount=*/0, sourceDims, rewriter.getContext()),
+  return AffineMap::get(/*dimCount=*/resultRank,
+                        /*symbolCount=*/0, sourceDims, rewriter.getContext());
+}
 
-      // Output indexing map.
-      rewriter.getMultiDimIdentityMap(resultRank)};
+// Broadcast the source value to all the outer dimensions of the result value.
+// If required, the element type is expanded using an arith.extsi operation.
+static mlir::Value linalgBroadcastAndMaybeExtSI(PatternRewriter &rewriter,
+                                                Location loc, Value source,
+                                                Value result) {
+  ShapedType resultTy = cast<ShapedType>(result.getType());
+  const int64_t resultRank = resultTy.getRank();
+  // Creating maps for the input and output of the broacast-like generic op.
+  SmallVector<AffineMap, 2> indexingMaps;
+  indexingMaps.push_back(getBroadcastingMap(rewriter, source, result));
+  indexingMaps.push_back(rewriter.getMultiDimIdentityMap(resultRank));
 
   // Build the broadcast-like operation as a linalg.generic.
   return rewriter
@@ -517,14 +523,6 @@ class DepthwiseConvConverter
                                weightShape[2], weightShape[3]},
                               resultETy);
 
-    // Broadcast the initial value to the output tensor before convolving.
-    SmallVector<AffineMap, 4> indexingMaps;
-    indexingMaps.push_back(AffineMap::get(
-        /*dimCount=*/resultRank, /*symbolCount=*/0,
-        {rewriter.getAffineDimExpr(3)}, rewriter.getContext()));
-    indexingMaps.push_back(rewriter.getMultiDimIdentityMap(resultRank));
-    indexingMaps.push_back(rewriter.getMultiDimIdentityMap(resultRank));
-
     auto resultZeroAttr = rewriter.getZeroAttr(resultETy);
     Value emptyTensor = rewriter.create<tensor::EmptyOp>(
         loc, linalgConvTy.getShape(), resultETy, filteredDims);
@@ -536,6 +534,13 @@ class DepthwiseConvConverter
 
     Value biasEmptyTensor = rewriter.create<tensor::EmptyOp>(
         loc, resultTy.getShape(), resultETy, filteredDims);
+
+    // Broadcast the initial value to the output tensor before convolving.
+    SmallVector<AffineMap, 4> indexingMaps;
+    indexingMaps.push_back(getBroadcastingMap(rewriter, bias, biasEmptyTensor));
+    indexingMaps.push_back(rewriter.getMultiDimIdentityMap(resultRank));
+    indexingMaps.push_back(rewriter.getMultiDimIdentityMap(resultRank));
+
     if (!isQuantized) {
       Value conv = rewriter
                        .create<linalg::DepthwiseConv2DNhwcHwcmOp>(
diff --git a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
index 0e21e96cc3fbb..e9acda657b3c2 100644
--- a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
+++ b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
@@ -33,6 +33,7 @@ using namespace mlir;
 
 namespace {
 
+// Return true if value represents a zero constant.
 static bool isZeroConstant(Value val) {
   auto constant = val.getDefiningOp<arith::ConstantOp>();
   if (!constant)
@@ -46,6 +47,17 @@ static bool isZeroConstant(Value val) {
       .Default([](auto) { return false; });
 }
 
+static LogicalResult storeLoadPreconditions(PatternRewriter &rewriter,
+                                            Operation *op, VectorType vecTy) {
+  // Validate only vector as the basic vector store and load ops guarantee
+  // XeGPU-compatible memref source.
+  unsigned vecRank = vecTy.getRank();
+  if (!(vecRank == 1 || vecRank == 2))
+    return rewriter.notifyMatchFailure(op, "Expects 1D or 2D vector");
+
+  return success();
+}
+
 static LogicalResult transferPreconditions(PatternRewriter &rewriter,
                                            VectorTransferOpInterface xferOp) {
   if (xferOp.getMask())
@@ -55,11 +67,13 @@ static LogicalResult transferPreconditions(PatternRewriter &rewriter,
   auto srcTy = dyn_cast<MemRefType>(xferOp.getShapedType());
   if (!srcTy)
     return rewriter.notifyMatchFailure(xferOp, "Expects memref source");
+
+  // Perform common data transfer checks.
   VectorType vecTy = xferOp.getVectorType();
-  unsigned vecRank = vecTy.getRank();
-  if (!(vecRank == 1 || vecRank == 2))
-    return rewriter.notifyMatchFailure(xferOp, "Expects 1D or 2D vector");
+  if (failed(storeLoadPreconditions(rewriter, xferOp, vecTy)))
+    return failure();
 
+  // Validate further transfer op semantics.
   SmallVector<int64_t> strides;
   int64_t offset;
   if (failed(getStridesAndOffset(srcTy, strides, offset)) ||
@@ -67,6 +81,7 @@ static LogicalResult transferPreconditions(PatternRewriter &rewriter,
     return rewriter.notifyMatchFailure(
         xferOp, "Buffer must be contiguous in the innermost dimension");
 
+  unsigned vecRank = vecTy.getRank();
   AffineMap map = xferOp.getPermutationMap();
   if (!map.isProjectedPermutation(/*allowZeroInResults=*/false))
     return rewriter.notifyMatchFailure(xferOp, "Unsupported permutation map");
@@ -232,6 +247,66 @@ struct TransferWriteLowering
   }
 };
 
+struct LoadLowering : public OpRewritePattern<vector::LoadOp> {
+  using OpRewritePattern<vector::LoadOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(vector::LoadOp loadOp,
+                                PatternRewriter &rewriter) const override {
+    Location loc = loadOp.getLoc();
+
+    VectorType vecTy = loadOp.getResult().getType();
+    if (failed(storeLoadPreconditions(rewriter, loadOp, vecTy)))
+      return failure();
+
+    auto descType = xegpu::TensorDescType::get(
+        vecTy.getShape(), vecTy.getElementType(), /*array_length=*/1,
+        /*boundary_check=*/true, xegpu::MemorySpace::Global);
+    xegpu::CreateNdDescOp ndDesc = createNdDescriptor(
+        rewriter, loc, descType, loadOp.getBase(), loadOp.getIndices());
+
+    // By default, no specific caching policy is assigned.
+    xegpu::CachePolicyAttr hint = nullptr;
+    auto loadNdOp = rewriter.create<xegpu::LoadNdOp>(
+        loc, vecTy, ndDesc, /*packed=*/nullptr, /*transpose=*/nullptr,
+        /*l1_hint=*/hint,
+        /*l2_hint=*/hint, /*l3_hint=*/hint);
+    rewriter.replaceOp(loadOp, loadNdOp);
+
+    return success();
+  }
+};
+
+struct StoreLowering : public OpRewritePattern<vector::StoreOp> {
+  using OpRewritePattern<vector::StoreOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(vector::StoreOp storeOp,
+                                PatternRewriter &rewriter) const override {
+    Location loc = storeOp.getLoc();
+
+    TypedValue<VectorType> vector = storeOp.getValueToStore();
+    VectorType vecTy = vector.getType();
+    if (failed(storeLoadPreconditions(rewriter, storeOp, vecTy)))
+      return failure();
+
+    auto descType =
+        xegpu::TensorDescType::get(vecTy.getShape(), vecTy.getElementType(),
+                                   /*array_length=*/1, /*boundary_check=*/true,
+                                   xegpu::MemorySpace::Global);
+    xegpu::CreateNdDescOp ndDesc = createNdDescriptor(
+        rewriter, loc, descType, storeOp.getBase(), storeOp.getIndices());
+
+    // By default, no specific caching policy is assigned.
+    xegpu::CachePolicyAttr hint = nullptr;
+    auto storeNdOp =
+        rewriter.create<xegpu::StoreNdOp>(loc, vector, ndDesc,
+                                          /*l1_hint=*/hint,
+                                          /*l2_hint=*/hint, /*l3_hint=*/hint);
+    rewriter.replaceOp(storeOp, storeNdOp);
+
+    return success();
+  }
+};
+
 struct ConvertVectorToXeGPUPass
     : public impl::ConvertVectorToXeGPUBase<ConvertVectorToXeGPUPass> {
   void runOnOperation() override {
@@ -247,8 +322,8 @@ struct ConvertVectorToXeGPUPass
 
 void mlir::populateVectorToXeGPUConversionPatterns(
     RewritePatternSet &patterns) {
-  patterns.add<TransferReadLowering, TransferWriteLowering>(
-      patterns.getContext());
+  patterns.add<TransferReadLowering, TransferWriteLowering, LoadLowering,
+               StoreLowering>(patterns.getContext());
 }
 
 std::unique_ptr<Pass> mlir::createConvertVectorToXeGPUPass() {
diff --git a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
index 6bb8dfecba0ec..71e9648a5e00f 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
@@ -1223,19 +1223,8 @@ static Operation *vectorizeAffineLoad(AffineLoadOp loadOp,
   LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ permutationMap: ");
   LLVM_DEBUG(permutationMap.print(dbgs()));
 
-  // Make sure that the in_bounds attribute corresponding to a broadcast dim
-  // is set to `true` - that's required by the xfer Op.
-  // FIXME: We're not veryfying whether the corresponding access is in bounds.
-  // TODO: Use masking instead.
-  SmallVector<unsigned> broadcastedDims = permutationMap.getBroadcastDims();
-  SmallVector<bool> inBounds(vectorType.getRank(), false);
-
-  for (auto idx : broadcastedDims)
-    inBounds[idx] = true;
-
   auto transfer = state.builder.create<vector::TransferReadOp>(
-      loadOp.getLoc(), vectorType, loadOp.getMemRef(), indices, permutationMap,
-      inBounds);
+      loadOp.getLoc(), vectorType, loadOp.getMemRef(), indices, permutationMap);
 
   // Register replacement for future uses in the scope.
   state.registerOpVectorReplacement(loadOp, transfer);
diff --git a/mlir/lib/Dialect/Arith/Utils/Utils.cpp b/mlir/lib/Dialect/Arith/Utils/Utils.cpp
index c0aa16cc0da40..67dcce454f028 100644
--- a/mlir/lib/Dialect/Arith/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Arith/Utils/Utils.cpp
@@ -370,6 +370,7 @@ std::optional<FloatType> parseFloatType(MLIRContext *ctx, StringRef name) {
       .Case("f8E5M2FNUZ", b.getFloat8E5M2FNUZType())
       .Case("f8E4M3FNUZ", b.getFloat8E4M3FNUZType())
       .Case("f8E3M4", b.getFloat8E3M4Type())
+      .Case("f8E8M0FNU", b.getFloat8E8M0FNUType())
       .Case("bf16", b.getBF16Type())
       .Case("f16", b.getF16Type())
       .Case("f32", b.getF32Type())
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 0561c364c7d59..fb7024a14f8d4 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -3010,8 +3010,16 @@ LogicalResult AtomicRMWOp::verify() {
   auto valType = getVal().getType();
   if (getBinOp() == AtomicBinOp::fadd || getBinOp() == AtomicBinOp::fsub ||
       getBinOp() == AtomicBinOp::fmin || getBinOp() == AtomicBinOp::fmax) {
-    if (!mlir::LLVM::isCompatibleFloatingPointType(valType))
+    if (isCompatibleVectorType(valType)) {
+      if (isScalableVectorType(valType))
+        return emitOpError("expected LLVM IR fixed vector type");
+      Type elemType = getVectorElementType(valType);
+      if (!isCompatibleFloatingPointType(elemType))
+        return emitOpError(
+            "expected LLVM IR floating point type for vector element");
+    } else if (!isCompatibleFloatingPointType(valType)) {
       return emitOpError("expected LLVM IR floating point type");
+    }
   } else if (getBinOp() == AtomicBinOp::xchg) {
     DataLayout dataLayout = DataLayout::closest(*this);
     if (!isTypeCompatibleWithAtomicOp(valType, dataLayout))
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/DIScopeForLLVMFuncOp.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/DIScopeForLLVMFuncOp.cpp
index 2cfaffa7c8efc..b276e06f93c61 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/DIScopeForLLVMFuncOp.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/DIScopeForLLVMFuncOp.cpp
@@ -91,6 +91,10 @@ struct DIScopeForLLVMFuncOp
     Location loc = module.getLoc();
 
     MLIRContext *context = &getContext();
+    if (!context->getLoadedDialect<LLVM::LLVMDialect>()) {
+      emitError(loc, "LLVM dialect is not loaded.");
+      return signalPassFailure();
+    }
 
     // To find a DICompileUnitAttr attached to a parent (the module for
     // example), otherwise create a default one.
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index 7e91562cacb49..8e7621754f76b 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -248,6 +248,11 @@ void transform::ApplyTilingCanonicalizationPatternsOp::populatePatterns(
   linalg::populateLinalgTilingCanonicalizationPatterns(patterns);
 }
 
+void transform::ApplyFoldAddIntoDestPatternsOp::populatePatterns(
+    RewritePatternSet &patterns) {
+  linalg::populateFoldAddIntoDestPatterns(patterns);
+}
+
 //===----------------------------------------------------------------------===//
 // BufferizeToAllocationOp
 //===----------------------------------------------------------------------===//
@@ -3420,11 +3425,11 @@ struct VectorizationPattern : public RewritePattern {
         flatten1DDepthwiseConv(flattenConv) {}
   LogicalResult matchAndRewrite(Operation *op,
                                 PatternRewriter &rewriter) const override {
-    LinalgOp linalgOp = dyn_cast<LinalgOp>(op);
-    if (!linalgOp)
-      return rewriter.notifyMatchFailure(op, "expected Linalg Op");
-    return vectorize(rewriter, linalgOp, /*inputVectorSizes=*/{},
-                     /*scalableVecDims=*/{}, vectorizeNDExtract,
+    if (!linalg::hasVectorizationImpl(op))
+      return rewriter.notifyMatchFailure(op,
+                                         "Unsupported Op, cannot vectorize");
+    return vectorize(rewriter, op, /*inputVectorSizes=*/{},
+                     /*inputScalableVecDims=*/{}, vectorizeNDExtract,
                      flatten1DDepthwiseConv);
   }
 
@@ -3505,8 +3510,7 @@ DiagnosedSilenceableFailure transform::VectorizeOp::apply(
 
   // TODO: Check that the correct number of vectorSizes was provided.
   for (Operation *target : targets) {
-    if (!isa<linalg::LinalgOp, tensor::PadOp, tensor::PackOp, tensor::UnPackOp>(
-            target)) {
+    if (!linalg::hasVectorizationImpl(target)) {
       return mlir::emitSilenceableFailure(target->getLoc())
              << "Unsupported Op, cannot vectorize";
     }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
index 47af392def94a..b3cd5537aad9b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
@@ -13,6 +13,7 @@ add_mlir_dialect_library(MLIRLinalgTransforms
   ElementwiseToLinalg.cpp
   EliminateEmptyTensors.cpp
   EraseUnusedOperandsAndResults.cpp
+  FoldAddIntoDest.cpp
   FusePadOpWithLinalgProducer.cpp
   Fusion.cpp
   Generalization.cpp
diff --git a/mlir/lib/Dialect/Linalg/Transforms/FoldAddIntoDest.cpp b/mlir/lib/Dialect/Linalg/Transforms/FoldAddIntoDest.cpp
new file mode 100644
index 0000000000000..e940b0787043e
--- /dev/null
+++ b/mlir/lib/Dialect/Linalg/Transforms/FoldAddIntoDest.cpp
@@ -0,0 +1,150 @@
+//===- FoldAddIntoDest.cpp ---------------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Linalg/IR/Linalg.h"
+#include "mlir/Dialect/Linalg/IR/LinalgInterfaces.h"
+#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
+#include "mlir/IR/Dominance.h"
+#include "mlir/Interfaces/DestinationStyleOpInterface.h"
+
+using namespace mlir;
+
+// Determine whether the value is defined to be zero.
+static bool isDefinedAsZero(Value val) {
+  if (!val)
+    return false;
+
+  // Check whether val is a constant scalar / vector splat / tensor splat float
+  // or integer zero.
+  if (matchPattern(val, m_AnyZeroFloat()) || matchPattern(val, m_Zero()))
+    return true;
+
+  return TypeSwitch<Operation *, bool>(val.getDefiningOp())
+      .Case<linalg::FillOp, linalg::CopyOp>([&](auto op) {
+        return op && op.getInputs().size() == 1 &&
+               isDefinedAsZero(op.getInputs()[0]);
+      })
+      .Default([&](auto) { return false; });
+}
+
+/// Replace a linalg.add with one operand the single user of a contraction,
+/// which has a zero-filled, "identity-mapped" destination and is dominated by
+/// the `other` operand, by the contraction with `other` as its dest.
+///
+/// As an example, the following pseudo-code will be rewritten
+///   %cst = arith.constant 0.000000e+00
+///   %empty = tensor.empty()
+///   %zeroed = linalg.fill ins(%cst : f32) outs(%empty : !type) -> !type
+///   %C = linalg.matmul ins(%A, %B) outs(%zeroed)
+///   %empty2 = tensor.empty()
+///   %zeroed2 = linalg.fill ins(%cst : f32) outs(%empty2 : !type) -> !type
+///   %F = linalg.matmul ins(%D, %E) outs(%zeroed2)
+///   %out = linalg.add ins(%C, %F) outs(%empty)
+/// to:
+///   %cst = arith.constant 0.000000e+00
+///   %empty = tensor.empty()
+///   %zeroed = linalg.fill ins(%cst : f32) outs(%empty : !type) -> !type
+///   %C = linalg.matmul ins(%A, %B) outs(%zeroed)
+///   %out = linalg.matmul ins(%D, %E) outs(%C)
+///
+struct FoldAddIntoDest final : public OpRewritePattern<linalg::AddOp> {
+  using OpRewritePattern<linalg::AddOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(linalg::AddOp addOp,
+                                PatternRewriter &rewriter) const override {
+    // For now, pattern only applies on tensor types (memref support is TODO).
+    if (!addOp.hasPureTensorSemantics())
+      return failure();
+
+    Value dominatingOperand = nullptr;
+    linalg::LinalgOp dominatedOp = nullptr;
+    { // We will forget about which operand was left or right after this block.
+      Value lhs = addOp.getInputs()[0];
+      Value rhs = addOp.getInputs()[1];
+
+      // Can only put one of addOp's operands in the dest/out arg of the other's
+      // defining op based on suitable dominance.
+      // TODO: Can be generalized to move ops around as long as that still
+      //       respects use-def chains and doesn't affect side-effects.
+      if (auto rhsOp = rhs.getDefiningOp<linalg::LinalgOp>()) {
+        DominanceInfo domInfo(rhsOp);
+        if (domInfo.properlyDominates(lhs, rhsOp)) {
+          dominatingOperand = lhs;
+          dominatedOp = rhsOp;
+        }
+      }
+      if (auto lhsOp = lhs.getDefiningOp<linalg::LinalgOp>()) {
+        DominanceInfo domInfo(lhsOp);
+        if (domInfo.properlyDominates(rhs, lhsOp)) {
+          dominatingOperand = rhs;
+          dominatedOp = lhsOp;
+        }
+      }
+      if (!dominatingOperand || !dominatedOp)
+        return failure();
+      // NB: As linalg.add's generalisation ignores the out argument in its
+      //     region there is no need to perform checks on addOp's out argument.
+    }
+
+    // When dominated op is a contraction we know it accumulates on its out arg.
+    // E.g., AddOp is not a contraction and hence ignores its out arg's value.
+    // TODO: Generalize check to also pass in case of other LinalgOps that
+    //       accumulate on their out arg but are not (binary) contraction ops.
+    auto dominatedDestOp =
+        dyn_cast<DestinationStyleOpInterface>((Operation *)dominatedOp);
+    if (dominatedOp->getNumResults() != 1 ||
+        !linalg::isaContractionOpInterface(dominatedOp) ||
+        (!dominatedDestOp || dominatedDestOp.getNumDpsInits() != 1))
+      return rewriter.notifyMatchFailure(
+          dominatedOp, "expected dominated op to be single-result "
+                       "destination-passing contraction");
+
+    // To change the contraction's result, `addOp` must be its only user.
+    if (!dominatedOp->getResult(0).hasOneUse())
+      return rewriter.notifyMatchFailure(
+          dominatedOp,
+          "expected linalg.add to be single user of contraction's result");
+
+    // As `dominatedOp` was already accumulating on its out argument, it is only
+    // safe to no longer use its current out arg when it is the additive ident.
+    auto *destOperand = dominatedDestOp.getDpsInitOperand(0);
+    if (!isDefinedAsZero(destOperand->get()))
+      return rewriter.notifyMatchFailure(
+          dominatedOp, "expected dominated op's dest to be additive zero");
+    // TODO: If the other op is a contraction and has additive ident as dest, we
+    // can swap the dests and achieve the proper sum, given suitable dominance.
+
+    // As an operand to `addOp`, `dominatingOperand` has an identity affine_map.
+    // Hence, we can only substitute `dominatingOperand` for the dest of the
+    // contraction when dest's indexing_map corresponds to an identity map
+    // w.r.t. just the dimensions of dest, i.e. is an ordered projection.
+    SmallVector<AffineMap> indexMaps = dominatedOp.getIndexingMapsArray();
+    int prevDimPos = -1;
+    for (auto expr : indexMaps[destOperand->getOperandNumber()].getResults()) {
+      auto dim = dyn_cast<AffineDimExpr>(expr);
+      if (!dim || prevDimPos > static_cast<int>(dim.getPosition()))
+        return rewriter.notifyMatchFailure(
+            dominatedOp, "expected index_map for contraction's dest to be an "
+                         "ordered projection");
+      prevDimPos = dim.getPosition();
+    }
+
+    // Replace the additive-ident, i.e. zero, out arg of the dominated op by the
+    // dominating summand. This makes the dominated op's result the sum of both
+    // of addOp's arguments - therefore we replace addOp and it uses by it.
+    rewriter.modifyOpInPlace(
+        dominatedOp, [&]() { dominatedOp->setOperand(2, dominatingOperand); });
+    rewriter.replaceAllOpUsesWith(addOp, dominatedOp->getResult(0));
+    return success();
+  }
+};
+
+void linalg::populateFoldAddIntoDestPatterns(RewritePatternSet &patterns) {
+  // Replace linalg.add when destination passing suffices for achieving the sum.
+  patterns.add<FoldAddIntoDest>(patterns.getContext());
+}
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
index e0dea8e78d55c..0fe096863d7b0 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
@@ -1021,8 +1021,11 @@ LogicalResult ExtractSliceOfPadTensorSwapPattern::matchAndRewrite(
   return success();
 }
 
-/// Returns a tensor.pad op if padding value is set. Otherwise, returns the
-/// source directly. The method assumes that the `packOp` has static shapes.
+/// If padding value is set, returns a tensor.pad Op for the source tensor,
+/// with the output shape matching the output of `packOp`. Otherwise, returns
+/// the source directly.
+///
+/// This method assumes that all outer dims for this pack Op are 1.
 static Value getPackOpSourceOrPaddedSource(OpBuilder &builder,
                                            tensor::PackOp packOp) {
   Value input = packOp.getSource();
@@ -1038,26 +1041,48 @@ static Value getPackOpSourceOrPaddedSource(OpBuilder &builder,
   ShapedType inputType = packOp.getSourceType();
   int64_t inputRank = inputType.getRank();
 
-  SmallVector<int64_t> paddedShape;
   DenseMap<int64_t, OpFoldResult> tileAndPosMapping =
       packOp.getDimAndTileMapping();
-  for (int64_t dim = 0; dim < inputRank; ++dim) {
-    int64_t size = inputType.getDimSize(dim);
-    if (!tileAndPosMapping.count(dim)) {
-      paddedShape.push_back(size);
+
+  // The sizes of dynamic tiles
+  SmallVector<Value> dynamicTileSizes;
+
+  // Collect dims for the padded shape.
+  SmallVector<int64_t> paddedShape;
+  for (int64_t dimIdx = 0; dimIdx < inputRank; ++dimIdx) {
+    // 1. Non-tiled outer dims.
+    // These dims should be 1 and we simply preserve them.
+    if (!tileAndPosMapping.count(dimIdx)) {
+      int64_t inputDimSize = inputType.getDimSize(dimIdx);
+      assert(inputDimSize == 1 &&
+             "with all outer dims == 1, this non-tiled input dim should be 1!");
+      paddedShape.push_back(inputDimSize);
+      continue;
+    }
+
+    // 2. Tiled outer dims
+    // As all outer dims == 1, it is safe to use the tile size for the padded
+    // shape.
+    OpFoldResult tileSizeForDim = tileAndPosMapping.lookup(dimIdx);
+
+    // 2.1 Static tile sizes
+    std::optional<int64_t> cstTileSize = getConstantIntValue(tileSizeForDim);
+    if (cstTileSize.has_value()) {
+      paddedShape.push_back(cstTileSize.value());
       continue;
     }
 
-    // The size is less than or equal to tileSize because outer dims are all 1s.
-    std::optional<int64_t> tileSize =
-        getConstantIntValue(tileAndPosMapping.lookup(dim));
-    assert(tileSize.has_value() && "dynamic inner tile size is not supported");
-    paddedShape.push_back(tileSize.value());
+    // 2.2 Dynamic tile sizes
+    paddedShape.push_back(ShapedType::kDynamic);
+
+    // Get the value that holds the dynamic size.
+    dynamicTileSizes.push_back(llvm::dyn_cast<Value>(tileSizeForDim));
   }
   auto resultType =
       RankedTensorType::get(paddedShape, inputType.getElementType());
   return tensor::createPadHighOp(resultType, input, packOp.getPaddingValue(),
-                                 /*nofold=*/false, loc, builder);
+                                 /*nofold=*/false, loc, builder,
+                                 dynamicTileSizes);
 }
 
 // Normalizes a permutation on a higher rank space to its actual size, e.g.
@@ -1120,12 +1145,6 @@ getPackUnpackRankReducedPerm(ArrayRef<int64_t> shape,
 
 LogicalResult GeneralizeOuterUnitDimsPackOpPattern::matchAndRewrite(
     tensor::PackOp packOp, PatternRewriter &rewriter) const {
-  if (llvm::any_of(packOp.getMixedTiles(),
-                   [](OpFoldResult tile) { return tile.is<Value>(); })) {
-    return rewriter.notifyMatchFailure(packOp,
-                                       "require inner tile sizes being static");
-  }
-
   // TODO: support the case that outer dimensions are not all 1s. A
   // tensor.expand_shape will be generated in this case.
   if (llvm::any_of(packOp.getTiledOuterDims(),
@@ -1147,12 +1166,15 @@ LogicalResult GeneralizeOuterUnitDimsPackOpPattern::matchAndRewrite(
   SmallVector<OpFoldResult> readOffsets(srcRank, zeroIdxAttr);
   SmallVector<OpFoldResult> readStrides(srcRank, oneIdxAttr);
   SmallVector<OpFoldResult> readSizes;
-  SmallVector<int64_t> readShape;
+  SmallVector<OpFoldResult> transShapeForEmpty;
+  SmallVector<int64_t> readShapeForExtractSlice;
   for (auto i : llvm::seq<unsigned>(0, srcRank)) {
     if (dimAndTileMapping.count(i)) {
-      readShape.push_back(getConstantIntValue(dimAndTileMapping[i])
-                              .value_or(ShapedType::kDynamic));
+      readShapeForExtractSlice.push_back(
+          getConstantIntValue(dimAndTileMapping[i])
+              .value_or(ShapedType::kDynamic));
       readSizes.push_back(dimAndTileMapping[i]);
+      transShapeForEmpty.push_back(dimAndTileMapping[i]);
       continue;
     }
     if (ShapedType::isDynamic(inputShape[i])) {
@@ -1161,12 +1183,14 @@ LogicalResult GeneralizeOuterUnitDimsPackOpPattern::matchAndRewrite(
     } else {
       readSizes.push_back(rewriter.getIndexAttr(inputShape[i]));
     }
-    if (inputShape[i] != 1)
-      readShape.push_back(inputShape[i]);
+    if (inputShape[i] != 1) {
+      readShapeForExtractSlice.push_back(inputShape[i]);
+      transShapeForEmpty.push_back(rewriter.getIndexAttr(inputShape[i]));
+    }
   }
 
   Type elemType = packOp.getSourceType().getElementType();
-  auto readType = RankedTensorType::get(readShape, elemType);
+  auto readType = RankedTensorType::get(readShapeForExtractSlice, elemType);
 
   Value tile = rewriter.create<tensor::ExtractSliceOp>(
       loc, readType, input, readOffsets, readSizes, readStrides);
@@ -1178,10 +1202,10 @@ LogicalResult GeneralizeOuterUnitDimsPackOpPattern::matchAndRewrite(
   LLVM_DEBUG(DBGS() << "Pack permutation: " << packOp << "\n";
              llvm::interleaveComma(perm, DBGS() << "perm: "); DBGSNL(););
 
-  SmallVector<int64_t> transpShape = readShape;
-  applyPermutationToVector<int64_t>(transpShape, perm);
+  applyPermutationToVector<OpFoldResult>(transShapeForEmpty, perm);
 
-  Value empty = rewriter.create<tensor::EmptyOp>(loc, transpShape, elemType);
+  Value empty =
+      rewriter.create<tensor::EmptyOp>(loc, transShapeForEmpty, elemType);
   auto transposedOp =
       rewriter.create<linalg::TransposeOp>(loc, tile, empty, perm);
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index ca85f4b9b9c15..09c6b2683b438 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -1380,17 +1380,8 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
 
     SmallVector<Value> indices(linalgOp.getShape(opOperand).size(), zero);
 
-    // Make sure that the in_bounds attribute corresponding to a broadcast dim
-    // is `true`
-    SmallVector<unsigned> broadcastedDims = readMap.getBroadcastDims();
-    SmallVector<bool> inBounds(readType.getRank(), false);
-
-    for (auto idx : broadcastedDims)
-      inBounds[idx] = true;
-
     Operation *read = rewriter.create<vector::TransferReadOp>(
-        loc, readType, opOperand->get(), indices, readMap,
-        ArrayRef<bool>(inBounds));
+        loc, readType, opOperand->get(), indices, readMap);
     read = state.maskOperation(rewriter, read, linalgOp, indexingMap);
     Value readValue = read->getResult(0);
 
@@ -2092,6 +2083,10 @@ LogicalResult mlir::linalg::vectorizeOpPrecondition(
     Operation *op, ArrayRef<int64_t> inputVectorSizes,
     ArrayRef<bool> inputScalableVecDims, bool vectorizeNDExtract,
     bool flatten1DDepthwiseConv) {
+
+  if (!hasVectorizationImpl(op))
+    return failure();
+
   if (failed(vectorizeScalableVectorPrecondition(op, inputVectorSizes,
                                                  inputScalableVecDims)))
     return failure();
@@ -2129,6 +2124,11 @@ static void convertAffineApply(RewriterBase &rewriter, LinalgOp linalgOp) {
   }
 }
 
+bool mlir::linalg::hasVectorizationImpl(Operation *op) {
+  return isa<linalg::LinalgOp, tensor::PadOp, tensor::PackOp, tensor::UnPackOp>(
+      op);
+}
+
 /// Emit a suitable vector form for an operation. If provided,
 /// `inputVectorSizes` are used to vectorize this operation.
 /// `inputVectorSizes` must match the rank of the iteration space of the
diff --git a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
index 68edd45448ee5..7321b19068016 100644
--- a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
+++ b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
@@ -15,6 +15,7 @@
 #include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/Interfaces/ViewLikeInterface.h"
 #include "llvm/ADT/STLExtras.h"
 
 namespace mlir {
@@ -193,15 +194,13 @@ MemrefValue skipFullyAliasingOperations(MemrefValue source) {
   return source;
 }
 
-MemrefValue skipSubViewsAndCasts(MemrefValue source) {
+MemrefValue skipViewLikeOps(MemrefValue source) {
   while (auto op = source.getDefiningOp()) {
-    if (auto subView = dyn_cast<memref::SubViewOp>(op)) {
-      source = cast<MemrefValue>(subView.getSource());
-    } else if (auto cast = dyn_cast<memref::CastOp>(op)) {
-      source = cast.getSource();
-    } else {
-      return source;
+    if (auto viewLike = dyn_cast<ViewLikeOpInterface>(op)) {
+      source = cast<MemrefValue>(viewLike.getViewSource());
+      continue;
     }
+    return source;
   }
   return source;
 }
diff --git a/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp b/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
index 5396531922aab..0f5fa61879b71 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
@@ -21,6 +21,7 @@
 #include "mlir/Dialect/Vector/Utils/VectorUtils.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/Interfaces/ValueBoundsOpInterface.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include <type_traits>
@@ -67,6 +68,10 @@ class InsertSliceOfTransferWriteOpFolder final
 
   LogicalResult matchAndRewrite(tensor::InsertSliceOp insertSliceOp,
                                 PatternRewriter &rewriter) const override;
+
+private:
+  static bool
+  doesTransferWriteCoverInsertSlice(vector::TransferWriteOp writeOp);
 };
 } // namespace
 
@@ -136,6 +141,10 @@ LogicalResult InsertSliceOfTransferWriteOpFolder::matchAndRewrite(
   if (failed(preconditionResult))
     return preconditionResult;
 
+  if (!doesTransferWriteCoverInsertSlice(writeOp))
+    return rewriter.notifyMatchFailure(
+        insertSliceOp, "transfer_write does not cover insert_slice");
+
   SmallVector<Value> indices(writeOp.getIndices().begin(),
                              writeOp.getIndices().end());
   SmallVector<Value> sourceIndices;
@@ -154,6 +163,17 @@ LogicalResult InsertSliceOfTransferWriteOpFolder::matchAndRewrite(
   return success();
 }
 
+bool InsertSliceOfTransferWriteOpFolder::doesTransferWriteCoverInsertSlice(
+    vector::TransferWriteOp writeOp) {
+  if (writeOp.getShapedType().hasStaticShape())
+    return llvm::equal(writeOp.getVectorType().getShape(),
+                       writeOp.getShapedType().getShape());
+
+  // TODO: Use ValueBoundsConstraintSet for dynamic shapes.
+
+  return false;
+}
+
 template <typename OpTy>
 struct InsertSliceOfInsertSliceFolder : public OpRewritePattern<OpTy> {
   using OpRewritePattern<OpTy>::OpRewritePattern;
diff --git a/mlir/lib/Dialect/Tensor/Utils/Utils.cpp b/mlir/lib/Dialect/Tensor/Utils/Utils.cpp
index a0d8a08fc6ba4..e0b91f323b0e6 100644
--- a/mlir/lib/Dialect/Tensor/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Tensor/Utils/Utils.cpp
@@ -16,28 +16,48 @@
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Interfaces/ValueBoundsOpInterface.h"
 
 using namespace mlir;
 using namespace mlir::tensor;
 
-PadOp mlir::tensor::createPadHighOp(RankedTensorType type, Value source,
+PadOp mlir::tensor::createPadHighOp(RankedTensorType resType, Value source,
                                     Value pad, bool nofold, Location loc,
-                                    OpBuilder &b) {
-  SmallVector<OpFoldResult> low(type.getRank(), b.getIndexAttr(0));
-  SmallVector<OpFoldResult> high(type.getRank(), b.getIndexAttr(0));
-  for (const auto &en : enumerate(type.getShape())) {
-    // Pad only the static dimensions of the result tensor type.
-    if (ShapedType::isDynamic(en.value()))
+                                    OpBuilder &b,
+                                    SmallVector<Value> dynOutDims) {
+
+  assert((resType.getNumDynamicDims() == dynOutDims.size()) ||
+         dynOutDims.empty() &&
+             "Either none or all output dynamic dims must be specified!");
+
+  // Init "low" and "high" padding values ("low" is kept as is, "high" is
+  // computed below).
+  SmallVector<OpFoldResult> low(resType.getRank(), b.getIndexAttr(0));
+  SmallVector<OpFoldResult> high(resType.getRank(), b.getIndexAttr(0));
+
+  size_t outDimIdx = 0;
+
+  for (const auto [idx, val] : enumerate(resType.getShape())) {
+    bool isDimDynamic = ShapedType::isDynamic(val);
+    bool updatePadHigh = !isDimDynamic || !dynOutDims.empty();
+
+    // Keep the default padding width (i.e. "0") when the output dim is dynamic
+    // and no actual output sizes have been provided.
+    if (!updatePadHigh)
       continue;
-    // Compute the padding width.
-    AffineExpr d0;
-    bindDims(b.getContext(), d0);
-    OpFoldResult sz = tensor::getMixedSize(b, loc, source, en.index());
-    high[en.index()] =
-        affine::makeComposedFoldedAffineApply(b, loc, en.value() - d0, {sz});
+
+    // Compute the padding width: resDim - sourceDim.
+    AffineExpr d0, d1;
+    bindDims(b.getContext(), d0, d1);
+    OpFoldResult sourceDim = tensor::getMixedSize(b, loc, source, idx);
+    OpFoldResult outDim = isDimDynamic ? OpFoldResult(dynOutDims[outDimIdx++])
+                                       : OpFoldResult(b.getIndexAttr(val));
+
+    high[idx] = affine::makeComposedFoldedAffineApply(b, loc, d0 - d1,
+                                                      {outDim, sourceDim});
   }
-  return b.create<PadOp>(loc, type, source, low, high, pad, nofold);
+  return b.create<PadOp>(loc, resType, source, low, high, pad, nofold);
 }
 
 SmallVector<Value> mlir::tensor::createDynamicDimValues(OpBuilder &b,
diff --git a/mlir/lib/Dialect/Transform/Interfaces/TransformInterfaces.cpp b/mlir/lib/Dialect/Transform/Interfaces/TransformInterfaces.cpp
index 91702ce7cc42b..fdd968238667e 100644
--- a/mlir/lib/Dialect/Transform/Interfaces/TransformInterfaces.cpp
+++ b/mlir/lib/Dialect/Transform/Interfaces/TransformInterfaces.cpp
@@ -330,7 +330,7 @@ void transform::TransformState::forgetMapping(Value opHandle,
   for (Operation *op : mappings.direct[opHandle])
     dropMappingEntry(mappings.reverse, op, opHandle);
   mappings.direct.erase(opHandle);
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
   // Payload IR is removed from the mapping. This invalidates the respective
   // iterators.
   mappings.incrementTimestamp(opHandle);
@@ -342,7 +342,7 @@ void transform::TransformState::forgetMapping(Value opHandle,
     for (Value resultHandle : resultHandles) {
       Mappings &localMappings = getMapping(resultHandle);
       dropMappingEntry(localMappings.values, resultHandle, opResult);
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
       // Payload IR is removed from the mapping. This invalidates the respective
       // iterators.
       mappings.incrementTimestamp(resultHandle);
@@ -358,7 +358,7 @@ void transform::TransformState::forgetValueMapping(
   for (Value payloadValue : mappings.reverseValues[valueHandle])
     dropMappingEntry(mappings.reverseValues, payloadValue, valueHandle);
   mappings.values.erase(valueHandle);
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
   // Payload IR is removed from the mapping. This invalidates the respective
   // iterators.
   mappings.incrementTimestamp(valueHandle);
@@ -372,7 +372,7 @@ void transform::TransformState::forgetValueMapping(
       dropMappingEntry(localMappings.direct, opHandle, payloadOp);
       dropMappingEntry(localMappings.reverse, payloadOp, opHandle);
 
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
       // Payload IR is removed from the mapping. This invalidates the respective
       // iterators.
       localMappings.incrementTimestamp(opHandle);
@@ -452,7 +452,7 @@ transform::TransformState::replacePayloadValue(Value value, Value replacement) {
     // between the handles and the IR objects
     if (!replacement) {
       dropMappingEntry(mappings.values, handle, value);
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
       // Payload IR is removed from the mapping. This invalidates the respective
       // iterators.
       mappings.incrementTimestamp(handle);
@@ -804,7 +804,7 @@ checkRepeatedConsumptionInOperand(ArrayRef<T> payload,
 void transform::TransformState::compactOpHandles() {
   for (Value handle : opHandlesToCompact) {
     Mappings &mappings = getMapping(handle, /*allowOutOfScope=*/true);
-#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
     if (llvm::find(mappings.direct[handle], nullptr) !=
         mappings.direct[handle].end())
       // Payload IR is removed from the mapping. This invalidates the respective
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index cac6b95545704..dc92bea09dc16 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -3947,10 +3947,6 @@ verifyTransferOp(VectorTransferOpInterface op, ShapedType shapedType,
                            "as permutation_map results: ")
            << AffineMapAttr::get(permutationMap)
            << " vs inBounds of size: " << inBounds.size();
-  for (unsigned int i = 0, e = permutationMap.getNumResults(); i < e; ++i)
-    if (isa<AffineConstantExpr>(permutationMap.getResult(i)) &&
-        !llvm::cast<BoolAttr>(inBounds.getValue()[i]).getValue())
-      return op->emitOpError("requires broadcast dimensions to be in-bounds");
 
   return success();
 }
@@ -4138,22 +4134,43 @@ static LogicalResult foldTransferInBoundsAttribute(TransferOp op) {
   bool changed = false;
   SmallVector<bool, 4> newInBounds;
   newInBounds.reserve(op.getTransferRank());
+  // Idxs of non-bcast dims - used when analysing bcast dims.
+  SmallVector<unsigned> nonBcastDims;
+
+  // 1. Process non-broadcast dims
   for (unsigned i = 0; i < op.getTransferRank(); ++i) {
-    // Already marked as in-bounds, nothing to see here.
+    // 1.1. Already marked as in-bounds, nothing to see here.
     if (op.isDimInBounds(i)) {
       newInBounds.push_back(true);
       continue;
     }
-    // Currently out-of-bounds, check whether we can statically determine it is
-    // inBounds.
+    // 1.2. Currently out-of-bounds, check whether we can statically determine
+    // it is inBounds.
+    bool inBounds = false;
     auto dimExpr = dyn_cast<AffineDimExpr>(permutationMap.getResult(i));
-    assert(dimExpr && "Broadcast dims must be in-bounds");
-    auto inBounds =
-        isInBounds(op, /*resultIdx=*/i, /*indicesIdx=*/dimExpr.getPosition());
+    if (dimExpr) {
+      inBounds = isInBounds(op, /*resultIdx=*/i,
+                            /*indicesIdx=*/dimExpr.getPosition());
+      nonBcastDims.push_back(i);
+    }
+
     newInBounds.push_back(inBounds);
     // We commit the pattern if it is "more inbounds".
     changed |= inBounds;
   }
+
+  // 2. Handle broadcast dims
+  // If all non-broadcast dims are "in bounds", then all bcast dims should be
+  // "in bounds" as well.
+  bool allNonBcastDimsInBounds = llvm::all_of(
+      nonBcastDims, [&newInBounds](unsigned idx) { return newInBounds[idx]; });
+  if (allNonBcastDimsInBounds) {
+    for (size_t idx : permutationMap.getBroadcastDims()) {
+      changed |= !newInBounds[idx];
+      newInBounds[idx] = true;
+    }
+  }
+
   if (!changed)
     return failure();
   // OpBuilder is only used as a helper to build an I64ArrayAttr.
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
index 4c93d3841bf87..e05c801121ffc 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
@@ -105,8 +105,7 @@ void TransferOptimization::deadStoreOp(vector::TransferWriteOp write) {
                     << "\n");
   llvm::SmallVector<Operation *, 8> blockingAccesses;
   Operation *firstOverwriteCandidate = nullptr;
-  Value source =
-      memref::skipSubViewsAndCasts(cast<MemrefValue>(write.getSource()));
+  Value source = memref::skipViewLikeOps(cast<MemrefValue>(write.getSource()));
   llvm::SmallVector<Operation *, 32> users(source.getUsers().begin(),
                                            source.getUsers().end());
   llvm::SmallDenseSet<Operation *, 32> processed;
@@ -115,7 +114,7 @@ void TransferOptimization::deadStoreOp(vector::TransferWriteOp write) {
     // If the user has already been processed skip.
     if (!processed.insert(user).second)
       continue;
-    if (isa<memref::SubViewOp, memref::CastOp>(user)) {
+    if (isa<ViewLikeOpInterface>(user)) {
       users.append(user->getUsers().begin(), user->getUsers().end());
       continue;
     }
@@ -192,8 +191,7 @@ void TransferOptimization::storeToLoadForwarding(vector::TransferReadOp read) {
                     << "\n");
   SmallVector<Operation *, 8> blockingWrites;
   vector::TransferWriteOp lastwrite = nullptr;
-  Value source =
-      memref::skipSubViewsAndCasts(cast<MemrefValue>(read.getSource()));
+  Value source = memref::skipViewLikeOps(cast<MemrefValue>(read.getSource()));
   llvm::SmallVector<Operation *, 32> users(source.getUsers().begin(),
                                            source.getUsers().end());
   llvm::SmallDenseSet<Operation *, 32> processed;
@@ -202,7 +200,7 @@ void TransferOptimization::storeToLoadForwarding(vector::TransferReadOp read) {
     // If the user has already been processed skip.
     if (!processed.insert(user).second)
       continue;
-    if (isa<memref::SubViewOp, memref::CollapseShapeOp, memref::CastOp>(user)) {
+    if (isa<ViewLikeOpInterface>(user)) {
       users.append(user->getUsers().begin(), user->getUsers().end());
       continue;
     }
diff --git a/mlir/lib/Dialect/XeGPU/IR/CMakeLists.txt b/mlir/lib/Dialect/XeGPU/IR/CMakeLists.txt
index a0ce7f9706cef..242a97ccfdf6d 100644
--- a/mlir/lib/Dialect/XeGPU/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/XeGPU/IR/CMakeLists.txt
@@ -12,7 +12,9 @@ add_mlir_dialect_library(MLIRXeGPUDialect
 
   LINK_LIBS PUBLIC
   MLIRArithDialect
+  MLIRArithUtils
   MLIRDialectUtils
   MLIRIR
   MLIRViewLikeInterface
+  MLIRVectorDialect
 )
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp
index 1dfbaed454c19..eb01b15de75c6 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp
@@ -55,6 +55,77 @@ ScatterTensorDescAttr::get(mlir::MLIRContext *context,
   return Base::get(context, scopeAttr, chunkSizeAttr);
 }
 
+//===----------------------------------------------------------------------===//
+// XeGPU_SGMapAttr
+//===----------------------------------------------------------------------===//
+namespace {
+template <typename T, unsigned N>
+LogicalResult parseIntArrayField(::mlir::AsmParser &parser,
+                                 llvm::SmallVector<T, N> &result,
+                                 llvm::StringRef fieldName) {
+  if (failed(parser.parseKeyword(fieldName))) {
+    parser.emitError(parser.getCurrentLocation(),
+                     "unexpected field name. Expected " + fieldName + ".");
+    return failure();
+  }
+
+  if (failed(parser.parseEqual())) {
+    parser.emitError(parser.getCurrentLocation(), "expected '=' sign.");
+    return failure();
+  }
+
+  auto elemParser = [&]() -> llvm::ParseResult {
+    uint32_t elem = 0;
+    auto res = parser.parseInteger(elem);
+    result.push_back(elem);
+    return res;
+  };
+
+  return parser.parseCommaSeparatedList(AsmParser::Delimiter::Square,
+                                        elemParser, fieldName);
+}
+} // namespace
+
+mlir::Attribute SGMapAttr::parse(::mlir::AsmParser &parser,
+                                 ::mlir::Type attrType) {
+  if (failed(parser.parseLess()))
+    return {};
+
+  llvm::SmallVector<uint32_t, 2> wi_layout, wi_data;
+  if (failed(parseIntArrayField(parser, wi_layout, "wi_layout")))
+    return {};
+
+  if (failed(parser.parseComma()))
+    return {};
+
+  if (failed(parseIntArrayField(parser, wi_data, "wi_data")))
+    return {};
+
+  return SGMapAttr::getChecked(
+      [&]() { return parser.emitError(parser.getNameLoc()); },
+      parser.getContext(), wi_layout, wi_data);
+}
+
+void SGMapAttr::print(::mlir::AsmPrinter &printer) const {
+  printer << "<";
+  printer.printKeywordOrString("wi_layout");
+  printer << " = [" << getWiLayout() << "], ";
+  printer.printKeywordOrString("wi_data");
+  printer << " = [" << getWiData() << "]";
+  printer << ">";
+}
+
+LogicalResult
+SGMapAttr::verify(llvm::function_ref<mlir::InFlightDiagnostic()> emitError,
+                  llvm::ArrayRef<uint32_t> wi_layout,
+                  llvm::ArrayRef<uint32_t> wi_data) {
+  if (wi_layout.size() != 2)
+    return emitError() << "expected wi_layout of size 2";
+  if (wi_data.size() != 2)
+    return emitError() << "expected wi_data of size 2";
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // XeGPU_TensorDescType
 //===----------------------------------------------------------------------===//
@@ -63,6 +134,7 @@ mlir::Type TensorDescType::parse(::mlir::AsmParser &parser) {
   llvm::SmallVector<int64_t> shape;
   mlir::Type elementType;
   mlir::FailureOr<mlir::Attribute> encoding;
+  mlir::FailureOr<mlir::Attribute> sg_map;
 
   // Parse literal '<'
   if (parser.parseLess())
@@ -81,14 +153,22 @@ mlir::Type TensorDescType::parse(::mlir::AsmParser &parser) {
   }
 
   // parse optional attributes
-  if (mlir::succeeded(parser.parseOptionalComma())) {
-    encoding = mlir::FieldParser<mlir::Attribute>::parse(parser);
-    if (mlir::failed(encoding)) {
-      parser.emitError(
-          parser.getCurrentLocation(),
-          "Failed to parse the attribute field for TensorDescType.\n");
-      return {};
+  while (mlir::succeeded(parser.parseOptionalComma())) {
+    mlir::Attribute attr;
+    ParseResult res = parser.parseAttribute(attr);
+    if (mlir::succeeded(res)) {
+      if (mlir::isa<SGMapAttr>(attr)) {
+        sg_map = attr;
+        continue;
+      }
+      if (mlir::isa<BlockTensorDescAttr, ScatterTensorDescAttr>(attr)) {
+        encoding = attr;
+        continue;
+      }
     }
+    parser.emitError(parser.getCurrentLocation(),
+                     "Failed to parse the attribute.\n");
+    return {};
   }
 
   // Parse literal '>'
@@ -96,7 +176,8 @@ mlir::Type TensorDescType::parse(::mlir::AsmParser &parser) {
     return {};
 
   return TensorDescType::get(parser.getContext(), shape, elementType,
-                             encoding.value_or(mlir::Attribute()));
+                             encoding.value_or(mlir::Attribute()),
+                             sg_map.value_or(mlir::Attribute()));
 }
 
 void TensorDescType::print(::mlir::AsmPrinter &printer) const {
@@ -116,25 +197,30 @@ void TensorDescType::print(::mlir::AsmPrinter &printer) const {
   if (auto encoding = getEncoding())
     printer << ", " << encoding;
 
+  if (auto sg_map = getSgMap())
+    printer << ", " << sg_map;
+
   printer << ">";
 }
 
 TensorDescType TensorDescType::get(llvm::ArrayRef<int64_t> shape,
                                    mlir::Type elementType, int array_length,
                                    bool boundary_check,
-                                   MemorySpace memory_space) {
+                                   MemorySpace memory_space,
+                                   mlir::Attribute sg_map) {
   auto context = elementType.getContext();
   auto attr = BlockTensorDescAttr::get(context, memory_space, array_length,
                                        boundary_check);
-  return Base::get(context, shape, elementType, attr);
+  return Base::get(context, shape, elementType, attr, sg_map);
 }
 
 TensorDescType TensorDescType::get(llvm::ArrayRef<int64_t> shape,
                                    mlir::Type elementType, int chunk_size,
-                                   MemorySpace memory_space) {
+                                   MemorySpace memory_space,
+                                   mlir::Attribute sg_map) {
   auto context = elementType.getContext();
   auto attr = ScatterTensorDescAttr::get(context, memory_space, chunk_size);
-  return Base::get(context, shape, elementType, attr);
+  return Base::get(context, shape, elementType, attr, sg_map);
 }
 
 } // namespace xegpu
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
index 1a7a6b3478409..5bd3c370e3859 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/Dialect/XeGPU/IR/XeGPU.h"
 #include "mlir/IR/Builders.h"
@@ -308,6 +309,24 @@ LogicalResult UpdateNdOffsetOp::verify() {
 // XeGPU_CreateDescOp
 //===----------------------------------------------------------------------===//
 
+void CreateDescOp::build(OpBuilder &builder, OperationState &state,
+                         TensorDescType TensorDesc, Value source,
+                         llvm::ArrayRef<OpFoldResult> offsets) {
+  auto loc = source.getLoc();
+  int64_t size = static_cast<int64_t>(offsets.size());
+  auto type = VectorType::get(size, builder.getIndexType());
+  auto values = getValueOrCreateConstantIndexOp(builder, loc, offsets);
+  auto offset = builder.create<vector::FromElementsOp>(loc, type, values);
+  build(builder, state, TensorDesc, source, offset);
+}
+
+void CreateDescOp::build(OpBuilder &builder, OperationState &state,
+                         TensorDescType TensorDesc, Value source,
+                         llvm::ArrayRef<int64_t> offsets) {
+  auto ofrs = getAsIndexOpFoldResult(builder.getContext(), offsets);
+  build(builder, state, TensorDesc, source, ofrs);
+}
+
 LogicalResult CreateDescOp::verify() {
   auto tdescTy = getTensorDescType();
 
@@ -473,6 +492,29 @@ LogicalResult StoreScatterOp::verify() {
 
   return success();
 }
+
+//===----------------------------------------------------------------------===//
+// XeGPU_UpdateOffsetOp
+//===----------------------------------------------------------------------===//
+void UpdateOffsetOp::build(OpBuilder &builder, OperationState &state,
+                           mlir::Value tensorDesc,
+                           llvm::ArrayRef<OpFoldResult> offsets) {
+  auto tdescTy = mlir::dyn_cast<TensorDescType>(tensorDesc.getType());
+  assert(tdescTy && "Expecting the source is a TensorDescType value.");
+  auto loc = tensorDesc.getLoc();
+  int64_t size = static_cast<int64_t>(offsets.size());
+  auto type = VectorType::get({size}, builder.getIndexType());
+  auto values = getValueOrCreateConstantIndexOp(builder, loc, offsets);
+  auto offset = builder.create<vector::FromElementsOp>(loc, type, values);
+  build(builder, state, tdescTy, tensorDesc, offset);
+}
+
+void UpdateOffsetOp::build(OpBuilder &builder, OperationState &state,
+                           Value tensorDesc, llvm::ArrayRef<int64_t> offsets) {
+  auto ofrs = getAsIndexOpFoldResult(builder.getContext(), offsets);
+  build(builder, state, tensorDesc, ofrs);
+}
+
 //===----------------------------------------------------------------------===//
 // XeGPU_DpasOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp
index e293ad6143fdb..51fe49a3c0bf8 100644
--- a/mlir/lib/IR/AsmPrinter.cpp
+++ b/mlir/lib/IR/AsmPrinter.cpp
@@ -2095,6 +2095,11 @@ void AsmPrinter::Impl::printLocationInternal(LocationAttr loc, bool pretty,
             [&](Location loc) { printLocationInternal(loc, pretty); },
             [&]() { os << ", "; });
         os << ']';
+      })
+      .Default([&](LocationAttr loc) {
+        // Assumes that this is a dialect-specific attribute and prints it
+        // directly.
+        printAttribute(loc);
       });
 }
 
@@ -2663,6 +2668,7 @@ void AsmPrinter::Impl::printTypeImpl(Type type) {
       .Case<Float8E4M3FNUZType>([&](Type) { os << "f8E4M3FNUZ"; })
       .Case<Float8E4M3B11FNUZType>([&](Type) { os << "f8E4M3B11FNUZ"; })
       .Case<Float8E3M4Type>([&](Type) { os << "f8E3M4"; })
+      .Case<Float8E8M0FNUType>([&](Type) { os << "f8E8M0FNU"; })
       .Case<BFloat16Type>([&](Type) { os << "bf16"; })
       .Case<Float16Type>([&](Type) { os << "f16"; })
       .Case<FloatTF32Type>([&](Type) { os << "tf32"; })
diff --git a/mlir/lib/IR/Builders.cpp b/mlir/lib/IR/Builders.cpp
index 7aed415343e55..a9bc3c0ef65a2 100644
--- a/mlir/lib/IR/Builders.cpp
+++ b/mlir/lib/IR/Builders.cpp
@@ -74,6 +74,10 @@ FloatType Builder::getFloat8E3M4Type() {
   return FloatType::getFloat8E3M4(context);
 }
 
+FloatType Builder::getFloat8E8M0FNUType() {
+  return FloatType::getFloat8E8M0FNU(context);
+}
+
 FloatType Builder::getBF16Type() { return FloatType::getBF16(context); }
 
 FloatType Builder::getF16Type() { return FloatType::getF16(context); }
diff --git a/mlir/lib/IR/BuiltinTypes.cpp b/mlir/lib/IR/BuiltinTypes.cpp
index 782a32b307468..25e9f80c9963c 100644
--- a/mlir/lib/IR/BuiltinTypes.cpp
+++ b/mlir/lib/IR/BuiltinTypes.cpp
@@ -121,6 +121,8 @@ const llvm::fltSemantics &FloatType::getFloatSemantics() {
     return APFloat::Float8E4M3B11FNUZ();
   if (llvm::isa<Float8E3M4Type>(*this))
     return APFloat::Float8E3M4();
+  if (llvm::isa<Float8E8M0FNUType>(*this))
+    return APFloat::Float8E8M0FNU();
   if (llvm::isa<BFloat16Type>(*this))
     return APFloat::BFloat();
   if (llvm::isa<Float16Type>(*this))
diff --git a/mlir/lib/IR/Location.cpp b/mlir/lib/IR/Location.cpp
index c548bbe4b6c86..dbd84912a8657 100644
--- a/mlir/lib/IR/Location.cpp
+++ b/mlir/lib/IR/Location.cpp
@@ -38,34 +38,20 @@ void BuiltinDialect::registerLocationAttributes() {
 //===----------------------------------------------------------------------===//
 
 WalkResult LocationAttr::walk(function_ref<WalkResult(Location)> walkFn) {
-  if (walkFn(*this).wasInterrupted())
-    return WalkResult::interrupt();
-
-  return TypeSwitch<LocationAttr, WalkResult>(*this)
-      .Case([&](CallSiteLoc callLoc) -> WalkResult {
-        if (callLoc.getCallee()->walk(walkFn).wasInterrupted())
-          return WalkResult::interrupt();
-        return callLoc.getCaller()->walk(walkFn);
-      })
-      .Case([&](FusedLoc fusedLoc) -> WalkResult {
-        for (Location subLoc : fusedLoc.getLocations())
-          if (subLoc->walk(walkFn).wasInterrupted())
-            return WalkResult::interrupt();
-        return WalkResult::advance();
-      })
-      .Case([&](NameLoc nameLoc) -> WalkResult {
-        return nameLoc.getChildLoc()->walk(walkFn);
-      })
-      .Case([&](OpaqueLoc opaqueLoc) -> WalkResult {
-        return opaqueLoc.getFallbackLocation()->walk(walkFn);
-      })
-      .Default(WalkResult::advance());
+  AttrTypeWalker walker;
+  // Walk locations, but skip any other attribute.
+  walker.addWalk([&](Attribute attr) {
+    if (auto loc = llvm::dyn_cast<LocationAttr>(attr))
+      return walkFn(loc);
+
+    return WalkResult::skip();
+  });
+  return walker.walk<WalkOrder::PreOrder>(*this);
 }
 
 /// Methods for support type inquiry through isa, cast, and dyn_cast.
 bool LocationAttr::classof(Attribute attr) {
-  return llvm::isa<CallSiteLoc, FileLineColLoc, FusedLoc, NameLoc, OpaqueLoc,
-                   UnknownLoc>(attr);
+  return attr.hasTrait<AttributeTrait::IsLocation>();
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/IR/MLIRContext.cpp b/mlir/lib/IR/MLIRContext.cpp
index f45de17dd2491..f05666fcde207 100644
--- a/mlir/lib/IR/MLIRContext.cpp
+++ b/mlir/lib/IR/MLIRContext.cpp
@@ -231,6 +231,7 @@ class MLIRContextImpl {
   Float8E4M3FNUZType f8E4M3FNUZTy;
   Float8E4M3B11FNUZType f8E4M3B11FNUZTy;
   Float8E3M4Type f8E3M4Ty;
+  Float8E8M0FNUType f8E8M0FNUTy;
   BFloat16Type bf16Ty;
   Float16Type f16Ty;
   FloatTF32Type tf32Ty;
@@ -326,6 +327,7 @@ MLIRContext::MLIRContext(const DialectRegistry &registry, Threading setting)
   impl->f8E4M3FNUZTy = TypeUniquer::get<Float8E4M3FNUZType>(this);
   impl->f8E4M3B11FNUZTy = TypeUniquer::get<Float8E4M3B11FNUZType>(this);
   impl->f8E3M4Ty = TypeUniquer::get<Float8E3M4Type>(this);
+  impl->f8E8M0FNUTy = TypeUniquer::get<Float8E8M0FNUType>(this);
   impl->bf16Ty = TypeUniquer::get<BFloat16Type>(this);
   impl->f16Ty = TypeUniquer::get<Float16Type>(this);
   impl->tf32Ty = TypeUniquer::get<FloatTF32Type>(this);
@@ -1049,6 +1051,9 @@ Float8E4M3B11FNUZType Float8E4M3B11FNUZType::get(MLIRContext *context) {
 Float8E3M4Type Float8E3M4Type::get(MLIRContext *context) {
   return context->getImpl().f8E3M4Ty;
 }
+Float8E8M0FNUType Float8E8M0FNUType::get(MLIRContext *context) {
+  return context->getImpl().f8E8M0FNUTy;
+}
 BFloat16Type BFloat16Type::get(MLIRContext *context) {
   return context->getImpl().bf16Ty;
 }
diff --git a/mlir/lib/IR/Types.cpp b/mlir/lib/IR/Types.cpp
index efefbc299a91f..e190902b2e489 100644
--- a/mlir/lib/IR/Types.cpp
+++ b/mlir/lib/IR/Types.cpp
@@ -49,6 +49,9 @@ bool Type::isFloat8E4M3FNUZ() const {
 bool Type::isFloat8E4M3B11FNUZ() const {
   return llvm::isa<Float8E4M3B11FNUZType>(*this);
 }
+bool Type::isFloat8E8M0FNU() const {
+  return llvm::isa<Float8E8M0FNUType>(*this);
+}
 bool Type::isFloat8E3M4() const { return llvm::isa<Float8E3M4Type>(*this); }
 bool Type::isBF16() const { return llvm::isa<BFloat16Type>(*this); }
 bool Type::isF16() const { return llvm::isa<Float16Type>(*this); }
diff --git a/mlir/lib/Interfaces/Utils/InferIntRangeCommon.cpp b/mlir/lib/Interfaces/Utils/InferIntRangeCommon.cpp
index ca3631d53bda9..ec9ed87723e1c 100644
--- a/mlir/lib/Interfaces/Utils/InferIntRangeCommon.cpp
+++ b/mlir/lib/Interfaces/Utils/InferIntRangeCommon.cpp
@@ -444,10 +444,10 @@ mlir::intrange::inferRemU(ArrayRef<ConstantIntRanges> argRanges) {
 
   unsigned width = rhsMin.getBitWidth();
   APInt umin = APInt::getZero(width);
-  APInt umax = APInt::getMaxValue(width);
+  // Remainder can't be larger than either of its arguments.
+  APInt umax = llvm::APIntOps::umin((rhsMax - 1), lhs.umax());
 
   if (!rhsMin.isZero()) {
-    umax = rhsMax - 1;
     // Special case: sweeping out a contiguous range in N/[modulus]
     if (rhsMin == rhsMax) {
       const APInt &lhsMin = lhs.umin(), &lhsMax = lhs.umax();
diff --git a/mlir/lib/Pass/Pass.cpp b/mlir/lib/Pass/Pass.cpp
index 83b46e8f5680a..6fd51c1e3cb53 100644
--- a/mlir/lib/Pass/Pass.cpp
+++ b/mlir/lib/Pass/Pass.cpp
@@ -732,7 +732,7 @@ void OpToOpPassAdaptor::runOnOperationImpl(bool verifyPasses) {
         unsigned initGeneration = mgr->impl->initializationGeneration;
         if (failed(runPipeline(*mgr, &op, am.nest(&op), verifyPasses,
                                initGeneration, instrumentor, &parentInfo)))
-          return signalPassFailure();
+          signalPassFailure();
       }
     }
   }
@@ -799,7 +799,8 @@ void OpToOpPassAdaptor::runOnOperationAsyncImpl(bool verifyPasses) {
   // An atomic failure variable for the async executors.
   std::vector<std::atomic<bool>> activePMs(asyncExecutors.size());
   std::fill(activePMs.begin(), activePMs.end(), false);
-  auto processFn = [&](OpPMInfo &opInfo) {
+  std::atomic<bool> hasFailure = false;
+  parallelForEach(context, opInfos, [&](OpPMInfo &opInfo) {
     // Find an executor for this operation.
     auto it = llvm::find_if(activePMs, [](std::atomic<bool> &isActive) {
       bool expectedInactive = false;
@@ -812,14 +813,15 @@ void OpToOpPassAdaptor::runOnOperationAsyncImpl(bool verifyPasses) {
     LogicalResult pipelineResult = runPipeline(
         pm, opInfo.op, opInfo.am, verifyPasses,
         pm.impl->initializationGeneration, instrumentor, &parentInfo);
+    if (failed(pipelineResult))
+      hasFailure.store(true);
 
     // Reset the active bit for this pass manager.
     activePMs[pmIndex].store(false);
-    return pipelineResult;
-  };
+  });
 
   // Signal a failure if any of the executors failed.
-  if (failed(failableParallelForEach(context, opInfos, processFn)))
+  if (hasFailure)
     signalPassFailure();
 }
 
diff --git a/mlir/lib/TableGen/Attribute.cpp b/mlir/lib/TableGen/Attribute.cpp
index 57c77c74106b9..de930cb400703 100644
--- a/mlir/lib/TableGen/Attribute.cpp
+++ b/mlir/lib/TableGen/Attribute.cpp
@@ -229,7 +229,7 @@ bool EnumAttr::genSpecializedAttr() const {
   return def->getValueAsBit("genSpecializedAttr");
 }
 
-llvm::Record *EnumAttr::getBaseAttrClass() const {
+const llvm::Record *EnumAttr::getBaseAttrClass() const {
   return def->getValueAsDef("baseAttrClass");
 }
 
diff --git a/mlir/lib/TableGen/Operator.cpp b/mlir/lib/TableGen/Operator.cpp
index 76af82a827da1..6a33ff5ecd672 100644
--- a/mlir/lib/TableGen/Operator.cpp
+++ b/mlir/lib/TableGen/Operator.cpp
@@ -208,7 +208,7 @@ StringRef Operator::getResultName(int index) const {
 }
 
 auto Operator::getResultDecorators(int index) const -> var_decorator_range {
-  Record *result =
+  const Record *result =
       cast<DefInit>(def.getValueAsDag("results")->getArg(index))->getDef();
   if (!result->isSubClassOf("OpVariable"))
     return var_decorator_range(nullptr, nullptr);
@@ -246,7 +246,7 @@ StringRef Operator::getArgName(int index) const {
 }
 
 auto Operator::getArgDecorators(int index) const -> var_decorator_range {
-  Record *arg =
+  const Record *arg =
       cast<DefInit>(def.getValueAsDag("arguments")->getArg(index))->getDef();
   if (!arg->isSubClassOf("OpVariable"))
     return var_decorator_range(nullptr, nullptr);
@@ -572,7 +572,7 @@ void Operator::populateOpStructure() {
     if (!argDefInit)
       PrintFatalError(def.getLoc(),
                       Twine("undefined type for argument #") + Twine(i));
-    Record *argDef = argDefInit->getDef();
+    const Record *argDef = argDefInit->getDef();
     if (argDef->isSubClassOf(opVarClass))
       argDef = argDef->getValueAsDef("constraint");
 
@@ -626,7 +626,8 @@ void Operator::populateOpStructure() {
   // elements.
   int operandIndex = 0, attrIndex = 0, propIndex = 0;
   for (unsigned i = 0; i != numArgs; ++i) {
-    Record *argDef = dyn_cast<DefInit>(argumentValues->getArg(i))->getDef();
+    const Record *argDef =
+        dyn_cast<DefInit>(argumentValues->getArg(i))->getDef();
     if (argDef->isSubClassOf(opVarClass))
       argDef = argDef->getValueAsDef("constraint");
 
@@ -708,7 +709,7 @@ void Operator::populateOpStructure() {
     // do further verification based on those verified facts. If you see this
     // error, fix the traits declaration order by checking the `dependentTraits`
     // field.
-    auto verifyTraitValidity = [&](Record *trait) {
+    auto verifyTraitValidity = [&](const Record *trait) {
       auto *dependentTraits = trait->getValueAsListInit("dependentTraits");
       for (auto *traitInit : *dependentTraits)
         if (!traitSet.contains(traitInit))
diff --git a/mlir/lib/TableGen/Pattern.cpp b/mlir/lib/TableGen/Pattern.cpp
index 1be0e744ffbc8..6437839ef2084 100644
--- a/mlir/lib/TableGen/Pattern.cpp
+++ b/mlir/lib/TableGen/Pattern.cpp
@@ -136,7 +136,8 @@ int DagNode::getNumReturnsOfNativeCode() const {
 llvm::StringRef DagNode::getSymbol() const { return node->getNameStr(); }
 
 Operator &DagNode::getDialectOp(RecordOperatorMap *mapper) const {
-  llvm::Record *opDef = cast<llvm::DefInit>(node->getOperator())->getDef();
+  const llvm::Record *opDef =
+      cast<llvm::DefInit>(node->getOperator())->getDef();
   auto [it, inserted] = mapper->try_emplace(opDef);
   if (inserted)
     it->second = std::make_unique<Operator>(opDef);
diff --git a/mlir/lib/TableGen/Predicate.cpp b/mlir/lib/TableGen/Predicate.cpp
index 3c3c475f4d365..0e38dab8491c0 100644
--- a/mlir/lib/TableGen/Predicate.cpp
+++ b/mlir/lib/TableGen/Predicate.cpp
@@ -79,7 +79,7 @@ const llvm::Record *CombinedPred::getCombinerDef() const {
   return def->getValueAsDef("kind");
 }
 
-std::vector<llvm::Record *> CombinedPred::getChildren() const {
+std::vector<const llvm::Record *> CombinedPred::getChildren() const {
   assert(def->getValue("children") &&
          "CombinedPred must have a value 'children'");
   return def->getValueAsListOfDefs("children");
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 9e5f800dca60b..19d80fbbd699b 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -2055,28 +2055,23 @@ convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp,
     isPostfixUpdate = atomicCaptureOp.getSecondOp() ==
                       atomicCaptureOp.getAtomicUpdateOp().getOperation();
     auto &innerOpList = atomicUpdateOp.getRegion().front().getOperations();
-    bool isRegionArgUsed{false};
     // Find the binary update operation that uses the region argument
     // and get the expression to update
-    for (Operation &innerOp : innerOpList) {
-      if (innerOp.getNumOperands() == 2) {
-        binop = convertBinOpToAtomic(innerOp);
-        if (!llvm::is_contained(innerOp.getOperands(),
-                                atomicUpdateOp.getRegion().getArgument(0)))
-          continue;
-        isRegionArgUsed = true;
-        isXBinopExpr =
-            innerOp.getNumOperands() > 0 &&
-            innerOp.getOperand(0) == atomicUpdateOp.getRegion().getArgument(0);
-        mlirExpr =
-            (isXBinopExpr ? innerOp.getOperand(1) : innerOp.getOperand(0));
-        break;
+    if (innerOpList.size() == 2) {
+      mlir::Operation &innerOp = *atomicUpdateOp.getRegion().front().begin();
+      if (!llvm::is_contained(innerOp.getOperands(),
+                              atomicUpdateOp.getRegion().getArgument(0))) {
+        return atomicUpdateOp.emitError(
+            "no atomic update operation with region argument"
+            " as operand found inside atomic.update region");
       }
+      binop = convertBinOpToAtomic(innerOp);
+      isXBinopExpr =
+          innerOp.getOperand(0) == atomicUpdateOp.getRegion().getArgument(0);
+      mlirExpr = (isXBinopExpr ? innerOp.getOperand(1) : innerOp.getOperand(0));
+    } else {
+      binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP;
     }
-    if (!isRegionArgUsed)
-      return atomicUpdateOp.emitError(
-          "no atomic update operation with region argument"
-          " as operand found inside atomic.update region");
   }
 
   llvm::Value *llvmExpr = moduleTranslation.lookupValue(mlirExpr);
diff --git a/mlir/lib/Tools/tblgen-lsp-server/TableGenServer.cpp b/mlir/lib/Tools/tblgen-lsp-server/TableGenServer.cpp
index f7cf4de58a0e0..5c59d94a061fa 100644
--- a/mlir/lib/Tools/tblgen-lsp-server/TableGenServer.cpp
+++ b/mlir/lib/Tools/tblgen-lsp-server/TableGenServer.cpp
@@ -22,6 +22,10 @@
 #include <optional>
 
 using namespace mlir;
+using llvm::Record;
+using llvm::RecordKeeper;
+using llvm::RecordVal;
+using llvm::SourceMgr;
 
 /// Returns the range of a lexical token given a SMLoc corresponding to the
 /// start of an token location. The range is computed heuristically, and
@@ -32,7 +36,7 @@ static SMRange convertTokenLocToRange(SMLoc loc) {
 
 /// Returns a language server uri for the given source location. `mainFileURI`
 /// corresponds to the uri for the main file of the source manager.
-static lsp::URIForFile getURIFromLoc(const llvm::SourceMgr &mgr, SMLoc loc,
+static lsp::URIForFile getURIFromLoc(const SourceMgr &mgr, SMLoc loc,
                                      const lsp::URIForFile &mainFileURI) {
   int bufferId = mgr.FindBufferContainingLoc(loc);
   if (bufferId == 0 || bufferId == static_cast<int>(mgr.getMainFileID()))
@@ -47,12 +51,12 @@ static lsp::URIForFile getURIFromLoc(const llvm::SourceMgr &mgr, SMLoc loc,
 }
 
 /// Returns a language server location from the given source range.
-static lsp::Location getLocationFromLoc(llvm::SourceMgr &mgr, SMRange loc,
+static lsp::Location getLocationFromLoc(SourceMgr &mgr, SMRange loc,
                                         const lsp::URIForFile &uri) {
   return lsp::Location(getURIFromLoc(mgr, loc.Start, uri),
                        lsp::Range(mgr, loc));
 }
-static lsp::Location getLocationFromLoc(llvm::SourceMgr &mgr, SMLoc loc,
+static lsp::Location getLocationFromLoc(SourceMgr &mgr, SMLoc loc,
                                         const lsp::URIForFile &uri) {
   return getLocationFromLoc(mgr, convertTokenLocToRange(loc), uri);
 }
@@ -61,7 +65,7 @@ static lsp::Location getLocationFromLoc(llvm::SourceMgr &mgr, SMLoc loc,
 static std::optional<lsp::Diagnostic>
 getLspDiagnoticFromDiag(const llvm::SMDiagnostic &diag,
                         const lsp::URIForFile &uri) {
-  auto *sourceMgr = const_cast<llvm::SourceMgr *>(diag.getSourceMgr());
+  auto *sourceMgr = const_cast<SourceMgr *>(diag.getSourceMgr());
   if (!sourceMgr || !diag.getLoc().isValid())
     return std::nullopt;
 
@@ -79,17 +83,17 @@ getLspDiagnoticFromDiag(const llvm::SMDiagnostic &diag,
 
   // Convert the severity for the diagnostic.
   switch (diag.getKind()) {
-  case llvm::SourceMgr::DK_Warning:
+  case SourceMgr::DK_Warning:
     lspDiag.severity = lsp::DiagnosticSeverity::Warning;
     break;
-  case llvm::SourceMgr::DK_Error:
+  case SourceMgr::DK_Error:
     lspDiag.severity = lsp::DiagnosticSeverity::Error;
     break;
-  case llvm::SourceMgr::DK_Note:
+  case SourceMgr::DK_Note:
     // Notes are emitted separately from the main diagnostic, so we just treat
     // them as remarks given that we can't determine the diagnostic to relate
     // them to.
-  case llvm::SourceMgr::DK_Remark:
+  case SourceMgr::DK_Remark:
     lspDiag.severity = lsp::DiagnosticSeverity::Information;
     break;
   }
@@ -100,16 +104,15 @@ getLspDiagnoticFromDiag(const llvm::SMDiagnostic &diag,
 
 /// Get the base definition of the given record value, or nullptr if one
 /// couldn't be found.
-static std::pair<const llvm::Record *, const llvm::RecordVal *>
-getBaseValue(const llvm::Record *record, const llvm::RecordVal *value) {
+static std::pair<const Record *, const RecordVal *>
+getBaseValue(const Record *record, const RecordVal *value) {
   if (value->isTemplateArg())
     return {nullptr, nullptr};
 
   // Find a base value for the field in the super classes of the given record.
   // On success, `record` is updated to the new parent record.
   StringRef valueName = value->getName();
-  auto findValueInSupers =
-      [&](const llvm::Record *&record) -> llvm::RecordVal * {
+  auto findValueInSupers = [&](const Record *&record) -> const RecordVal * {
     for (auto [parentRecord, loc] : record->getSuperClasses()) {
       if (auto *newBase = parentRecord->getValue(valueName)) {
         record = parentRecord;
@@ -120,8 +123,8 @@ getBaseValue(const llvm::Record *record, const llvm::RecordVal *value) {
   };
 
   // Try to find the lowest definition of the record value.
-  std::pair<const llvm::Record *, const llvm::RecordVal *> baseValue = {};
-  while (const llvm::RecordVal *newBase = findValueInSupers(record))
+  std::pair<const Record *, const RecordVal *> baseValue = {};
+  while (const RecordVal *newBase = findValueInSupers(record))
     baseValue = {record, newBase};
 
   // Check that the base isn't the same as the current value (e.g. if the value
@@ -140,15 +143,15 @@ namespace {
 /// contains the definition of the symbol, the location of the symbol, and any
 /// recorded references.
 struct TableGenIndexSymbol {
-  TableGenIndexSymbol(const llvm::Record *record)
+  TableGenIndexSymbol(const Record *record)
       : definition(record),
         defLoc(convertTokenLocToRange(record->getLoc().front())) {}
-  TableGenIndexSymbol(const llvm::RecordVal *value)
+  TableGenIndexSymbol(const RecordVal *value)
       : definition(value), defLoc(convertTokenLocToRange(value->getLoc())) {}
   virtual ~TableGenIndexSymbol() = default;
 
   // The main definition of the symbol.
-  PointerUnion<const llvm::Record *, const llvm::RecordVal *> definition;
+  PointerUnion<const Record *, const RecordVal *> definition;
 
   /// The source location of the definition.
   SMRange defLoc;
@@ -158,37 +161,33 @@ struct TableGenIndexSymbol {
 };
 /// This class represents a single record symbol.
 struct TableGenRecordSymbol : public TableGenIndexSymbol {
-  TableGenRecordSymbol(const llvm::Record *record)
-      : TableGenIndexSymbol(record) {}
+  TableGenRecordSymbol(const Record *record) : TableGenIndexSymbol(record) {}
   ~TableGenRecordSymbol() override = default;
 
   static bool classof(const TableGenIndexSymbol *symbol) {
-    return symbol->definition.is<const llvm::Record *>();
+    return symbol->definition.is<const Record *>();
   }
 
   /// Return the value of this symbol.
-  const llvm::Record *getValue() const {
-    return definition.get<const llvm::Record *>();
-  }
+  const Record *getValue() const { return definition.get<const Record *>(); }
 };
 /// This class represents a single record value symbol.
 struct TableGenRecordValSymbol : public TableGenIndexSymbol {
-  TableGenRecordValSymbol(const llvm::Record *record,
-                          const llvm::RecordVal *value)
+  TableGenRecordValSymbol(const Record *record, const RecordVal *value)
       : TableGenIndexSymbol(value), record(record) {}
   ~TableGenRecordValSymbol() override = default;
 
   static bool classof(const TableGenIndexSymbol *symbol) {
-    return symbol->definition.is<const llvm::RecordVal *>();
+    return symbol->definition.is<const RecordVal *>();
   }
 
   /// Return the value of this symbol.
-  const llvm::RecordVal *getValue() const {
-    return definition.get<const llvm::RecordVal *>();
+  const RecordVal *getValue() const {
+    return definition.get<const RecordVal *>();
   }
 
   /// The parent record of this symbol.
-  const llvm::Record *record;
+  const Record *record;
 };
 
 /// This class provides an index for definitions/uses within a TableGen
@@ -199,7 +198,7 @@ class TableGenIndex {
   TableGenIndex() : intervalMap(allocator) {}
 
   /// Initialize the index with the given RecordKeeper.
-  void initialize(const llvm::RecordKeeper &records);
+  void initialize(const RecordKeeper &records);
 
   /// Lookup a symbol for the given location. Returns nullptr if no symbol could
   /// be found. If provided, `overlappedRange` is set to the range that the
@@ -217,15 +216,15 @@ class TableGenIndex {
       llvm::IntervalMapHalfOpenInfo<const char *>>;
 
   /// Get or insert a symbol for the given record.
-  TableGenIndexSymbol *getOrInsertDef(const llvm::Record *record) {
+  TableGenIndexSymbol *getOrInsertDef(const Record *record) {
     auto it = defToSymbol.try_emplace(record, nullptr);
     if (it.second)
       it.first->second = std::make_unique<TableGenRecordSymbol>(record);
     return &*it.first->second;
   }
   /// Get or insert a symbol for the given record value.
-  TableGenIndexSymbol *getOrInsertDef(const llvm::Record *record,
-                                      const llvm::RecordVal *value) {
+  TableGenIndexSymbol *getOrInsertDef(const Record *record,
+                                      const RecordVal *value) {
     auto it = defToSymbol.try_emplace(value, nullptr);
     if (it.second) {
       it.first->second =
@@ -246,7 +245,7 @@ class TableGenIndex {
 };
 } // namespace
 
-void TableGenIndex::initialize(const llvm::RecordKeeper &records) {
+void TableGenIndex::initialize(const RecordKeeper &records) {
   intervalMap.clear();
   defToSymbol.clear();
 
@@ -282,7 +281,7 @@ void TableGenIndex::initialize(const llvm::RecordKeeper &records) {
       llvm::make_pointee_range(llvm::make_second_range(records.getClasses()));
   auto defs =
       llvm::make_pointee_range(llvm::make_second_range(records.getDefs()));
-  for (const llvm::Record &def : llvm::concat<llvm::Record>(classes, defs)) {
+  for (const Record &def : llvm::concat<Record>(classes, defs)) {
     auto *sym = getOrInsertDef(&def);
     insertRef(sym, sym->defLoc, /*isDef=*/true);
 
@@ -293,7 +292,7 @@ void TableGenIndex::initialize(const llvm::RecordKeeper &records) {
       insertRef(sym, loc);
 
     // Add definitions for any values.
-    for (const llvm::RecordVal &value : def.getValues()) {
+    for (const RecordVal &value : def.getValues()) {
       auto *sym = getOrInsertDef(&def, &value);
       insertRef(sym, sym->defLoc, /*isDef=*/true);
       for (SMRange refLoc : value.getReferenceLocs())
@@ -359,13 +358,12 @@ class TableGenTextFile {
 
   std::optional<lsp::Hover> findHover(const lsp::URIForFile &uri,
                                       const lsp::Position &hoverPos);
-  lsp::Hover buildHoverForRecord(const llvm::Record *record,
+  lsp::Hover buildHoverForRecord(const Record *record,
                                  const SMRange &hoverRange);
-  lsp::Hover buildHoverForTemplateArg(const llvm::Record *record,
-                                      const llvm::RecordVal *value,
+  lsp::Hover buildHoverForTemplateArg(const Record *record,
+                                      const RecordVal *value,
                                       const SMRange &hoverRange);
-  lsp::Hover buildHoverForField(const llvm::Record *record,
-                                const llvm::RecordVal *value,
+  lsp::Hover buildHoverForField(const Record *record, const RecordVal *value,
                                 const SMRange &hoverRange);
 
 private:
@@ -383,10 +381,10 @@ class TableGenTextFile {
   std::vector<std::string> includeDirs;
 
   /// The source manager containing the contents of the input file.
-  llvm::SourceMgr sourceMgr;
+  SourceMgr sourceMgr;
 
   /// The record keeper containing the parsed tablegen constructs.
-  std::unique_ptr<llvm::RecordKeeper> recordKeeper;
+  std::unique_ptr<RecordKeeper> recordKeeper;
 
   /// The index of the parsed file.
   TableGenIndex index;
@@ -430,8 +428,8 @@ void TableGenTextFile::initialize(const lsp::URIForFile &uri,
                                   int64_t newVersion,
                                   std::vector<lsp::Diagnostic> &diagnostics) {
   version = newVersion;
-  sourceMgr = llvm::SourceMgr();
-  recordKeeper = std::make_unique<llvm::RecordKeeper>();
+  sourceMgr = SourceMgr();
+  recordKeeper = std::make_unique<RecordKeeper>();
 
   // Build a buffer for this file.
   auto memBuffer = llvm::MemoryBuffer::getMemBuffer(contents, uri.file());
@@ -442,7 +440,7 @@ void TableGenTextFile::initialize(const lsp::URIForFile &uri,
   sourceMgr.setIncludeDirs(includeDirs);
   sourceMgr.AddNewSourceBuffer(std::move(memBuffer), SMLoc());
 
-  // This class provides a context argument for the llvm::SourceMgr diagnostic
+  // This class provides a context argument for the SourceMgr diagnostic
   // handler.
   struct DiagHandlerContext {
     std::vector<lsp::Diagnostic> &diagnostics;
@@ -543,13 +541,13 @@ TableGenTextFile::findHover(const lsp::URIForFile &uri,
   // Build hover for a RecordVal, which is either a template argument or a
   // field.
   auto *recordVal = cast<TableGenRecordValSymbol>(symbol);
-  const llvm::RecordVal *value = recordVal->getValue();
+  const RecordVal *value = recordVal->getValue();
   if (value->isTemplateArg())
     return buildHoverForTemplateArg(recordVal->record, value, hoverRange);
   return buildHoverForField(recordVal->record, value, hoverRange);
 }
 
-lsp::Hover TableGenTextFile::buildHoverForRecord(const llvm::Record *record,
+lsp::Hover TableGenTextFile::buildHoverForRecord(const Record *record,
                                                  const SMRange &hoverRange) {
   lsp::Hover hover(lsp::Range(sourceMgr, hoverRange));
   {
@@ -570,7 +568,7 @@ lsp::Hover TableGenTextFile::buildHoverForRecord(const llvm::Record *record,
     auto printAndFormatField = [&](StringRef fieldName) {
       // Check that the record actually has the given field, and that it's a
       // string.
-      const llvm::RecordVal *value = record->getValue(fieldName);
+      const RecordVal *value = record->getValue(fieldName);
       if (!value || !value->getValue())
         return;
       auto *stringValue = dyn_cast<llvm::StringInit>(value->getValue());
@@ -593,10 +591,8 @@ lsp::Hover TableGenTextFile::buildHoverForRecord(const llvm::Record *record,
   return hover;
 }
 
-lsp::Hover
-TableGenTextFile::buildHoverForTemplateArg(const llvm::Record *record,
-                                           const llvm::RecordVal *value,
-                                           const SMRange &hoverRange) {
+lsp::Hover TableGenTextFile::buildHoverForTemplateArg(
+    const Record *record, const RecordVal *value, const SMRange &hoverRange) {
   lsp::Hover hover(lsp::Range(sourceMgr, hoverRange));
   {
     llvm::raw_string_ostream hoverOS(hover.contents.value);
@@ -609,8 +605,8 @@ TableGenTextFile::buildHoverForTemplateArg(const llvm::Record *record,
   return hover;
 }
 
-lsp::Hover TableGenTextFile::buildHoverForField(const llvm::Record *record,
-                                                const llvm::RecordVal *value,
+lsp::Hover TableGenTextFile::buildHoverForField(const Record *record,
+                                                const RecordVal *value,
                                                 const SMRange &hoverRange) {
   lsp::Hover hover(lsp::Range(sourceMgr, hoverRange));
   {
diff --git a/mlir/lib/Transforms/RemoveDeadValues.cpp b/mlir/lib/Transforms/RemoveDeadValues.cpp
index 1cc0096805616..3de4fb75ed831 100644
--- a/mlir/lib/Transforms/RemoveDeadValues.cpp
+++ b/mlir/lib/Transforms/RemoveDeadValues.cpp
@@ -576,6 +576,8 @@ void RemoveDeadValues::runOnOperation() {
   // all symbol ops present in the IR are function-like, and all symbol user ops
   // present in the IR are call-like.
   WalkResult acceptableIR = module->walk([&](Operation *op) {
+    if (op == module)
+      return WalkResult::advance();
     if (isa<BranchOpInterface>(op) ||
         (isa<SymbolOpInterface>(op) && !isa<FunctionOpInterface>(op)) ||
         (isa<SymbolUserOpInterface>(op) && !isa<CallOpInterface>(op))) {
diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index a5872cd3d3069..97dd3ab1f4829 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -2466,13 +2466,8 @@ LogicalResult OperationConverter::convertOperations(ArrayRef<Operation *> ops) {
   // legalized.
   finalize(rewriter);
 
-  // After a successful conversion, apply rewrites if this is not an analysis
-  // conversion.
-  if (mode == OpConversionMode::Analysis) {
-    rewriterImpl.undoRewrites();
-  } else {
-    rewriterImpl.applyRewrites();
-  }
+  // After a successful conversion, apply rewrites.
+  rewriterImpl.applyRewrites();
 
   // Gather all unresolved materializations.
   SmallVector<UnrealizedConversionCastOp> allCastOps;
@@ -3215,13 +3210,78 @@ LogicalResult mlir::applyFullConversion(Operation *op,
 //===----------------------------------------------------------------------===//
 // Analysis Conversion
 
+/// Find a common IsolatedFromAbove ancestor of the given ops. If at least one
+/// op is a top-level module op (which is expected to be isolated from above),
+/// return that op.
+static Operation *findCommonAncestor(ArrayRef<Operation *> ops) {
+  // Check if there is a top-level operation within `ops`. If so, return that
+  // op.
+  for (Operation *op : ops) {
+    if (!op->getParentOp()) {
+#ifndef NDEBUG
+      assert(op->hasTrait<OpTrait::IsIsolatedFromAbove>() &&
+             "expected top-level op to be isolated from above");
+      for (Operation *other : ops)
+        assert(op->isAncestor(other) &&
+               "expected ops to have a common ancestor");
+#endif // NDEBUG
+      return op;
+    }
+  }
+
+  // No top-level op. Find a common ancestor.
+  Operation *commonAncestor =
+      ops.front()->getParentWithTrait<OpTrait::IsIsolatedFromAbove>();
+  for (Operation *op : ops.drop_front()) {
+    while (!commonAncestor->isProperAncestor(op)) {
+      commonAncestor =
+          commonAncestor->getParentWithTrait<OpTrait::IsIsolatedFromAbove>();
+      assert(commonAncestor &&
+             "expected to find a common isolated from above ancestor");
+    }
+  }
+
+  return commonAncestor;
+}
+
 LogicalResult mlir::applyAnalysisConversion(
     ArrayRef<Operation *> ops, ConversionTarget &target,
     const FrozenRewritePatternSet &patterns, ConversionConfig config) {
+#ifndef NDEBUG
+  if (config.legalizableOps)
+    assert(config.legalizableOps->empty() && "expected empty set");
+#endif // NDEBUG
+
+  // Clone closted common ancestor that is isolated from above.
+  Operation *commonAncestor = findCommonAncestor(ops);
+  IRMapping mapping;
+  Operation *clonedAncestor = commonAncestor->clone(mapping);
+  // Compute inverse IR mapping.
+  DenseMap<Operation *, Operation *> inverseOperationMap;
+  for (auto &it : mapping.getOperationMap())
+    inverseOperationMap[it.second] = it.first;
+
+  // Convert the cloned operations. The original IR will remain unchanged.
+  SmallVector<Operation *> opsToConvert = llvm::map_to_vector(
+      ops, [&](Operation *op) { return mapping.lookup(op); });
   OperationConverter opConverter(target, patterns, config,
                                  OpConversionMode::Analysis);
-  return opConverter.convertOperations(ops);
+  LogicalResult status = opConverter.convertOperations(opsToConvert);
+
+  // Remap `legalizableOps`, so that they point to the original ops and not the
+  // cloned ops.
+  if (config.legalizableOps) {
+    DenseSet<Operation *> originalLegalizableOps;
+    for (Operation *op : *config.legalizableOps)
+      originalLegalizableOps.insert(inverseOperationMap[op]);
+    *config.legalizableOps = std::move(originalLegalizableOps);
+  }
+
+  // Erase the cloned IR.
+  clonedAncestor->erase();
+  return status;
 }
+
 LogicalResult
 mlir::applyAnalysisConversion(Operation *op, ConversionTarget &target,
                               const FrozenRewritePatternSet &patterns,
diff --git a/mlir/python/mlir/_mlir_libs/_mlir/ir.pyi b/mlir/python/mlir/_mlir_libs/_mlir/ir.pyi
index 41ed84e046725..fb7efb8cd28a5 100644
--- a/mlir/python/mlir/_mlir_libs/_mlir/ir.pyi
+++ b/mlir/python/mlir/_mlir_libs/_mlir/ir.pyi
@@ -117,6 +117,7 @@ __all__ = [
     "Float8E4M3Type",
     "Float8E5M2FNUZType",
     "Float8E5M2Type",
+    "Float8E8M0FNUType",
     "FloatAttr",
     "FloatTF32Type",
     "FloatType",
@@ -1660,6 +1661,19 @@ class Float8E5M2Type(FloatType):
     @property
     def typeid(self) -> TypeID: ...
 
+class Float8E8M0FNUType(FloatType):
+    static_typeid: ClassVar[TypeID]
+    @staticmethod
+    def get(context: Context | None = None) -> Float8E8M0FNUType:
+        """
+        Create a float8_e8m0fnu type.
+        """
+    @staticmethod
+    def isinstance(other: Type) -> bool: ...
+    def __init__(self, cast_from_type: Type) -> None: ...
+    @property
+    def typeid(self) -> TypeID: ...
+
 class FloatAttr(Attribute):
     static_typeid: ClassVar[TypeID]
     @staticmethod
diff --git a/mlir/python/mlir/dialects/tensor.py b/mlir/python/mlir/dialects/tensor.py
index 0b30d10209908..146b5f85d07f5 100644
--- a/mlir/python/mlir/dialects/tensor.py
+++ b/mlir/python/mlir/dialects/tensor.py
@@ -1,6 +1,7 @@
 #  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 #  See https://llvm.org/LICENSE.txt for license information.
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+from typing import Optional
 
 from ._tensor_ops_gen import *
 from ._tensor_ops_gen import _Dialect
@@ -25,6 +26,7 @@ def __init__(
         sizes: Sequence[Union[int, Value]],
         element_type: Type,
         *,
+        encoding: Optional[Attribute] = None,
         loc=None,
         ip=None,
     ):
@@ -40,7 +42,7 @@ def __init__(
             else:
                 static_sizes.append(ShapedType.get_dynamic_size())
                 dynamic_sizes.append(s)
-        result_type = RankedTensorType.get(static_sizes, element_type)
+        result_type = RankedTensorType.get(static_sizes, element_type, encoding)
         super().__init__(result_type, dynamic_sizes, loc=loc, ip=ip)
 
 
@@ -48,11 +50,14 @@ def empty(
     sizes: Sequence[Union[int, Value]],
     element_type: Type,
     *,
+    encoding: Optional[Attribute] = None,
     loc=None,
     ip=None,
 ) -> _ods_cext.ir.Value:
     return _get_op_result_or_op_results(
-        EmptyOp(sizes=sizes, element_type=element_type, loc=loc, ip=ip)
+        EmptyOp(
+            sizes=sizes, element_type=element_type, encoding=encoding, loc=loc, ip=ip
+        )
     )
 
 
diff --git a/mlir/python/mlir/extras/types.py b/mlir/python/mlir/extras/types.py
index 5b24a6d526f2f..34eee1edb57ff 100644
--- a/mlir/python/mlir/extras/types.py
+++ b/mlir/python/mlir/extras/types.py
@@ -20,6 +20,7 @@
     Float8E4M3FNType,
     Float8E4M3Type,
     Float8E5M2Type,
+    Float8E8M0FNUType,
     FunctionType,
     IndexType,
     IntegerType,
@@ -80,6 +81,7 @@ def ui(width):
 f4E2M1FN = lambda: Float4E2M1FNType.get()
 f6E2M3FN = lambda: Float6E2M3FNType.get()
 f6E3M2FN = lambda: Float6E3M2FNType.get()
+f8E8M0FNU = lambda: Float8E8M0FNUType.get()
 
 none = lambda: NoneType.get()
 
diff --git a/mlir/test/Conversion/MemRefToLLVM/convert-static-memref-ops.mlir b/mlir/test/Conversion/MemRefToLLVM/convert-static-memref-ops.mlir
index 96cf2264e9c2f..4f4ce0d0ed978 100644
--- a/mlir/test/Conversion/MemRefToLLVM/convert-static-memref-ops.mlir
+++ b/mlir/test/Conversion/MemRefToLLVM/convert-static-memref-ops.mlir
@@ -95,6 +95,22 @@ func.func @static_alloca() -> memref<32x18xf32> {
 
 // -----
 
+// CHECK-LABEL: func @static_alloca_zero()
+func.func @static_alloca_zero() -> memref<32x0x18xf32> {
+// CHECK: %[[sz1:.*]] = llvm.mlir.constant(32 : index) : i64
+// CHECK: %[[sz2:.*]] = llvm.mlir.constant(0 : index) : i64
+// CHECK: %[[sz3:.*]] = llvm.mlir.constant(18 : index) : i64
+// CHECK: %[[st1:.*]] = llvm.mlir.constant(1 : index) : i64
+// CHECK: %[[st2:.*]] = llvm.mlir.constant(0 : index) : i64
+// CHECK: %[[num_elems:.*]] = llvm.mlir.constant(0 : index) : i64
+// CHECK: %[[allocated:.*]] = llvm.alloca %[[num_elems]] x f32 : (i64) -> !llvm.ptr
+ %0 = memref.alloca() : memref<32x0x18xf32>
+
+ return %0 : memref<32x0x18xf32>
+}
+
+// -----
+
 // CHECK-LABEL: func @static_dealloc
 func.func @static_dealloc(%static: memref<10x8xf32>) {
 // CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-named.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-named.mlir
index 1e360e0b7c22a..4b93e0a3cd3f3 100644
--- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-named.mlir
+++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-named.mlir
@@ -715,6 +715,22 @@ func.func @depthwise_conv(%arg0 : tensor<1x7x5x3xf32>, %arg1 : tensor<3x1x3x11xf
 
 // -----
 
+// CHECK: #[[$MAP0:.*]] = affine_map<(d0, d1, d2, d3) -> (0)>
+// CHECK: #[[$MAP1:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
+
+// CHECK-LABEL: @depthwise_conv_scalar_bias
+func.func @depthwise_conv_scalar_bias(%arg0 : tensor<1x7x5x3xf32>, %arg1 : tensor<3x1x3x11xf32>, %arg2 : tensor<1xf32>) -> () {
+  // CHECK: [[BIAS:%.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP1]], #[[$MAP1]]], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%arg2, %{{.*}} : tensor<1xf32>, tensor<1x5x5x33xf32>) outs(%{{.*}} : tensor<1x5x5x33xf32>) {
+  // CHECK: ^bb0(%[[ARG3:[0-9a-zA-Z_]+]]: f32, %[[ARG4:[0-9a-zA-Z_]+]]: f32, %{{.*}}: f32):
+  // CHECK:   [[ADD:%.+]] = arith.addf %[[ARG3]], %[[ARG4]] : f32
+  // CHECK:   linalg.yield [[ADD]] : f32
+  // CHECK: } -> tensor<1x5x5x33xf32>
+  %2 = tosa.depthwise_conv2d %arg0, %arg1, %arg2 { pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>, dilation = array<i64: 1, 1> } : (tensor<1x7x5x3xf32>, tensor<3x1x3x11xf32>, tensor<1xf32>)  -> tensor<1x5x5x33xf32>
+  return
+}
+
+// -----
+
 // CHECK: #[[$MAP0:.*]] = affine_map<(d0, d1, d2, d3) -> (d3)>
 // CHECK: #[[$MAP1:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
 
@@ -853,6 +869,20 @@ func.func @conv3d_f32(%input: tensor<1x49x48x47x27xf32>, %weights: tensor<28x3x4
 
 // -----
 
+// CHECK: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (0)>
+// CHECK: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3, d4)>
+
+// CHECK-LABEL: @conv3d_scalar_bias_f32
+func.func @conv3d_scalar_bias_f32(%input: tensor<1x49x48x47x27xf32>, %weights: tensor<28x3x4x5x27xf32>, %bias: tensor<1xf32>) -> () {
+  // CHECK:  %[[INIT:.+]] = tensor.empty() : tensor<1x47x45x43x28xf32>
+  // CHECK:      %[[BROADCAST:.+]] = linalg.generic
+  // CHECK-SAME: {indexing_maps = [#[[$MAP1]], #[[$MAP2]]], iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel"]}
+  %0 = tosa.conv3d %input, %weights, %bias {pad = array<i64: 0, 0, 0, 0, 0, 0>, stride = array<i64: 1, 1, 1>, dilation = array<i64: 1, 1, 1>} : (tensor<1x49x48x47x27xf32>, tensor<28x3x4x5x27xf32>, tensor<1xf32>)  -> tensor<1x47x45x43x28xf32>
+  return
+}
+
+// -----
+
 // CHECK: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d4)>
 // CHECK: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3, d4)>
 
diff --git a/mlir/test/Conversion/VectorToSCF/vector-to-scf.mlir b/mlir/test/Conversion/VectorToSCF/vector-to-scf.mlir
index 30df419822994..c55a0c558bc2f 100644
--- a/mlir/test/Conversion/VectorToSCF/vector-to-scf.mlir
+++ b/mlir/test/Conversion/VectorToSCF/vector-to-scf.mlir
@@ -133,7 +133,7 @@ func.func @materialize_read(%M: index, %N: index, %O: index, %P: index) {
     affine.for %i1 = 0 to %N {
       affine.for %i2 = 0 to %O {
         affine.for %i3 = 0 to %P step 5 {
-          %f = vector.transfer_read %A[%i0, %i1, %i2, %i3], %f0 {in_bounds = [false, true, false], permutation_map = affine_map<(d0, d1, d2, d3) -> (d3, 0, d0)>} : memref<?x?x?x?xf32>, vector<5x4x3xf32>
+          %f = vector.transfer_read %A[%i0, %i1, %i2, %i3], %f0 {permutation_map = affine_map<(d0, d1, d2, d3) -> (d3, 0, d0)>} : memref<?x?x?x?xf32>, vector<5x4x3xf32>
           // Add a dummy use to prevent dead code elimination from removing
           // transfer read ops.
           "dummy_use"(%f) : (vector<5x4x3xf32>) -> ()
@@ -507,7 +507,7 @@ func.func @transfer_read_with_tensor(%arg: tensor<f32>) -> vector<1xf32> {
     // CHECK-NEXT: %[[RESULT:.*]] = vector.broadcast %[[EXTRACTED]] : f32 to vector<1xf32>
     // CHECK-NEXT: return %[[RESULT]] : vector<1xf32>
     %f0 = arith.constant 0.0 : f32
-    %0 = vector.transfer_read %arg[], %f0 {in_bounds = [true], permutation_map = affine_map<()->(0)>} :
+    %0 = vector.transfer_read %arg[], %f0 {permutation_map = affine_map<()->(0)>} :
       tensor<f32>, vector<1xf32>
     return %0: vector<1xf32>
 }
@@ -746,7 +746,7 @@ func.func @cannot_lower_transfer_read_with_leading_scalable(%arg0: memref<?x4xf3
 func.func @does_not_crash_on_unpack_one_dim(%subview:  memref<1x1x1x1xi32>, %mask: vector<1x1xi1>) -> vector<1x1x1x1xi32> {
   %c0 = arith.constant 0 : index
   %c0_i32 = arith.constant 0 : i32
-  %3 = vector.transfer_read %subview[%c0, %c0, %c0, %c0], %c0_i32, %mask {in_bounds = [false, true, true, false], permutation_map = #map1}
+  %3 = vector.transfer_read %subview[%c0, %c0, %c0, %c0], %c0_i32, %mask {permutation_map = #map1}
           : memref<1x1x1x1xi32>, vector<1x1x1x1xi32>
   return %3 : vector<1x1x1x1xi32>
 }
diff --git a/mlir/test/Conversion/VectorToXeGPU/load-to-xegpu.mlir b/mlir/test/Conversion/VectorToXeGPU/load-to-xegpu.mlir
new file mode 100644
index 0000000000000..e2a506f8ad5ab
--- /dev/null
+++ b/mlir/test/Conversion/VectorToXeGPU/load-to-xegpu.mlir
@@ -0,0 +1,105 @@
+// RUN: mlir-opt %s -convert-vector-to-xegpu -split-input-file | FileCheck %s
+
+func.func @load_1D_vector(%source: memref<8x16x32xf32>, %offset: index) -> vector<8xf32> {
+  %0 = vector.load %source[%offset, %offset, %offset]
+    : memref<8x16x32xf32>, vector<8xf32>
+  return %0 : vector<8xf32>
+}
+
+// CHECK-LABEL: @load_1D_vector(
+// CHECK-SAME:  %[[SRC:.+]]: memref<8x16x32xf32>,
+// CHECK-SAME:  %[[OFFSET:.+]]: index
+// CHECK:       %[[DESC:.+]] = xegpu.create_nd_tdesc
+// CHECK-SAME:    %[[SRC]][%[[OFFSET]], %[[OFFSET]], %[[OFFSET]]]
+// CHECK-SAME:    memref<8x16x32xf32> -> !xegpu.tensor_desc<8xf32,
+// CHECK-SAME:    boundary_check = true
+// CHECK:       %[[VEC:.+]] = xegpu.load_nd %[[DESC]]{{.*}}-> vector<8xf32>
+// CHECK:       return %[[VEC]]
+
+// -----
+
+func.func @load_2D_vector(%source: memref<8x16x32xf32>,
+    %offset: index) -> vector<8x16xf32> {
+  %0 = vector.load %source[%offset, %offset, %offset]
+    : memref<8x16x32xf32>, vector<8x16xf32>
+  return %0 : vector<8x16xf32>
+}
+
+// CHECK-LABEL: @load_2D_vector(
+// CHECK-SAME:  %[[SRC:.+]]: memref<8x16x32xf32>,
+// CHECK-SAME:  %[[OFFSET:.+]]: index
+// CHECK:       %[[DESC:.+]] = xegpu.create_nd_tdesc
+// CHECK-SAME:    %[[SRC]][%[[OFFSET]], %[[OFFSET]], %[[OFFSET]]]
+// CHECK-SAME:    memref<8x16x32xf32> -> !xegpu.tensor_desc<8x16xf32,
+// CHECK-SAME:    boundary_check = true
+// CHECK:       %[[VEC:.+]] = xegpu.load_nd %[[DESC]]{{.*}}-> vector<8x16xf32>
+// CHECK:       return %[[VEC]]
+
+// -----
+
+func.func @load_dynamic_source(%source: memref<?x?x?xf32>,
+    %offset: index) -> vector<8x16xf32> {
+  %0 = vector.load %source[%offset, %offset, %offset]
+    : memref<?x?x?xf32>, vector<8x16xf32>
+  return %0 : vector<8x16xf32>
+}
+
+// CHECK-LABEL: @load_dynamic_source(
+// CHECK-SAME:  %[[SRC:.+]]: memref<?x?x?xf32>,
+// CHECK-SAME:  %[[OFFSET:.+]]: index
+// CHECK-DAG:   %[[C0:.+]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C1:.+]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C2:.+]] = arith.constant 2 : index
+// CHECK-DAG:   %[[DIM_0:.+]] = memref.dim %[[SRC]], %[[C0]]
+// CHECK-DAG:   %[[DIM_1:.+]] = memref.dim %[[SRC]], %[[C1]]
+// CHECK-DAG:   %[[DIM_2:.+]] = memref.dim %[[SRC]], %[[C2]]
+// CHECK:       %[[DIM_0_STRIDE:.+]] = arith.muli %[[DIM_2]], %[[DIM_1]]
+// CHECK:       %[[DESC:.+]] = xegpu.create_nd_tdesc %[[SRC]][%[[OFFSET]], %[[OFFSET]], %[[OFFSET]]]
+// CHECK-SAME:    [%[[DIM_0]], %[[DIM_1]], %[[DIM_2]]], [%[[DIM_0_STRIDE]], %[[DIM_2]], 1]
+// CHECK-SAME:    memref<?x?x?xf32> -> !xegpu.tensor_desc<8x16xf32,
+// CHECK-SAME:    boundary_check = true
+// CHECK:       %[[VEC:.+]] = xegpu.load_nd %[[DESC]]{{.*}}-> vector<8x16xf32>
+// CHECK:       return %[[VEC]]
+
+// -----
+
+func.func @load_out_of_bounds(%source: memref<7x15xf32>,
+    %offset: index) -> vector<8x16xf32> {
+  %0 = vector.load %source[%offset, %offset]
+    : memref<7x15xf32>, vector<8x16xf32>
+  return %0 : vector<8x16xf32>
+}
+
+// CHECK-LABEL: @load_out_of_bounds(
+// CHECK-SAME:  %[[SRC:.+]]: memref<7x15xf32>,
+// CHECK-SAME:  %[[OFFSET:.+]]: index
+// CHECK:       %[[DESC:.+]] = xegpu.create_nd_tdesc
+// CHECK-SAME:    %[[SRC]][%[[OFFSET]], %[[OFFSET]]]
+// CHECK-SAME:    memref<7x15xf32> -> !xegpu.tensor_desc<8x16xf32,
+// CHECK-SAME:    boundary_check = true
+// CHECK:       %[[VEC:.+]] = xegpu.load_nd %[[DESC]]{{.*}}-> vector<8x16xf32>
+// CHECK:       return %[[VEC]]
+
+// -----
+
+func.func @no_load_high_dim_vector(%source: memref<16x32x64xf32>,
+    %offset: index) -> vector<8x16x32xf32> {
+  %0 = vector.load %source[%offset, %offset, %offset]
+    : memref<16x32x64xf32>, vector<8x16x32xf32>
+  return %0 : vector<8x16x32xf32>
+}
+
+// CHECK-LABEL: @no_load_high_dim_vector(
+// CHECK:       vector.load
+
+// -----
+
+func.func @no_load_zero_dim_vector(%source: memref<64xf32>,
+    %offset: index) -> vector<f32> {
+  %0 = vector.load %source[%offset]
+    : memref<64xf32>, vector<f32>
+  return %0 : vector<f32>
+}
+
+// CHECK-LABEL: @no_load_zero_dim_vector(
+// CHECK:       vector.load
diff --git a/mlir/test/Conversion/VectorToXeGPU/store-to-xegpu.mlir b/mlir/test/Conversion/VectorToXeGPU/store-to-xegpu.mlir
new file mode 100644
index 0000000000000..3d45407c2486d
--- /dev/null
+++ b/mlir/test/Conversion/VectorToXeGPU/store-to-xegpu.mlir
@@ -0,0 +1,106 @@
+// RUN: mlir-opt %s -convert-vector-to-xegpu -split-input-file | FileCheck %s
+
+func.func @store_1D_vector(%vec: vector<8xf32>,
+    %source: memref<8x16x32xf32>, %offset: index) {
+  vector.store %vec, %source[%offset, %offset, %offset]
+    : memref<8x16x32xf32>, vector<8xf32>
+  return
+}
+
+// CHECK-LABEL: @store_1D_vector(
+// CHECK-SAME:  %[[VEC:.+]]: vector<8xf32>,
+// CHECK-SAME:  %[[SRC:.+]]: memref<8x16x32xf32>,
+// CHECK-SAME:  %[[OFFSET:.+]]: index
+// CHECK:       %[[DESC:.+]] = xegpu.create_nd_tdesc
+// CHECK-SAME:    %[[SRC]][%[[OFFSET]], %[[OFFSET]], %[[OFFSET]]]
+// CHECK-SAME:    memref<8x16x32xf32> -> !xegpu.tensor_desc<8xf32,
+// CHECK-SAME:    boundary_check = true
+// CHECK:       xegpu.store_nd %[[VEC]], %[[DESC]] : vector<8xf32>
+
+// -----
+
+func.func @store_2D_vector(%vec: vector<8x16xf32>,
+    %source: memref<8x16x32xf32>, %offset: index) {
+  vector.store %vec, %source[%offset, %offset, %offset]
+    : memref<8x16x32xf32>, vector<8x16xf32>
+  return
+}
+
+// CHECK-LABEL: @store_2D_vector(
+// CHECK-SAME:  %[[VEC:.+]]: vector<8x16xf32>,
+// CHECK-SAME:  %[[SRC:.+]]: memref<8x16x32xf32>,
+// CHECK-SAME:  %[[OFFSET:.+]]: index
+// CHECK:       %[[DESC:.+]] = xegpu.create_nd_tdesc
+// CHECK-SAME:    %[[SRC]][%[[OFFSET]], %[[OFFSET]], %[[OFFSET]]]
+// CHECK-SAME:    memref<8x16x32xf32> -> !xegpu.tensor_desc<8x16xf32,
+// CHECK-SAME:    boundary_check = true
+// CHECK:       xegpu.store_nd %[[VEC]], %[[DESC]] : vector<8x16xf32>
+
+// -----
+
+func.func @store_dynamic_source(%vec: vector<8x16xf32>,
+    %source: memref<?x?x?xf32>, %offset: index) {
+  vector.store %vec, %source[%offset, %offset, %offset]
+    : memref<?x?x?xf32>, vector<8x16xf32>
+  return
+}
+
+// CHECK-LABEL: @store_dynamic_source(
+// CHECK-SAME:  %[[VEC:.+]]: vector<8x16xf32>,
+// CHECK-SAME:  %[[SRC:.+]]: memref<?x?x?xf32>,
+// CHECK-SAME:  %[[OFFSET:.+]]: index
+// CHECK-DAG:   %[[C0:.+]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C1:.+]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C2:.+]] = arith.constant 2 : index
+// CHECK-DAG:   %[[DIM_0:.+]] = memref.dim %[[SRC]], %[[C0]]
+// CHECK-DAG:   %[[DIM_1:.+]] = memref.dim %[[SRC]], %[[C1]]
+// CHECK-DAG:   %[[DIM_2:.+]] = memref.dim %[[SRC]], %[[C2]]
+// CHECK:       %[[DIM_0_STRIDE:.+]] = arith.muli %[[DIM_2]], %[[DIM_1]]
+// CHECK:       %[[DESC:.+]] = xegpu.create_nd_tdesc %[[SRC]][%[[OFFSET]], %[[OFFSET]], %[[OFFSET]]]
+// CHECK-SAME:    [%[[DIM_0]], %[[DIM_1]], %[[DIM_2]]], [%[[DIM_0_STRIDE]], %[[DIM_2]], 1]
+// CHECK-SAME:    memref<?x?x?xf32> -> !xegpu.tensor_desc<8x16xf32,
+// CHECK-SAME:    boundary_check = true
+// CHECK:       xegpu.store_nd %[[VEC]], %[[DESC]] : vector<8x16xf32>
+
+// -----
+
+func.func @store_out_of_bounds(%vec: vector<8x16xf32>,
+    %source: memref<7x64xf32>, %offset: index) {
+  vector.store %vec, %source[%offset, %offset]
+    : memref<7x64xf32>, vector<8x16xf32>
+  return
+}
+
+// CHECK-LABEL:   @store_out_of_bounds(
+// CHECK-SAME:  %[[VEC:.+]]: vector<8x16xf32>,
+// CHECK-SAME:  %[[SRC:.+]]: memref<7x64xf32>,
+// CHECK-SAME:  %[[OFFSET:.+]]: index
+// CHECK:       %[[DESC:.+]] = xegpu.create_nd_tdesc
+// CHECK-SAME:    %[[SRC]][%[[OFFSET]], %[[OFFSET]]]
+// CHECK-SAME:    memref<7x64xf32> -> !xegpu.tensor_desc<8x16xf32,
+// CHECK-SAME:    boundary_check = true
+// CHECK:       xegpu.store_nd %[[VEC]], %[[DESC]] : vector<8x16xf32>
+
+// -----
+
+func.func @no_store_high_dim_vector(%vec: vector<8x16x32xf32>,
+    %source: memref<16x32x64xf32>, %offset: index) {
+  vector.store %vec, %source[%offset, %offset, %offset]
+    : memref<16x32x64xf32>, vector<8x16x32xf32>
+  return
+}
+
+// CHECK-LABEL: @no_store_high_dim_vector(
+// CHECK:       vector.store
+
+// -----
+
+func.func @no_store_zero_dim_vector(%vec: vector<f32>,
+    %source: memref<64xf32>, %offset: index) {
+  vector.store %vec, %source[%offset]
+    : memref<64xf32>, vector<f32>
+  return
+}
+
+// CHECK-LABEL: @no_store_zero_dim_vector(
+// CHECK:       vector.store
diff --git a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir
index 0a077624d18f8..9244604128cb7 100644
--- a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir
+++ b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir
@@ -22,7 +22,7 @@ func.func @vec1d_1(%A : memref<?x?xf32>, %B : memref<?x?x?xf32>) {
 // CHECK-NEXT: %{{.*}} = affine.apply #[[$map_id1]](%[[C0]])
 // CHECK-NEXT: %{{.*}} = affine.apply #[[$map_id1]](%[[C0]])
 // CHECK-NEXT: %{{.*}} = arith.constant 0.0{{.*}}: f32
-// CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {in_bounds = [true], permutation_map = #[[$map_proj_d0d1_0]]} : memref<?x?xf32>, vector<128xf32>
+// CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {permutation_map = #[[$map_proj_d0d1_0]]} : memref<?x?xf32>, vector<128xf32>
    affine.for %i0 = 0 to %M { // vectorized due to scalar -> vector
      %a0 = affine.load %A[%c0, %c0] : memref<?x?xf32>
    }
@@ -425,7 +425,7 @@ func.func @vec_rejected_8(%A : memref<?x?xf32>, %B : memref<?x?x?xf32>) {
 // CHECK:     %{{.*}} = affine.apply #[[$map_id1]](%{{.*}})
 // CHECK:     %{{.*}} = affine.apply #[[$map_id1]](%{{.*}})
 // CHECK:     %{{.*}} = arith.constant 0.0{{.*}}: f32
-// CHECK:     {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {in_bounds = [true], permutation_map = #[[$map_proj_d0d1_0]]} : memref<?x?xf32>, vector<128xf32>
+// CHECK:     {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {permutation_map = #[[$map_proj_d0d1_0]]} : memref<?x?xf32>, vector<128xf32>
    affine.for %i17 = 0 to %M { // not vectorized, the 1-D pattern that matched %{{.*}} in DFS post-order prevents vectorizing %{{.*}}
      affine.for %i18 = 0 to %M { // vectorized due to scalar -> vector
        %a18 = affine.load %A[%c0, %c0] : memref<?x?xf32>
@@ -459,7 +459,7 @@ func.func @vec_rejected_9(%A : memref<?x?xf32>, %B : memref<?x?x?xf32>) {
 // CHECK:      %{{.*}} = affine.apply #[[$map_id1]](%{{.*}})
 // CHECK-NEXT: %{{.*}} = affine.apply #[[$map_id1]](%{{.*}})
 // CHECK-NEXT: %{{.*}} = arith.constant 0.0{{.*}}: f32
-// CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {in_bounds = [true], permutation_map = #[[$map_proj_d0d1_0]]} : memref<?x?xf32>, vector<128xf32>
+// CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {permutation_map = #[[$map_proj_d0d1_0]]} : memref<?x?xf32>, vector<128xf32>
    affine.for %i17 = 0 to %M { // not vectorized, the 1-D pattern that matched %i18 in DFS post-order prevents vectorizing %{{.*}}
      affine.for %i18 = 0 to %M { // vectorized due to scalar -> vector
        %a18 = affine.load %A[%c0, %c0] : memref<?x?xf32>
diff --git a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_2d.mlir b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_2d.mlir
index eb5120a49e3d4..83916e755363b 100644
--- a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_2d.mlir
+++ b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_2d.mlir
@@ -123,8 +123,8 @@ func.func @vectorize_matmul(%arg0: memref<?x?xf32>, %arg1: memref<?x?xf32>, %arg
   //      VECT:  affine.for %[[I2:.*]] = #[[$map_id1]](%[[C0]]) to #[[$map_id1]](%[[M]]) step 4 {
   // VECT-NEXT:    affine.for %[[I3:.*]] = #[[$map_id1]](%[[C0]]) to #[[$map_id1]](%[[N]]) step 8 {
   // VECT-NEXT:      affine.for %[[I4:.*]] = #[[$map_id1]](%[[C0]]) to #[[$map_id1]](%[[K]]) {
-  //      VECT:        %[[A:.*]] = vector.transfer_read %{{.*}}[%[[I4]], %[[I3]]], %{{.*}} {in_bounds = [true, false], permutation_map = #[[$map_proj_d0d1_zerod1]]} : memref<?x?xf32>, vector<4x8xf32>
-  //      VECT:        %[[B:.*]] = vector.transfer_read %{{.*}}[%[[I2]], %[[I4]]], %{{.*}} {in_bounds = [false, true], permutation_map = #[[$map_proj_d0d1_d0zero]]} : memref<?x?xf32>, vector<4x8xf32>
+  //      VECT:        %[[A:.*]] = vector.transfer_read %{{.*}}[%[[I4]], %[[I3]]], %{{.*}} {permutation_map = #[[$map_proj_d0d1_zerod1]]} : memref<?x?xf32>, vector<4x8xf32>
+  //      VECT:        %[[B:.*]] = vector.transfer_read %{{.*}}[%[[I2]], %[[I4]]], %{{.*}} {permutation_map = #[[$map_proj_d0d1_d0zero]]} : memref<?x?xf32>, vector<4x8xf32>
   // VECT-NEXT:        %[[C:.*]] = arith.mulf %[[B]], %[[A]] : vector<4x8xf32>
   //      VECT:        %[[D:.*]] = vector.transfer_read %{{.*}}[%[[I2]], %[[I3]]], %{{.*}} : memref<?x?xf32>, vector<4x8xf32>
   // VECT-NEXT:        %[[E:.*]] = arith.addf %[[D]], %[[C]] : vector<4x8xf32>
diff --git a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_affine_apply.mlir b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_affine_apply.mlir
index 16ade6455d697..15a7133cf0f65 100644
--- a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_affine_apply.mlir
+++ b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_affine_apply.mlir
@@ -141,7 +141,7 @@ func.func @affine_map_with_expr_2(%arg0: memref<8x12x16xf32>, %arg1: memref<8x24
 // CHECK-NEXT:       %[[S1:.*]] = affine.apply #[[$MAP_ID4]](%[[ARG3]], %[[ARG4]], %[[I0]])
 // CHECK-NEXT:       %[[S2:.*]] = affine.apply #[[$MAP_ID5]](%[[ARG3]], %[[ARG4]], %[[I0]])
 // CHECK-NEXT:       %[[CST:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK-NEXT:       %[[S3:.*]] = vector.transfer_read %[[ARG0]][%[[S0]], %[[S1]], %[[S2]]], %[[CST]] {in_bounds = [true], permutation_map = #[[$MAP_ID6]]} : memref<8x12x16xf32>, vector<8xf32>
+// CHECK-NEXT:       %[[S3:.*]] = vector.transfer_read %[[ARG0]][%[[S0]], %[[S1]], %[[S2]]], %[[CST]] {permutation_map = #[[$MAP_ID6]]} : memref<8x12x16xf32>, vector<8xf32>
 // CHECK-NEXT:       vector.transfer_write %[[S3]], %[[ARG1]][%[[ARG3]], %[[ARG4]], %[[ARG5]]] : vector<8xf32>, memref<8x24x48xf32>
 // CHECK-NEXT:     }
 // CHECK-NEXT:   }
diff --git a/mlir/test/Dialect/Arith/int-range-interface.mlir b/mlir/test/Dialect/Arith/int-range-interface.mlir
index 48fdb1cdced4d..4b04229e5db52 100644
--- a/mlir/test/Dialect/Arith/int-range-interface.mlir
+++ b/mlir/test/Dialect/Arith/int-range-interface.mlir
@@ -266,6 +266,19 @@ func.func @remui_base(%arg0 : index, %arg1 : index ) -> i1 {
     func.return %3 : i1
 }
 
+// CHECK-LABEL: func @remui_base_maybe_zero
+// CHECK: %[[true:.*]] = arith.constant true
+// CHECK: return %[[true]]
+func.func @remui_base_maybe_zero(%arg0 : index, %arg1 : index ) -> i1 {
+    %c4 = arith.constant 4 : index
+    %c5 = arith.constant 5 : index    
+
+    %0 = arith.minui %arg1, %c4 : index
+    %1 = arith.remui %arg0, %0 : index
+    %2 = arith.cmpi ult, %1, %c5 : index
+    func.return %2 : i1
+}    
+
 // CHECK-LABEL: func @remsi_base
 // CHECK: %[[ret:.*]] = arith.cmpi sge
 // CHECK: return %[[ret]]
diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir
index 9388d7ef24936..5677d7ff41202 100644
--- a/mlir/test/Dialect/LLVMIR/invalid.mlir
+++ b/mlir/test/Dialect/LLVMIR/invalid.mlir
@@ -643,6 +643,21 @@ func.func @atomicrmw_expected_float(%i32_ptr : !llvm.ptr, %i32 : i32) {
 
 // -----
 
+func.func @atomicrmw_scalable_vector(%ptr : !llvm.ptr, %f32_vec : vector<[2]xf32>) {
+  // expected-error@+1 {{'val' must be floating point LLVM type or LLVM pointer type or signless integer or LLVM dialect-compatible fixed-length vector type}}
+  %0 = llvm.atomicrmw fadd %ptr, %f32_vec unordered : !llvm.ptr, vector<[2]xf32>
+  llvm.return
+}
+// -----
+
+func.func @atomicrmw_vector_expected_float(%ptr : !llvm.ptr, %i32_vec : vector<3xi32>) {
+  // expected-error@+1 {{expected LLVM IR floating point type for vector element}}
+  %0 = llvm.atomicrmw fadd %ptr, %i32_vec unordered : !llvm.ptr, vector<3xi32>
+  llvm.return
+}
+
+// -----
+
 func.func @atomicrmw_unexpected_xchg_type(%i1_ptr : !llvm.ptr, %i1 : i1) {
   // expected-error@+1 {{unexpected LLVM IR type for 'xchg' bin_op}}
   %0 = llvm.atomicrmw xchg %i1_ptr, %i1 unordered : !llvm.ptr, i1
diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
index 62f1de2b7fe7d..3062cdc38c0ab 100644
--- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir
+++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
@@ -420,11 +420,13 @@ func.func @atomic_store(%val : f32, %large_val : i256, %ptr : !llvm.ptr) {
 }
 
 // CHECK-LABEL: @atomicrmw
-func.func @atomicrmw(%ptr : !llvm.ptr, %val : f32) {
+func.func @atomicrmw(%ptr : !llvm.ptr, %f32 : f32, %f16_vec : vector<2xf16>) {
   // CHECK: llvm.atomicrmw fadd %{{.*}}, %{{.*}} monotonic : !llvm.ptr, f32
-  %0 = llvm.atomicrmw fadd %ptr, %val monotonic : !llvm.ptr, f32
+  %0 = llvm.atomicrmw fadd %ptr, %f32 monotonic : !llvm.ptr, f32
   // CHECK: llvm.atomicrmw volatile fsub %{{.*}}, %{{.*}} syncscope("singlethread") monotonic {alignment = 16 : i64} : !llvm.ptr, f32
-  %1 = llvm.atomicrmw volatile fsub %ptr, %val syncscope("singlethread") monotonic {alignment = 16 : i64} : !llvm.ptr, f32
+  %1 = llvm.atomicrmw volatile fsub %ptr, %f32 syncscope("singlethread") monotonic {alignment = 16 : i64} : !llvm.ptr, f32
+  // CHECK: llvm.atomicrmw fmin %{{.*}}, %{{.*}} monotonic : !llvm.ptr, vector<2xf16>
+  %2 = llvm.atomicrmw fmin %ptr, %f16_vec monotonic : !llvm.ptr, vector<2xf16>
   llvm.return
 }
 
diff --git a/mlir/test/Dialect/Linalg/fold-add-into-dest.mlir b/mlir/test/Dialect/Linalg/fold-add-into-dest.mlir
new file mode 100644
index 0000000000000..d8e92e40739dc
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/fold-add-into-dest.mlir
@@ -0,0 +1,329 @@
+// RUN: mlir-opt %s -transform-interpreter -cse -split-input-file | FileCheck %s
+
+!type = tensor<2048x2048xf32>
+func.func @fold_add_on_two_matmuls(%arg0: !type, %arg1: !type) -> !type {
+  %0 = arith.constant dense<1.111111e+00> : !type
+  %cst = arith.constant 0.000000e+00 : f32
+  %1 = tensor.empty() : !type
+  %2 = linalg.fill ins(%cst : f32) outs(%1 : !type) -> !type
+  %3 = linalg.matmul ins(%arg0, %0 : !type, !type) outs(%2 : !type) -> !type
+  %4 = tensor.empty() : !type
+  %5 = linalg.fill ins(%cst : f32) outs(%1 : !type) -> !type
+  %6 = linalg.matmul ins(%arg1, %0 : !type, !type) outs(%5 : !type) -> !type
+  %7 = linalg.add ins(%3, %6 : !type, !type) outs(%1 : !type) -> !type
+  return %7 : !type
+}
+
+// CHECK-LABEL: func.func @fold_add_on_two_matmuls(
+// CHECK-SAME: %[[ARG0:.*]]: {{.*}}, %[[ARG1:.*]]: {{.*}})
+// CHECK-NEXT: %[[DENSE:.*]] = arith.constant dense<1.11
+// CHECK-NEXT: %[[ZERO:.*]] = arith.constant 0.000000e+00
+// CHECK-NEXT: %[[EMPTY:.*]] = tensor.empty()
+// CHECK-NEXT: %[[FILLED:.*]] = linalg.fill ins(%[[ZERO]] : {{.*}}) outs(%[[EMPTY]] : {{.*}})
+// CHECK-NEXT: %[[ACC:.+]] = linalg.matmul ins(%[[ARG0]], %[[DENSE]] : {{.*}}) outs(%[[FILLED]] : {{.*}})
+// CHECK-NEXT: %[[RES:.+]] = linalg.matmul ins(%[[ARG1]], %[[DENSE]] : {{.*}}) outs(%[[ACC]] : {{.*}})
+// CHECK-NOT: linalg.add
+// CHECK-NEXT: return %[[RES]]
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_patterns to %func {
+      transform.apply_patterns.linalg.fold_add_into_dest
+    } : !transform.any_op
+    transform.yield
+  }
+}
+// -----
+
+!type = tensor<2048x2048xf32>
+func.func @expect_no_fold_of_add_as_orig_dest_not_additive_zero(%arg0: !type, %arg1: !type) -> !type {
+  %0 = arith.constant dense<1.111111e+00> : !type
+  %cst = arith.constant 0.000000e+00 : f32
+  %1 = tensor.empty() : !type
+  %2 = linalg.fill ins(%cst : f32) outs(%1 : !type) -> !type
+  %3 = linalg.matmul ins(%arg0, %0 : !type, !type) outs(%2 : !type) -> !type
+  %4 = linalg.matmul ins(%arg1, %0 : !type, !type) outs(%0 : !type) -> !type
+  %5 = linalg.add ins(%3, %4 : !type, !type) outs(%1 : !type) -> !type
+  return %5 : !type
+}
+
+// CHECK-LABEL: func.func @expect_no_fold_of_add_as_orig_dest_not_additive_zero
+// CHECK: linalg.fill
+// CHECK-NEXT: linalg.matmul
+// CHECK-NEXT: linalg.matmul
+// CHECK-NEXT: linalg.add
+// CHECK-NEXT: return
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_patterns to %func {
+      transform.apply_patterns.linalg.fold_add_into_dest
+    } : !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+!type = tensor<2048x2048xf32>
+func.func @expect_no_fold_of_add_as_contraction_result_has_multiple_users(%arg0: !type, %arg1: !type) -> (!type, !type) {
+  %0 = arith.constant dense<1.111111e+00> : !type
+  %cst = arith.constant 0.000000e+00 : f32
+  %1 = tensor.empty() : !type
+  %2 = linalg.fill ins(%cst : f32) outs(%1 : !type) -> !type
+  %3 = linalg.matmul ins(%arg0, %0 : !type, !type) outs(%2 : !type) -> !type
+  %4 = linalg.matmul ins(%arg1, %0 : !type, !type) outs(%0 : !type) -> !type
+  %5 = linalg.add ins(%3, %4 : !type, !type) outs(%1 : !type) -> !type
+  %6 = linalg.mul ins(%4, %arg0 : !type, !type) outs(%1 : !type) -> !type
+  return %5, %6 : !type, !type
+}
+
+// CHECK-LABEL: func.func @expect_no_fold_of_add_as_contraction_result_has_multiple_users
+// CHECK: linalg.fill
+// CHECK-NEXT: linalg.matmul
+// CHECK-NEXT: linalg.matmul
+// CHECK-NEXT: linalg.add
+// CHECK-NEXT: linalg.mul
+// CHECK-NEXT: return
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_patterns to %func {
+      transform.apply_patterns.linalg.fold_add_into_dest
+    } : !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+!type = tensor<2048x2048xf32>
+func.func @fold_add_on_matmul_and_func_arg(%arg0: !type, %arg1: !type) -> !type {
+  %0 = arith.constant dense<1.111111e+00> : !type
+  %cst = arith.constant 0.000000e+00 : f32
+  %1 = tensor.empty() : !type
+  %2 = linalg.fill ins(%cst : f32) outs(%1 : !type) -> !type
+  %3 = linalg.matmul ins(%arg0, %0 : !type, !type) outs(%2 : !type) -> !type
+  %5 = linalg.add ins(%3, %arg1 : !type, !type) outs(%1 : !type) -> !type
+  return %5 : !type
+}
+
+// CHECK-LABEL: func.func @fold_add_on_matmul_and_func_arg
+// CHECK: %[[RES:.+]] = linalg.matmul
+// CHECK-NOT: linalg.add
+// CHECK-NEXT: return %[[RES]]
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_patterns to %func {
+      transform.apply_patterns.linalg.fold_add_into_dest
+    } : !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+!type = tensor<2048x2048xf32>
+func.func @expect_no_fold_of_add_as_operands_do_not_dominate_each_other(%arg0: !type, %arg1: !type) -> !type {
+  %0 = arith.constant dense<1.111111e+00> : !type
+  %cst = arith.constant 0.000000e+00 : f32
+  %1 = tensor.empty() : !type
+  %2 = linalg.fill ins(%cst : f32) outs(%1 : !type) -> !type
+  %3 = linalg.matmul ins(%arg0, %0 : !type, !type) outs(%2 : !type) -> !type
+  %4 = linalg.add ins(%3, %3 : !type, !type) outs(%1 : !type) -> !type
+  return %4 : !type
+}
+
+// CHECK-LABEL: func.func @expect_no_fold_of_add_as_operands_do_not_dominate_each_other
+// CHECK: linalg.fill
+// CHECK-NEXT: linalg.matmul
+// CHECK-NEXT: linalg.add
+// CHECK-NEXT: return
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_patterns to %func {
+      transform.apply_patterns.linalg.fold_add_into_dest
+    } : !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+!type = tensor<2048x2048xf32>
+func.func @fold_add_on_transposed_matmuls(%arg0: !type, %arg1: !type) -> !type {
+  %0 = arith.constant dense<1.111111e+00> : !type
+  %cst = arith.constant 0.000000e+00 : f32
+  %1 = tensor.empty() : !type
+  %2 = linalg.fill ins(%cst : f32) outs(%1 : !type) -> !type
+  %3 = linalg.matmul_transpose_a ins(%arg0, %0 : !type, !type) outs(%2 : !type) -> !type
+  %4 = linalg.matmul_transpose_b ins(%arg1, %0 : !type, !type) outs(%2 : !type) -> !type
+  %5 = linalg.add ins(%3, %4 : !type, !type) outs(%1 : !type) -> !type
+  return %5 : !type
+}
+
+// CHECK-LABEL: func.func @fold_add_on_transposed_matmuls
+// CHECK: %[[ACC:.+]] = linalg.matmul_transpose_a
+// CHECK-NEXT: %[[RES:.+]] = linalg.matmul_transpose_b ins({{.+}}) outs(%[[ACC]]
+// CHECK-NOT: linalg.add
+// CHECK-NEXT: return %[[RES]]
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_patterns to %func {
+      transform.apply_patterns.linalg.fold_add_into_dest
+    } : !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+!type = tensor<2048x2048xf32>
+func.func @expect_no_fold_of_add_as_dominated_op_is_not_a_contraction(%arg0: !type, %arg1: !type) -> !type {
+  %0 = arith.constant dense<1.111111e+00> : !type
+  %cst = arith.constant 0.000000e+00 : f32
+  %1 = tensor.empty() : !type
+  %2 = linalg.fill ins(%cst : f32) outs(%1 : !type) -> !type
+  %3 = linalg.matmul ins(%arg0, %0 : !type, !type) outs(%2 : !type) -> !type
+  %4 = linalg.sub ins(%arg1, %0 : !type, !type) outs(%2 : !type) -> !type
+  %5 = linalg.add ins(%3, %4 : !type, !type) outs(%1 : !type) -> !type
+  return %5 : !type
+}
+
+// CHECK-LABEL: func.func @expect_no_fold_of_add_as_dominated_op_is_not_a_contraction
+// CHECK: linalg.fill
+// CHECK-NEXT: linalg.matmul
+// CHECK-NEXT: linalg.sub
+// CHECK-NEXT: linalg.add
+// CHECK-NEXT: return
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_patterns to %func {
+      transform.apply_patterns.linalg.fold_add_into_dest
+    } : !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+#map0 = affine_map<(d0, d1, d2) -> (d0, d2)>
+#map1 = affine_map<(d0, d1, d2) -> (d2, d1)>
+#map2 = affine_map<(d0, d1, d2) -> (d1, d0)>  // NB: not an ordered projection
+
+!type = tensor<2048x2048xf32>
+func.func @expect_no_fold_of_add_as_dest_accumulation_is_not_identity_mapped(%arg0: !type, %arg1: !type) -> !type {
+  %0 = arith.constant dense<1.111111e+00> : !type
+  %cst = arith.constant 0.000000e+00 : f32
+  %1 = tensor.empty() : !type
+  %2 = linalg.fill ins(%cst : f32) outs(%1 : !type) -> !type
+  %3 = linalg.generic { indexing_maps = [#map0, #map1, #map2],
+                        iterator_types = ["parallel", "parallel", "reduction"] }
+    ins(%arg0, %0: !type, !type) outs(%2: !type) {
+      ^bb0(%a: f32, %b: f32, %c: f32):
+        %5 = arith.mulf %a, %b : f32
+        %6 = arith.addf %c, %5 : f32
+        linalg.yield %6 : f32
+  } -> !type
+  %4 = linalg.add ins(%3, %arg1 : !type, !type) outs(%1 : !type) -> !type
+  return %4 : !type
+}
+
+// CHECK-LABEL: func.func @expect_no_fold_of_add_as_dest_accumulation_is_not_identity_mapped
+// CHECK: linalg.fill
+// CHECK-NEXT: linalg.generic
+// CHECK: linalg.add
+// CHECK-NEXT: return
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_patterns to %func {
+      transform.apply_patterns.linalg.fold_add_into_dest
+    } : !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+#map0 = affine_map<(d0, d1, d2) -> (d0, d2)>
+#map1 = affine_map<(d0, d1, d2) -> (d2, d1)>
+#map2 = affine_map<(d0, d1, d2) -> (d0, d1)>  // NB: is an ordered projection
+
+!type = tensor<2048x2048xf32>
+func.func @fold_add_on_a_generic_and_an_argument(%arg0: !type, %arg1: !type) -> !type {
+  %0 = arith.constant dense<1.111111e+00> : !type
+  %cst = arith.constant 0.000000e+00 : f32
+  %1 = tensor.empty() : !type
+  %2 = linalg.fill ins(%cst : f32) outs(%1 : !type) -> !type
+  %3 = linalg.generic { indexing_maps = [#map0, #map1, #map2],
+                        iterator_types = ["parallel", "parallel", "reduction"] }
+    ins(%arg0, %0: !type, !type) outs(%2: !type) {
+      ^bb0(%a: f32, %b: f32, %c: f32):
+        %5 = arith.mulf %a, %b : f32
+        %6 = arith.addf %c, %5 : f32
+        linalg.yield %6 : f32
+  } -> !type
+  %4 = linalg.add ins(%3, %arg1 : !type, !type) outs(%1 : !type) -> !type
+  return %4 : !type
+}
+
+// CHECK-LABEL: func.func @fold_add_on_a_generic_and_an_argument
+// CHECK: linalg.generic
+// CHECK-NOT: linalg.add
+// CHECK: return
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_patterns to %func {
+      transform.apply_patterns.linalg.fold_add_into_dest
+    } : !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+memref.global "private" constant @big_const : memref<2048x2048xf32> = dense<1.11111104> {alignment = 64 : i64}
+func.func @expect_no_fold_due_to_no_memref_support(%arg0: memref<2048x2048xf32>, %arg1: memref<2048x2048xf32>) -> memref<2048x2048xf32> {
+  %cst = arith.constant 0.000000e+00 : f32
+  %0 = memref.get_global @big_const  : memref<2048x2048xf32>
+  %alloc = memref.alloc() {alignment = 64 : i64} : memref<2048x2048xf32>
+  %alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<2048x2048xf32>
+  linalg.fill ins(%cst : f32) outs(%alloc_0 : memref<2048x2048xf32>)
+  linalg.matmul ins(%arg0, %0 : memref<2048x2048xf32>, memref<2048x2048xf32>) outs(%alloc_0 : memref<2048x2048xf32>)
+  linalg.fill ins(%cst : f32) outs(%alloc : memref<2048x2048xf32>)
+  linalg.matmul ins(%arg1, %0 : memref<2048x2048xf32>, memref<2048x2048xf32>) outs(%alloc : memref<2048x2048xf32>)
+  linalg.add ins(%alloc_0, %alloc : memref<2048x2048xf32>, memref<2048x2048xf32>) outs(%alloc : memref<2048x2048xf32>)
+  memref.dealloc %alloc_0 : memref<2048x2048xf32>
+  return %alloc : memref<2048x2048xf32>
+}
+
+// CHECK-LABEL: func.func @expect_no_fold_due_to_no_memref_support
+// CHECK: linalg.matmul
+// CHECK: linalg.matmul
+// CHECK: linalg.add
+// CHECK: return
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_patterns to %func {
+      transform.apply_patterns.linalg.fold_add_into_dest
+    } : !transform.any_op
+    transform.yield
+  }
+}
diff --git a/mlir/test/Dialect/Linalg/generalize-tensor-pack.mlir b/mlir/test/Dialect/Linalg/generalize-tensor-pack.mlir
index 7d87a0994004f..7f6b5e279f685 100644
--- a/mlir/test/Dialect/Linalg/generalize-tensor-pack.mlir
+++ b/mlir/test/Dialect/Linalg/generalize-tensor-pack.mlir
@@ -19,11 +19,14 @@ func.func @simple_KCRS_to_KCRSsr(%arg0: tensor<1x1x32x8xf32>, %arg1: tensor<1x1x
 
 // -----
 
-func.func @simple_pad_and_pack(%input: tensor<5x1xf32>, %output: tensor<1x1x8x2xf32>, %pad: f32) -> tensor<1x1x8x2xf32> {
+func.func @simple_pad_and_pack_static_tiles(%input: tensor<5x1xf32>, %output: tensor<1x1x8x2xf32>, %pad: f32) -> tensor<1x1x8x2xf32> {
   %0 = tensor.pack %input padding_value(%pad : f32) inner_dims_pos = [0, 1] inner_tiles = [8, 2] into %output : tensor<5x1xf32> -> tensor<1x1x8x2xf32>
   return %0 : tensor<1x1x8x2xf32>
 }
-// CHECK-LABEL: func.func @simple_pad_and_pack
+// CHECK: #[[$ATTR_0:.+]] = affine_map<()[s0] -> (s0 - 5)>
+// CHECK: #[[$ATTR_1:.+]] = affine_map<()[s0] -> (s0 - 1)>
+
+// CHECK-LABEL: func.func @simple_pad_and_pack_static_tiles
 // CHECK-SAME:    %[[SRC:[a-zA-Z0-9]+]]
 // CHECK-SAME:    %[[DEST:[a-zA-Z0-9]+]]
 // CHECK-SAME:    %[[PAD_VAL:[a-zA-Z0-9]+]]
@@ -34,6 +37,84 @@ func.func @simple_pad_and_pack(%input: tensor<5x1xf32>, %output: tensor<1x1x8x2x
 // CHECK-SAME:      [0, 0, 0, 0] [1, 1, 8, 2] [1, 1, 1, 1]
 // CHECK:         return %[[INSERT]]
 
+/// Same as example above, but with 1 dynamic tile size.
+
+func.func @simple_pad_and_pack_dynamic_tile(%input: tensor<5x1xf32>, %output: tensor<1x1x?x2xf32>, %pad: f32, %high: index) -> tensor<1x1x?x2xf32> {
+  %0 = tensor.pack %input padding_value(%pad : f32) inner_dims_pos = [0, 1] inner_tiles = [%high, 2] into %output : tensor<5x1xf32> -> tensor<1x1x?x2xf32>
+  return %0 : tensor<1x1x?x2xf32>
+}
+
+// CHECK-LABEL:   func.func @simple_pad_and_pack_dynamic_tile(
+// CHECK-SAME:      %[[SRC:[a-zA-Z0-9]+]]
+// CHECK-SAME:      %[[DEST:[a-zA-Z0-9]+]]
+// CHECK-SAME:      %[[PAD_VAL:[a-zA-Z0-9]+]]
+// CHECK-SAME:      %[[HIGH_VAL:[a-zA-Z0-9]+]]: index) -> tensor<1x1x?x2xf32> {
+// CHECK:           %[[C2:.*]] = arith.constant 2 : index
+// CHECK:           %[[PAD_HIGH:.*]] = affine.apply #[[$ATTR_0]](){{\[}}%[[HIGH_VAL]]]
+// CHECK:           %[[PAD:.*]] = tensor.pad %[[SRC]] low[0, 0] high{{\[}}%[[PAD_HIGH]], 1] {
+// CHECK:             tensor.yield %[[PAD_VAL]] : f32
+// CHECK-NOT:       linalg.transpose
+// CHECK:           %[[SLICE:.*]] = tensor.extract_slice %[[VAL_10:.*]][0, 0] {{\[}}%[[HIGH_VAL]], 2] [1, 1] : tensor<?x2xf32> to tensor<?x2xf32>
+// CHECK:           %[[DIM:.*]] = tensor.dim %[[DEST]], %[[C2]] : tensor<1x1x?x2xf32>
+// CHECK:           %[[RES:.*]] = tensor.insert_slice %[[SLICE]] into %[[DEST]][0, 0, 0, 0] [1, 1, %[[DIM]], 2] [1, 1, 1, 1] : tensor<?x2xf32> into tensor<1x1x?x2xf32>
+// CHECK:           return %[[RES]] : tensor<1x1x?x2xf32>
+
+/// Same as example above, but with 1 scalable tile size.
+
+/// NOTE: For this example to make sense in practice, the "?" in the output shape
+///       should effectively be 8 * vector.vscale (and that's what tensor.dim
+///       below should return).
+
+func.func @simple_pad_and_pack_scalable_tile(%input: tensor<5x1xf32>, %output: tensor<1x1x?x2xf32>, %pad: f32) -> tensor<1x1x?x2xf32> {
+  %c8 = arith.constant 8 : index
+  %vscale = vector.vscale
+  %c8_vscale = arith.muli %vscale, %c8 : index
+  %0 = tensor.pack %input padding_value(%pad : f32) inner_dims_pos = [0, 1] inner_tiles = [%c8_vscale, 2] into %output : tensor<5x1xf32> -> tensor<1x1x?x2xf32>
+  return %0 : tensor<1x1x?x2xf32>
+}
+
+// CHECK-LABEL:   func.func @simple_pad_and_pack_scalable_tile(
+// CHECK-SAME:      %[[SRC:[a-zA-Z0-9]+]]: tensor<5x1xf32>,
+// CHECK-SAME:      %[[DEST:[a-zA-Z0-9]+]]: tensor<1x1x?x2xf32>,
+// CHECK-SAME:      %[[PAD_VAL:[a-zA-Z0-9]+]]: f32) -> tensor<1x1x?x2xf32> {
+// CHECK-DAG:       %[[C2:.+]] = arith.constant 2 : index
+// CHECK-DAG:       %[[C8:.+]] = arith.constant 8 : index
+// CHECK-DAG:       %[[VS:.+]] = vector.vscale
+// CHECK:           %[[C8_VS:.+]] = arith.muli %[[VS]], %[[C8]] : index
+// CHECK:           %[[PAD_HIGH:.*]] = affine.apply #[[$ATTR_0]](){{\[}}%[[C8_VS]]]
+// CHECK:           %[[PAD:.+]] = tensor.pad %[[SRC]] low[0, 0] high{{\[}}%[[PAD_HIGH]], 1] {
+// CHECK:             tensor.yield %[[PAD_VAL]] : f32
+// CHECK-NOT:       linalg.transpose
+// CHECK:           %[[SLICE:.+]] = tensor.extract_slice %[[PAD:.+]][0, 0] {{\[}}%[[C8_VS]], 2] [1, 1] : tensor<?x2xf32> to tensor<?x2xf32>
+// CHECK:           %[[DIM:.+]] = tensor.dim %[[DEST]], %[[C2]] : tensor<1x1x?x2xf32>
+// CHECK:           %[[RES:.+]] = tensor.insert_slice %[[SLICE]] into %[[DEST]][0, 0, 0, 0] [1, 1, %[[DIM]], 2] [1, 1, 1, 1] : tensor<?x2xf32> into tensor<1x1x?x2xf32>
+// CHECK:           return %[[RES]] : tensor<1x1x?x2xf32>
+
+/// Same as example above, but with both tile sizes dynamic.
+
+func.func @simple_pad_and_pack_dynamic_tiles(%input: tensor<5x1xf32>, %output: tensor<1x1x?x?xf32>, %pad: f32, %high_1: index, %high_2: index) -> tensor<1x1x?x?xf32> {
+  %0 = tensor.pack %input padding_value(%pad : f32) inner_dims_pos = [0, 1] inner_tiles = [%high_1, %high_2] into %output : tensor<5x1xf32> -> tensor<1x1x?x?xf32>
+  return %0 : tensor<1x1x?x?xf32>
+}
+// CHECK-LABEL:   func.func @simple_pad_and_pack_dynamic_tiles(
+// CHECK-SAME:      %[[SRC:[a-zA-Z0-9]+]]: tensor<5x1xf32>,
+// CHECK-SAME:      %[[DEST:[a-zA-Z0-9]+]]: tensor<1x1x?x?xf32>,
+// CHECK-SAME:      %[[PAD_VAL:[a-zA-Z0-9]+]]: f32,
+// CHECK-SAME:      %[[HIGH_VAL_1:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[HIGH_VAL_2:[a-zA-Z0-9]+]]: index) -> tensor<1x1x?x?xf32> {
+// CHECK:           %[[C3:.*]] = arith.constant 3 : index
+// CHECK:           %[[C2:.*]] = arith.constant 2 : index
+// CHECK:           %[[PAD_HIGH_1:.*]] = affine.apply #[[$ATTR_0]](){{\[}}%[[HIGH_VAL_1]]]
+// CHECK:           %[[PAD_HIGH_2:.*]] = affine.apply #[[$ATTR_1]](){{\[}}%[[HIGH_VAL_2]]]
+// CHECK:           %[[PAD:.*]] = tensor.pad %[[SRC]] low[0, 0] high{{\[}}%[[PAD_HIGH_1]], %[[PAD_HIGH_2]]] {
+// CHECK:             tensor.yield %[[PAD_VAL]] : f32
+// CHECK-NOT:       linalg.transpose
+// CHECK:           %[[SLICE:.*]] = tensor.extract_slice %[[PAD:.*]][0, 0] {{\[}}%[[HIGH_VAL_1]], %[[HIGH_VAL_2]]] [1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
+// CHECK:           %[[DIM_1:.*]] = tensor.dim %[[DEST]], %[[C2]] : tensor<1x1x?x?xf32>
+// CHECK:           %[[DIM_2:.*]] = tensor.dim %[[DEST]], %[[C3]] : tensor<1x1x?x?xf32>
+// CHECK:           %[[RES:.*]] = tensor.insert_slice %[[SLICE]] into %[[DEST]][0, 0, 0, 0] [1, 1, %[[DIM_1]], %[[DIM_2]]] [1, 1, 1, 1] : tensor<?x?xf32> into tensor<1x1x?x?xf32>
+// CHECK:           return %[[RES]] : tensor<1x1x?x?xf32>
+
 // -----
 
 func.func @simple_NC_to_CNnc(%arg0: tensor<32x8xf32>, %arg1: tensor<1x1x32x8xf32>) -> tensor<1x1x32x8xf32>{
diff --git a/mlir/test/Dialect/Linalg/hoisting.mlir b/mlir/test/Dialect/Linalg/hoisting.mlir
index 44c15c272bb3e..241b8a486c012 100644
--- a/mlir/test/Dialect/Linalg/hoisting.mlir
+++ b/mlir/test/Dialect/Linalg/hoisting.mlir
@@ -200,7 +200,7 @@ func.func @hoist_vector_transfer_pairs_in_affine_loops(%memref0: memref<64x64xi3
   affine.for %arg3 = 0 to 64 {
     affine.for %arg4 = 0 to 64 step 16 {
       affine.for %arg5 = 0 to 64 {
-        %0 = vector.transfer_read %memref0[%arg3, %arg5], %c0_i32 {in_bounds = [true], permutation_map = affine_map<(d0, d1) -> (0)>} : memref<64x64xi32>, vector<16xi32>
+        %0 = vector.transfer_read %memref0[%arg3, %arg5], %c0_i32 {permutation_map = affine_map<(d0, d1) -> (0)>} : memref<64x64xi32>, vector<16xi32>
         %1 = vector.transfer_read %memref1[%arg5, %arg4], %c0_i32 : memref<64x64xi32>, vector<16xi32>
         %2 = vector.transfer_read %memref2[%arg3, %arg4], %c0_i32 : memref<64x64xi32>, vector<16xi32>
         %3 = arith.muli %0, %1 : vector<16xi32>
diff --git a/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir b/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir
index 9a43d43cd9460..e7beb72547112 100644
--- a/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir
@@ -1091,9 +1091,15 @@ module attributes {transform.with_named_sequence} {
 func.func private @make_vector() -> tensor<12x13xf32>
 
 // CHECK-LABEL: func @pad_and_insert_slice_dest
+//  CHECK-SAME:     %[[ARG0:.*]]: tensor<1x5x6xf32>
 // Check the insert slice is not rewritten if the padded result is used by the destination operand.
+//   CHECK-NOT:   tensor.pad
+//       CHECK:   %[[EMPTY:.*]] = tensor.empty() : tensor<1x12x13xf32>
+//       CHECK:   %[[WRITE_1:.*]] = vector.transfer_write %{{.*}}, %[[EMPTY]]{{.*}} : vector<1x12x13xf32>, tensor<1x12x13xf32>
+//       CHECK:   %[[READ:.*]]  = vector.transfer_read %[[ARG0:.*]]{{.*}} : tensor<1x5x6xf32>, vector<1x5x6xf32>
+//       CHECK:   %[[WRITE_2:.*]] = vector.transfer_write %[[READ]], %[[WRITE_1]]{{.*}} : vector<1x5x6xf32>, tensor<1x12x13xf32>
 //       CHECK:   %[[T1:.*]] = call @make_vector() : () -> tensor<12x13xf32>
-//       CHECK:   = tensor.insert_slice %[[T1]] into
+//       CHECK:   tensor.insert_slice %[[T1]] into %[[WRITE_2]]
 func.func @pad_and_insert_slice_dest(
     %arg0: tensor<1x5x6xf32>) -> tensor<1x12x13xf32> {
   %c5 = arith.constant 5.0 : f32
@@ -1110,7 +1116,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %3 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %4 = transform.get_parent_op %3 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
-    %5 = transform.structured.vectorize_children_and_apply_patterns %4 : (!transform.any_op) -> !transform.any_op
+    %5 = transform.structured.vectorize_children_and_apply_patterns %4  { vectorize_padding } : (!transform.any_op) -> !transform.any_op
     transform.yield
   }
 }
@@ -2004,3 +2010,68 @@ module attributes {transform.with_named_sequence} {
 // CHECK:           %[[VAL_8:.*]] = vector.transpose %[[VAL_7]], [1, 2, 3, 0] : vector<1x1x12x197xf32> to vector<1x12x197x1xf32>
 // CHECK:           %[[VAL_9:.*]] = vector.transfer_write %[[VAL_8]], %[[VAL_3]]{{\[}}%[[VAL_2]], %[[VAL_2]], %[[VAL_2]], %[[VAL_2]]] {in_bounds = [true, true, true, true]} : vector<1x12x197x1xf32>, tensor<1x12x197x1xf32>
 // CHECK:           return %[[VAL_9]] : tensor<1x12x197x1xf32>
+
+// -----
+
+// Input identical as the test in vectorization.mlir. Output is different -
+// vector sizes are inferred (rather than user-specified) and hence _no_
+// masking was used.
+
+func.func @test_vectorize_pack(%arg0: tensor<32x8x16xf32>, %arg1: tensor<4x1x32x16x2xf32>) -> tensor<4x1x32x16x2xf32> {
+  %pack = tensor.pack %arg0 outer_dims_perm = [1, 2, 0] inner_dims_pos = [2, 1] inner_tiles = [16, 2] into %arg1 : tensor<32x8x16xf32> -> tensor<4x1x32x16x2xf32>
+  return %pack : tensor<4x1x32x16x2xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["tensor.pack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// CHECK-LABEL:   func.func @test_vectorize_pack(
+// CHECK-SAME:      %[[VAL_0:.*]]: tensor<32x8x16xf32>,
+// CHECK-SAME:      %[[VAL_1:.*]]: tensor<4x1x32x16x2xf32>) -> tensor<4x1x32x16x2xf32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]] = vector.transfer_read %[[VAL_0]]{{\[}}%[[VAL_3]], %[[VAL_3]], %[[VAL_3]]], %[[VAL_2]] {in_bounds = [true, true, true]} : tensor<32x8x16xf32>, vector<32x8x16xf32>
+// CHECK:           %[[VAL_5:.*]] = vector.shape_cast %[[VAL_4]] : vector<32x8x16xf32> to vector<32x4x2x1x16xf32>
+// CHECK:           %[[VAL_6:.*]] = vector.transpose %[[VAL_5]], [1, 3, 0, 4, 2] : vector<32x4x2x1x16xf32> to vector<4x1x32x16x2xf32>
+// CHECK:           %[[VAL_7:.*]] = tensor.empty() : tensor<4x1x32x16x2xf32>
+// CHECK:           %[[VAL_8:.*]] = vector.transfer_write %[[VAL_6]], %[[VAL_7]]{{\[}}%[[VAL_3]], %[[VAL_3]], %[[VAL_3]], %[[VAL_3]], %[[VAL_3]]] {in_bounds = [true, true, true, true, true]} : vector<4x1x32x16x2xf32>, tensor<4x1x32x16x2xf32>
+// CHECK:           return %[[VAL_8]] : tensor<4x1x32x16x2xf32>
+
+// -----
+
+// Input identical as the test in vectorization.mlir. Output is different -
+// vector sizes are inferred (rather than user-specified) and hence _no_
+// masking was used.
+
+func.func @test_vectorize_padded_pack(%arg0: tensor<32x7x15xf32>, %arg1: tensor<32x4x1x16x2xf32>) -> tensor<32x4x1x16x2xf32> {
+  %pad = arith.constant 0.000000e+00 : f32
+  %pack = tensor.pack %arg0 padding_value(%pad : f32) inner_dims_pos = [2, 1] inner_tiles = [16, 2] into %arg1 : tensor<32x7x15xf32> -> tensor<32x4x1x16x2xf32>
+  return %pack : tensor<32x4x1x16x2xf32>
+}
+
+// CHECK-LABEL:   func.func @test_vectorize_padded_pack(
+// CHECK-SAME:      %[[VAL_0:.*]]: tensor<32x7x15xf32>,
+// CHECK-SAME:      %[[VAL_1:.*]]: tensor<32x4x1x16x2xf32>) -> tensor<32x4x1x16x2xf32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]] = vector.transfer_read %[[VAL_0]]{{\[}}%[[VAL_3]], %[[VAL_3]], %[[VAL_3]]], %[[VAL_2]] {in_bounds = [true, false, false]} : tensor<32x7x15xf32>, vector<32x8x16xf32>
+// CHECK:           %[[VAL_5:.*]] = vector.shape_cast %[[VAL_4]] : vector<32x8x16xf32> to vector<32x4x2x1x16xf32>
+// CHECK:           %[[VAL_6:.*]] = vector.transpose %[[VAL_5]], [0, 1, 3, 4, 2] : vector<32x4x2x1x16xf32> to vector<32x4x1x16x2xf32>
+// CHECK:           %[[VAL_7:.*]] = tensor.empty() : tensor<32x4x1x16x2xf32>
+// CHECK:           %[[VAL_8:.*]] = vector.transfer_write %[[VAL_6]], %[[VAL_7]]{{\[}}%[[VAL_3]], %[[VAL_3]], %[[VAL_3]], %[[VAL_3]], %[[VAL_3]]] {in_bounds = [true, true, true, true, true]} : vector<32x4x1x16x2xf32>, tensor<32x4x1x16x2xf32>
+// CHECK:           return %[[VAL_8]] : tensor<32x4x1x16x2xf32>
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["tensor.pack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
diff --git a/mlir/test/Dialect/Linalg/vectorization.mlir b/mlir/test/Dialect/Linalg/vectorization.mlir
index 0e2b2458d29cd..0f2abe06569d6 100644
--- a/mlir/test/Dialect/Linalg/vectorization.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization.mlir
@@ -130,7 +130,7 @@ func.func @vectorize_dynamic_1d_broadcast(%arg0: tensor<?xf32>,
 // CHECK-LABEL:   @vectorize_dynamic_1d_broadcast
 // CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
 // CHECK:           %[[VAL_4:.*]] = tensor.dim %{{.*}}, %[[VAL_3]] : tensor<?xf32>
-// CHECK:           %[[VAL_7:.*]] = vector.transfer_read %{{.*}} {in_bounds = {{.*}}, permutation_map = #{{.*}}} : tensor<?xf32>, vector<4xf32>
+// CHECK:           %[[VAL_7:.*]] = vector.transfer_read %{{.*}} {permutation_map = #{{.*}}} : tensor<?xf32>, vector<4xf32>
 // CHECK:           %[[VAL_9:.*]] = vector.create_mask %[[VAL_4]] : vector<4xi1>
 // CHECK:           %[[VAL_10:.*]] = vector.mask %[[VAL_9]] { vector.transfer_read %{{.*}} {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
 // CHECK:           %[[VAL_12:.*]] = vector.mask %[[VAL_9]] { vector.transfer_read %{{.*}} {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
@@ -666,6 +666,10 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
+// Input identical as the test in vectorization-with-patterns.mlir. Output is
+// different - vector sizes are inferred (rather than user-specified) and hence
+// masking was used.
+
 func.func @test_vectorize_pack(%arg0: tensor<32x8x16xf32>, %arg1: tensor<4x1x32x16x2xf32>) -> tensor<4x1x32x16x2xf32> {
   %pack = tensor.pack %arg0 outer_dims_perm = [1, 2, 0] inner_dims_pos = [2, 1] inner_tiles = [16, 2] into %arg1 : tensor<32x8x16xf32> -> tensor<4x1x32x16x2xf32>
   return %pack : tensor<4x1x32x16x2xf32>
@@ -692,6 +696,10 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
+// Input identical as the test in vectorization-with-patterns.mlir. Output is
+// different - vector sizes are inferred (rather than user-specified) and hence
+// masking was used.
+
 func.func @test_vectorize_padded_pack(%arg0: tensor<32x7x15xf32>, %arg1: tensor<32x4x1x16x2xf32>) -> tensor<32x4x1x16x2xf32> {
   %pad = arith.constant 0.000000e+00 : f32
   %pack = tensor.pack %arg0 padding_value(%pad : f32) inner_dims_pos = [2, 1] inner_tiles = [16, 2] into %arg1 : tensor<32x7x15xf32> -> tensor<32x4x1x16x2xf32>
diff --git a/mlir/test/Dialect/Tensor/fold-tensor-subset-ops.mlir b/mlir/test/Dialect/Tensor/fold-tensor-subset-ops.mlir
index 1a84e14104932..988b5d835c16e 100644
--- a/mlir/test/Dialect/Tensor/fold-tensor-subset-ops.mlir
+++ b/mlir/test/Dialect/Tensor/fold-tensor-subset-ops.mlir
@@ -144,8 +144,6 @@ func.func @transfer_read_of_extract_slice_swappy_rank_reducing(%t : tensor<?x?x?
 
 // -----
 
-//   CHECK-DAG: #[[MAP1:.+]] = affine_map<()[s0, s1] -> (s0 + s1)>
-
 //       CHECK: func @fold_vector_transfer_write_with_rank_reduced_insert_slice
 //  CHECK-SAME:    %[[ARG0:[a-zA-Z0-9]+]]: tensor<?x?x?xf32>
 //  CHECK-SAME:    %[[ARG1:[a-zA-Z0-9]+]]: vector<4xf32>
@@ -155,6 +153,7 @@ func.func @transfer_read_of_extract_slice_swappy_rank_reducing(%t : tensor<?x?x?
 //  CHECK-SAME:    %[[ARG5:[a-zA-Z0-9]+]]: index
 //  CHECK-SAME:    %[[ARG6:[a-zA-Z0-9]+]]: index
 //  CHECK-SAME:    %[[ARG7:[a-zA-Z0-9]+]]: index
+//  CHECK-SAME:    %[[ARG8:[a-zA-Z0-9]+]]: tensor<?x?xf32>
 func.func @fold_vector_transfer_write_with_rank_reduced_insert_slice(
     %arg0 : tensor<?x?x?xf32>,
     %arg1 : vector<4xf32>, %arg2: index, %arg3 : index, %arg4 : index,
@@ -162,11 +161,8 @@ func.func @fold_vector_transfer_write_with_rank_reduced_insert_slice(
     %st : tensor<?x?xf32>) -> tensor<?x?x?xf32> {
   %cst = arith.constant 0.0 : f32
 
-//   CHECK-NOT:    insert_slice
-//   CHECK-DAG:    %[[C0:.+]] = arith.constant 0 : index
-//   CHECK-DAG:    %[[IDX0:.+]] = affine.apply #[[MAP1]]()[%[[ARG2]], %[[ARG6]]]
-//   CHECK-DAG:    %[[IDX1:.+]] = affine.apply #[[MAP1]]()[%[[ARG3]], %[[ARG7]]]
-//   CHECK-DAG:    vector.transfer_write %[[ARG1]], %[[ARG0]][%[[C0]], %[[IDX0]], %[[IDX1]]] {in_bounds = [true]} : vector<4xf32>, tensor<?x?x?xf32
+  //   CHECK-DAG:  %[[r1:.*]] = vector.transfer_write %[[ARG1]], %[[ARG8]][%[[ARG6]], %[[ARG7]]] {in_bounds = [true]} : vector<4xf32>, tensor<?x?xf32>
+  //   CHECK-DAG:  %[[r2:.*]] = tensor.insert_slice %[[r1]] into %[[ARG0]][0, %[[ARG2]], %[[ARG3]]] [1, %[[ARG4]], %[[ARG5]]] [1, 1, 1] : tensor<?x?xf32> into tensor<?x?x?xf32>
   %0 = vector.transfer_write %arg1, %st[%arg6, %arg7] {in_bounds = [true]}
       : vector<4xf32>, tensor<?x?xf32>
   %1 = tensor.insert_slice %0 into %arg0[0, %arg2, %arg3] [1, %arg4, %arg5] [1, 1, 1]
@@ -176,9 +172,6 @@ func.func @fold_vector_transfer_write_with_rank_reduced_insert_slice(
 
 // -----
 
-//   CHECK-DAG: #[[MAP1:.+]] = affine_map<()[s0, s1] -> (s0 + s1)>
-//   CHECK-DAG: #[[MAP2:.+]] = affine_map<(d0, d1, d2) -> (d1)>
-
 //       CHECK: func @fold_vector_transfer_write_with_inner_rank_reduced_insert_slice
 //  CHECK-SAME:    %[[ARG0:[a-zA-Z0-9]+]]: tensor<?x?x?xf32>
 //  CHECK-SAME:    %[[ARG1:[a-zA-Z0-9]+]]: vector<4xf32>
@@ -188,6 +181,7 @@ func.func @fold_vector_transfer_write_with_rank_reduced_insert_slice(
 //  CHECK-SAME:    %[[ARG5:[a-zA-Z0-9]+]]: index
 //  CHECK-SAME:    %[[ARG6:[a-zA-Z0-9]+]]: index
 //  CHECK-SAME:    %[[ARG7:[a-zA-Z0-9]+]]: index
+//  CHECK-SAME:    %[[ARG8:[a-zA-Z0-9]+]]: tensor<?x?xf32>
 func.func @fold_vector_transfer_write_with_inner_rank_reduced_insert_slice(
     %arg0 : tensor<?x?x?xf32>,
     %arg1 : vector<4xf32>, %arg2: index, %arg3 : index, %arg4 : index,
@@ -195,12 +189,8 @@ func.func @fold_vector_transfer_write_with_inner_rank_reduced_insert_slice(
     %st : tensor<?x?xf32>) -> tensor<?x?x?xf32> {
   %cst = arith.constant 0.0 : f32
 
-  //   CHECK-NOT: insert_slice
-  //   CHECK-DAG:  %[[C0:.+]] = arith.constant 0 : index
-  //   CHECK-DAG:  %[[IDX0:.+]] = affine.apply #[[MAP1]]()[%[[ARG2]], %[[ARG6]]]
-  //   CHECK-DAG:  %[[IDX1:.+]] = affine.apply #[[MAP1]]()[%[[ARG3]], %[[ARG7]]]
-  //   CHECK-DAG:  vector.transfer_write %[[ARG1]], %[[ARG0]][%[[IDX0]], %[[IDX1]], %[[C0]]]
-  //  CHECK-SAME:    {in_bounds = [true], permutation_map = #[[MAP2]]} : vector<4xf32>, tensor<?x?x?xf32
+  //   CHECK-DAG:  %[[r1:.*]] = vector.transfer_write %[[ARG1]], %[[ARG8]][%[[ARG6]], %[[ARG7]]] {in_bounds = [true]} : vector<4xf32>, tensor<?x?xf32>
+  //   CHECK-DAG:  %[[r2:.*]] = tensor.insert_slice %[[r1]] into %[[ARG0]][%[[ARG2]], %[[ARG3]], 0] [%[[ARG4]], %[[ARG5]], 1] [1, 1, 1] : tensor<?x?xf32> into tensor<?x?x?xf32>
   %0 = vector.transfer_write %arg1, %st[%arg6, %arg7] {in_bounds = [true]}
       : vector<4xf32>, tensor<?x?xf32>
   %1 = tensor.insert_slice %0 into %arg0[%arg2, %arg3, 0] [%arg4, %arg5, 1] [1, 1, 1]
@@ -226,6 +216,24 @@ func.func @insert_slice_of_transfer_write(%t1 : tensor<?x12xf32>, %v : vector<5x
 
 // -----
 
+// This test is negative since `transfer_write` only
+// writes to `5x6` of the `100x100` elements of `%arg3`
+// CHECK-LABEL: func @insert_slice_of_transfer_write_overwrite_all(
+//  CHECK-SAME:     %[[arg0:.*]]: tensor<1000x1000xf32>, %[[arg1:.*]]: vector<5x6xf32>, %[[arg2:.*]]: index, %[[arg3:.*]]: tensor<100x100xf32>
+func.func @insert_slice_of_transfer_write_overwrite_all(%arg0: tensor<1000x1000xf32>, %arg1: vector<5x6xf32>, %arg2: index, %arg3: tensor<100x100xf32>) -> tensor<1000x1000xf32> {
+  %c0 = arith.constant 0 : index
+
+//       CHECK:   %[[c0:.*]] = arith.constant 0 : index
+//       CHECK:   %[[r1:.*]] = vector.transfer_write %[[arg1]], %[[arg3]][%[[c0]], %[[c0]]] {in_bounds = [true, true]} : vector<5x6xf32>, tensor<100x100xf32>
+//       CHECK:   %[[r2:.*]] = tensor.insert_slice %[[r1]] into %[[arg0]][3, %[[arg2]]] [100, 100] [1, 1] : tensor<100x100xf32> into tensor<1000x1000xf32>
+//       CHECK:   return %[[r2]] : tensor<1000x1000xf32>
+  %0 = vector.transfer_write %arg1, %arg3[%c0, %c0] {in_bounds = [true, true]} : vector<5x6xf32>, tensor<100x100xf32>
+  %inserted_slice = tensor.insert_slice %0 into %arg0[3, %arg2] [100, 100] [1, 1] : tensor<100x100xf32> into tensor<1000x1000xf32>
+  return %inserted_slice : tensor<1000x1000xf32>
+}
+
+// -----
+
 //   CHECK-DAG: #[[$d0d2:.+]] = affine_map<(d0, d1, d2) -> (d0, d2)>
 
 // CHECK-LABEL: func @insert_slice_of_transfer_write_swappy_rank_extending(
diff --git a/mlir/test/Dialect/Vector/invalid.mlir b/mlir/test/Dialect/Vector/invalid.mlir
index e2bc5ef6128e7..36d04bb77e3b9 100644
--- a/mlir/test/Dialect/Vector/invalid.mlir
+++ b/mlir/test/Dialect/Vector/invalid.mlir
@@ -454,6 +454,15 @@ func.func @test_vector.transfer_read(%arg0: memref<?x?xf32>) {
 
 // -----
 
+func.func @test_vector.transfer_read(%arg0: memref<?x?xf32>) {
+  %c3 = arith.constant 3 : index
+  %cst = arith.constant 3.0 : f32
+  // expected-error@+1 {{requires a projected permutation_map (at most one dim or the zero constant can appear in each result)}}
+  %0 = vector.transfer_read %arg0[%c3, %c3], %cst {permutation_map = affine_map<(d0, d1)->(1)>} : memref<?x?xf32>, vector<128xf32>
+}
+
+// -----
+
 func.func @test_vector.transfer_read(%arg0: memref<?x?x?xf32>) {
   %c3 = arith.constant 3 : index
   %cst = arith.constant 3.0 : f32
@@ -505,16 +514,6 @@ func.func @test_vector.transfer_read(%arg0: memref<?x?xvector<2x3xf32>>) {
 
 // -----
 
-func.func @test_vector.transfer_read(%arg0: memref<?x?xvector<2x3xf32>>) {
-  %c3 = arith.constant 3 : index
-  %f0 = arith.constant 0.0 : f32
-  %vf0 = vector.splat %f0 : vector<2x3xf32>
-  // expected-error@+1 {{requires broadcast dimensions to be in-bounds}}
-  %0 = vector.transfer_read %arg0[%c3, %c3], %vf0 {in_bounds = [false, true], permutation_map = affine_map<(d0, d1)->(0, d1)>} : memref<?x?xvector<2x3xf32>>, vector<1x1x2x3xf32>
-}
-
-// -----
-
 func.func @test_vector.transfer_read(%arg0: memref<?x?xvector<2x3xf32>>) {
   %c3 = arith.constant 3 : index
   %f0 = arith.constant 0.0 : f32
@@ -618,6 +617,15 @@ func.func @test_vector.transfer_write(%arg0: memref<?x?xf32>) {
 
 // -----
 
+func.func @test_vector.transfer_write(%arg0: memref<?x?xf32>) {
+  %c3 = arith.constant 3 : index
+  %cst = arith.constant dense<3.0> : vector<128 x f32>
+  // expected-error@+1 {{requires a projected permutation_map (at most one dim or the zero constant can appear in each result)}}
+  vector.transfer_write %cst, %arg0[%c3, %c3] {permutation_map = affine_map<(d0, d1)->(1)>} : vector<128xf32>, memref<?x?xf32>
+}
+
+// -----
+
 func.func @test_vector.transfer_write(%arg0: memref<?x?x?xf32>) {
   %c3 = arith.constant 3 : index
   %cst = arith.constant dense<3.0> : vector<3 x 7 x f32>
diff --git a/mlir/test/Dialect/Vector/ops.mlir b/mlir/test/Dialect/Vector/ops.mlir
index 08d1a189231bc..3baacba9b6124 100644
--- a/mlir/test/Dialect/Vector/ops.mlir
+++ b/mlir/test/Dialect/Vector/ops.mlir
@@ -70,7 +70,7 @@ func.func @vector_transfer_ops(%arg0: memref<?x?xf32>,
   // CHECK: vector.transfer_read %{{.*}}[%[[C3]], %[[C3]]], %{{.*}}, %{{.*}} : memref<?x?xf32>, vector<5xf32>
   %8 = vector.transfer_read %arg0[%c3, %c3], %f0, %m : memref<?x?xf32>, vector<5xf32>
   // CHECK: vector.transfer_read %{{.*}}[%[[C3]], %[[C3]], %[[C3]]], %{{.*}}, %{{.*}} : memref<?x?x?xf32>, vector<5x4x8xf32>
-  %9 = vector.transfer_read %arg4[%c3, %c3, %c3], %f0, %m2 {in_bounds = [false, false, true], permutation_map = affine_map<(d0, d1, d2)->(d1, d0, 0)>} : memref<?x?x?xf32>, vector<5x4x8xf32>
+  %9 = vector.transfer_read %arg4[%c3, %c3, %c3], %f0, %m2 {permutation_map = affine_map<(d0, d1, d2)->(d1, d0, 0)>} : memref<?x?x?xf32>, vector<5x4x8xf32>
 
   // CHECK: vector.transfer_write
   vector.transfer_write %0, %arg0[%c3, %c3] {permutation_map = affine_map<(d0, d1)->(d0)>} : vector<128xf32>, memref<?x?xf32>
diff --git a/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir b/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir
index 15000d706adfc..0feaf690af251 100644
--- a/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir
+++ b/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir
@@ -327,13 +327,11 @@ func.func @masked_permutation_xfer_read_fixed_width(
   %c0 = arith.constant 0 : index
   %3 = vector.mask %mask {
     vector.transfer_read %dest[%c0, %c0], %cst {
-      in_bounds = [false, true, false],
       permutation_map = affine_map<(d0, d1) -> (d1, 0, d0)>
     } : tensor<?x1xf32>, vector<1x4x4xf32>
   } : vector<4x1xi1> -> vector<1x4x4xf32>
 
   "test.some_use"(%3) : (vector<1x4x4xf32>) -> ()
-
   return
 }
 
diff --git a/mlir/test/Dialect/Vector/vector-transfer-unroll.mlir b/mlir/test/Dialect/Vector/vector-transfer-unroll.mlir
index 75c5ad26fcf23..5dd65ea132d08 100644
--- a/mlir/test/Dialect/Vector/vector-transfer-unroll.mlir
+++ b/mlir/test/Dialect/Vector/vector-transfer-unroll.mlir
@@ -207,7 +207,7 @@ func.func @transfer_read_unroll_permutation(%mem : memref<6x4xf32>) -> vector<4x
 func.func @transfer_read_unroll_broadcast(%mem : memref<6x4xf32>) -> vector<6x4xf32> {
   %c0 = arith.constant 0 : index
   %cf0 = arith.constant 0.0 : f32
-  %res = vector.transfer_read %mem[%c0, %c0], %cf0 {in_bounds = [true, false], permutation_map = #map0} : memref<6x4xf32>, vector<6x4xf32>
+  %res = vector.transfer_read %mem[%c0, %c0], %cf0 {permutation_map = #map0} : memref<6x4xf32>, vector<6x4xf32>
   return %res : vector<6x4xf32>
 }
 
@@ -234,7 +234,7 @@ func.func @transfer_read_unroll_broadcast(%mem : memref<6x4xf32>) -> vector<6x4x
 func.func @transfer_read_unroll_broadcast_permuation(%mem : memref<6x4xf32>) -> vector<4x6xf32> {
   %c0 = arith.constant 0 : index
   %cf0 = arith.constant 0.0 : f32
-  %res = vector.transfer_read %mem[%c0, %c0], %cf0 {in_bounds = [true, false], permutation_map = #map0} : memref<6x4xf32>, vector<4x6xf32>
+  %res = vector.transfer_read %mem[%c0, %c0], %cf0 {permutation_map = #map0} : memref<6x4xf32>, vector<4x6xf32>
   return %res : vector<4x6xf32>
 }
 
diff --git a/mlir/test/Dialect/Vector/vector-transferop-opt.mlir b/mlir/test/Dialect/Vector/vector-transferop-opt.mlir
index 07e6647533c6f..f4f7fb1ba0304 100644
--- a/mlir/test/Dialect/Vector/vector-transferop-opt.mlir
+++ b/mlir/test/Dialect/Vector/vector-transferop-opt.mlir
@@ -229,7 +229,7 @@ func.func @forward_dead_store_negative(%arg0: i1, %arg1 : memref<4x4xf32>,
 // final `vector.transfer_write` should be preserved as:
 //         vector.transfer_write %2, %subview
 
-// CHECK-LABEL:  func.func @collapse_shape
+// CHECK-LABEL:  func.func @collapse_shape_and_read_from_source
 //       CHECK:    scf.for {{.*}} {
 //       CHECK:      vector.transfer_read
 //       CHECK:      vector.transfer_write
@@ -237,7 +237,7 @@ func.func @forward_dead_store_negative(%arg0: i1, %arg1 : memref<4x4xf32>,
 //       CHECK:      vector.transfer_read
 //       CHECK:      vector.transfer_write
 
-func.func @collapse_shape(%in_0: memref<1x20x1xi32>, %vec: vector<4xi32>) {
+func.func @collapse_shape_and_read_from_source(%in_0: memref<1x20x1xi32>, %vec: vector<4xi32>) {
   %c0_i32 = arith.constant 0 : i32
   %c0 = arith.constant 0 : index
   %c4 = arith.constant 4 : index
@@ -257,6 +257,98 @@ func.func @collapse_shape(%in_0: memref<1x20x1xi32>, %vec: vector<4xi32>) {
   return
 }
 
+// The same regression test for expand_shape.
+
+// CHECK-LABEL:  func.func @expand_shape_and_read_from_source
+//       CHECK:    scf.for {{.*}} {
+//       CHECK:      vector.transfer_read
+//       CHECK:      vector.transfer_write
+//       CHECK:      vector.transfer_write
+//       CHECK:      vector.transfer_read
+//       CHECK:      vector.transfer_write
+
+func.func @expand_shape_and_read_from_source(%in_0: memref<20xi32>, %vec: vector<1x4x1xi32>) {
+  %c0_i32 = arith.constant 0 : i32
+  %c0 = arith.constant 0 : index
+  %c4 = arith.constant 4 : index
+  %c20 = arith.constant 20 : index
+
+  %alloca = memref.alloca() {alignment = 64 : i64} : memref<4xi32>
+  %expand_shape = memref.expand_shape %alloca [[0, 1, 2]] output_shape [1, 4, 1] : memref<4xi32> into memref<1x4x1xi32>
+  scf.for %arg0 = %c0 to %c20 step %c4 {
+    %subview = memref.subview %in_0[%arg0] [4] [1] : memref<20xi32> to memref<4xi32, strided<[1], offset: ?>>
+    %1 = vector.transfer_read %subview[%c0], %c0_i32 {in_bounds = [true]} : memref<4xi32, strided<[1], offset: ?>>, vector<4xi32>
+    // $alloca and $expand_shape alias
+    vector.transfer_write %1, %alloca[%c0] {in_bounds = [true]} : vector<4xi32>, memref<4xi32>
+    vector.transfer_write %vec, %expand_shape[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x4x1xi32>, memref<1x4x1xi32>
+    %2 = vector.transfer_read %alloca[%c0], %c0_i32 {in_bounds = [true]} : memref<4xi32>, vector<4xi32>
+    vector.transfer_write %2, %subview[%c0] {in_bounds = [true]} : vector<4xi32>, memref<4xi32, strided<[1], offset: ?>>
+  }
+  return
+}
+
+// The same regression test, but the initial write is to the collapsed memref,
+// and the subsequent unforwardable read is from the collapse shape.
+
+// CHECK-LABEL:  func.func @collapse_shape_and_read_from_collapse
+//       CHECK:    scf.for {{.*}} {
+//       CHECK:      vector.transfer_read
+//       CHECK:      vector.transfer_write
+//       CHECK:      vector.transfer_write
+//       CHECK:      vector.transfer_read
+//       CHECK:      vector.transfer_write
+
+func.func @collapse_shape_and_read_from_collapse(%in_0: memref<20xi32>, %vec: vector<1x4x1xi32>) {
+  %c0_i32 = arith.constant 0 : i32
+  %c0 = arith.constant 0 : index
+  %c4 = arith.constant 4 : index
+  %c20 = arith.constant 20 : index
+
+  %alloca = memref.alloca() {alignment = 64 : i64} : memref<1x4x1xi32>
+  %collapse_shape = memref.collapse_shape %alloca [[0, 1, 2]] : memref<1x4x1xi32> into memref<4xi32>
+  scf.for %arg0 = %c0 to %c20 step %c4 {
+    %subview = memref.subview %in_0[%arg0] [4] [1] : memref<20xi32> to memref<4xi32, strided<[1], offset: ?>>
+    %1 = vector.transfer_read %subview[%c0], %c0_i32 {in_bounds = [true]} : memref<4xi32, strided<[1], offset: ?>>, vector<4xi32>
+    vector.transfer_write %1, %collapse_shape[%c0] {in_bounds = [true]} : vector<4xi32>, memref<4xi32>
+    // $alloca and $collapse_shape alias
+    vector.transfer_write %vec, %alloca[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x4x1xi32>, memref<1x4x1xi32>
+    %2 = vector.transfer_read %collapse_shape[%c0], %c0_i32 {in_bounds = [true]} : memref<4xi32>, vector<4xi32>
+    vector.transfer_write %2, %subview[%c0] {in_bounds = [true]} : vector<4xi32>, memref<4xi32, strided<[1], offset: ?>>
+  }
+  return
+}
+
+// The same test except writing to the expanded source first (same as the
+// previous collapse test but for expand).
+
+// CHECK-LABEL:  func.func @expand_shape_and_read_from_expand
+//       CHECK:    scf.for {{.*}} {
+//       CHECK:      vector.transfer_read
+//       CHECK:      vector.transfer_write
+//       CHECK:      vector.transfer_write
+//       CHECK:      vector.transfer_read
+//       CHECK:      vector.transfer_write
+
+func.func @expand_shape_and_read_from_expand(%in_0: memref<1x20x1xi32>, %vec: vector<4xi32>) {
+  %c0_i32 = arith.constant 0 : i32
+  %c0 = arith.constant 0 : index
+  %c4 = arith.constant 4 : index
+  %c20 = arith.constant 20 : index
+
+  %alloca = memref.alloca() {alignment = 64 : i64} : memref<4xi32>
+  %expand_shape = memref.expand_shape %alloca [[0, 1, 2]] output_shape [1, 4, 1] : memref<4xi32> into memref<1x4x1xi32>
+  scf.for %arg0 = %c0 to %c20 step %c4 {
+    %subview = memref.subview %in_0[0, %arg0, 0] [1, 4, 1] [1, 1, 1] : memref<1x20x1xi32> to memref<1x4x1xi32, strided<[20, 1, 1], offset: ?>>
+    %1 = vector.transfer_read %subview[%c0, %c0, %c0], %c0_i32 {in_bounds = [true, true, true]} : memref<1x4x1xi32, strided<[20, 1, 1], offset: ?>>, vector<1x4x1xi32>
+    vector.transfer_write %1, %expand_shape[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x4x1xi32>, memref<1x4x1xi32>
+    // $alloca and $expand_shape alias
+    vector.transfer_write %vec, %alloca[%c0] {in_bounds = [true]} : vector<4xi32>, memref<4xi32>
+    %2 = vector.transfer_read %expand_shape[%c0, %c0, %c0], %c0_i32 {in_bounds = [true, true, true]} : memref<1x4x1xi32>, vector<1x4x1xi32>
+    vector.transfer_write %2, %subview[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x4x1xi32>, memref<1x4x1xi32, strided<[20, 1, 1], offset: ?>>
+  }
+  return
+}
+
 // CHECK-LABEL: func @forward_dead_store_dynamic_same_index
 //   CHECK-NOT:   vector.transfer_write
 //   CHECK-NOT:   vector.transfer_read
diff --git a/mlir/test/Dialect/XeGPU/XeGPUOps.mlir b/mlir/test/Dialect/XeGPU/XeGPUOps.mlir
index c1126efb6046d..a4587faa3345c 100644
--- a/mlir/test/Dialect/XeGPU/XeGPUOps.mlir
+++ b/mlir/test/Dialect/XeGPU/XeGPUOps.mlir
@@ -13,6 +13,14 @@ gpu.func @test_create_nd_tdesc_vc_1(%src: memref<24x32xf32>) {
   gpu.return
 }
 
+// CHECK: gpu.func @test_create_nd_tdesc_with_sg_map(%[[arg0:.*]]: memref<24x32xf32>) {
+gpu.func @test_create_nd_tdesc_with_sg_map(%src: memref<24x32xf32>) {
+  // CHECK: %[[REG:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<8x16xf32, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>>
+  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> ->
+    !xegpu.tensor_desc<8x16xf32, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>>
+  gpu.return
+}
+
 // CHECK: gpu.func @test_create_nd_tdesc_vc_2(%[[arg0:.*]]: ui64, %[[arg1:.*]]: index, %[[arg2:.*]]: index, %[[arg3:.*]]: index, %[[arg4:.*]]: index) {
 gpu.func @test_create_nd_tdesc_vc_2(%src: ui64, %w : index, %h : index, %x : index, %y : index) {
   //CHECK: %[[C:.*]] = arith.constant 1 : index
@@ -43,6 +51,13 @@ gpu.func @test_create_nd_tdesc_vc_5(%src: memref<2x24x32xf32, 3>) {
   gpu.return
 }
 
+// CHECK: gpu.func @test_create_nd_tdesc_vc_6(%[[arg0:.*]]: memref<24x32xf32>) {
+gpu.func @test_create_nd_tdesc_vc_6(%src: memref<24x32xf32>) {
+  // CHECK: %[[REG:.*]] = xegpu.create_nd_tdesc %[[arg0]][0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<24x16xf32, #xegpu.block_tdesc_attr<array_length = 2 : i64>, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>
+  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<24x16xf32, #xegpu.block_tdesc_attr<array_length = 2>, #xegpu.sg_map<wi_layout = [1, 16], wi_data = [1, 1]>>
+  gpu.return
+}
+
 // CHECK: gpu.func @test_prefetch_nd_vc(%[[arg0:.*]]: memref<24x32xf16>) {
 gpu.func @test_prefetch_nd_vc(%src: memref<24x32xf16>) {
   // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %[[arg0]][0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<8x16xf16>
@@ -104,22 +119,37 @@ gpu.func @test_create_update_nd_tdesc_vc(%src: memref<24x32xf32>) {
 
 // CHECK: gpu.func @test_create_tdesc_vc(%[[arg0:.*]]: ui64) {
 gpu.func @test_create_tdesc_vc(%src: ui64) {
-  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %arg0 [0, 8, 16, 24] : ui64 -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
-  %1 = xegpu.create_tdesc %src[0, 8, 16, 24] : ui64  -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
+  //CHECK: %[[cst:.*]] = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
+  %1 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex>  -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
   gpu.return
 }
 
 // CHECK: gpu.func @test_create_tdesc_vc_1(%[[arg0:.*]]: memref<?xf32, 3>) {
 gpu.func @test_create_tdesc_vc_1(%src: memref<?xf32, 3>) {
-  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %arg0 [0, 8, 16, 24] : memref<?xf32, 3> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<memory_space =  slm, chunk_size = 2 : i64>>
-  %1 = xegpu.create_tdesc %src[0, 8, 16, 24] : memref<?xf32, 3>  -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<memory_space = slm, chunk_size = 2>>
+  //CHECK: %[[cst:.*]] = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : memref<?xf32, 3>, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<memory_space =  slm, chunk_size = 2 : i64>>
+  %1 = xegpu.create_tdesc %src, %0 : memref<?xf32, 3>, vector<4xindex>  -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<memory_space = slm, chunk_size = 2>>
+  gpu.return
+}
+
+// CHECK: gpu.func @test_create_tdesc_vc_with_sg_map(%[[arg0:.*]]: ui64) {
+gpu.func @test_create_tdesc_vc_with_sg_map(%src: ui64) {
+  //CHECK: %[[cst:.*]] = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 1]>>
+  %1 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>, #xegpu.sg_map<wi_layout = [4, 1], wi_data = [1, 1]>>
   gpu.return
 }
 
 // CHECK: gpu.func @test_prefetch_vc(%[[arg0:.*]]: ui64) {
 gpu.func @test_prefetch_vc(%src: ui64) {
-  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %arg0 [0, 8, 16, 24] : ui64 -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
-  %1 = xegpu.create_tdesc %src[0, 8, 16, 24] : ui64  -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
+  //CHECK: %[[cst:.*]] = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
+  %1 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
   // CHECK: xegpu.prefetch %[[R0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
   xegpu.prefetch %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}>: !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
   gpu.return
@@ -129,8 +159,10 @@ gpu.func @test_prefetch_vc(%src: ui64) {
 gpu.func @test_load_gather_vc(%src: ui64) {
   //CHECK: %[[cst:.*]] = arith.constant dense<true> : vector<4xi1>
   %0 = arith.constant dense<1>: vector<4xi1>
-  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %arg0 [0, 8, 16, 24] : ui64 -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
-  %1 = xegpu.create_tdesc %src[0, 8, 16, 24] : ui64  -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
+  //CHECK: %[[c2:.*]] = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  %c = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[c2]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
+  %1 = xegpu.create_tdesc %src, %c : ui64, vector<4xindex>  -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
   //CHECK: %[[R1:.*]] = xegpu.load %[[R0]], %[[cst]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose}>
   //CHECK-SAME: !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<4xi1> -> vector<2x4xf32>
   %2 = xegpu.load %1, %0 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose}>
@@ -144,8 +176,10 @@ gpu.func @test_store_scatter_vc(%src: ui64) {
   %0 = arith.constant dense<1>: vector<4xi1>
   //CHECK: %[[c1:.*]] = arith.constant dense<2.900000e+00> : vector<2x4xf32>
   %1 = arith.constant dense<2.9>: vector<2x4xf32>
-  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %arg0 [0, 8, 16, 24] : ui64 -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
-  %2 = xegpu.create_tdesc %src[0, 8, 16, 24] : ui64  -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
+  //CHECK: %[[c2:.*]] = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  %c = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[c2]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
+  %2 = xegpu.create_tdesc %src, %c : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
   //CHECK: xegpu.store %[[c1]], %[[R0]], %[[c0]] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>, transpose}>
   //CHECK-SAME: vector<2x4xf32>, !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<4xi1>
   xegpu.store %1, %2, %0 <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>, transpose}>
@@ -155,10 +189,14 @@ gpu.func @test_store_scatter_vc(%src: ui64) {
 
 // CHECK: gpu.func @test_create_update_tdesc_vc(%[[arg0:.*]]: ui64) {
 gpu.func @test_create_update_tdesc_vc(%src: ui64) {
-  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %arg0 [0, 8, 16, 24] : ui64 -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
-  %1 = xegpu.create_tdesc %src[0, 8, 16, 24]: ui64  -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
-  //CHECK: %[[R1:.*]] = xegpu.update_offset %[[R0]], [32, 32, 32, 32] : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
-  %2 = xegpu.update_offset %1, [32, 32, 32, 32] : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
+  //CHECK: %[[cst:.*]] = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
+  //CHECK: %[[st:.*]] = arith.constant dense<32> : vector<4xindex>
+  //CHECK: %[[R1:.*]] = xegpu.update_offset %[[R0]], %[[st]] : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<4xindex>
+  %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  %1 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
+  %s = arith.constant dense<[32, 32, 32, 32]> : vector<4xindex>
+  %2 = xegpu.update_offset %1, %s : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>, vector<4xindex>
   gpu.return
 }
 
@@ -179,8 +217,10 @@ gpu.func @test_dpas_vc_with_packed_b(%a : vector<8x16xf16>, %b: vector<8x16x2xf1
 
 // CHECK: gpu.func @test_atomic_rmw(%[[arg0:.*]]: ui64, %[[arg1:.*]]: vector<16xf32>, %[[arg2:.*]]: vector<16xi1>)
 gpu.func @test_atomic_rmw(%src: ui64, %value : vector<16xf32>, %mask : vector<16xi1>) {
-  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]] [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] : ui64 -> !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>
-  %1 = xegpu.create_tdesc %src[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]: ui64 -> !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>
+  //CHECK: %[[c:.*]] = arith.constant dense<[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]> : vector<16xindex>
+  %c = arith.constant dense<[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]> : vector<16xindex>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[c]] : ui64, vector<16xindex> -> !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>
+  %1 = xegpu.create_tdesc %src, %c: ui64, vector<16xindex> -> !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>
   //CHECK: %[[R1:.*]] = xegpu.atomic_rmw addf %[[R0]], %[[arg2]], %[[arg1]] : !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>, vector<16xi1>, vector<16xf32> -> vector<16xf32>
   xegpu.atomic_rmw addf %1, %mask, %value: !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>, vector<16xi1>, vector<16xf32> -> vector<16xf32>
   gpu.return
diff --git a/mlir/test/Dialect/XeGPU/invalid.mlir b/mlir/test/Dialect/XeGPU/invalid.mlir
index 193dae352e370..f8a0d95bd70a2 100644
--- a/mlir/test/Dialect/XeGPU/invalid.mlir
+++ b/mlir/test/Dialect/XeGPU/invalid.mlir
@@ -39,8 +39,9 @@ func.func @test_prefetch_nd_vc_1(%src: memref<24x32xf16>) {
 
 // -----
 func.func @test_prefetch_nd_vc_2(%src: memref<24xf16>) {
-  %1 = xegpu.create_tdesc %src[0, 1, 2, 3, 4, 5, 6, 7]
-        : memref<24xf16> -> !xegpu.tensor_desc<8xf16, #xegpu.scatter_tdesc_attr<>>
+  %0 = arith.constant dense<[0, 1, 2, 3, 4, 5, 6, 7]> : vector<8xindex>
+  %1 = xegpu.create_tdesc %src, %0 : memref<24xf16>, vector<8xindex>
+                -> !xegpu.tensor_desc<8xf16, #xegpu.scatter_tdesc_attr<>>
   // expected-error@+1 {{Expects a non-scattered TensorDesc}}
   xegpu.prefetch_nd %1 <{l1_hint = #xegpu.cache_hint<cached>}>
         : !xegpu.tensor_desc<8xf16, #xegpu.scatter_tdesc_attr<>>
@@ -58,8 +59,9 @@ func.func @test_load_nd_vc_1(%src: memref<8x16xf16>) {
 
 // -----
 func.func @test_load_nd_vc_2(%src: memref<16xf16>) {
-  %1 = xegpu.create_tdesc %src[0, 2, 4, 6, 8, 10, 12, 14]
-        : memref<16xf16> -> !xegpu.tensor_desc<8x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
+  %0 = arith.constant dense<[0, 2, 4, 6, 8, 10, 12, 14]> : vector<8xindex>
+  %1 = xegpu.create_tdesc %src, %0 : memref<16xf16>, vector<8xindex>
+          -> !xegpu.tensor_desc<8x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
   // expected-error@+1 {{Expects a non-scattered TensorDesc.}}
   %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>}>
       : !xegpu.tensor_desc<8x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2>> -> vector<8x2xf16>
@@ -86,9 +88,10 @@ func.func @test_store_nd_vc_1(%dst: memref<24x32xf16>) {
 
 // -----
 func.func @test_store_nd_vc_2(%dst: memref<16xf16>) {
+  %0 = arith.constant dense<[0, 2, 4, 6, 8, 10, 12, 14]> : vector<8xindex>
   %1 = arith.constant dense<1.0>: vector<8x2xf16>
-  %2 = xegpu.create_tdesc %dst[0, 2, 4, 6, 8, 10, 12, 14]
-        : memref<16xf16> -> !xegpu.tensor_desc<8x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
+  %2 = xegpu.create_tdesc %dst, %0 : memref<16xf16>, vector<8xindex>
+            -> !xegpu.tensor_desc<8x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
   // expected-error@+1 {{Expects a non-scattered TensorDesc}}
   xegpu.store_nd %1, %2 <{l1_hint = #xegpu.cache_hint<streaming>}>
         : vector<8x2xf16>, !xegpu.tensor_desc<8x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
@@ -97,8 +100,9 @@ func.func @test_store_nd_vc_2(%dst: memref<16xf16>) {
 
 // -----
 func.func @test_update_nd_offset_1(%dst: memref<16xf16>) {
-  %1 = xegpu.create_tdesc %dst[0, 2, 4, 6, 8, 10, 12, 14]
-        : memref<16xf16> -> !xegpu.tensor_desc<8x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
+  %0 = arith.constant dense<[0, 2, 4, 6, 8, 10, 12, 14]> : vector<8xindex>
+  %1 = xegpu.create_tdesc %dst, %0 : memref<16xf16>, vector<8xindex>
+            -> !xegpu.tensor_desc<8x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
   // expected-error@+1 {{Expects a non-scattered TensorDesc}}
   xegpu.update_nd_offset %1, [0, 2] : !xegpu.tensor_desc<8x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
   return
@@ -106,24 +110,27 @@ func.func @test_update_nd_offset_1(%dst: memref<16xf16>) {
 
 // -----
 func.func @test_create_tdesc_vc_1(%src: ui64) {
+  %0 = arith.constant dense<[0, 2, 4, 6, 8, 10, 12, 14]> : vector<8xindex>
   // expected-error@+1 {{Expects a scattered TensorDesc}}
-  %1 = xegpu.create_tdesc %src[0, 2, 4, 6, 8, 10, 12, 14]
-        : ui64 -> !xegpu.tensor_desc<8xf16>
+  %1 = xegpu.create_tdesc %src, %0 : ui64, vector<8xindex> -> !xegpu.tensor_desc<8xf16>
   return
 }
 
 // -----
 func.func @test_create_tdesc_vc_2(%src: ui64) {
+  %0 = arith.constant dense<[0, 2, 4, 6, 8, 10, 12, 14]> : vector<8xindex>
   // expected-error@+1 {{Incorrect TensorDesc shape}}
-  %1 = xegpu.create_tdesc %src[0, 2, 4, 6, 8, 10, 12, 14] {chunk_size = 2}
-        : ui64 -> !xegpu.tensor_desc<8x4xf16, #xegpu.scatter_tdesc_attr<>>
+  %1 = xegpu.create_tdesc %src, %0 : ui64, vector<8xindex>
+          -> !xegpu.tensor_desc<8x4xf16, #xegpu.scatter_tdesc_attr<>>
   return
 }
 
 // -----
 func.func @test_create_tdesc_vc_1(%src: memref<?xf32>) {
+  %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
   // expected-error@+1 {{Memory space mismatch}}
-  %1 = xegpu.create_tdesc %src[0, 8, 16, 24] : memref<?xf32>  -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<memory_space = slm, chunk_size = 2>>
+  %1 = xegpu.create_tdesc %src, %0 : memref<?xf32>, vector<4xindex>
+          -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<memory_space = slm, chunk_size = 2>>
   return
 }
 
@@ -137,7 +144,9 @@ func.func @test_prefetch_vc_1(%src: memref<24x32xf16>) {
 
 // -----
 func.func @test_prefetch_vc_2(%src: ui64) {
-  %1 = xegpu.create_tdesc %src[0, 8, 16, 24] : ui64  -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
+  %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
+  %1 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex>
+          -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
   // expected-error@+1 {{invlid l1_hint: #xegpu.cache_hint<write_back>}}
   xegpu.prefetch %1 <{l1_hint = #xegpu.cache_hint<write_back>}>: !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
   return
@@ -155,8 +164,9 @@ func.func @test_load_gather_vc_1(%src: memref<24x32xf16>) {
 
 // -----
 func.func @test_load_gather_vc_2(%src: ui64) {
+  %cst = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
   %0 = arith.constant dense<1>: vector<4xi1>
-  %1 = xegpu.create_tdesc %src[0, 8, 16, 24] : ui64
+  %1 = xegpu.create_tdesc %src, %cst : ui64, vector<4xindex>
         -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
   // expected-error@+1 {{invlid l1_hint: #xegpu.cache_hint<write_back>}}
   %2 = xegpu.load %1, %0 <{l1_hint = #xegpu.cache_hint<write_back>}>
@@ -178,10 +188,11 @@ func.func @test_store_scatter_vc_1(%src: memref<24x32xf32>) {
 
 // -----
 func.func @test_store_scatter_vc_2(%src: ui64) {
+  %cst = arith.constant dense<[0, 8, 16, 24]>: vector<4xindex>
   %0 = arith.constant dense<1>: vector<4xi1>
   %1 = arith.constant dense<2.9>: vector<4x2xf32>
-  %2 = xegpu.create_tdesc %src[0, 8, 16, 24]
-          : ui64 -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
+  %2 = xegpu.create_tdesc %src, %cst : ui64, vector<4xindex>
+              -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
   // expected-error@+1 {{invlid l1_hint: #xegpu.cache_hint<streaming>}}
   xegpu.store %1, %2, %0 <{l1_hint = #xegpu.cache_hint<streaming>}> : vector<4x2xf32>,
           !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>, vector<4xi1>
@@ -204,7 +215,8 @@ func.func @test_dpas_vc_2(%a : vector<8x8x2xf16>, %b: vector<8x16x2xf16>) {
 
 // -----
 func.func @test_atomic_rmw(%src: ui64, %value : vector<16x4xf32>, %mask : vector<16xi1>) {
-  %1 = xegpu.create_tdesc %src[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] : ui64 -> !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<chunk_size = 8>>
+  %0 = arith.constant dense<[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]> : vector<16xindex>
+  %1 = xegpu.create_tdesc %src, %0 : ui64, vector<16xindex> -> !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<chunk_size = 8>>
   // expected-error@+1 {{failed to verify that all of {tensorDesc, value, result} have same shape}}
   xegpu.atomic_rmw addf %1, %mask, %value: !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<chunk_size = 8>>, vector<16xi1>, vector<16x4xf32> -> vector<16x8xf32>
   return
diff --git a/mlir/test/IR/attribute.mlir b/mlir/test/IR/attribute.mlir
index 31a4663f72e6e..a62de3f5004d7 100644
--- a/mlir/test/IR/attribute.mlir
+++ b/mlir/test/IR/attribute.mlir
@@ -76,6 +76,10 @@ func.func @float_attrs_pass() {
     // CHECK: float_attr = 2.000000e+00 : f8E3M4
     float_attr = 2. : f8E3M4
   } : () -> ()
+  "test.float_attrs"() {
+    // CHECK: float_attr = 2.000000e+00 : f8E8M0FNU
+    float_attr = 2. : f8E8M0FNU
+  } : () -> ()
   "test.float_attrs"() {
     // CHECK: float_attr = 2.000000e+00 : f16
     float_attr = 2. : f16
diff --git a/mlir/test/IR/invalid-locations.mlir b/mlir/test/IR/invalid-locations.mlir
index bb8006485c5be..78f3def9380ae 100644
--- a/mlir/test/IR/invalid-locations.mlir
+++ b/mlir/test/IR/invalid-locations.mlir
@@ -94,13 +94,6 @@ func.func @location_fused_missing_r_square() {
 
 // -----
 
-func.func @location_invalid_alias() {
-  // expected-error@+1 {{expected location, but found dialect attribute: '#foo.loc'}}
-  return loc(#foo.loc)
-}
-
-// -----
-
 func.func @location_invalid_alias() {
   // expected-error@+1 {{operation location alias was never defined}}
   return loc(#invalid_alias)
diff --git a/mlir/test/IR/locations.mlir b/mlir/test/IR/locations.mlir
index 8d7c7e4f13ed4..0c6426ebec874 100644
--- a/mlir/test/IR/locations.mlir
+++ b/mlir/test/IR/locations.mlir
@@ -89,3 +89,10 @@ func.func @optional_location_specifier() {
   test.attr_with_loc("foo" loc("foo_loc"))
   return
 }
+
+// CHECK-LABEL: @dialect_location
+// CHECK: test.attr_with_loc("dialectLoc" loc(#test.custom_location<"foo.mlir" * 32>))
+func.func @dialect_location() {
+  test.attr_with_loc("dialectLoc" loc(#test.custom_location<"foo.mlir"*32>))
+  return
+}
diff --git a/mlir/test/IR/pretty-locations.mlir b/mlir/test/IR/pretty-locations.mlir
index e9337b5bef37b..598bebeb83aeb 100644
--- a/mlir/test/IR/pretty-locations.mlir
+++ b/mlir/test/IR/pretty-locations.mlir
@@ -24,6 +24,9 @@ func.func @inline_notation() -> i32 {
   affine.if #set0(%2) {
   } loc(fused<"myPass">["foo", "foo2"])
 
+  // CHECK: "foo.op"() : () -> () #test.custom_location<"foo.mlir" * 1234>
+  "foo.op"() : () -> () loc(#test.custom_location<"foo.mlir" * 1234>)
+
   // CHECK: return %0 : i32 [unknown]
   return %1 : i32 loc(unknown)
 }
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-collapse-tensor.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-collapse-tensor.mlir
index 734e09b7ed103..af4feb356ddf5 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/test-collapse-tensor.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-collapse-tensor.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt %s \
 // RUN: -one-shot-bufferize="bufferize-function-boundaries" \
-// RUN: -finalizing-bufferize -buffer-deallocation-pipeline -convert-bufferization-to-memref \
+// RUN: -buffer-deallocation-pipeline -convert-bufferization-to-memref \
 // RUN: -convert-scf-to-cf -expand-strided-metadata -lower-affine -convert-cf-to-llvm -convert-arith-to-llvm \
 // RUN: -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \
 // RUN: mlir-cpu-runner -e main -entry-point-result=void \
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-expand-tensor.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-expand-tensor.mlir
index 45283e173c9f0..cab049cb6791b 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/test-expand-tensor.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-expand-tensor.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt %s \
 // RUN: -one-shot-bufferize="bufferize-function-boundaries" \
-// RUN: -finalizing-bufferize -buffer-deallocation-pipeline -convert-bufferization-to-memref \
+// RUN: -buffer-deallocation-pipeline -convert-bufferization-to-memref \
 // RUN: -convert-scf-to-cf -expand-strided-metadata -lower-affine -convert-cf-to-llvm -convert-arith-to-llvm \
 // RUN: -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \
 // RUN: mlir-cpu-runner -e main -entry-point-result=void \
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-padtensor.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-padtensor.mlir
index 23a07464bb5be..a5cf6c6c3031e 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/test-padtensor.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-padtensor.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt %s -test-linalg-transform-patterns=test-linalg-to-vector-patterns \
 // RUN: -one-shot-bufferize="bufferize-function-boundaries" \
-// RUN: -finalizing-bufferize -buffer-deallocation-pipeline -convert-bufferization-to-memref \
+// RUN: -buffer-deallocation-pipeline -convert-bufferization-to-memref \
 // RUN: -convert-linalg-to-loops -convert-scf-to-cf -expand-strided-metadata \
 // RUN: -lower-affine -convert-arith-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \
 // RUN: mlir-cpu-runner -e main -entry-point-result=void \
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert-multiple-uses.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert-multiple-uses.mlir
index 01a0ba26fd7cd..402014106adf1 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert-multiple-uses.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert-multiple-uses.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt %s \
 // RUN: -one-shot-bufferize="bufferize-function-boundaries" \
-// RUN: -finalizing-bufferize -buffer-deallocation-pipeline -convert-bufferization-to-memref \
+// RUN: -buffer-deallocation-pipeline -convert-bufferization-to-memref \
 // RUN: -convert-linalg-to-loops -convert-scf-to-cf -expand-strided-metadata  \
 // RUN: -lower-affine -convert-arith-to-llvm --finalize-memref-to-llvm \
 // RUN: -convert-func-to-llvm -reconcile-unrealized-casts | \
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert.mlir
index 73d4aff73fb7a..7949ec7c46a8a 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt %s \
 // RUN: -one-shot-bufferize="bufferize-function-boundaries" \
-// RUN: -finalizing-bufferize -buffer-deallocation-pipeline -convert-bufferization-to-memref \
+// RUN: -buffer-deallocation-pipeline -convert-bufferization-to-memref \
 // RUN: -convert-linalg-to-loops -convert-scf-to-cf -expand-strided-metadata \
 // RUN: -lower-affine -convert-arith-to-llvm --finalize-memref-to-llvm \
 // RUN: -convert-func-to-llvm -reconcile-unrealized-casts | \
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-e2e.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-e2e.mlir
index ff9ddedf91e17..b627d3bf9cd41 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-e2e.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-e2e.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt %s \
 // RUN: -one-shot-bufferize="bufferize-function-boundaries" \
-// RUN: -finalizing-bufferize -buffer-deallocation-pipeline -convert-bufferization-to-memref -convert-linalg-to-loops \
+// RUN: -buffer-deallocation-pipeline -convert-bufferization-to-memref -convert-linalg-to-loops \
 // RUN: -convert-arith-to-llvm -convert-scf-to-cf -convert-cf-to-llvm --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \
 // RUN: mlir-cpu-runner -e main -entry-point-result=void \
 // RUN:   -shared-libs=%mlir_runner_utils \
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-matmul.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-matmul.mlir
index 698191577efe3..04e64251b98d6 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-matmul.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-matmul.mlir
@@ -1,7 +1,7 @@
 // UNSUPPORTED: asan
 // RUN: mlir-opt %s -test-transform-dialect-erase-schedule \
 // RUN: -one-shot-bufferize="bufferize-function-boundaries" \
-// RUN: -finalizing-bufferize -buffer-deallocation-pipeline -convert-bufferization-to-memref -convert-linalg-to-loops -convert-scf-to-cf \
+// RUN: -buffer-deallocation-pipeline -convert-bufferization-to-memref -convert-linalg-to-loops -convert-scf-to-cf \
 // RUN: -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \
 // RUN: mlir-cpu-runner -e main -entry-point-result=void \
 // RUN:   -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils \
@@ -9,7 +9,7 @@
 
 // RUN: mlir-opt %s -transform-interpreter -test-transform-dialect-erase-schedule \
 // RUN: -one-shot-bufferize="bufferize-function-boundaries" \
-// RUN: -finalizing-bufferize -convert-linalg-to-loops -convert-scf-to-cf -convert-scf-to-cf \
+// RUN: -convert-linalg-to-loops -convert-scf-to-cf -convert-scf-to-cf \
 // RUN:  -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \
 // RUN: mlir-cpu-runner -e main -entry-point-result=void \
 // RUN:   -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/python/test_output.py b/mlir/test/Integration/Dialect/SparseTensor/python/test_output.py
index 544273eb18835..b1438bd71055d 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/python/test_output.py
+++ b/mlir/test/Integration/Dialect/SparseTensor/python/test_output.py
@@ -129,6 +129,10 @@ def main():
         prop = st.LevelProperty
         levels = [
             [builder(fmt.compressed, [prop.non_unique]), builder(fmt.singleton)],
+            [
+                builder(fmt.compressed, [prop.non_unique]),
+                builder(fmt.singleton, [prop.soa]),
+            ],
             [builder(fmt.dense), builder(fmt.compressed)],
             [builder(fmt.dense), builder(fmt.loose_compressed)],
             [builder(fmt.compressed), builder(fmt.compressed)],
@@ -179,7 +183,7 @@ def main():
         build_compile_and_run_output(attr, compiler, expected(2))
         count = count + 1
 
-    # CHECK: Passed 17 tests
+    # CHECK: Passed 21 tests
     print("Passed", count, "tests")
 
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-1d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-1d.mlir
index 12b0511d486ea..8a98d39e657f2 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-1d.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-1d.mlir
@@ -82,7 +82,7 @@ func.func @transfer_read_1d_broadcast(
     %A : memref<?x?xf32>, %base1 : index, %base2 : index) {
   %fm42 = arith.constant -42.0: f32
   %f = vector.transfer_read %A[%base1, %base2], %fm42
-      {in_bounds = [true], permutation_map = affine_map<(d0, d1) -> (0)>}
+      {permutation_map = affine_map<(d0, d1) -> (0)>}
       : memref<?x?xf32>, vector<9xf32>
   vector.print %f: vector<9xf32>
   return
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-2d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-2d.mlir
index 9f8849fa9a148..cb8a8ce8ab0b0 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-2d.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-2d.mlir
@@ -57,7 +57,7 @@ func.func @transfer_read_2d_mask_broadcast(
   %fm42 = arith.constant -42.0: f32
   %mask = arith.constant dense<[1, 0, 1, 0, 1, 1, 1, 0, 1]> : vector<9xi1>
   %f = vector.transfer_read %A[%base1, %base2], %fm42, %mask
-      {in_bounds = [true, false], permutation_map = affine_map<(d0, d1) -> (0, d1)>} :
+      {permutation_map = affine_map<(d0, d1) -> (0, d1)>} :
     memref<?x?xf32>, vector<4x9xf32>
   vector.print %f: vector<4x9xf32>
   return
@@ -69,7 +69,7 @@ func.func @transfer_read_2d_mask_transpose_broadcast_last_dim(
   %fm42 = arith.constant -42.0: f32
   %mask = arith.constant dense<[1, 0, 1, 1]> : vector<4xi1>
   %f = vector.transfer_read %A[%base1, %base2], %fm42, %mask
-      {in_bounds = [false, true], permutation_map = affine_map<(d0, d1) -> (d1, 0)>} :
+      {permutation_map = affine_map<(d0, d1) -> (d1, 0)>} :
     memref<?x?xf32>, vector<4x9xf32>
   vector.print %f: vector<4x9xf32>
   return
@@ -91,7 +91,7 @@ func.func @transfer_read_2d_broadcast(
     %A : memref<?x?xf32>, %base1: index, %base2: index) {
   %fm42 = arith.constant -42.0: f32
   %f = vector.transfer_read %A[%base1, %base2], %fm42
-      {in_bounds = [false, true], permutation_map = affine_map<(d0, d1) -> (d1, 0)>} :
+      {permutation_map = affine_map<(d0, d1) -> (d1, 0)>} :
     memref<?x?xf32>, vector<4x9xf32>
   vector.print %f: vector<4x9xf32>
   return
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-3d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-3d.mlir
index 466afeec459b4..4aecca3d6891e 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-3d.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-3d.mlir
@@ -32,7 +32,7 @@ func.func @transfer_read_3d_broadcast(%A : memref<?x?x?x?xf32>,
                                  %o: index, %a: index, %b: index, %c: index) {
   %fm42 = arith.constant -42.0: f32
   %f = vector.transfer_read %A[%o, %a, %b, %c], %fm42
-      {in_bounds = [false, true, false], permutation_map = affine_map<(d0, d1, d2, d3) -> (d1, 0, d3)>}
+      {permutation_map = affine_map<(d0, d1, d2, d3) -> (d1, 0, d3)>}
       : memref<?x?x?x?xf32>, vector<2x5x3xf32>
   vector.print %f: vector<2x5x3xf32>
   return
@@ -43,7 +43,7 @@ func.func @transfer_read_3d_mask_broadcast(
   %fm42 = arith.constant -42.0: f32
   %mask = arith.constant dense<[0, 1]> : vector<2xi1>
   %f = vector.transfer_read %A[%o, %a, %b, %c], %fm42, %mask
-      {in_bounds = [false, true, true], permutation_map = affine_map<(d0, d1, d2, d3) -> (d1, 0, 0)>}
+      {permutation_map = affine_map<(d0, d1, d2, d3) -> (d1, 0, 0)>}
       : memref<?x?x?x?xf32>, vector<2x5x3xf32>
   vector.print %f: vector<2x5x3xf32>
   return
diff --git a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/dump-ptx.mlir b/mlir/test/Integration/GPU/CUDA/dump-ptx.mlir
similarity index 100%
rename from mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/dump-ptx.mlir
rename to mlir/test/Integration/GPU/CUDA/dump-ptx.mlir
diff --git a/mlir/test/Pass/crash-recovery-dynamic-failure.mlir b/mlir/test/Pass/crash-recovery-dynamic-failure.mlir
index cea4f6a79963b..6d2bd69639108 100644
--- a/mlir/test/Pass/crash-recovery-dynamic-failure.mlir
+++ b/mlir/test/Pass/crash-recovery-dynamic-failure.mlir
@@ -15,4 +15,4 @@ module @inner_mod1 {
 // REPRO_LOCAL_DYNAMIC_FAILURE: module @inner_mod1
 // REPRO_LOCAL_DYNAMIC_FAILURE: module @foo {
 
-// REPRO_LOCAL_DYNAMIC_FAILURE: pipeline: "builtin.module(builtin.module(test-pass-failure))"
+// REPRO_LOCAL_DYNAMIC_FAILURE: pipeline: "builtin.module(builtin.module(test-pass-failure{gen-diagnostics=false}))"
diff --git a/mlir/test/Pass/full_diagnostics.mlir b/mlir/test/Pass/full_diagnostics.mlir
new file mode 100644
index 0000000000000..881260ce60d32
--- /dev/null
+++ b/mlir/test/Pass/full_diagnostics.mlir
@@ -0,0 +1,17 @@
+// RUN: mlir-opt %s -pass-pipeline='builtin.module(func.func(test-pass-failure{gen-diagnostics}))' -verify-diagnostics
+
+// Test that all errors are reported.
+// expected-error@below {{illegal operation}}
+func.func @TestAlwaysIllegalOperationPass1() {
+  return
+}
+
+// expected-error@below {{illegal operation}}
+func.func @TestAlwaysIllegalOperationPass2() {
+  return
+}
+
+// expected-error@below {{illegal operation}}
+func.func @TestAlwaysIllegalOperationPass3() {
+  return
+}
diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir
index 007284d0ca443..c884f83cb4d32 100644
--- a/mlir/test/Target/LLVMIR/llvmir.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir.mlir
@@ -72,6 +72,9 @@ llvm.mlir.global internal @f8E5M2FNUZ_global_as_i8(1.5 : f8E5M2FNUZ) : i8
 // CHECK: @f8E4M3B11FNUZ_global_as_i8 = internal global i8 92
 llvm.mlir.global internal @f8E4M3B11FNUZ_global_as_i8(1.5 : f8E4M3B11FNUZ) : i8
 
+// CHECK: @f8E8M0FNU_global_as_i8 = internal global i8 127
+llvm.mlir.global internal @f8E8M0FNU_global_as_i8(1.0 : f8E8M0FNU) : i8
+
 // CHECK: @bf16_global_as_i16 = internal global i16 16320
 llvm.mlir.global internal @bf16_global_as_i16(1.5 : bf16) : i16
 
@@ -1496,7 +1499,8 @@ llvm.func @elements_constant_3d_array() -> !llvm.array<2 x array<2 x array<2 x i
 // CHECK-LABEL: @atomicrmw
 llvm.func @atomicrmw(
     %f32_ptr : !llvm.ptr, %f32 : f32,
-    %i32_ptr : !llvm.ptr, %i32 : i32) {
+    %i32_ptr : !llvm.ptr, %i32 : i32,
+    %f16_vec_ptr : !llvm.ptr, %f16_vec : vector<2xf16>) {
   // CHECK: atomicrmw fadd ptr %{{.*}}, float %{{.*}} monotonic
   %0 = llvm.atomicrmw fadd %f32_ptr, %f32 monotonic : !llvm.ptr, f32
   // CHECK: atomicrmw fsub ptr %{{.*}}, float %{{.*}} monotonic
@@ -1535,11 +1539,19 @@ llvm.func @atomicrmw(
   %17 = llvm.atomicrmw usub_cond %i32_ptr, %i32 monotonic : !llvm.ptr, i32
   // CHECK: atomicrmw usub_sat ptr %{{.*}}, i32 %{{.*}} monotonic
   %18 = llvm.atomicrmw usub_sat %i32_ptr, %i32 monotonic : !llvm.ptr, i32
+  // CHECK: atomicrmw fadd ptr %{{.*}}, <2 x half> %{{.*}} monotonic
+  %19 = llvm.atomicrmw fadd %f16_vec_ptr, %f16_vec monotonic : !llvm.ptr, vector<2xf16>
+  // CHECK: atomicrmw fsub ptr %{{.*}}, <2 x half> %{{.*}} monotonic
+  %20 = llvm.atomicrmw fsub %f16_vec_ptr, %f16_vec monotonic : !llvm.ptr, vector<2xf16>
+  // CHECK: atomicrmw fmax ptr %{{.*}}, <2 x half> %{{.*}} monotonic
+  %21 = llvm.atomicrmw fmax %f16_vec_ptr, %f16_vec monotonic : !llvm.ptr, vector<2xf16>
+  // CHECK: atomicrmw fmin ptr %{{.*}}, <2 x half> %{{.*}} monotonic
+  %22 = llvm.atomicrmw fmin %f16_vec_ptr, %f16_vec monotonic : !llvm.ptr, vector<2xf16>
 
   // CHECK: atomicrmw volatile
   // CHECK-SAME:  syncscope("singlethread")
   // CHECK-SAME:  align 8
-  %19 = llvm.atomicrmw volatile udec_wrap %i32_ptr, %i32 syncscope("singlethread") monotonic {alignment = 8 : i64} : !llvm.ptr, i32
+  %23 = llvm.atomicrmw volatile udec_wrap %i32_ptr, %i32 syncscope("singlethread") monotonic {alignment = 8 : i64} : !llvm.ptr, i32
   llvm.return
 }
 
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index 95e12e5bc4e74..5d76e87472dfe 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -1450,6 +1450,120 @@ llvm.func @omp_atomic_update(%x:!llvm.ptr, %expr: i32, %xbool: !llvm.ptr, %exprb
 
 // -----
 
+//CHECK: %[[X_NEW_VAL:.*]] = alloca { float, float }, align 8
+//CHECK: {{.*}} = alloca { float, float }, i64 1, align 8
+//CHECK: %[[ORIG_VAL:.*]] = alloca { float, float }, i64 1, align 8
+
+//CHECK: br label %entry
+
+//CHECK: entry:
+//CHECK: %[[ATOMIC_TEMP_LOAD:.*]] = alloca { float, float }, align 8
+//CHECK: call void @__atomic_load(i64 8, ptr %[[ORIG_VAL]], ptr %[[ATOMIC_TEMP_LOAD]], i32 0)
+//CHECK: %[[PHI_NODE_ENTRY_1:.*]] = load { float, float }, ptr %[[ATOMIC_TEMP_LOAD]], align 8
+//CHECK: br label %.atomic.cont
+
+//CHECK: .atomic.cont
+//CHECK: %[[VAL_4:.*]] = phi { float, float } [ %[[PHI_NODE_ENTRY_1]], %entry ], [ %{{.*}}, %.atomic.cont ]
+//CHECK: %[[VAL_5:.*]] = extractvalue { float, float } %[[VAL_4]], 0
+//CHECK: %[[VAL_6:.*]] = extractvalue { float, float } %[[VAL_4]], 1
+//CHECK: %[[VAL_7:.*]] = fadd contract float %[[VAL_5]], 1.000000e+00
+//CHECK: %[[VAL_8:.*]] = fadd contract float %[[VAL_6]], 1.000000e+00
+//CHECK: %[[VAL_9:.*]] = insertvalue { float, float } undef, float %[[VAL_7]], 0
+//CHECK: %[[VAL_10:.*]] = insertvalue { float, float } %[[VAL_9]], float %[[VAL_8]], 1
+//CHECK: store { float, float } %[[VAL_10]], ptr %[[X_NEW_VAL]], align 4
+//CHECK: %[[VAL_11:.*]] = call i1 @__atomic_compare_exchange(i64 8, ptr %[[ORIG_VAL]], ptr %[[ATOMIC_TEMP_LOAD]], ptr %[[X_NEW_VAL]], i32 2, i32 2)
+//CHECK: %[[VAL_12:.*]] = load { float, float }, ptr %[[ATOMIC_TEMP_LOAD]], align 4
+//CHECK: br i1 %[[VAL_11]], label %.atomic.exit, label %.atomic.cont
+
+llvm.func @_QPomp_atomic_update_complex() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x !llvm.struct<(f32, f32)> {bindc_name = "ib"} : (i64) -> !llvm.ptr
+    %2 = llvm.mlir.constant(1 : i64) : i64
+    %3 = llvm.alloca %2 x !llvm.struct<(f32, f32)> {bindc_name = "ia"} : (i64) -> !llvm.ptr
+    %4 = llvm.mlir.constant(1.000000e+00 : f32) : f32
+    %5 = llvm.mlir.undef : !llvm.struct<(f32, f32)>
+    %6 = llvm.insertvalue %4, %5[0] : !llvm.struct<(f32, f32)>
+    %7 = llvm.insertvalue %4, %6[1] : !llvm.struct<(f32, f32)>
+    omp.atomic.update %3 : !llvm.ptr {
+    ^bb0(%arg0: !llvm.struct<(f32, f32)>):
+      %8 = llvm.extractvalue %arg0[0] : !llvm.struct<(f32, f32)>
+      %9 = llvm.extractvalue %arg0[1] : !llvm.struct<(f32, f32)>
+      %10 = llvm.extractvalue %7[0] : !llvm.struct<(f32, f32)>
+      %11 = llvm.extractvalue %7[1] : !llvm.struct<(f32, f32)>
+      %12 = llvm.fadd %8, %10  {fastmathFlags = #llvm.fastmath<contract>} : f32
+      %13 = llvm.fadd %9, %11  {fastmathFlags = #llvm.fastmath<contract>} : f32
+      %14 = llvm.mlir.undef : !llvm.struct<(f32, f32)>
+      %15 = llvm.insertvalue %12, %14[0] : !llvm.struct<(f32, f32)>
+      %16 = llvm.insertvalue %13, %15[1] : !llvm.struct<(f32, f32)>
+      omp.yield(%16 : !llvm.struct<(f32, f32)>)
+    }
+   llvm.return
+}
+
+// -----
+
+//CHECK: %[[X_NEW_VAL:.*]] = alloca { float, float }, align 8
+//CHECK: %[[VAL_1:.*]] = alloca { float, float }, i64 1, align 8
+//CHECK: %[[ORIG_VAL:.*]] = alloca { float, float }, i64 1, align 8
+//CHECK: store { float, float } { float 2.000000e+00, float 2.000000e+00 }, ptr %[[ORIG_VAL]], align 4
+//CHECK: br label %entry
+
+//CHECK: entry:							; preds = %0
+//CHECK: %[[ATOMIC_TEMP_LOAD:.*]] = alloca { float, float }, align 8
+//CHECK: call void @__atomic_load(i64 8, ptr %[[ORIG_VAL]], ptr %[[ATOMIC_TEMP_LOAD]], i32 0)
+//CHECK: %[[PHI_NODE_ENTRY_1:.*]] = load { float, float }, ptr %[[ATOMIC_TEMP_LOAD]], align 8
+//CHECK: br label %.atomic.cont
+
+//CHECK: .atomic.cont
+//CHECK: %[[VAL_4:.*]] = phi { float, float } [ %[[PHI_NODE_ENTRY_1]], %entry ], [ %{{.*}}, %.atomic.cont ]
+//CHECK: %[[VAL_5:.*]] = extractvalue { float, float } %[[VAL_4]], 0
+//CHECK: %[[VAL_6:.*]] = extractvalue { float, float } %[[VAL_4]], 1
+//CHECK: %[[VAL_7:.*]] = fadd contract float %[[VAL_5]], 1.000000e+00
+//CHECK: %[[VAL_8:.*]] = fadd contract float %[[VAL_6]], 1.000000e+00
+//CHECK: %[[VAL_9:.*]] = insertvalue { float, float } undef, float %[[VAL_7]], 0
+//CHECK: %[[VAL_10:.*]] = insertvalue { float, float } %[[VAL_9]], float %[[VAL_8]], 1
+//CHECK: store { float, float } %[[VAL_10]], ptr %[[X_NEW_VAL]], align 4 
+//CHECK: %[[VAL_11:.*]] = call i1 @__atomic_compare_exchange(i64 8, ptr %[[ORIG_VAL]], ptr %[[ATOMIC_TEMP_LOAD]], ptr %[[X_NEW_VAL]], i32 2, i32 2)
+//CHECK: %[[VAL_12:.*]] = load { float, float }, ptr %[[ATOMIC_TEMP_LOAD]], align 4
+//CHECK: br i1 %[[VAL_11]], label %.atomic.exit, label %.atomic.cont
+//CHECK: .atomic.exit
+//CHECK: store { float, float } %[[VAL_10]], ptr %[[VAL_1]], align 4
+
+llvm.func @_QPomp_atomic_capture_complex() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x !llvm.struct<(f32, f32)> {bindc_name = "ib"} : (i64) -> !llvm.ptr
+    %2 = llvm.mlir.constant(1 : i64) : i64
+    %3 = llvm.alloca %2 x !llvm.struct<(f32, f32)> {bindc_name = "ia"} : (i64) -> !llvm.ptr
+    %4 = llvm.mlir.constant(1.000000e+00 : f32) : f32
+    %5 = llvm.mlir.constant(2.000000e+00 : f32) : f32
+    %6 = llvm.mlir.undef : !llvm.struct<(f32, f32)>
+    %7 = llvm.insertvalue %5, %6[0] : !llvm.struct<(f32, f32)>
+    %8 = llvm.insertvalue %5, %7[1] : !llvm.struct<(f32, f32)>
+    llvm.store %8, %3 : !llvm.struct<(f32, f32)>, !llvm.ptr
+    %9 = llvm.mlir.undef : !llvm.struct<(f32, f32)>
+    %10 = llvm.insertvalue %4, %9[0] : !llvm.struct<(f32, f32)>
+    %11 = llvm.insertvalue %4, %10[1] : !llvm.struct<(f32, f32)>
+    omp.atomic.capture {
+      omp.atomic.update %3 : !llvm.ptr {
+      ^bb0(%arg0: !llvm.struct<(f32, f32)>):
+        %12 = llvm.extractvalue %arg0[0] : !llvm.struct<(f32, f32)>
+        %13 = llvm.extractvalue %arg0[1] : !llvm.struct<(f32, f32)>
+        %14 = llvm.extractvalue %11[0] : !llvm.struct<(f32, f32)>
+        %15 = llvm.extractvalue %11[1] : !llvm.struct<(f32, f32)>
+        %16 = llvm.fadd %12, %14  {fastmathFlags = #llvm.fastmath<contract>} : f32
+        %17 = llvm.fadd %13, %15  {fastmathFlags = #llvm.fastmath<contract>} : f32
+        %18 = llvm.mlir.undef : !llvm.struct<(f32, f32)>
+        %19 = llvm.insertvalue %16, %18[0] : !llvm.struct<(f32, f32)>
+        %20 = llvm.insertvalue %17, %19[1] : !llvm.struct<(f32, f32)>
+        omp.yield(%20 : !llvm.struct<(f32, f32)>)
+      }
+      omp.atomic.read %1 = %3 : !llvm.ptr, !llvm.struct<(f32, f32)>
+    }
+    llvm.return
+}
+
+// -----
+
 // Checking an order-dependent operation when the order is `expr binop x`
 // CHECK-LABEL: @omp_atomic_update_ordering
 // CHECK-SAME: (ptr %[[x:.*]], i32 %[[expr:.*]])
diff --git a/mlir/test/Transforms/remove-dead-values.mlir b/mlir/test/Transforms/remove-dead-values.mlir
index a31cb97cb5712..9f2be3331b6b4 100644
--- a/mlir/test/Transforms/remove-dead-values.mlir
+++ b/mlir/test/Transforms/remove-dead-values.mlir
@@ -1,13 +1,25 @@
 // RUN: mlir-opt %s -remove-dead-values -split-input-file -verify-diagnostics | FileCheck %s
 
 // The IR remains untouched because of the presence of a non-function-like
-// symbol op (module @dont_touch_unacceptable_ir).
+// symbol op inside the module (const @__dont_touch_unacceptable_ir).
 //
+module {
 // expected-error @+1 {{cannot optimize an IR with non-function symbol ops, non-call symbol user ops or branch ops}}
-module @dont_touch_unacceptable_ir {
-  func.func @has_cleanable_simple_op(%arg0 : i32) {
-    %non_live = arith.addi %arg0, %arg0 : i32
-    return
+  memref.global "private" constant @__dont_touch_unacceptable_ir : memref<i32> = dense<0>
+  func.func @main(%arg0: i32) -> i32 {
+    return %arg0 : i32
+  }
+}
+
+// -----
+
+// Dead values are removed from the IR even if the module has a name
+//
+module @named_module_acceptable {
+  func.func @main(%arg0: tensor<10xf32>) -> tensor<10xf32> {
+    %0 = tensor.empty() : tensor<10xbf16>
+    // CHECK-NOT: tensor.empty
+    return %arg0 : tensor<10xf32>
   }
 }
 
diff --git a/mlir/test/lib/Dialect/Test/TestAttrDefs.td b/mlir/test/lib/Dialect/Test/TestAttrDefs.td
index b3b94bd0ffea3..907184b2e1ce4 100644
--- a/mlir/test/lib/Dialect/Test/TestAttrDefs.td
+++ b/mlir/test/lib/Dialect/Test/TestAttrDefs.td
@@ -27,6 +27,8 @@ include "mlir/IR/OpAsmInterface.td"
 class Test_Attr<string name, list<Trait> traits = []>
     : AttrDef<Test_Dialect, name, traits>;
 
+class Test_LocAttr<string name> : LocationAttrDef<Test_Dialect, name, []>;
+
 def SimpleAttrA : Test_Attr<"SimpleA"> {
   let mnemonic = "smpla";
 }
@@ -377,4 +379,14 @@ def NestedPolynomialAttr2 : Test_Attr<"NestedPolynomialAttr2"> {
 }
 
 
+// Test custom location handling.
+def TestCustomLocationAttr : Test_LocAttr<"TestCustomLocation"> {
+  let mnemonic = "custom_location";
+  let parameters = (ins "mlir::StringAttr":$file, "unsigned":$line);
+
+  // Choose a silly separator token so we know it's hitting this code path
+  // and not another.
+  let assemblyFormat = "`<` $file `*` $line `>`";
+}
+
 #endif // TEST_ATTRDEFS
diff --git a/mlir/test/lib/Pass/TestPassManager.cpp b/mlir/test/lib/Pass/TestPassManager.cpp
index ee32bec0c79bd..7afe2109f04db 100644
--- a/mlir/test/lib/Pass/TestPassManager.cpp
+++ b/mlir/test/lib/Pass/TestPassManager.cpp
@@ -151,11 +151,21 @@ struct TestCrashRecoveryPass
 struct TestFailurePass : public PassWrapper<TestFailurePass, OperationPass<>> {
   MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestFailurePass)
 
-  void runOnOperation() final { signalPassFailure(); }
+  TestFailurePass() = default;
+  TestFailurePass(const TestFailurePass &other) : PassWrapper(other) {}
+
+  void runOnOperation() final {
+    signalPassFailure();
+    if (genDiagnostics)
+      mlir::emitError(getOperation()->getLoc(), "illegal operation");
+  }
   StringRef getArgument() const final { return "test-pass-failure"; }
   StringRef getDescription() const final {
     return "Test a pass in the pass manager that always fails";
   }
+
+  Option<bool> genDiagnostics{*this, "gen-diagnostics",
+                              llvm::cl::desc("Generate a diagnostic message")};
 };
 
 /// A test pass that creates an invalid operation in a function body.
diff --git a/mlir/test/python/dialects/sparse_tensor/dialect.py b/mlir/test/python/dialects/sparse_tensor/dialect.py
index 3cc4575eb3e24..656979f3d9a1d 100644
--- a/mlir/test/python/dialects/sparse_tensor/dialect.py
+++ b/mlir/test/python/dialects/sparse_tensor/dialect.py
@@ -1,7 +1,7 @@
 # RUN: %PYTHON %s | FileCheck %s
 
 from mlir.ir import *
-from mlir.dialects import sparse_tensor as st
+from mlir.dialects import sparse_tensor as st, tensor
 import textwrap
 
 
@@ -225,3 +225,21 @@ def testEncodingAttrOnTensorType():
         # CHECK: #sparse_tensor.encoding<{ map = (d0) -> (d0 : compressed), posWidth = 64, crdWidth = 32 }>
         print(tt.encoding)
         assert tt.encoding == encoding
+
+
+# CHECK-LABEL: TEST: testEncodingEmptyTensor
+@run
+def testEncodingEmptyTensor():
+    with Context(), Location.unknown():
+        module = Module.create()
+        with InsertionPoint(module.body):
+            levels = [st.LevelFormat.compressed]
+            ordering = AffineMap.get_permutation([0])
+            encoding = st.EncodingAttr.get(levels, ordering, ordering, 32, 32)
+            tensor.empty((1024,), F32Type.get(), encoding=encoding)
+
+        # CHECK: #sparse = #sparse_tensor.encoding<{ map = (d0) -> (d0 : compressed), posWidth = 32, crdWidth = 32 }>
+        # CHECK: module {
+        # CHECK:   %[[VAL_0:.*]] = tensor.empty() : tensor<1024xf32, #sparse>
+        # CHECK: }
+        print(module)
diff --git a/mlir/test/python/ir/builtin_types.py b/mlir/test/python/ir/builtin_types.py
index 6154a6ff9e9ae..48ddc8359ca0a 100644
--- a/mlir/test/python/ir/builtin_types.py
+++ b/mlir/test/python/ir/builtin_types.py
@@ -133,6 +133,8 @@ def testFloatTypeSubclasses():
     # CHECK: True
     print(isinstance(Type.parse("f8E5M2FNUZ", ctx), FloatType))
     # CHECK: True
+    print(isinstance(Type.parse("f8E8M0FNU", ctx), FloatType))
+    # CHECK: True
     print(isinstance(Type.parse("f16", ctx), FloatType))
     # CHECK: True
     print(isinstance(Type.parse("bf16", ctx), FloatType))
@@ -259,6 +261,8 @@ def testFloatType():
         print("float:", Float8E4M3FNUZType.get())
         # CHECK: float: f8E4M3B11FNUZ
         print("float:", Float8E4M3B11FNUZType.get())
+        # CHECK: float: f8E8M0FNU
+        print("float:", Float8E8M0FNUType.get())
         # CHECK: float: bf16
         print("float:", BF16Type.get())
         # CHECK: float: f16
@@ -631,6 +635,7 @@ def testTypeIDs():
             (Float8E4M3FNUZType, Float8E4M3FNUZType.get()),
             (Float8E4M3B11FNUZType, Float8E4M3B11FNUZType.get()),
             (Float8E5M2FNUZType, Float8E5M2FNUZType.get()),
+            (Float8E8M0FNUType, Float8E8M0FNUType.get()),
             (BF16Type, BF16Type.get()),
             (F16Type, F16Type.get()),
             (F32Type, F32Type.get()),
@@ -659,6 +664,7 @@ def testTypeIDs():
         # CHECK: Float8E4M3FNUZType(f8E4M3FNUZ)
         # CHECK: Float8E4M3B11FNUZType(f8E4M3B11FNUZ)
         # CHECK: Float8E5M2FNUZType(f8E5M2FNUZ)
+        # CHECK: Float8E8M0FNUType(f8E8M0FNU)
         # CHECK: BF16Type(bf16)
         # CHECK: F16Type(f16)
         # CHECK: F32Type(f32)
@@ -761,6 +767,9 @@ def print_downcasted(typ):
         # CHECK: Float8E5M2FNUZType
         # CHECK: Float8E5M2FNUZType(f8E5M2FNUZ)
         print_downcasted(Float8E5M2FNUZType.get())
+        # CHECK: Float8E8M0FNUType
+        # CHECK: Float8E8M0FNUType(f8E8M0FNU)
+        print_downcasted(Float8E8M0FNUType.get())
         # CHECK: BF16Type
         # CHECK: BF16Type(bf16)
         print_downcasted(BF16Type.get())
diff --git a/mlir/test/tblgen-to-irdl/TestDialect.td b/mlir/test/tblgen-to-irdl/TestDialect.td
index 1ba84a5d3683d..7f4815d865b60 100644
--- a/mlir/test/tblgen-to-irdl/TestDialect.td
+++ b/mlir/test/tblgen-to-irdl/TestDialect.td
@@ -106,6 +106,17 @@ def Test_OrOp : Test_Op<"or"> {
 // CHECK-NEXT:    irdl.operands(%[[v3]])
 // CHECK-NEXT:  }
 
+// Check regions are converted correctly.
+def Test_RegionsOp : Test_Op<"regions"> {
+  let regions = (region AnyRegion:$any_region,
+                        SizedRegion<1>:$single_block_region);
+}
+// CHECK-LABEL: irdl.operation @regions {
+// CHECK-NEXT:    %[[v0:[^ ]*]] = irdl.region
+// CHECK-NEXT:    %[[v1:[^ ]*]] = irdl.region with size 1
+// CHECK-NEXT:    irdl.regions(%[[v0]], %[[v1]])
+// CHECK-NEXT:  }
+
 // Check that various types are converted correctly.
 def Test_TypesOp : Test_Op<"types"> {
   let arguments = (ins I32:$a,
diff --git a/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp b/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
index 1a248c3a16647..6a39424bd463f 100644
--- a/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
+++ b/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
@@ -22,6 +22,8 @@
 
 using namespace mlir;
 using namespace mlir::tblgen;
+using llvm::Record;
+using llvm::RecordKeeper;
 
 //===----------------------------------------------------------------------===//
 // Utility Functions
@@ -30,14 +32,14 @@ using namespace mlir::tblgen;
 /// Find all the AttrOrTypeDef for the specified dialect. If no dialect
 /// specified and can only find one dialect's defs, use that.
 static void collectAllDefs(StringRef selectedDialect,
-                           ArrayRef<const llvm::Record *> records,
+                           ArrayRef<const Record *> records,
                            SmallVectorImpl<AttrOrTypeDef> &resultDefs) {
   // Nothing to do if no defs were found.
   if (records.empty())
     return;
 
   auto defs = llvm::map_range(
-      records, [&](const llvm::Record *rec) { return AttrOrTypeDef(rec); });
+      records, [&](const Record *rec) { return AttrOrTypeDef(rec); });
   if (selectedDialect.empty()) {
     // If a dialect was not specified, ensure that all found defs belong to the
     // same dialect.
@@ -690,15 +692,14 @@ class DefGenerator {
   bool emitDefs(StringRef selectedDialect);
 
 protected:
-  DefGenerator(ArrayRef<const llvm::Record *> defs, raw_ostream &os,
+  DefGenerator(ArrayRef<const Record *> defs, raw_ostream &os,
                StringRef defType, StringRef valueType, bool isAttrGenerator)
       : defRecords(defs), os(os), defType(defType), valueType(valueType),
         isAttrGenerator(isAttrGenerator) {
     // Sort by occurrence in file.
-    llvm::sort(defRecords,
-               [](const llvm::Record *lhs, const llvm::Record *rhs) {
-                 return lhs->getID() < rhs->getID();
-               });
+    llvm::sort(defRecords, [](const Record *lhs, const Record *rhs) {
+      return lhs->getID() < rhs->getID();
+    });
   }
 
   /// Emit the list of def type names.
@@ -707,7 +708,7 @@ class DefGenerator {
   void emitParsePrintDispatch(ArrayRef<AttrOrTypeDef> defs);
 
   /// The set of def records to emit.
-  std::vector<const llvm::Record *> defRecords;
+  std::vector<const Record *> defRecords;
   /// The attribute or type class to emit.
   /// The stream to emit to.
   raw_ostream &os;
@@ -722,13 +723,13 @@ class DefGenerator {
 
 /// A specialized generator for AttrDefs.
 struct AttrDefGenerator : public DefGenerator {
-  AttrDefGenerator(const llvm::RecordKeeper &records, raw_ostream &os)
+  AttrDefGenerator(const RecordKeeper &records, raw_ostream &os)
       : DefGenerator(records.getAllDerivedDefinitionsIfDefined("AttrDef"), os,
                      "Attr", "Attribute", /*isAttrGenerator=*/true) {}
 };
 /// A specialized generator for TypeDefs.
 struct TypeDefGenerator : public DefGenerator {
-  TypeDefGenerator(const llvm::RecordKeeper &records, raw_ostream &os)
+  TypeDefGenerator(const RecordKeeper &records, raw_ostream &os)
       : DefGenerator(records.getAllDerivedDefinitionsIfDefined("TypeDef"), os,
                      "Type", "Type", /*isAttrGenerator=*/false) {}
 };
@@ -1030,9 +1031,9 @@ bool DefGenerator::emitDefs(StringRef selectedDialect) {
 
 /// Find all type constraints for which a C++ function should be generated.
 static std::vector<Constraint>
-getAllTypeConstraints(const llvm::RecordKeeper &records) {
+getAllTypeConstraints(const RecordKeeper &records) {
   std::vector<Constraint> result;
-  for (const llvm::Record *def :
+  for (const Record *def :
        records.getAllDerivedDefinitionsIfDefined("TypeConstraint")) {
     // Ignore constraints defined outside of the top-level file.
     if (llvm::SrcMgr.FindBufferContainingLoc(def->getLoc()[0]) !=
@@ -1047,7 +1048,7 @@ getAllTypeConstraints(const llvm::RecordKeeper &records) {
   return result;
 }
 
-static void emitTypeConstraintDecls(const llvm::RecordKeeper &records,
+static void emitTypeConstraintDecls(const RecordKeeper &records,
                                     raw_ostream &os) {
   static const char *const typeConstraintDecl = R"(
 bool {0}(::mlir::Type type);
@@ -1057,7 +1058,7 @@ bool {0}(::mlir::Type type);
     os << strfmt(typeConstraintDecl, *constr.getCppFunctionName());
 }
 
-static void emitTypeConstraintDefs(const llvm::RecordKeeper &records,
+static void emitTypeConstraintDefs(const RecordKeeper &records,
                                    raw_ostream &os) {
   static const char *const typeConstraintDef = R"(
 bool {0}(::mlir::Type type) {
@@ -1088,13 +1089,13 @@ static llvm::cl::opt<std::string>
 
 static mlir::GenRegistration
     genAttrDefs("gen-attrdef-defs", "Generate AttrDef definitions",
-                [](const llvm::RecordKeeper &records, raw_ostream &os) {
+                [](const RecordKeeper &records, raw_ostream &os) {
                   AttrDefGenerator generator(records, os);
                   return generator.emitDefs(attrDialect);
                 });
 static mlir::GenRegistration
     genAttrDecls("gen-attrdef-decls", "Generate AttrDef declarations",
-                 [](const llvm::RecordKeeper &records, raw_ostream &os) {
+                 [](const RecordKeeper &records, raw_ostream &os) {
                    AttrDefGenerator generator(records, os);
                    return generator.emitDecls(attrDialect);
                  });
@@ -1110,13 +1111,13 @@ static llvm::cl::opt<std::string>
 
 static mlir::GenRegistration
     genTypeDefs("gen-typedef-defs", "Generate TypeDef definitions",
-                [](const llvm::RecordKeeper &records, raw_ostream &os) {
+                [](const RecordKeeper &records, raw_ostream &os) {
                   TypeDefGenerator generator(records, os);
                   return generator.emitDefs(typeDialect);
                 });
 static mlir::GenRegistration
     genTypeDecls("gen-typedef-decls", "Generate TypeDef declarations",
-                 [](const llvm::RecordKeeper &records, raw_ostream &os) {
+                 [](const RecordKeeper &records, raw_ostream &os) {
                    TypeDefGenerator generator(records, os);
                    return generator.emitDecls(typeDialect);
                  });
@@ -1124,14 +1125,14 @@ static mlir::GenRegistration
 static mlir::GenRegistration
     genTypeConstrDefs("gen-type-constraint-defs",
                       "Generate type constraint definitions",
-                      [](const llvm::RecordKeeper &records, raw_ostream &os) {
+                      [](const RecordKeeper &records, raw_ostream &os) {
                         emitTypeConstraintDefs(records, os);
                         return false;
                       });
 static mlir::GenRegistration
     genTypeConstrDecls("gen-type-constraint-decls",
                        "Generate type constraint declarations",
-                       [](const llvm::RecordKeeper &records, raw_ostream &os) {
+                       [](const RecordKeeper &records, raw_ostream &os) {
                          emitTypeConstraintDecls(records, os);
                          return false;
                        });
diff --git a/mlir/tools/mlir-tblgen/BytecodeDialectGen.cpp b/mlir/tools/mlir-tblgen/BytecodeDialectGen.cpp
index 964b33a9fa41f..86ebaf2cf27df 100644
--- a/mlir/tools/mlir-tblgen/BytecodeDialectGen.cpp
+++ b/mlir/tools/mlir-tblgen/BytecodeDialectGen.cpp
@@ -18,11 +18,10 @@
 
 using namespace llvm;
 
-static llvm::cl::OptionCategory dialectGenCat("Options for -gen-bytecode");
-static llvm::cl::opt<std::string>
-    selectedBcDialect("bytecode-dialect",
-                      llvm::cl::desc("The dialect to gen for"),
-                      llvm::cl::cat(dialectGenCat), llvm::cl::CommaSeparated);
+static cl::OptionCategory dialectGenCat("Options for -gen-bytecode");
+static cl::opt<std::string>
+    selectedBcDialect("bytecode-dialect", cl::desc("The dialect to gen for"),
+                      cl::cat(dialectGenCat), cl::CommaSeparated);
 
 namespace {
 
@@ -32,14 +31,14 @@ class Generator {
   Generator(raw_ostream &output) : output(output) {}
 
   /// Returns whether successfully emitted attribute/type parsers.
-  void emitParse(StringRef kind, Record &x);
+  void emitParse(StringRef kind, const Record &x);
 
   /// Returns whether successfully emitted attribute/type printers.
   void emitPrint(StringRef kind, StringRef type,
-                 ArrayRef<std::pair<int64_t, Record *>> vec);
+                 ArrayRef<std::pair<int64_t, const Record *>> vec);
 
   /// Emits parse dispatch table.
-  void emitParseDispatch(StringRef kind, ArrayRef<Record *> vec);
+  void emitParseDispatch(StringRef kind, ArrayRef<const Record *> vec);
 
   /// Emits print dispatch table.
   void emitPrintDispatch(StringRef kind, ArrayRef<std::string> vec);
@@ -51,8 +50,9 @@ class Generator {
                        StringRef failure, mlir::raw_indented_ostream &ios);
 
   /// Emits print instructions.
-  void emitPrintHelper(Record *memberRec, StringRef kind, StringRef parent,
-                       StringRef name, mlir::raw_indented_ostream &ios);
+  void emitPrintHelper(const Record *memberRec, StringRef kind,
+                       StringRef parent, StringRef name,
+                       mlir::raw_indented_ostream &ios);
 
   raw_ostream &output;
 };
@@ -75,7 +75,7 @@ static std::string capitalize(StringRef str) {
 }
 
 /// Return the C++ type for the given record.
-static std::string getCType(Record *def) {
+static std::string getCType(const Record *def) {
   std::string format = "{0}";
   if (def->isSubClassOf("Array")) {
     def = def->getValueAsDef("elemT");
@@ -92,7 +92,8 @@ static std::string getCType(Record *def) {
   return formatv(format.c_str(), cType.str());
 }
 
-void Generator::emitParseDispatch(StringRef kind, ArrayRef<Record *> vec) {
+void Generator::emitParseDispatch(StringRef kind,
+                                  ArrayRef<const Record *> vec) {
   mlir::raw_indented_ostream os(output);
   char const *head =
       R"(static {0} read{0}(MLIRContext* context, DialectBytecodeReader &reader))";
@@ -126,7 +127,7 @@ void Generator::emitParseDispatch(StringRef kind, ArrayRef<Record *> vec) {
   os << "return " << capitalize(kind) << "();\n";
 }
 
-void Generator::emitParse(StringRef kind, Record &x) {
+void Generator::emitParse(StringRef kind, const Record &x) {
   if (x.getNameInitAsString() == "ReservedOrDead")
     return;
 
@@ -159,7 +160,7 @@ void printParseConditional(mlir::raw_indented_ostream &ios,
 
   auto parsedArgs =
       llvm::to_vector(make_filter_range(args, [](Init *const attr) {
-        Record *def = cast<DefInit>(attr)->getDef();
+        const Record *def = cast<DefInit>(attr)->getDef();
         if (def->isSubClassOf("Array"))
           return true;
         return !def->getValueAsString("cParser").empty();
@@ -168,12 +169,12 @@ void printParseConditional(mlir::raw_indented_ostream &ios,
   interleave(
       zip(parsedArgs, argNames),
       [&](std::tuple<llvm::Init *&, const std::string &> it) {
-        Record *attr = cast<DefInit>(std::get<0>(it))->getDef();
+        const Record *attr = cast<DefInit>(std::get<0>(it))->getDef();
         std::string parser;
         if (auto optParser = attr->getValueAsOptionalString("cParser")) {
           parser = *optParser;
         } else if (attr->isSubClassOf("Array")) {
-          Record *def = attr->getValueAsDef("elemT");
+          const Record *def = attr->getValueAsDef("elemT");
           bool composite = def->isSubClassOf("CompositeBytecode");
           if (!composite && def->isSubClassOf("AttributeKind"))
             parser = "succeeded($_reader.readAttributes($_var))";
@@ -212,7 +213,7 @@ void Generator::emitParseHelper(StringRef kind, StringRef returnType,
     DefInit *first = dyn_cast<DefInit>(arg);
     if (!first)
       PrintFatalError("Unexpected type for " + name);
-    Record *def = first->getDef();
+    const Record *def = first->getDef();
 
     // Create variable decls, if there are a block of same type then create
     // comma separated list of them.
@@ -237,12 +238,12 @@ void Generator::emitParseHelper(StringRef kind, StringRef returnType,
 
   // Emit list helper functions.
   for (auto [arg, name] : zip(args, argNames)) {
-    Record *attr = cast<DefInit>(arg)->getDef();
+    const Record *attr = cast<DefInit>(arg)->getDef();
     if (!attr->isSubClassOf("Array"))
       continue;
 
     // TODO: Dedupe readers.
-    Record *def = attr->getValueAsDef("elemT");
+    const Record *def = attr->getValueAsDef("elemT");
     if (!def->isSubClassOf("CompositeBytecode") &&
         (def->isSubClassOf("AttributeKind") || def->isSubClassOf("TypeKind")))
       continue;
@@ -293,7 +294,7 @@ void Generator::emitParseHelper(StringRef kind, StringRef returnType,
 }
 
 void Generator::emitPrint(StringRef kind, StringRef type,
-                          ArrayRef<std::pair<int64_t, Record *>> vec) {
+                          ArrayRef<std::pair<int64_t, const Record *>> vec) {
   if (type == "ReservedOrDead")
     return;
 
@@ -304,7 +305,7 @@ void Generator::emitPrint(StringRef kind, StringRef type,
   auto funScope = os.scope("{\n", "}\n\n");
 
   // Check that predicates specified if multiple bytecode instances.
-  for (llvm::Record *rec : make_second_range(vec)) {
+  for (const Record *rec : make_second_range(vec)) {
     StringRef pred = rec->getValueAsString("printerPredicate");
     if (vec.size() > 1 && pred.empty()) {
       for (auto [index, rec] : vec) {
@@ -333,7 +334,7 @@ void Generator::emitPrint(StringRef kind, StringRef type,
          llvm::zip(members->getArgs(), members->getArgNames())) {
       DefInit *def = dyn_cast<DefInit>(arg);
       assert(def);
-      Record *memberRec = def->getDef();
+      const Record *memberRec = def->getDef();
       emitPrintHelper(memberRec, kind, kind, name->getAsUnquotedString(), os);
     }
 
@@ -344,7 +345,7 @@ void Generator::emitPrint(StringRef kind, StringRef type,
   }
 }
 
-void Generator::emitPrintHelper(Record *memberRec, StringRef kind,
+void Generator::emitPrintHelper(const Record *memberRec, StringRef kind,
                                 StringRef parent, StringRef name,
                                 mlir::raw_indented_ostream &ios) {
   std::string getter;
@@ -362,7 +363,7 @@ void Generator::emitPrintHelper(Record *memberRec, StringRef kind,
   }
 
   if (memberRec->isSubClassOf("Array")) {
-    Record *def = memberRec->getValueAsDef("elemT");
+    const Record *def = memberRec->getValueAsDef("elemT");
     if (!def->isSubClassOf("CompositeBytecode")) {
       if (def->isSubClassOf("AttributeKind")) {
         ios << "writer.writeAttributes(" << getter << ");\n";
@@ -423,7 +424,7 @@ void Generator::emitPrintDispatch(StringRef kind, ArrayRef<std::string> vec) {
 namespace {
 /// Container of Attribute or Type for Dialect.
 struct AttrOrType {
-  std::vector<Record *> attr, type;
+  std::vector<const Record *> attr, type;
 };
 } // namespace
 
@@ -452,7 +453,7 @@ static bool emitBCRW(const RecordKeeper &records, raw_ostream &os) {
   auto it = dialectAttrOrType.front();
   Generator gen(os);
 
-  SmallVector<std::vector<Record *> *, 2> vecs;
+  SmallVector<std::vector<const Record *> *, 2> vecs;
   SmallVector<std::string, 2> kinds;
   vecs.push_back(&it.second.attr);
   kinds.push_back("attribute");
@@ -460,7 +461,8 @@ static bool emitBCRW(const RecordKeeper &records, raw_ostream &os) {
   kinds.push_back("type");
   for (auto [vec, kind] : zip(vecs, kinds)) {
     // Handle Attribute/Type emission.
-    std::map<std::string, std::vector<std::pair<int64_t, Record *>>> perType;
+    std::map<std::string, std::vector<std::pair<int64_t, const Record *>>>
+        perType;
     for (auto kt : llvm::enumerate(*vec))
       perType[getCType(kt.value())].emplace_back(kt.index(), kt.value());
     for (const auto &jt : perType) {
diff --git a/mlir/tools/mlir-tblgen/DialectGen.cpp b/mlir/tools/mlir-tblgen/DialectGen.cpp
index 2412876958a0c..55c3d9da25900 100644
--- a/mlir/tools/mlir-tblgen/DialectGen.cpp
+++ b/mlir/tools/mlir-tblgen/DialectGen.cpp
@@ -30,6 +30,8 @@
 
 using namespace mlir;
 using namespace mlir::tblgen;
+using llvm::Record;
+using llvm::RecordKeeper;
 
 static llvm::cl::OptionCategory dialectGenCat("Options for -gen-dialect-*");
 llvm::cl::opt<std::string>
@@ -39,8 +41,8 @@ llvm::cl::opt<std::string>
 /// Utility iterator used for filtering records for a specific dialect.
 namespace {
 using DialectFilterIterator =
-    llvm::filter_iterator<ArrayRef<llvm::Record *>::iterator,
-                          std::function<bool(const llvm::Record *)>>;
+    llvm::filter_iterator<ArrayRef<Record *>::iterator,
+                          std::function<bool(const Record *)>>;
 } // namespace
 
 static void populateDiscardableAttributes(
@@ -62,8 +64,8 @@ static void populateDiscardableAttributes(
 /// the given dialect.
 template <typename T>
 static iterator_range<DialectFilterIterator>
-filterForDialect(ArrayRef<llvm::Record *> records, Dialect &dialect) {
-  auto filterFn = [&](const llvm::Record *record) {
+filterForDialect(ArrayRef<Record *> records, Dialect &dialect) {
+  auto filterFn = [&](const Record *record) {
     return T(record).getDialect() == dialect;
   };
   return {DialectFilterIterator(records.begin(), records.end(), filterFn),
@@ -295,11 +297,10 @@ static void emitDialectDecl(Dialect &dialect, raw_ostream &os) {
        << "::" << dialect.getCppClassName() << ")\n";
 }
 
-static bool emitDialectDecls(const llvm::RecordKeeper &recordKeeper,
-                             raw_ostream &os) {
-  emitSourceFileHeader("Dialect Declarations", os, recordKeeper);
+static bool emitDialectDecls(const RecordKeeper &records, raw_ostream &os) {
+  emitSourceFileHeader("Dialect Declarations", os, records);
 
-  auto dialectDefs = recordKeeper.getAllDerivedDefinitions("Dialect");
+  auto dialectDefs = records.getAllDerivedDefinitions("Dialect");
   if (dialectDefs.empty())
     return false;
 
@@ -340,8 +341,7 @@ static const char *const dialectDestructorStr = R"(
 
 )";
 
-static void emitDialectDef(Dialect &dialect,
-                           const llvm::RecordKeeper &recordKeeper,
+static void emitDialectDef(Dialect &dialect, const RecordKeeper &records,
                            raw_ostream &os) {
   std::string cppClassName = dialect.getCppClassName();
 
@@ -389,11 +389,10 @@ static void emitDialectDef(Dialect &dialect,
     os << llvm::formatv(dialectDestructorStr, cppClassName);
 }
 
-static bool emitDialectDefs(const llvm::RecordKeeper &recordKeeper,
-                            raw_ostream &os) {
-  emitSourceFileHeader("Dialect Definitions", os, recordKeeper);
+static bool emitDialectDefs(const RecordKeeper &records, raw_ostream &os) {
+  emitSourceFileHeader("Dialect Definitions", os, records);
 
-  auto dialectDefs = recordKeeper.getAllDerivedDefinitions("Dialect");
+  auto dialectDefs = records.getAllDerivedDefinitions("Dialect");
   if (dialectDefs.empty())
     return false;
 
@@ -401,7 +400,7 @@ static bool emitDialectDefs(const llvm::RecordKeeper &recordKeeper,
   std::optional<Dialect> dialect = findDialectToGenerate(dialects);
   if (!dialect)
     return true;
-  emitDialectDef(*dialect, recordKeeper, os);
+  emitDialectDef(*dialect, records, os);
   return false;
 }
 
@@ -411,12 +410,12 @@ static bool emitDialectDefs(const llvm::RecordKeeper &recordKeeper,
 
 static mlir::GenRegistration
     genDialectDecls("gen-dialect-decls", "Generate dialect declarations",
-                    [](const llvm::RecordKeeper &records, raw_ostream &os) {
+                    [](const RecordKeeper &records, raw_ostream &os) {
                       return emitDialectDecls(records, os);
                     });
 
 static mlir::GenRegistration
     genDialectDefs("gen-dialect-defs", "Generate dialect definitions",
-                   [](const llvm::RecordKeeper &records, raw_ostream &os) {
+                   [](const RecordKeeper &records, raw_ostream &os) {
                      return emitDialectDefs(records, os);
                    });
diff --git a/mlir/tools/mlir-tblgen/DirectiveCommonGen.cpp b/mlir/tools/mlir-tblgen/DirectiveCommonGen.cpp
index 26de95fcc5773..09c068fb76693 100644
--- a/mlir/tools/mlir-tblgen/DirectiveCommonGen.cpp
+++ b/mlir/tools/mlir-tblgen/DirectiveCommonGen.cpp
@@ -42,7 +42,7 @@ using llvm::RecordKeeper;
 // Clause record in OMP.td. This name can be used to specify the type of the
 // OpenMP operation's operand. The allowedClauseValues field provides the list
 // of ClauseValues which are part of the enumeration.
-static bool emitDecls(const RecordKeeper &recordKeeper, llvm::StringRef dialect,
+static bool emitDecls(const RecordKeeper &records, llvm::StringRef dialect,
                       raw_ostream &os) {
   // A dialect must be selected for the generated attributes.
   if (dialect.empty()) {
@@ -51,10 +51,10 @@ static bool emitDecls(const RecordKeeper &recordKeeper, llvm::StringRef dialect,
   }
 
   const auto directiveLanguages =
-      recordKeeper.getAllDerivedDefinitions("DirectiveLanguage");
+      records.getAllDerivedDefinitions("DirectiveLanguage");
   assert(!directiveLanguages.empty() && "DirectiveLanguage missing.");
 
-  for (const Clause c : recordKeeper.getAllDerivedDefinitions("Clause")) {
+  for (const Clause c : records.getAllDerivedDefinitions("Clause")) {
     const auto &clauseVals = c.getClauseVals();
     if (clauseVals.empty())
       continue;
diff --git a/mlir/tools/mlir-tblgen/EnumPythonBindingGen.cpp b/mlir/tools/mlir-tblgen/EnumPythonBindingGen.cpp
index 79249944e484f..3f660ae151c74 100644
--- a/mlir/tools/mlir-tblgen/EnumPythonBindingGen.cpp
+++ b/mlir/tools/mlir-tblgen/EnumPythonBindingGen.cpp
@@ -21,6 +21,9 @@
 
 using namespace mlir;
 using namespace mlir::tblgen;
+using llvm::formatv;
+using llvm::Record;
+using llvm::RecordKeeper;
 
 /// File header and includes.
 constexpr const char *fileHeader = R"Py(
@@ -42,44 +45,42 @@ static std::string makePythonEnumCaseName(StringRef name) {
 
 /// Emits the Python class for the given enum.
 static void emitEnumClass(EnumAttr enumAttr, raw_ostream &os) {
-  os << llvm::formatv("class {0}({1}):\n", enumAttr.getEnumClassName(),
-                      enumAttr.isBitEnum() ? "IntFlag" : "IntEnum");
+  os << formatv("class {0}({1}):\n", enumAttr.getEnumClassName(),
+                enumAttr.isBitEnum() ? "IntFlag" : "IntEnum");
   if (!enumAttr.getSummary().empty())
-    os << llvm::formatv("    \"\"\"{0}\"\"\"\n", enumAttr.getSummary());
+    os << formatv("    \"\"\"{0}\"\"\"\n", enumAttr.getSummary());
   os << "\n";
 
   for (const EnumAttrCase &enumCase : enumAttr.getAllCases()) {
-    os << llvm::formatv(
-        "    {0} = {1}\n", makePythonEnumCaseName(enumCase.getSymbol()),
-        enumCase.getValue() >= 0 ? std::to_string(enumCase.getValue())
-                                 : "auto()");
+    os << formatv("    {0} = {1}\n",
+                  makePythonEnumCaseName(enumCase.getSymbol()),
+                  enumCase.getValue() >= 0 ? std::to_string(enumCase.getValue())
+                                           : "auto()");
   }
 
   os << "\n";
 
   if (enumAttr.isBitEnum()) {
-    os << llvm::formatv("    def __iter__(self):\n"
-                        "        return iter([case for case in type(self) if "
-                        "(self & case) is case])\n");
-    os << llvm::formatv("    def __len__(self):\n"
-                        "        return bin(self).count(\"1\")\n");
+    os << formatv("    def __iter__(self):\n"
+                  "        return iter([case for case in type(self) if "
+                  "(self & case) is case])\n");
+    os << formatv("    def __len__(self):\n"
+                  "        return bin(self).count(\"1\")\n");
     os << "\n";
   }
 
-  os << llvm::formatv("    def __str__(self):\n");
+  os << formatv("    def __str__(self):\n");
   if (enumAttr.isBitEnum())
-    os << llvm::formatv("        if len(self) > 1:\n"
-                        "            return \"{0}\".join(map(str, self))\n",
-                        enumAttr.getDef().getValueAsString("separator"));
+    os << formatv("        if len(self) > 1:\n"
+                  "            return \"{0}\".join(map(str, self))\n",
+                  enumAttr.getDef().getValueAsString("separator"));
   for (const EnumAttrCase &enumCase : enumAttr.getAllCases()) {
-    os << llvm::formatv("        if self is {0}.{1}:\n",
-                        enumAttr.getEnumClassName(),
-                        makePythonEnumCaseName(enumCase.getSymbol()));
-    os << llvm::formatv("            return \"{0}\"\n", enumCase.getStr());
+    os << formatv("        if self is {0}.{1}:\n", enumAttr.getEnumClassName(),
+                  makePythonEnumCaseName(enumCase.getSymbol()));
+    os << formatv("            return \"{0}\"\n", enumCase.getStr());
   }
-  os << llvm::formatv(
-      "        raise ValueError(\"Unknown {0} enum entry.\")\n\n\n",
-      enumAttr.getEnumClassName());
+  os << formatv("        raise ValueError(\"Unknown {0} enum entry.\")\n\n\n",
+                enumAttr.getEnumClassName());
   os << "\n";
 }
 
@@ -105,15 +106,13 @@ static bool emitAttributeBuilder(const EnumAttr &enumAttr, raw_ostream &os) {
     return true;
   }
 
-  os << llvm::formatv("@register_attribute_builder(\"{0}\")\n",
-                      enumAttr.getAttrDefName());
-  os << llvm::formatv("def _{0}(x, context):\n",
-                      enumAttr.getAttrDefName().lower());
-  os << llvm::formatv(
-      "    return "
-      "_ods_ir.IntegerAttr.get(_ods_ir.IntegerType.get_signless({0}, "
-      "context=context), int(x))\n\n",
-      bitwidth);
+  os << formatv("@register_attribute_builder(\"{0}\")\n",
+                enumAttr.getAttrDefName());
+  os << formatv("def _{0}(x, context):\n", enumAttr.getAttrDefName().lower());
+  os << formatv("    return "
+                "_ods_ir.IntegerAttr.get(_ods_ir.IntegerType.get_signless({0}, "
+                "context=context), int(x))\n\n",
+                bitwidth);
   return false;
 }
 
@@ -123,27 +122,26 @@ static bool emitAttributeBuilder(const EnumAttr &enumAttr, raw_ostream &os) {
 static bool emitDialectEnumAttributeBuilder(StringRef attrDefName,
                                             StringRef formatString,
                                             raw_ostream &os) {
-  os << llvm::formatv("@register_attribute_builder(\"{0}\")\n", attrDefName);
-  os << llvm::formatv("def _{0}(x, context):\n", attrDefName.lower());
-  os << llvm::formatv("    return "
-                      "_ods_ir.Attribute.parse(f'{0}', context=context)\n\n",
-                      formatString);
+  os << formatv("@register_attribute_builder(\"{0}\")\n", attrDefName);
+  os << formatv("def _{0}(x, context):\n", attrDefName.lower());
+  os << formatv("    return "
+                "_ods_ir.Attribute.parse(f'{0}', context=context)\n\n",
+                formatString);
   return false;
 }
 
 /// Emits Python bindings for all enums in the record keeper. Returns
 /// `false` on success, `true` on failure.
-static bool emitPythonEnums(const llvm::RecordKeeper &recordKeeper,
-                            raw_ostream &os) {
+static bool emitPythonEnums(const RecordKeeper &records, raw_ostream &os) {
   os << fileHeader;
-  for (const llvm::Record *it :
-       recordKeeper.getAllDerivedDefinitionsIfDefined("EnumAttrInfo")) {
+  for (const Record *it :
+       records.getAllDerivedDefinitionsIfDefined("EnumAttrInfo")) {
     EnumAttr enumAttr(*it);
     emitEnumClass(enumAttr, os);
     emitAttributeBuilder(enumAttr, os);
   }
-  for (const llvm::Record *it :
-       recordKeeper.getAllDerivedDefinitionsIfDefined("EnumAttr")) {
+  for (const Record *it :
+       records.getAllDerivedDefinitionsIfDefined("EnumAttr")) {
     AttrOrTypeDef attr(&*it);
     if (!attr.getMnemonic()) {
       llvm::errs() << "enum case " << attr
@@ -156,11 +154,11 @@ static bool emitPythonEnums(const llvm::RecordKeeper &recordKeeper,
     if (assemblyFormat == "`<` $value `>`") {
       emitDialectEnumAttributeBuilder(
           attr.getName(),
-          llvm::formatv("#{0}.{1}<{{str(x)}>", dialect, mnemonic).str(), os);
+          formatv("#{0}.{1}<{{str(x)}>", dialect, mnemonic).str(), os);
     } else if (assemblyFormat == "$value") {
       emitDialectEnumAttributeBuilder(
           attr.getName(),
-          llvm::formatv("#{0}<{1} {{str(x)}>", dialect, mnemonic).str(), os);
+          formatv("#{0}<{1} {{str(x)}>", dialect, mnemonic).str(), os);
     } else {
       llvm::errs()
           << "unsupported assembly format for python enum bindings generation";
diff --git a/mlir/tools/mlir-tblgen/EnumsGen.cpp b/mlir/tools/mlir-tblgen/EnumsGen.cpp
index 95767a29b9c3c..47ffd291cae68 100644
--- a/mlir/tools/mlir-tblgen/EnumsGen.cpp
+++ b/mlir/tools/mlir-tblgen/EnumsGen.cpp
@@ -26,10 +26,9 @@
 using llvm::formatv;
 using llvm::isDigit;
 using llvm::PrintFatalError;
-using llvm::raw_ostream;
 using llvm::Record;
 using llvm::RecordKeeper;
-using llvm::StringRef;
+using namespace mlir;
 using mlir::tblgen::Attribute;
 using mlir::tblgen::EnumAttr;
 using mlir::tblgen::EnumAttrCase;
@@ -139,7 +138,7 @@ inline ::llvm::raw_ostream &operator<<(::llvm::raw_ostream &p, {0} value) {{
     // is not a power of two (i.e. not a single bit case) and not a known case.
   } else if (enumAttr.isBitEnum()) {
     // Process the known multi-bit cases that use valid keywords.
-    llvm::SmallVector<EnumAttrCase *> validMultiBitCases;
+    SmallVector<EnumAttrCase *> validMultiBitCases;
     for (auto [index, caseVal] : llvm::enumerate(cases)) {
       uint64_t value = caseVal.getValue();
       if (value && !llvm::has_single_bit(value) && !nonKeywordCases.test(index))
@@ -476,7 +475,7 @@ static void emitSpecializedAttrDef(const Record &enumDef, raw_ostream &os) {
   EnumAttr enumAttr(enumDef);
   StringRef enumName = enumAttr.getEnumClassName();
   StringRef attrClassName = enumAttr.getSpecializedAttrClassName();
-  llvm::Record *baseAttrDef = enumAttr.getBaseAttrClass();
+  const Record *baseAttrDef = enumAttr.getBaseAttrClass();
   Attribute baseAttr(baseAttrDef);
 
   // Emit classof method
@@ -565,7 +564,7 @@ static void emitEnumDecl(const Record &enumDef, raw_ostream &os) {
   StringRef underlyingToSymFnName = enumAttr.getUnderlyingToSymbolFnName();
   auto enumerants = enumAttr.getAllCases();
 
-  llvm::SmallVector<StringRef, 2> namespaces;
+  SmallVector<StringRef, 2> namespaces;
   llvm::SplitString(cppNamespace, namespaces, "::");
 
   for (auto ns : namespaces)
@@ -642,11 +641,11 @@ class {1} : public ::mlir::{2} {
   emitDenseMapInfo(qualName, underlyingType, cppNamespace, os);
 }
 
-static bool emitEnumDecls(const RecordKeeper &recordKeeper, raw_ostream &os) {
-  llvm::emitSourceFileHeader("Enum Utility Declarations", os, recordKeeper);
+static bool emitEnumDecls(const RecordKeeper &records, raw_ostream &os) {
+  llvm::emitSourceFileHeader("Enum Utility Declarations", os, records);
 
   for (const Record *def :
-       recordKeeper.getAllDerivedDefinitionsIfDefined("EnumAttrInfo"))
+       records.getAllDerivedDefinitionsIfDefined("EnumAttrInfo"))
     emitEnumDecl(*def, os);
 
   return false;
@@ -656,7 +655,7 @@ static void emitEnumDef(const Record &enumDef, raw_ostream &os) {
   EnumAttr enumAttr(enumDef);
   StringRef cppNamespace = enumAttr.getCppNamespace();
 
-  llvm::SmallVector<StringRef, 2> namespaces;
+  SmallVector<StringRef, 2> namespaces;
   llvm::SplitString(cppNamespace, namespaces, "::");
 
   for (auto ns : namespaces)
@@ -680,11 +679,11 @@ static void emitEnumDef(const Record &enumDef, raw_ostream &os) {
   os << "\n";
 }
 
-static bool emitEnumDefs(const RecordKeeper &recordKeeper, raw_ostream &os) {
-  llvm::emitSourceFileHeader("Enum Utility Definitions", os, recordKeeper);
+static bool emitEnumDefs(const RecordKeeper &records, raw_ostream &os) {
+  llvm::emitSourceFileHeader("Enum Utility Definitions", os, records);
 
   for (const Record *def :
-       recordKeeper.getAllDerivedDefinitionsIfDefined("EnumAttrInfo"))
+       records.getAllDerivedDefinitionsIfDefined("EnumAttrInfo"))
     emitEnumDef(*def, os);
 
   return false;
diff --git a/mlir/tools/mlir-tblgen/FormatGen.cpp b/mlir/tools/mlir-tblgen/FormatGen.cpp
index 7540e584b8fac..d145f3e5a23dd 100644
--- a/mlir/tools/mlir-tblgen/FormatGen.cpp
+++ b/mlir/tools/mlir-tblgen/FormatGen.cpp
@@ -13,6 +13,7 @@
 
 using namespace mlir;
 using namespace mlir::tblgen;
+using llvm::SourceMgr;
 
 //===----------------------------------------------------------------------===//
 // FormatToken
@@ -26,14 +27,14 @@ SMLoc FormatToken::getLoc() const {
 // FormatLexer
 //===----------------------------------------------------------------------===//
 
-FormatLexer::FormatLexer(llvm::SourceMgr &mgr, SMLoc loc)
+FormatLexer::FormatLexer(SourceMgr &mgr, SMLoc loc)
     : mgr(mgr), loc(loc),
       curBuffer(mgr.getMemoryBuffer(mgr.getMainFileID())->getBuffer()),
       curPtr(curBuffer.begin()) {}
 
 FormatToken FormatLexer::emitError(SMLoc loc, const Twine &msg) {
-  mgr.PrintMessage(loc, llvm::SourceMgr::DK_Error, msg);
-  llvm::SrcMgr.PrintMessage(this->loc, llvm::SourceMgr::DK_Note,
+  mgr.PrintMessage(loc, SourceMgr::DK_Error, msg);
+  llvm::SrcMgr.PrintMessage(this->loc, SourceMgr::DK_Note,
                             "in custom assembly format for this operation");
   return formToken(FormatToken::error, loc.getPointer());
 }
@@ -44,10 +45,10 @@ FormatToken FormatLexer::emitError(const char *loc, const Twine &msg) {
 
 FormatToken FormatLexer::emitErrorAndNote(SMLoc loc, const Twine &msg,
                                           const Twine &note) {
-  mgr.PrintMessage(loc, llvm::SourceMgr::DK_Error, msg);
-  llvm::SrcMgr.PrintMessage(this->loc, llvm::SourceMgr::DK_Note,
+  mgr.PrintMessage(loc, SourceMgr::DK_Error, msg);
+  llvm::SrcMgr.PrintMessage(this->loc, SourceMgr::DK_Note,
                             "in custom assembly format for this operation");
-  mgr.PrintMessage(loc, llvm::SourceMgr::DK_Note, note);
+  mgr.PrintMessage(loc, SourceMgr::DK_Note, note);
   return formToken(FormatToken::error, loc.getPointer());
 }
 
diff --git a/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp b/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp
index 5560298831865..9e19f479d673a 100644
--- a/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp
+++ b/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp
@@ -177,9 +177,8 @@ static LogicalResult emitOneBuilder(const Record &record, raw_ostream &os) {
 
 // Emit all builders.  Returns false on success because of the generator
 // registration requirements.
-static bool emitBuilders(const RecordKeeper &recordKeeper, raw_ostream &os) {
-  for (const Record *def :
-       recordKeeper.getAllDerivedDefinitions("LLVM_OpBase")) {
+static bool emitBuilders(const RecordKeeper &records, raw_ostream &os) {
+  for (const Record *def : records.getAllDerivedDefinitions("LLVM_OpBase")) {
     if (failed(emitOneBuilder(*def, os)))
       return true;
   }
@@ -305,15 +304,14 @@ static LogicalResult emitOneMLIRBuilder(const Record &record, raw_ostream &os,
 
 // Emit all intrinsic MLIR builders. Returns false on success because of the
 // generator registration requirements.
-static bool emitIntrMLIRBuilders(const RecordKeeper &recordKeeper,
-                                 raw_ostream &os) {
+static bool emitIntrMLIRBuilders(const RecordKeeper &records, raw_ostream &os) {
   // Emit condition to check if "llvmEnumName" matches the intrinsic id.
   auto emitIntrCond = [](const Record &record) {
     return "intrinsicID == llvm::Intrinsic::" +
            record.getValueAsString("llvmEnumName");
   };
   for (const Record *def :
-       recordKeeper.getAllDerivedDefinitions("LLVM_IntrOpBase")) {
+       records.getAllDerivedDefinitions("LLVM_IntrOpBase")) {
     if (failed(emitOneMLIRBuilder(*def, os, emitIntrCond)))
       return true;
   }
@@ -322,15 +320,13 @@ static bool emitIntrMLIRBuilders(const RecordKeeper &recordKeeper,
 
 // Emit all op builders. Returns false on success because of the
 // generator registration requirements.
-static bool emitOpMLIRBuilders(const RecordKeeper &recordKeeper,
-                               raw_ostream &os) {
+static bool emitOpMLIRBuilders(const RecordKeeper &records, raw_ostream &os) {
   // Emit condition to check if "llvmInstName" matches the instruction opcode.
   auto emitOpcodeCond = [](const Record &record) {
     return "inst->getOpcode() == llvm::Instruction::" +
            record.getValueAsString("llvmInstName");
   };
-  for (const Record *def :
-       recordKeeper.getAllDerivedDefinitions("LLVM_OpBase")) {
+  for (const Record *def : records.getAllDerivedDefinitions("LLVM_OpBase")) {
     if (failed(emitOneMLIRBuilder(*def, os, emitOpcodeCond)))
       return true;
   }
@@ -411,8 +407,7 @@ class LLVMCEnumAttr : public tblgen::EnumAttr {
 // Emits conversion function "LLVMClass convertEnumToLLVM(Enum)" and containing
 // switch-based logic to convert from the MLIR LLVM dialect enum attribute case
 // (Enum) to the corresponding LLVM API enumerant
-static void emitOneEnumToConversion(const llvm::Record *record,
-                                    raw_ostream &os) {
+static void emitOneEnumToConversion(const Record *record, raw_ostream &os) {
   LLVMEnumAttr enumAttr(record);
   StringRef llvmClass = enumAttr.getLLVMClassName();
   StringRef cppClassName = enumAttr.getEnumClassName();
@@ -441,8 +436,7 @@ static void emitOneEnumToConversion(const llvm::Record *record,
 // Emits conversion function "LLVMClass convertEnumToLLVM(Enum)" and containing
 // switch-based logic to convert from the MLIR LLVM dialect enum attribute case
 // (Enum) to the corresponding LLVM API C-style enumerant
-static void emitOneCEnumToConversion(const llvm::Record *record,
-                                     raw_ostream &os) {
+static void emitOneCEnumToConversion(const Record *record, raw_ostream &os) {
   LLVMCEnumAttr enumAttr(record);
   StringRef llvmClass = enumAttr.getLLVMClassName();
   StringRef cppClassName = enumAttr.getEnumClassName();
@@ -472,8 +466,7 @@ static void emitOneCEnumToConversion(const llvm::Record *record,
 // Emits conversion function "Enum convertEnumFromLLVM(LLVMClass)" and
 // containing switch-based logic to convert from the LLVM API enumerant to MLIR
 // LLVM dialect enum attribute (Enum).
-static void emitOneEnumFromConversion(const llvm::Record *record,
-                                      raw_ostream &os) {
+static void emitOneEnumFromConversion(const Record *record, raw_ostream &os) {
   LLVMEnumAttr enumAttr(record);
   StringRef llvmClass = enumAttr.getLLVMClassName();
   StringRef cppClassName = enumAttr.getEnumClassName();
@@ -508,8 +501,7 @@ static void emitOneEnumFromConversion(const llvm::Record *record,
 // Emits conversion function "Enum convertEnumFromLLVM(LLVMEnum)" and
 // containing switch-based logic to convert from the LLVM API C-style enumerant
 // to MLIR LLVM dialect enum attribute (Enum).
-static void emitOneCEnumFromConversion(const llvm::Record *record,
-                                       raw_ostream &os) {
+static void emitOneCEnumFromConversion(const Record *record, raw_ostream &os) {
   LLVMCEnumAttr enumAttr(record);
   StringRef llvmClass = enumAttr.getLLVMClassName();
   StringRef cppClassName = enumAttr.getEnumClassName();
@@ -540,17 +532,15 @@ static void emitOneCEnumFromConversion(const llvm::Record *record,
 // Emits conversion functions between MLIR enum attribute case and corresponding
 // LLVM API enumerants for all registered LLVM dialect enum attributes.
 template <bool ConvertTo>
-static bool emitEnumConversionDefs(const RecordKeeper &recordKeeper,
+static bool emitEnumConversionDefs(const RecordKeeper &records,
                                    raw_ostream &os) {
-  for (const Record *def :
-       recordKeeper.getAllDerivedDefinitions("LLVM_EnumAttr"))
+  for (const Record *def : records.getAllDerivedDefinitions("LLVM_EnumAttr"))
     if (ConvertTo)
       emitOneEnumToConversion(def, os);
     else
       emitOneEnumFromConversion(def, os);
 
-  for (const Record *def :
-       recordKeeper.getAllDerivedDefinitions("LLVM_CEnumAttr"))
+  for (const Record *def : records.getAllDerivedDefinitions("LLVM_CEnumAttr"))
     if (ConvertTo)
       emitOneCEnumToConversion(def, os);
     else
@@ -566,10 +556,9 @@ static void emitOneIntrinsic(const Record &record, raw_ostream &os) {
 
 // Emit the list of LLVM IR intrinsics identifiers that are convertible to a
 // matching MLIR LLVM dialect intrinsic operation.
-static bool emitConvertibleIntrinsics(const RecordKeeper &recordKeeper,
+static bool emitConvertibleIntrinsics(const RecordKeeper &records,
                                       raw_ostream &os) {
-  for (const Record *def :
-       recordKeeper.getAllDerivedDefinitions("LLVM_IntrOpBase"))
+  for (const Record *def : records.getAllDerivedDefinitions("LLVM_IntrOpBase"))
     emitOneIntrinsic(*def, os);
 
   return false;
diff --git a/mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp b/mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp
index 1e3cd8b86d567..411a98a48bfb2 100644
--- a/mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp
+++ b/mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp
@@ -24,6 +24,11 @@
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
 
+using llvm::Record;
+using llvm::RecordKeeper;
+using llvm::Regex;
+using namespace mlir;
+
 static llvm::cl::OptionCategory intrinsicGenCat("Intrinsics Generator Options");
 
 static llvm::cl::opt<std::string>
@@ -54,14 +59,14 @@ static llvm::cl::opt<std::string> aliasAnalysisRegexp(
 using IndicesTy = llvm::SmallBitVector;
 
 /// Return a CodeGen value type entry from a type record.
-static llvm::MVT::SimpleValueType getValueType(const llvm::Record *rec) {
+static llvm::MVT::SimpleValueType getValueType(const Record *rec) {
   return (llvm::MVT::SimpleValueType)rec->getValueAsDef("VT")->getValueAsInt(
       "Value");
 }
 
 /// Return the indices of the definitions in a list of definitions that
 /// represent overloadable types
-static IndicesTy getOverloadableTypeIdxs(const llvm::Record &record,
+static IndicesTy getOverloadableTypeIdxs(const Record &record,
                                          const char *listName) {
   auto results = record.getValueAsListOfDefs(listName);
   IndicesTy overloadedOps(results.size());
@@ -87,13 +92,13 @@ namespace {
 /// the fields of the record.
 class LLVMIntrinsic {
 public:
-  LLVMIntrinsic(const llvm::Record &record) : record(record) {}
+  LLVMIntrinsic(const Record &record) : record(record) {}
 
   /// Get the name of the operation to be used in MLIR.  Uses the appropriate
   /// field if not empty, constructs a name by replacing underscores with dots
   /// in the record name otherwise.
   std::string getOperationName() const {
-    llvm::StringRef name = record.getValueAsString(fieldName);
+    StringRef name = record.getValueAsString(fieldName);
     if (!name.empty())
       return name.str();
 
@@ -101,8 +106,8 @@ class LLVMIntrinsic {
     assert(name.starts_with("int_") &&
            "LLVM intrinsic names are expected to start with 'int_'");
     name = name.drop_front(4);
-    llvm::SmallVector<llvm::StringRef, 8> chunks;
-    llvm::StringRef targetPrefix = record.getValueAsString("TargetPrefix");
+    SmallVector<StringRef, 8> chunks;
+    StringRef targetPrefix = record.getValueAsString("TargetPrefix");
     name.split(chunks, '_');
     auto *chunksBegin = chunks.begin();
     // Remove the target prefix from target specific intrinsics.
@@ -119,8 +124,8 @@ class LLVMIntrinsic {
   }
 
   /// Get the name of the record without the "intrinsic" prefix.
-  llvm::StringRef getProperRecordName() const {
-    llvm::StringRef name = record.getName();
+  StringRef getProperRecordName() const {
+    StringRef name = record.getName();
     assert(name.starts_with("int_") &&
            "LLVM intrinsic names are expected to start with 'int_'");
     return name.drop_front(4);
@@ -129,10 +134,9 @@ class LLVMIntrinsic {
   /// Get the number of operands.
   unsigned getNumOperands() const {
     auto operands = record.getValueAsListOfDefs(fieldOperands);
-    assert(llvm::all_of(operands,
-                        [](const llvm::Record *r) {
-                          return r->isSubClassOf("LLVMType");
-                        }) &&
+    assert(llvm::all_of(
+               operands,
+               [](const Record *r) { return r->isSubClassOf("LLVMType"); }) &&
            "expected operands to be of LLVM type");
     return operands.size();
   }
@@ -142,7 +146,7 @@ class LLVMIntrinsic {
   /// structure type.
   unsigned getNumResults() const {
     auto results = record.getValueAsListOfDefs(fieldResults);
-    for (const llvm::Record *r : results) {
+    for (const Record *r : results) {
       (void)r;
       assert(r->isSubClassOf("LLVMType") &&
              "expected operands to be of LLVM type");
@@ -155,7 +159,7 @@ class LLVMIntrinsic {
   bool hasSideEffects() const {
     return llvm::none_of(
         record.getValueAsListOfDefs(fieldTraits),
-        [](const llvm::Record *r) { return r->getName() == "IntrNoMem"; });
+        [](const Record *r) { return r->getName() == "IntrNoMem"; });
   }
 
   /// Return true if the intrinsic is commutative, i.e. has the respective
@@ -163,7 +167,7 @@ class LLVMIntrinsic {
   bool isCommutative() const {
     return llvm::any_of(
         record.getValueAsListOfDefs(fieldTraits),
-        [](const llvm::Record *r) { return r->getName() == "Commutative"; });
+        [](const Record *r) { return r->getName() == "Commutative"; });
   }
 
   IndicesTy getOverloadableOperandsIdxs() const {
@@ -181,7 +185,7 @@ class LLVMIntrinsic {
   const char *fieldResults = "RetTypes";
   const char *fieldTraits = "IntrProperties";
 
-  const llvm::Record &record;
+  const Record &record;
 };
 } // namespace
 
@@ -195,27 +199,26 @@ void printBracketedRange(const Range &range, llvm::raw_ostream &os) {
 
 /// Emits ODS (TableGen-based) code for `record` representing an LLVM intrinsic.
 /// Returns true on error, false on success.
-static bool emitIntrinsic(const llvm::Record &record, llvm::raw_ostream &os) {
+static bool emitIntrinsic(const Record &record, llvm::raw_ostream &os) {
   LLVMIntrinsic intr(record);
 
-  llvm::Regex accessGroupMatcher(accessGroupRegexp);
+  Regex accessGroupMatcher(accessGroupRegexp);
   bool requiresAccessGroup =
       !accessGroupRegexp.empty() && accessGroupMatcher.match(record.getName());
 
-  llvm::Regex aliasAnalysisMatcher(aliasAnalysisRegexp);
+  Regex aliasAnalysisMatcher(aliasAnalysisRegexp);
   bool requiresAliasAnalysis = !aliasAnalysisRegexp.empty() &&
                                aliasAnalysisMatcher.match(record.getName());
 
   // Prepare strings for traits, if any.
-  llvm::SmallVector<llvm::StringRef, 2> traits;
+  SmallVector<StringRef, 2> traits;
   if (intr.isCommutative())
     traits.push_back("Commutative");
   if (!intr.hasSideEffects())
     traits.push_back("NoMemoryEffect");
 
   // Prepare strings for operands.
-  llvm::SmallVector<llvm::StringRef, 8> operands(intr.getNumOperands(),
-                                                 "LLVM_Type");
+  SmallVector<StringRef, 8> operands(intr.getNumOperands(), "LLVM_Type");
   if (requiresAccessGroup)
     operands.push_back(
         "OptionalAttr<LLVM_AccessGroupArrayAttr>:$access_groups");
@@ -247,14 +250,13 @@ static bool emitIntrinsic(const llvm::Record &record, llvm::raw_ostream &os) {
 /// Traverses the list of TableGen definitions derived from the "Intrinsic"
 /// class and generates MLIR ODS definitions for those intrinsics that have
 /// the name matching the filter.
-static bool emitIntrinsics(const llvm::RecordKeeper &records,
-                           llvm::raw_ostream &os) {
+static bool emitIntrinsics(const RecordKeeper &records, llvm::raw_ostream &os) {
   llvm::emitSourceFileHeader("Operations for LLVM intrinsics", os, records);
   os << "include \"mlir/Dialect/LLVMIR/LLVMOpBase.td\"\n";
   os << "include \"mlir/Interfaces/SideEffectInterfaces.td\"\n\n";
 
   auto defs = records.getAllDerivedDefinitions("Intrinsic");
-  for (const llvm::Record *r : defs) {
+  for (const Record *r : defs) {
     if (!nameFilter.empty() && !r->getName().contains(nameFilter))
       continue;
     if (emitIntrinsic(*r, os))
diff --git a/mlir/tools/mlir-tblgen/OmpOpGen.cpp b/mlir/tools/mlir-tblgen/OmpOpGen.cpp
index c1e0e9fab6682..1c20a6a9bcf4e 100644
--- a/mlir/tools/mlir-tblgen/OmpOpGen.cpp
+++ b/mlir/tools/mlir-tblgen/OmpOpGen.cpp
@@ -210,7 +210,7 @@ static void verifyClause(const Record *op, const Record *clause) {
 ///         type.
 static StringRef translateArgumentType(ArrayRef<SMLoc> loc, StringInit *name,
                                        Init *init, int &nest, int &rank) {
-  Record *def = cast<DefInit>(init)->getDef();
+  const Record *def = cast<DefInit>(init)->getDef();
 
   llvm::StringSet<> superClasses;
   for (auto [sc, _] : def->getSuperClasses())
@@ -318,7 +318,7 @@ static void genOperandsDef(const Record *op, raw_ostream &os) {
     return;
 
   SmallVector<std::string> clauseNames;
-  for (Record *clause : op->getValueAsListOfDefs("clauseList"))
+  for (const Record *clause : op->getValueAsListOfDefs("clauseList"))
     clauseNames.push_back((extractOmpClauseName(clause) + "ClauseOps").str());
 
   StringRef opName = stripPrefixAndSuffix(
@@ -328,8 +328,8 @@ static void genOperandsDef(const Record *op, raw_ostream &os) {
 
 /// Verify that all properties of `OpenMP_Clause`s of records deriving from
 /// `OpenMP_Op`s have been inherited by the latter.
-static bool verifyDecls(const RecordKeeper &recordKeeper, raw_ostream &) {
-  for (const Record *op : recordKeeper.getAllDerivedDefinitions("OpenMP_Op")) {
+static bool verifyDecls(const RecordKeeper &records, raw_ostream &) {
+  for (const Record *op : records.getAllDerivedDefinitions("OpenMP_Op")) {
     for (const Record *clause : op->getValueAsListOfDefs("clauseList"))
       verifyClause(op, clause);
   }
@@ -341,16 +341,15 @@ static bool verifyDecls(const RecordKeeper &recordKeeper, raw_ostream &) {
 /// `OpenMP_Clause` definitions and aggregate them into operation-specific
 /// structures according to the `clauses` argument of each definition deriving
 /// from `OpenMP_Op`.
-static bool genClauseOps(const RecordKeeper &recordKeeper, raw_ostream &os) {
+static bool genClauseOps(const RecordKeeper &records, raw_ostream &os) {
   mlir::tblgen::NamespaceEmitter ns(os, "mlir::omp");
-  for (const Record *clause :
-       recordKeeper.getAllDerivedDefinitions("OpenMP_Clause"))
+  for (const Record *clause : records.getAllDerivedDefinitions("OpenMP_Clause"))
     genClauseOpsStruct(clause, os);
 
   // Produce base mixin class.
   os << baseMixinClass;
 
-  for (const Record *op : recordKeeper.getAllDerivedDefinitions("OpenMP_Op"))
+  for (const Record *op : records.getAllDerivedDefinitions("OpenMP_Op"))
     genOperandsDef(op, os);
 
   return false;
diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index cbe8a05595856..67df002ce3818 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -4498,7 +4498,7 @@ void OpOperandAdaptorEmitter::emitDef(
 
 /// Emit the class declarations or definitions for the given op defs.
 static void
-emitOpClasses(const RecordKeeper &recordKeeper,
+emitOpClasses(const RecordKeeper &records,
               const std::vector<const Record *> &defs, raw_ostream &os,
               const StaticVerifierFunctionEmitter &staticVerifierEmitter,
               bool emitDecl) {
@@ -4535,7 +4535,7 @@ emitOpClasses(const RecordKeeper &recordKeeper,
 }
 
 /// Emit the declarations for the provided op classes.
-static void emitOpClassDecls(const RecordKeeper &recordKeeper,
+static void emitOpClassDecls(const RecordKeeper &records,
                              const std::vector<const Record *> &defs,
                              raw_ostream &os) {
   // First emit forward declaration for each class, this allows them to refer
@@ -4550,37 +4550,37 @@ static void emitOpClassDecls(const RecordKeeper &recordKeeper,
   IfDefScope scope("GET_OP_CLASSES", os);
   if (defs.empty())
     return;
-  StaticVerifierFunctionEmitter staticVerifierEmitter(os, recordKeeper);
+  StaticVerifierFunctionEmitter staticVerifierEmitter(os, records);
   staticVerifierEmitter.collectOpConstraints(defs);
-  emitOpClasses(recordKeeper, defs, os, staticVerifierEmitter,
+  emitOpClasses(records, defs, os, staticVerifierEmitter,
                 /*emitDecl=*/true);
 }
 
 /// Emit the definitions for the provided op classes.
-static void emitOpClassDefs(const RecordKeeper &recordKeeper,
+static void emitOpClassDefs(const RecordKeeper &records,
                             ArrayRef<const Record *> defs, raw_ostream &os,
                             StringRef constraintPrefix = "") {
   if (defs.empty())
     return;
 
   // Generate all of the locally instantiated methods first.
-  StaticVerifierFunctionEmitter staticVerifierEmitter(os, recordKeeper,
+  StaticVerifierFunctionEmitter staticVerifierEmitter(os, records,
                                                       constraintPrefix);
   os << formatv(opCommentHeader, "Local Utility Method", "Definitions");
   staticVerifierEmitter.collectOpConstraints(defs);
   staticVerifierEmitter.emitOpConstraints(defs);
 
   // Emit the classes.
-  emitOpClasses(recordKeeper, defs, os, staticVerifierEmitter,
+  emitOpClasses(records, defs, os, staticVerifierEmitter,
                 /*emitDecl=*/false);
 }
 
 /// Emit op declarations for all op records.
-static bool emitOpDecls(const RecordKeeper &recordKeeper, raw_ostream &os) {
-  emitSourceFileHeader("Op Declarations", os, recordKeeper);
+static bool emitOpDecls(const RecordKeeper &records, raw_ostream &os) {
+  emitSourceFileHeader("Op Declarations", os, records);
 
-  std::vector<const Record *> defs = getRequestedOpDefinitions(recordKeeper);
-  emitOpClassDecls(recordKeeper, defs, os);
+  std::vector<const Record *> defs = getRequestedOpDefinitions(records);
+  emitOpClassDecls(records, defs, os);
 
   // If we are generating sharded op definitions, emit the sharded op
   // registration hooks.
@@ -4606,7 +4606,7 @@ static bool emitOpDecls(const RecordKeeper &recordKeeper, raw_ostream &os) {
 
 /// Generate the dialect op registration hook and the op class definitions for a
 /// shard of ops.
-static void emitOpDefShard(const RecordKeeper &recordKeeper,
+static void emitOpDefShard(const RecordKeeper &records,
                            ArrayRef<const Record *> defs,
                            const Dialect &dialect, unsigned shardIndex,
                            unsigned shardCount, raw_ostream &os) {
@@ -4640,14 +4640,14 @@ static void emitOpDefShard(const RecordKeeper &recordKeeper,
   os << "}\n";
 
   // Generate the per-shard op definitions.
-  emitOpClassDefs(recordKeeper, defs, os, indexStr);
+  emitOpClassDefs(records, defs, os, indexStr);
 }
 
 /// Emit op definitions for all op records.
-static bool emitOpDefs(const RecordKeeper &recordKeeper, raw_ostream &os) {
-  emitSourceFileHeader("Op Definitions", os, recordKeeper);
+static bool emitOpDefs(const RecordKeeper &records, raw_ostream &os) {
+  emitSourceFileHeader("Op Definitions", os, records);
 
-  std::vector<const Record *> defs = getRequestedOpDefinitions(recordKeeper);
+  std::vector<const Record *> defs = getRequestedOpDefinitions(records);
   SmallVector<ArrayRef<const Record *>, 4> shardedDefs;
   shardOpDefinitions(defs, shardedDefs);
 
@@ -4662,7 +4662,7 @@ static bool emitOpDefs(const RecordKeeper &recordKeeper, raw_ostream &os) {
     }
     {
       IfDefScope scope("GET_OP_CLASSES", os);
-      emitOpClassDefs(recordKeeper, defs, os);
+      emitOpClassDefs(records, defs, os);
     }
     return false;
   }
@@ -4671,7 +4671,7 @@ static bool emitOpDefs(const RecordKeeper &recordKeeper, raw_ostream &os) {
     return false;
   Dialect dialect = Operator(defs.front()).getDialect();
   for (auto [idx, value] : llvm::enumerate(shardedDefs)) {
-    emitOpDefShard(recordKeeper, value, dialect, idx, shardedDefs.size(), os);
+    emitOpDefShard(records, value, dialect, idx, shardedDefs.size(), os);
   }
   return false;
 }
diff --git a/mlir/tools/mlir-tblgen/OpDocGen.cpp b/mlir/tools/mlir-tblgen/OpDocGen.cpp
index bf759572d2501..ff3c6b16bb6eb 100644
--- a/mlir/tools/mlir-tblgen/OpDocGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDocGen.cpp
@@ -34,30 +34,30 @@
 #include <set>
 #include <string>
 
+using namespace llvm;
+using namespace mlir;
+using namespace mlir::tblgen;
+using mlir::tblgen::Operator;
+
 //===----------------------------------------------------------------------===//
 // Commandline Options
 //===----------------------------------------------------------------------===//
-static llvm::cl::OptionCategory
+static cl::OptionCategory
     docCat("Options for -gen-(attrdef|typedef|enum|op|dialect)-doc");
-llvm::cl::opt<std::string>
+cl::opt<std::string>
     stripPrefix("strip-prefix",
-                llvm::cl::desc("Strip prefix of the fully qualified names"),
-                llvm::cl::init("::mlir::"), llvm::cl::cat(docCat));
-llvm::cl::opt<bool> allowHugoSpecificFeatures(
+                cl::desc("Strip prefix of the fully qualified names"),
+                cl::init("::mlir::"), cl::cat(docCat));
+cl::opt<bool> allowHugoSpecificFeatures(
     "allow-hugo-specific-features",
-    llvm::cl::desc("Allows using features specific to Hugo"),
-    llvm::cl::init(false), llvm::cl::cat(docCat));
-
-using namespace llvm;
-using namespace mlir;
-using namespace mlir::tblgen;
-using mlir::tblgen::Operator;
+    cl::desc("Allows using features specific to Hugo"), cl::init(false),
+    cl::cat(docCat));
 
 void mlir::tblgen::emitSummary(StringRef summary, raw_ostream &os) {
   if (!summary.empty()) {
-    llvm::StringRef trimmed = summary.trim();
+    StringRef trimmed = summary.trim();
     char first = std::toupper(trimmed.front());
-    llvm::StringRef rest = trimmed.drop_front();
+    StringRef rest = trimmed.drop_front();
     os << "\n_" << first << rest << "_\n\n";
   }
 }
@@ -152,10 +152,10 @@ static void emitOpTraitsDoc(const Operator &op, raw_ostream &os) {
         effectName.consume_front("::");
         effectName.consume_front("mlir::");
         std::string effectStr;
-        llvm::raw_string_ostream os(effectStr);
+        raw_string_ostream os(effectStr);
         os << effectName << "{";
         auto list = trait.getDef().getValueAsListOfDefs("effects");
-        llvm::interleaveComma(list, os, [&](Record *rec) {
+        interleaveComma(list, os, [&](const Record *rec) {
           StringRef effect = rec->getValueAsString("effect");
           effect.consume_front("::");
           effect.consume_front("mlir::");
@@ -163,7 +163,7 @@ static void emitOpTraitsDoc(const Operator &op, raw_ostream &os) {
         });
         os << "}";
         effects.insert(backticks(effectStr));
-        name.append(llvm::formatv(" ({0})", traitName).str());
+        name.append(formatv(" ({0})", traitName).str());
       }
       interfaces.insert(backticks(name));
       continue;
@@ -172,15 +172,15 @@ static void emitOpTraitsDoc(const Operator &op, raw_ostream &os) {
     traits.insert(backticks(name));
   }
   if (!traits.empty()) {
-    llvm::interleaveComma(traits, os << "\nTraits: ");
+    interleaveComma(traits, os << "\nTraits: ");
     os << "\n";
   }
   if (!interfaces.empty()) {
-    llvm::interleaveComma(interfaces, os << "\nInterfaces: ");
+    interleaveComma(interfaces, os << "\nInterfaces: ");
     os << "\n";
   }
   if (!effects.empty()) {
-    llvm::interleaveComma(effects, os << "\nEffects: ");
+    interleaveComma(effects, os << "\nEffects: ");
     os << "\n";
   }
 }
@@ -196,7 +196,7 @@ static void emitOpDoc(const Operator &op, raw_ostream &os) {
   std::string classNameStr = op.getQualCppClassName();
   StringRef className = classNameStr;
   (void)className.consume_front(stripPrefix);
-  os << llvm::formatv("### `{0}` ({1})\n", op.getOperationName(), className);
+  os << formatv("### `{0}` ({1})\n", op.getOperationName(), className);
 
   // Emit the summary, syntax, and description if present.
   if (op.hasSummary())
@@ -282,12 +282,12 @@ static void emitSourceLink(StringRef inputFilename, raw_ostream &os) {
      << inputFromMlirInclude << ")\n\n";
 }
 
-static void emitOpDoc(const RecordKeeper &recordKeeper, raw_ostream &os) {
-  auto opDefs = getRequestedOpDefinitions(recordKeeper);
+static void emitOpDoc(const RecordKeeper &records, raw_ostream &os) {
+  auto opDefs = getRequestedOpDefinitions(records);
 
   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
-  emitSourceLink(recordKeeper.getInputFilename(), os);
-  for (const llvm::Record *opDef : opDefs)
+  emitSourceLink(records.getInputFilename(), os);
+  for (const Record *opDef : opDefs)
     emitOpDoc(Operator(opDef), os);
 }
 
@@ -339,7 +339,7 @@ static void emitAttrOrTypeDefAssemblyFormat(const AttrOrTypeDef &def,
 }
 
 static void emitAttrOrTypeDefDoc(const AttrOrTypeDef &def, raw_ostream &os) {
-  os << llvm::formatv("### {0}\n", def.getCppClassName());
+  os << formatv("### {0}\n", def.getCppClassName());
 
   // Emit the summary if present.
   if (def.hasSummary())
@@ -371,12 +371,12 @@ static void emitAttrOrTypeDefDoc(const AttrOrTypeDef &def, raw_ostream &os) {
   os << "\n";
 }
 
-static void emitAttrOrTypeDefDoc(const RecordKeeper &recordKeeper,
-                                 raw_ostream &os, StringRef recordTypeName) {
-  auto defs = recordKeeper.getAllDerivedDefinitions(recordTypeName);
+static void emitAttrOrTypeDefDoc(const RecordKeeper &records, raw_ostream &os,
+                                 StringRef recordTypeName) {
+  auto defs = records.getAllDerivedDefinitions(recordTypeName);
 
   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
-  for (const llvm::Record *def : defs)
+  for (const Record *def : defs)
     emitAttrOrTypeDefDoc(AttrOrTypeDef(def), os);
 }
 
@@ -385,7 +385,7 @@ static void emitAttrOrTypeDefDoc(const RecordKeeper &recordKeeper,
 //===----------------------------------------------------------------------===//
 
 static void emitEnumDoc(const EnumAttr &def, raw_ostream &os) {
-  os << llvm::formatv("### {0}\n", def.getEnumClassName());
+  os << formatv("### {0}\n", def.getEnumClassName());
 
   // Emit the summary if present.
   if (!def.getSummary().empty())
@@ -404,10 +404,9 @@ static void emitEnumDoc(const EnumAttr &def, raw_ostream &os) {
   os << "\n";
 }
 
-static void emitEnumDoc(const RecordKeeper &recordKeeper, raw_ostream &os) {
+static void emitEnumDoc(const RecordKeeper &records, raw_ostream &os) {
   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
-  for (const llvm::Record *def :
-       recordKeeper.getAllDerivedDefinitions("EnumAttr"))
+  for (const Record *def : records.getAllDerivedDefinitions("EnumAttr"))
     emitEnumDoc(EnumAttr(def), os);
 }
 
@@ -431,7 +430,7 @@ struct OpDocGroup {
 static void maybeNest(bool nest, llvm::function_ref<void(raw_ostream &os)> fn,
                       raw_ostream &os) {
   std::string str;
-  llvm::raw_string_ostream ss(str);
+  raw_string_ostream ss(str);
   fn(ss);
   for (StringRef x : llvm::split(str, "\n")) {
     if (nest && x.starts_with("#"))
@@ -507,7 +506,7 @@ static void emitDialectDoc(const Dialect &dialect, StringRef inputFilename,
   emitIfNotEmpty(dialect.getDescription(), os);
 
   // Generate a TOC marker except if description already contains one.
-  llvm::Regex r("^[[:space:]]*\\[TOC\\]$", llvm::Regex::RegexFlags::Newline);
+  Regex r("^[[:space:]]*\\[TOC\\]$", Regex::RegexFlags::Newline);
   if (!r.match(dialect.getDescription()))
     os << "[TOC]\n\n";
 
@@ -515,20 +514,19 @@ static void emitDialectDoc(const Dialect &dialect, StringRef inputFilename,
             os);
 }
 
-static bool emitDialectDoc(const RecordKeeper &recordKeeper, raw_ostream &os) {
-  auto dialectDefs = recordKeeper.getAllDerivedDefinitionsIfDefined("Dialect");
+static bool emitDialectDoc(const RecordKeeper &records, raw_ostream &os) {
+  auto dialectDefs = records.getAllDerivedDefinitionsIfDefined("Dialect");
   SmallVector<Dialect> dialects(dialectDefs.begin(), dialectDefs.end());
   std::optional<Dialect> dialect = findDialectToGenerate(dialects);
   if (!dialect)
     return true;
 
-  std::vector<const Record *> opDefs = getRequestedOpDefinitions(recordKeeper);
-  auto attrDefs = recordKeeper.getAllDerivedDefinitionsIfDefined("DialectAttr");
-  auto typeDefs = recordKeeper.getAllDerivedDefinitionsIfDefined("DialectType");
-  auto typeDefDefs = recordKeeper.getAllDerivedDefinitionsIfDefined("TypeDef");
-  auto attrDefDefs = recordKeeper.getAllDerivedDefinitionsIfDefined("AttrDef");
-  auto enumDefs =
-      recordKeeper.getAllDerivedDefinitionsIfDefined("EnumAttrInfo");
+  std::vector<const Record *> opDefs = getRequestedOpDefinitions(records);
+  auto attrDefs = records.getAllDerivedDefinitionsIfDefined("DialectAttr");
+  auto typeDefs = records.getAllDerivedDefinitionsIfDefined("DialectType");
+  auto typeDefDefs = records.getAllDerivedDefinitionsIfDefined("TypeDef");
+  auto attrDefDefs = records.getAllDerivedDefinitionsIfDefined("AttrDef");
+  auto enumDefs = records.getAllDerivedDefinitionsIfDefined("EnumAttrInfo");
 
   std::vector<Attribute> dialectAttrs;
   std::vector<AttrDef> dialectAttrDefs;
@@ -537,17 +535,15 @@ static bool emitDialectDoc(const RecordKeeper &recordKeeper, raw_ostream &os) {
   std::vector<TypeDef> dialectTypeDefs;
   std::vector<EnumAttr> dialectEnums;
 
-  llvm::SmallDenseSet<const Record *> seen;
-  auto addIfNotSeen = [&](const llvm::Record *record, const auto &def,
-                          auto &vec) {
+  SmallDenseSet<const Record *> seen;
+  auto addIfNotSeen = [&](const Record *record, const auto &def, auto &vec) {
     if (seen.insert(record).second) {
       vec.push_back(def);
       return true;
     }
     return false;
   };
-  auto addIfInDialect = [&](const llvm::Record *record, const auto &def,
-                            auto &vec) {
+  auto addIfInDialect = [&](const Record *record, const auto &def, auto &vec) {
     return def.getDialect() == *dialect && addIfNotSeen(record, def, vec);
   };
 
@@ -568,7 +564,7 @@ static bool emitDialectDoc(const RecordKeeper &recordKeeper, raw_ostream &os) {
     }
   }
   for (const Record *rec :
-       recordKeeper.getAllDerivedDefinitionsIfDefined("OpDocGroup")) {
+       records.getAllDerivedDefinitionsIfDefined("OpDocGroup")) {
     if (opDocGroup[rec].ops.empty())
       continue;
     opDocGroup[rec].summary = rec->getValueAsString("summary");
@@ -598,7 +594,7 @@ static bool emitDialectDoc(const RecordKeeper &recordKeeper, raw_ostream &os) {
             });
 
   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
-  emitDialectDoc(*dialect, recordKeeper.getInputFilename(), dialectAttrs,
+  emitDialectDoc(*dialect, records.getInputFilename(), dialectAttrs,
                  dialectAttrDefs, dialectOps, dialectTypes, dialectTypeDefs,
                  dialectEnums, os);
   return false;
diff --git a/mlir/tools/mlir-tblgen/OpFormatGen.cpp b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
index 7016fe41ca75d..c99c71572bec2 100644
--- a/mlir/tools/mlir-tblgen/OpFormatGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
@@ -28,6 +28,9 @@
 
 using namespace mlir;
 using namespace mlir::tblgen;
+using llvm::formatv;
+using llvm::Record;
+using llvm::StringMap;
 
 //===----------------------------------------------------------------------===//
 // VariableElement
@@ -404,7 +407,7 @@ struct OperationFormat {
   StringRef opCppClassName;
 
   /// A map of buildable types to indices.
-  llvm::MapVector<StringRef, int, llvm::StringMap<int>> buildableTypes;
+  llvm::MapVector<StringRef, int, StringMap<int>> buildableTypes;
 
   /// The index of the buildable type, if valid, for every operand and result.
   std::vector<TypeResolution> operandTypes, resultTypes;
@@ -891,8 +894,7 @@ static void genElementParserStorage(FormatElement *element, const Operator &op,
 
   } else if (auto *attr = dyn_cast<AttributeVariable>(element)) {
     const NamedAttribute *var = attr->getVar();
-    body << llvm::formatv("  {0} {1}Attr;\n", var->attr.getStorageType(),
-                          var->name);
+    body << formatv("  {0} {1}Attr;\n", var->attr.getStorageType(), var->name);
 
   } else if (auto *operand = dyn_cast<OperandVariable>(element)) {
     StringRef name = operand->getVar()->name;
@@ -910,31 +912,31 @@ static void genElementParserStorage(FormatElement *element, const Operator &op,
            << "  ::llvm::ArrayRef<::mlir::OpAsmParser::UnresolvedOperand> "
            << name << "Operands(&" << name << "RawOperand, 1);";
     }
-    body << llvm::formatv("  ::llvm::SMLoc {0}OperandsLoc;\n"
-                          "  (void){0}OperandsLoc;\n",
-                          name);
+    body << formatv("  ::llvm::SMLoc {0}OperandsLoc;\n"
+                    "  (void){0}OperandsLoc;\n",
+                    name);
 
   } else if (auto *region = dyn_cast<RegionVariable>(element)) {
     StringRef name = region->getVar()->name;
     if (region->getVar()->isVariadic()) {
-      body << llvm::formatv(
+      body << formatv(
           "  ::llvm::SmallVector<std::unique_ptr<::mlir::Region>, 2> "
           "{0}Regions;\n",
           name);
     } else {
-      body << llvm::formatv("  std::unique_ptr<::mlir::Region> {0}Region = "
-                            "std::make_unique<::mlir::Region>();\n",
-                            name);
+      body << formatv("  std::unique_ptr<::mlir::Region> {0}Region = "
+                      "std::make_unique<::mlir::Region>();\n",
+                      name);
     }
 
   } else if (auto *successor = dyn_cast<SuccessorVariable>(element)) {
     StringRef name = successor->getVar()->name;
     if (successor->getVar()->isVariadic()) {
-      body << llvm::formatv("  ::llvm::SmallVector<::mlir::Block *, 2> "
-                            "{0}Successors;\n",
-                            name);
+      body << formatv("  ::llvm::SmallVector<::mlir::Block *, 2> "
+                      "{0}Successors;\n",
+                      name);
     } else {
-      body << llvm::formatv("  ::mlir::Block *{0}Successor = nullptr;\n", name);
+      body << formatv("  ::mlir::Block *{0}Successor = nullptr;\n", name);
     }
 
   } else if (auto *dir = dyn_cast<TypeDirective>(element)) {
@@ -944,8 +946,8 @@ static void genElementParserStorage(FormatElement *element, const Operator &op,
       body << "  ::llvm::SmallVector<::mlir::Type, 1> " << name << "Types;\n";
     else
       body
-          << llvm::formatv("  ::mlir::Type {0}RawType{{};\n", name)
-          << llvm::formatv(
+          << formatv("  ::mlir::Type {0}RawType{{};\n", name)
+          << formatv(
                  "  ::llvm::ArrayRef<::mlir::Type> {0}Types(&{0}RawType, 1);\n",
                  name);
   } else if (auto *dir = dyn_cast<FunctionalTypeDirective>(element)) {
@@ -969,27 +971,27 @@ static void genCustomParameterParser(FormatElement *param, MethodBody &body) {
     StringRef name = operand->getVar()->name;
     ArgumentLengthKind lengthKind = getArgumentLengthKind(operand->getVar());
     if (lengthKind == ArgumentLengthKind::VariadicOfVariadic)
-      body << llvm::formatv("{0}OperandGroups", name);
+      body << formatv("{0}OperandGroups", name);
     else if (lengthKind == ArgumentLengthKind::Variadic)
-      body << llvm::formatv("{0}Operands", name);
+      body << formatv("{0}Operands", name);
     else if (lengthKind == ArgumentLengthKind::Optional)
-      body << llvm::formatv("{0}Operand", name);
+      body << formatv("{0}Operand", name);
     else
       body << formatv("{0}RawOperand", name);
 
   } else if (auto *region = dyn_cast<RegionVariable>(param)) {
     StringRef name = region->getVar()->name;
     if (region->getVar()->isVariadic())
-      body << llvm::formatv("{0}Regions", name);
+      body << formatv("{0}Regions", name);
     else
-      body << llvm::formatv("*{0}Region", name);
+      body << formatv("*{0}Region", name);
 
   } else if (auto *successor = dyn_cast<SuccessorVariable>(param)) {
     StringRef name = successor->getVar()->name;
     if (successor->getVar()->isVariadic())
-      body << llvm::formatv("{0}Successors", name);
+      body << formatv("{0}Successors", name);
     else
-      body << llvm::formatv("{0}Successor", name);
+      body << formatv("{0}Successor", name);
 
   } else if (auto *dir = dyn_cast<RefDirective>(param)) {
     genCustomParameterParser(dir->getArg(), body);
@@ -998,11 +1000,11 @@ static void genCustomParameterParser(FormatElement *param, MethodBody &body) {
     ArgumentLengthKind lengthKind;
     StringRef listName = getTypeListName(dir->getArg(), lengthKind);
     if (lengthKind == ArgumentLengthKind::VariadicOfVariadic)
-      body << llvm::formatv("{0}TypeGroups", listName);
+      body << formatv("{0}TypeGroups", listName);
     else if (lengthKind == ArgumentLengthKind::Variadic)
-      body << llvm::formatv("{0}Types", listName);
+      body << formatv("{0}Types", listName);
     else if (lengthKind == ArgumentLengthKind::Optional)
-      body << llvm::formatv("{0}Type", listName);
+      body << formatv("{0}Type", listName);
     else
       body << formatv("{0}RawType", listName);
 
@@ -1013,8 +1015,8 @@ static void genCustomParameterParser(FormatElement *param, MethodBody &body) {
     body << tgfmt(string->getValue(), &ctx);
 
   } else if (auto *property = dyn_cast<PropertyVariable>(param)) {
-    body << llvm::formatv("result.getOrAddProperties<Properties>().{0}",
-                          property->getVar()->name);
+    body << formatv("result.getOrAddProperties<Properties>().{0}",
+                    property->getVar()->name);
   } else {
     llvm_unreachable("unknown custom directive parameter");
   }
@@ -1037,24 +1039,24 @@ static void genCustomDirectiveParser(CustomDirective *dir, MethodBody &body,
       body << "    " << var->name
            << "OperandsLoc = parser.getCurrentLocation();\n";
       if (var->isOptional()) {
-        body << llvm::formatv(
+        body << formatv(
             "    ::std::optional<::mlir::OpAsmParser::UnresolvedOperand> "
             "{0}Operand;\n",
             var->name);
       } else if (var->isVariadicOfVariadic()) {
-        body << llvm::formatv("    "
-                              "::llvm::SmallVector<::llvm::SmallVector<::mlir::"
-                              "OpAsmParser::UnresolvedOperand>> "
-                              "{0}OperandGroups;\n",
-                              var->name);
+        body << formatv("    "
+                        "::llvm::SmallVector<::llvm::SmallVector<::mlir::"
+                        "OpAsmParser::UnresolvedOperand>> "
+                        "{0}OperandGroups;\n",
+                        var->name);
       }
     } else if (auto *dir = dyn_cast<TypeDirective>(param)) {
       ArgumentLengthKind lengthKind;
       StringRef listName = getTypeListName(dir->getArg(), lengthKind);
       if (lengthKind == ArgumentLengthKind::Optional) {
-        body << llvm::formatv("    ::mlir::Type {0}Type;\n", listName);
+        body << formatv("    ::mlir::Type {0}Type;\n", listName);
       } else if (lengthKind == ArgumentLengthKind::VariadicOfVariadic) {
-        body << llvm::formatv(
+        body << formatv(
             "    ::llvm::SmallVector<llvm::SmallVector<::mlir::Type>> "
             "{0}TypeGroups;\n",
             listName);
@@ -1064,7 +1066,7 @@ static void genCustomDirectiveParser(CustomDirective *dir, MethodBody &body,
       if (auto *operand = dyn_cast<OperandVariable>(input)) {
         if (!operand->getVar()->isOptional())
           continue;
-        body << llvm::formatv(
+        body << formatv(
             "    {0} {1}Operand = {1}Operands.empty() ? {0}() : "
             "{1}Operands[0];\n",
             "::std::optional<::mlir::OpAsmParser::UnresolvedOperand>",
@@ -1074,9 +1076,9 @@ static void genCustomDirectiveParser(CustomDirective *dir, MethodBody &body,
         ArgumentLengthKind lengthKind;
         StringRef listName = getTypeListName(type->getArg(), lengthKind);
         if (lengthKind == ArgumentLengthKind::Optional) {
-          body << llvm::formatv("    ::mlir::Type {0}Type = {0}Types.empty() ? "
-                                "::mlir::Type() : {0}Types[0];\n",
-                                listName);
+          body << formatv("    ::mlir::Type {0}Type = {0}Types.empty() ? "
+                          "::mlir::Type() : {0}Types[0];\n",
+                          listName);
         }
       }
     }
@@ -1101,23 +1103,23 @@ static void genCustomDirectiveParser(CustomDirective *dir, MethodBody &body,
     if (auto *attr = dyn_cast<AttributeVariable>(param)) {
       const NamedAttribute *var = attr->getVar();
       if (var->attr.isOptional() || var->attr.hasDefaultValue())
-        body << llvm::formatv("    if ({0}Attr)\n  ", var->name);
+        body << formatv("    if ({0}Attr)\n  ", var->name);
       if (useProperties) {
         body << formatv(
             "    result.getOrAddProperties<{1}::Properties>().{0} = {0}Attr;\n",
             var->name, opCppClassName);
       } else {
-        body << llvm::formatv("    result.addAttribute(\"{0}\", {0}Attr);\n",
-                              var->name);
+        body << formatv("    result.addAttribute(\"{0}\", {0}Attr);\n",
+                        var->name);
       }
     } else if (auto *operand = dyn_cast<OperandVariable>(param)) {
       const NamedTypeConstraint *var = operand->getVar();
       if (var->isOptional()) {
-        body << llvm::formatv("    if ({0}Operand.has_value())\n"
-                              "      {0}Operands.push_back(*{0}Operand);\n",
-                              var->name);
+        body << formatv("    if ({0}Operand.has_value())\n"
+                        "      {0}Operands.push_back(*{0}Operand);\n",
+                        var->name);
       } else if (var->isVariadicOfVariadic()) {
-        body << llvm::formatv(
+        body << formatv(
             "    for (const auto &subRange : {0}OperandGroups) {{\n"
             "      {0}Operands.append(subRange.begin(), subRange.end());\n"
             "      {0}OperandGroupSizes.push_back(subRange.size());\n"
@@ -1128,11 +1130,11 @@ static void genCustomDirectiveParser(CustomDirective *dir, MethodBody &body,
       ArgumentLengthKind lengthKind;
       StringRef listName = getTypeListName(dir->getArg(), lengthKind);
       if (lengthKind == ArgumentLengthKind::Optional) {
-        body << llvm::formatv("    if ({0}Type)\n"
-                              "      {0}Types.push_back({0}Type);\n",
-                              listName);
+        body << formatv("    if ({0}Type)\n"
+                        "      {0}Types.push_back({0}Type);\n",
+                        listName);
       } else if (lengthKind == ArgumentLengthKind::VariadicOfVariadic) {
-        body << llvm::formatv(
+        body << formatv(
             "    for (const auto &subRange : {0}TypeGroups)\n"
             "      {0}Types.append(subRange.begin(), subRange.end());\n",
             listName);
@@ -1460,9 +1462,9 @@ void OperationFormat::genElementParser(FormatElement *element, MethodBody &body,
       body << "  if (" << attrVar->getVar()->name << "Attr) {\n";
     } else if (auto *propVar = dyn_cast<PropertyVariable>(firstElement)) {
       genPropertyParser(propVar, body, opCppClassName, /*requireParse=*/false);
-      body << llvm::formatv("if ({0}PropParseResult.has_value() && "
-                            "succeeded(*{0}PropParseResult)) ",
-                            propVar->getVar()->name)
+      body << formatv("if ({0}PropParseResult.has_value() && "
+                      "succeeded(*{0}PropParseResult)) ",
+                      propVar->getVar()->name)
            << " {\n";
     } else if (auto *literal = dyn_cast<LiteralElement>(firstElement)) {
       body << "  if (::mlir::succeeded(parser.parseOptional";
@@ -1477,13 +1479,12 @@ void OperationFormat::genElementParser(FormatElement *element, MethodBody &body,
         genElementParser(regionVar, body, attrTypeCtx);
         body << "  if (!" << region->name << "Regions.empty()) {\n";
       } else {
-        body << llvm::formatv(optionalRegionParserCode, region->name);
+        body << formatv(optionalRegionParserCode, region->name);
         body << "  if (!" << region->name << "Region->empty()) {\n  ";
         if (hasImplicitTermTrait)
-          body << llvm::formatv(regionEnsureTerminatorParserCode, region->name);
+          body << formatv(regionEnsureTerminatorParserCode, region->name);
         else if (hasSingleBlockTrait)
-          body << llvm::formatv(regionEnsureSingleBlockParserCode,
-                                region->name);
+          body << formatv(regionEnsureSingleBlockParserCode, region->name);
       }
     } else if (auto *custom = dyn_cast<CustomDirective>(firstElement)) {
       body << "  if (auto optResult = [&]() -> ::mlir::OptionalParseResult {\n";
@@ -1575,26 +1576,26 @@ void OperationFormat::genElementParser(FormatElement *element, MethodBody &body,
     ArgumentLengthKind lengthKind = getArgumentLengthKind(operand->getVar());
     StringRef name = operand->getVar()->name;
     if (lengthKind == ArgumentLengthKind::VariadicOfVariadic)
-      body << llvm::formatv(variadicOfVariadicOperandParserCode, name);
+      body << formatv(variadicOfVariadicOperandParserCode, name);
     else if (lengthKind == ArgumentLengthKind::Variadic)
-      body << llvm::formatv(variadicOperandParserCode, name);
+      body << formatv(variadicOperandParserCode, name);
     else if (lengthKind == ArgumentLengthKind::Optional)
-      body << llvm::formatv(optionalOperandParserCode, name);
+      body << formatv(optionalOperandParserCode, name);
     else
       body << formatv(operandParserCode, name);
 
   } else if (auto *region = dyn_cast<RegionVariable>(element)) {
     bool isVariadic = region->getVar()->isVariadic();
-    body << llvm::formatv(isVariadic ? regionListParserCode : regionParserCode,
-                          region->getVar()->name);
+    body << formatv(isVariadic ? regionListParserCode : regionParserCode,
+                    region->getVar()->name);
     if (hasImplicitTermTrait)
-      body << llvm::formatv(isVariadic ? regionListEnsureTerminatorParserCode
-                                       : regionEnsureTerminatorParserCode,
-                            region->getVar()->name);
+      body << formatv(isVariadic ? regionListEnsureTerminatorParserCode
+                                 : regionEnsureTerminatorParserCode,
+                      region->getVar()->name);
     else if (hasSingleBlockTrait)
-      body << llvm::formatv(isVariadic ? regionListEnsureSingleBlockParserCode
-                                       : regionEnsureSingleBlockParserCode,
-                            region->getVar()->name);
+      body << formatv(isVariadic ? regionListEnsureSingleBlockParserCode
+                                 : regionEnsureSingleBlockParserCode,
+                      region->getVar()->name);
 
   } else if (auto *successor = dyn_cast<SuccessorVariable>(element)) {
     bool isVariadic = successor->getVar()->isVariadic();
@@ -1631,24 +1632,24 @@ void OperationFormat::genElementParser(FormatElement *element, MethodBody &body,
          << "    return ::mlir::failure();\n";
 
   } else if (isa<RegionsDirective>(element)) {
-    body << llvm::formatv(regionListParserCode, "full");
+    body << formatv(regionListParserCode, "full");
     if (hasImplicitTermTrait)
-      body << llvm::formatv(regionListEnsureTerminatorParserCode, "full");
+      body << formatv(regionListEnsureTerminatorParserCode, "full");
     else if (hasSingleBlockTrait)
-      body << llvm::formatv(regionListEnsureSingleBlockParserCode, "full");
+      body << formatv(regionListEnsureSingleBlockParserCode, "full");
 
   } else if (isa<SuccessorsDirective>(element)) {
-    body << llvm::formatv(successorListParserCode, "full");
+    body << formatv(successorListParserCode, "full");
 
   } else if (auto *dir = dyn_cast<TypeDirective>(element)) {
     ArgumentLengthKind lengthKind;
     StringRef listName = getTypeListName(dir->getArg(), lengthKind);
     if (lengthKind == ArgumentLengthKind::VariadicOfVariadic) {
-      body << llvm::formatv(variadicOfVariadicTypeParserCode, listName);
+      body << formatv(variadicOfVariadicTypeParserCode, listName);
     } else if (lengthKind == ArgumentLengthKind::Variadic) {
-      body << llvm::formatv(variadicTypeParserCode, listName);
+      body << formatv(variadicTypeParserCode, listName);
     } else if (lengthKind == ArgumentLengthKind::Optional) {
-      body << llvm::formatv(optionalTypeParserCode, listName);
+      body << formatv(optionalTypeParserCode, listName);
     } else {
       const char *parserCode =
           dir->shouldBeQualified() ? qualifiedTypeParserCode : typeParserCode;
@@ -1903,14 +1904,14 @@ void OperationFormat::genParserVariadicSegmentResolution(Operator &op,
       if (!operand.isVariadicOfVariadic())
         continue;
       if (op.getDialect().usePropertiesForAttributes()) {
-        body << llvm::formatv(
+        body << formatv(
             "  result.getOrAddProperties<{0}::Properties>().{1} = "
             "parser.getBuilder().getDenseI32ArrayAttr({2}OperandGroupSizes);\n",
             op.getCppClassName(),
             operand.constraint.getVariadicOfVariadicSegmentSizeAttr(),
             operand.name);
       } else {
-        body << llvm::formatv(
+        body << formatv(
             "  result.addAttribute(\"{0}\", "
             "parser.getBuilder().getDenseI32ArrayAttr({1}OperandGroupSizes));"
             "\n",
@@ -2160,7 +2161,7 @@ static void genCustomDirectiveParameterPrinter(FormatElement *element,
     if (var->isVariadic())
       body << name << "().getTypes()";
     else if (var->isOptional())
-      body << llvm::formatv("({0}() ? {0}().getType() : ::mlir::Type())", name);
+      body << formatv("({0}() ? {0}().getType() : ::mlir::Type())", name);
     else
       body << name << "().getType()";
 
@@ -2195,8 +2196,7 @@ static void genCustomDirectivePrinter(CustomDirective *customDir,
 static void genRegionPrinter(const Twine &regionName, MethodBody &body,
                              bool hasImplicitTermTrait) {
   if (hasImplicitTermTrait)
-    body << llvm::formatv(regionSingleBlockImplicitTerminatorPrinterCode,
-                          regionName);
+    body << formatv(regionSingleBlockImplicitTerminatorPrinterCode, regionName);
   else
     body << "  _odsPrinter.printRegion(" << regionName << ");\n";
 }
@@ -2220,12 +2220,12 @@ static MethodBody &genTypeOperandPrinter(FormatElement *arg, const Operator &op,
   auto *operand = dyn_cast<OperandVariable>(arg);
   auto *var = operand ? operand->getVar() : cast<ResultVariable>(arg)->getVar();
   if (var->isVariadicOfVariadic())
-    return body << llvm::formatv("{0}().join().getTypes()",
-                                 op.getGetterName(var->name));
+    return body << formatv("{0}().join().getTypes()",
+                           op.getGetterName(var->name));
   if (var->isVariadic())
     return body << op.getGetterName(var->name) << "().getTypes()";
   if (var->isOptional())
-    return body << llvm::formatv(
+    return body << formatv(
                "({0}() ? ::llvm::ArrayRef<::mlir::Type>({0}().getType()) : "
                "::llvm::ArrayRef<::mlir::Type>())",
                op.getGetterName(var->name));
@@ -2242,10 +2242,10 @@ static void genEnumAttrPrinter(const NamedAttribute *var, const Operator &op,
   const EnumAttr &enumAttr = cast<EnumAttr>(baseAttr);
   std::vector<EnumAttrCase> cases = enumAttr.getAllCases();
 
-  body << llvm::formatv(enumAttrBeginPrinterCode,
-                        (var->attr.isOptional() ? "*" : "") +
-                            op.getGetterName(var->name),
-                        enumAttr.getSymbolToStringFnName());
+  body << formatv(enumAttrBeginPrinterCode,
+                  (var->attr.isOptional() ? "*" : "") +
+                      op.getGetterName(var->name),
+                  enumAttr.getSymbolToStringFnName());
 
   // Get a string containing all of the cases that can't be represented with a
   // keyword.
@@ -2276,9 +2276,8 @@ static void genEnumAttrPrinter(const NamedAttribute *var, const Operator &op,
       if (nonKeywordCases.test(it.index()))
         continue;
       StringRef symbol = it.value().getSymbol();
-      body << llvm::formatv("    case {0}::{1}::{2}:\n", cppNamespace, enumName,
-                            llvm::isDigit(symbol.front()) ? ("_" + symbol)
-                                                          : symbol);
+      body << formatv("    case {0}::{1}::{2}:\n", cppNamespace, enumName,
+                      llvm::isDigit(symbol.front()) ? ("_" + symbol) : symbol);
     }
     body << "      _odsPrinter << caseValueStr;\n"
             "      break;\n"
@@ -2584,7 +2583,7 @@ void OperationFormat::genElementPrinter(FormatElement *element,
   } else if (auto *dir = dyn_cast<TypeDirective>(element)) {
     if (auto *operand = dyn_cast<OperandVariable>(dir->getArg())) {
       if (operand->getVar()->isVariadicOfVariadic()) {
-        body << llvm::formatv(
+        body << formatv(
             "  ::llvm::interleaveComma({0}().getTypes(), _odsPrinter, "
             "[&](::mlir::TypeRange types) {{ _odsPrinter << \"(\" << "
             "types << \")\"; });\n",
@@ -2710,7 +2709,7 @@ class OpFormatParser : public FormatParser {
   /// Verify the state of operation operands within the format.
   LogicalResult
   verifyOperands(SMLoc loc,
-                 llvm::StringMap<TypeResolutionInstance> &variableTyResolver);
+                 StringMap<TypeResolutionInstance> &variableTyResolver);
 
   /// Verify the state of operation regions within the format.
   LogicalResult verifyRegions(SMLoc loc);
@@ -2718,7 +2717,7 @@ class OpFormatParser : public FormatParser {
   /// Verify the state of operation results within the format.
   LogicalResult
   verifyResults(SMLoc loc,
-                llvm::StringMap<TypeResolutionInstance> &variableTyResolver);
+                StringMap<TypeResolutionInstance> &variableTyResolver);
 
   /// Verify the state of operation successors within the format.
   LogicalResult verifySuccessors(SMLoc loc);
@@ -2730,18 +2729,17 @@ class OpFormatParser : public FormatParser {
   /// resolution.
   void handleAllTypesMatchConstraint(
       ArrayRef<StringRef> values,
-      llvm::StringMap<TypeResolutionInstance> &variableTyResolver);
+      StringMap<TypeResolutionInstance> &variableTyResolver);
   /// Check for inferable type resolution given all operands, and or results,
   /// have the same type. If 'includeResults' is true, the results also have the
   /// same type as all of the operands.
   void handleSameTypesConstraint(
-      llvm::StringMap<TypeResolutionInstance> &variableTyResolver,
+      StringMap<TypeResolutionInstance> &variableTyResolver,
       bool includeResults);
   /// Check for inferable type resolution based on another operand, result, or
   /// attribute.
   void handleTypesMatchConstraint(
-      llvm::StringMap<TypeResolutionInstance> &variableTyResolver,
-      const llvm::Record &def);
+      StringMap<TypeResolutionInstance> &variableTyResolver, const Record &def);
 
   /// Returns an argument or attribute with the given name that has been seen
   /// within the format.
@@ -2794,9 +2792,9 @@ LogicalResult OpFormatParser::verify(SMLoc loc,
                           "custom assembly format");
 
   // Check for any type traits that we can use for inferring types.
-  llvm::StringMap<TypeResolutionInstance> variableTyResolver;
+  StringMap<TypeResolutionInstance> variableTyResolver;
   for (const Trait &trait : op.getTraits()) {
-    const llvm::Record &def = trait.getDef();
+    const Record &def = trait.getDef();
     if (def.isSubClassOf("AllTypesMatch")) {
       handleAllTypesMatchConstraint(def.getValueAsListOfStrings("values"),
                                     variableTyResolver);
@@ -2995,10 +2993,9 @@ OpFormatParser::verifyAttributeColonType(SMLoc loc,
       return false;
     // If we encounter `:`, the range is known to be invalid.
     (void)emitError(
-        loc,
-        llvm::formatv("format ambiguity caused by `:` literal found after "
-                      "attribute `{0}` which does not have a buildable type",
-                      cast<AttributeVariable>(base)->getVar()->name));
+        loc, formatv("format ambiguity caused by `:` literal found after "
+                     "attribute `{0}` which does not have a buildable type",
+                     cast<AttributeVariable>(base)->getVar()->name));
     return true;
   };
   return verifyAdjacentElements(isBase, isInvalid, elements);
@@ -3018,9 +3015,9 @@ OpFormatParser::verifyAttrDictRegion(SMLoc loc,
       return false;
     (void)emitErrorAndNote(
         loc,
-        llvm::formatv("format ambiguity caused by `attr-dict` directive "
-                      "followed by region `{0}`",
-                      region->getVar()->name),
+        formatv("format ambiguity caused by `attr-dict` directive "
+                "followed by region `{0}`",
+                region->getVar()->name),
         "try using `attr-dict-with-keyword` instead");
     return true;
   };
@@ -3028,7 +3025,7 @@ OpFormatParser::verifyAttrDictRegion(SMLoc loc,
 }
 
 LogicalResult OpFormatParser::verifyOperands(
-    SMLoc loc, llvm::StringMap<TypeResolutionInstance> &variableTyResolver) {
+    SMLoc loc, StringMap<TypeResolutionInstance> &variableTyResolver) {
   // Check that all of the operands are within the format, and their types can
   // be inferred.
   auto &buildableTypes = fmt.buildableTypes;
@@ -3093,7 +3090,7 @@ LogicalResult OpFormatParser::verifyRegions(SMLoc loc) {
 }
 
 LogicalResult OpFormatParser::verifyResults(
-    SMLoc loc, llvm::StringMap<TypeResolutionInstance> &variableTyResolver) {
+    SMLoc loc, StringMap<TypeResolutionInstance> &variableTyResolver) {
   // If we format all of the types together, there is nothing to check.
   if (fmt.allResultTypes)
     return success();
@@ -3197,7 +3194,7 @@ OpFormatParser::verifyOIListElements(SMLoc loc,
 
 void OpFormatParser::handleAllTypesMatchConstraint(
     ArrayRef<StringRef> values,
-    llvm::StringMap<TypeResolutionInstance> &variableTyResolver) {
+    StringMap<TypeResolutionInstance> &variableTyResolver) {
   for (unsigned i = 0, e = values.size(); i != e; ++i) {
     // Check to see if this value matches a resolved operand or result type.
     ConstArgument arg = findSeenArg(values[i]);
@@ -3213,7 +3210,7 @@ void OpFormatParser::handleAllTypesMatchConstraint(
 }
 
 void OpFormatParser::handleSameTypesConstraint(
-    llvm::StringMap<TypeResolutionInstance> &variableTyResolver,
+    StringMap<TypeResolutionInstance> &variableTyResolver,
     bool includeResults) {
   const NamedTypeConstraint *resolver = nullptr;
   int resolvedIt = -1;
@@ -3238,8 +3235,7 @@ void OpFormatParser::handleSameTypesConstraint(
 }
 
 void OpFormatParser::handleTypesMatchConstraint(
-    llvm::StringMap<TypeResolutionInstance> &variableTyResolver,
-    const llvm::Record &def) {
+    StringMap<TypeResolutionInstance> &variableTyResolver, const Record &def) {
   StringRef lhsName = def.getValueAsString("lhs");
   StringRef rhsName = def.getValueAsString("rhs");
   StringRef transformer = def.getValueAsString("transformer");
diff --git a/mlir/tools/mlir-tblgen/OpGenHelpers.cpp b/mlir/tools/mlir-tblgen/OpGenHelpers.cpp
index 702ea66432455..44dbacf19fffd 100644
--- a/mlir/tools/mlir-tblgen/OpGenHelpers.cpp
+++ b/mlir/tools/mlir-tblgen/OpGenHelpers.cpp
@@ -41,18 +41,18 @@ static std::string getOperationName(const Record &def) {
   auto opName = def.getValueAsString("opName");
   if (prefix.empty())
     return std::string(opName);
-  return std::string(llvm::formatv("{0}.{1}", prefix, opName));
+  return std::string(formatv("{0}.{1}", prefix, opName));
 }
 
 std::vector<const Record *>
-mlir::tblgen::getRequestedOpDefinitions(const RecordKeeper &recordKeeper) {
-  const Record *classDef = recordKeeper.getClass("Op");
+mlir::tblgen::getRequestedOpDefinitions(const RecordKeeper &records) {
+  const Record *classDef = records.getClass("Op");
   if (!classDef)
     PrintFatalError("ERROR: Couldn't find the 'Op' class!\n");
 
-  llvm::Regex includeRegex(opIncFilter), excludeRegex(opExcFilter);
+  Regex includeRegex(opIncFilter), excludeRegex(opExcFilter);
   std::vector<const Record *> defs;
-  for (const auto &def : recordKeeper.getDefs()) {
+  for (const auto &def : records.getDefs()) {
     if (!def.second->isSubClassOf(classDef))
       continue;
     // Include if no include filter or include filter matches.
@@ -70,7 +70,7 @@ mlir::tblgen::getRequestedOpDefinitions(const RecordKeeper &recordKeeper) {
 }
 
 bool mlir::tblgen::isPythonReserved(StringRef str) {
-  static llvm::StringSet<> reserved({
+  static StringSet<> reserved({
       "False",  "None",   "True",    "and",      "as",       "assert", "async",
       "await",  "break",  "class",   "continue", "def",      "del",    "elif",
       "else",   "except", "finally", "for",      "from",     "global", "if",
@@ -86,8 +86,8 @@ bool mlir::tblgen::isPythonReserved(StringRef str) {
 }
 
 void mlir::tblgen::shardOpDefinitions(
-    ArrayRef<const llvm::Record *> defs,
-    SmallVectorImpl<ArrayRef<const llvm::Record *>> &shardedDefs) {
+    ArrayRef<const Record *> defs,
+    SmallVectorImpl<ArrayRef<const Record *>> &shardedDefs) {
   assert(opShardCount > 0 && "expected a positive shard count");
   if (opShardCount == 1) {
     shardedDefs.push_back(defs);
diff --git a/mlir/tools/mlir-tblgen/OpGenHelpers.h b/mlir/tools/mlir-tblgen/OpGenHelpers.h
index b7c9fe3a7b799..62ae32cacc688 100644
--- a/mlir/tools/mlir-tblgen/OpGenHelpers.h
+++ b/mlir/tools/mlir-tblgen/OpGenHelpers.h
@@ -23,7 +23,7 @@ namespace tblgen {
 /// Returns all the op definitions filtered by the user. The filtering is via
 /// command-line option "op-include-regex" and "op-exclude-regex".
 std::vector<const llvm::Record *>
-getRequestedOpDefinitions(const llvm::RecordKeeper &recordKeeper);
+getRequestedOpDefinitions(const llvm::RecordKeeper &records);
 
 /// Checks whether `str` is a Python keyword or would shadow builtin function.
 /// Regenerate using python -c"print(set(sorted(__import__('keyword').kwlist)))"
diff --git a/mlir/tools/mlir-tblgen/OpInterfacesGen.cpp b/mlir/tools/mlir-tblgen/OpInterfacesGen.cpp
index 7c32c2549d788..1f1b1d9a34039 100644
--- a/mlir/tools/mlir-tblgen/OpInterfacesGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpInterfacesGen.cpp
@@ -23,6 +23,8 @@
 #include "llvm/TableGen/TableGenBackend.h"
 
 using namespace mlir;
+using llvm::Record;
+using llvm::RecordKeeper;
 using mlir::tblgen::Interface;
 using mlir::tblgen::InterfaceMethod;
 using mlir::tblgen::OpInterface;
@@ -61,14 +63,13 @@ static void emitMethodNameAndArgs(const InterfaceMethod &method,
 
 /// Get an array of all OpInterface definitions but exclude those subclassing
 /// "DeclareOpInterfaceMethods".
-static std::vector<const llvm::Record *>
-getAllInterfaceDefinitions(const llvm::RecordKeeper &recordKeeper,
-                           StringRef name) {
-  std::vector<const llvm::Record *> defs =
-      recordKeeper.getAllDerivedDefinitions((name + "Interface").str());
+static std::vector<const Record *>
+getAllInterfaceDefinitions(const RecordKeeper &records, StringRef name) {
+  std::vector<const Record *> defs =
+      records.getAllDerivedDefinitions((name + "Interface").str());
 
   std::string declareName = ("Declare" + name + "InterfaceMethods").str();
-  llvm::erase_if(defs, [&](const llvm::Record *def) {
+  llvm::erase_if(defs, [&](const Record *def) {
     // Ignore any "declare methods" interfaces.
     if (def->isSubClassOf(declareName))
       return true;
@@ -88,7 +89,7 @@ class InterfaceGenerator {
   bool emitInterfaceDocs();
 
 protected:
-  InterfaceGenerator(std::vector<const llvm::Record *> &&defs, raw_ostream &os)
+  InterfaceGenerator(std::vector<const Record *> &&defs, raw_ostream &os)
       : defs(std::move(defs)), os(os) {}
 
   void emitConceptDecl(const Interface &interface);
@@ -99,7 +100,7 @@ class InterfaceGenerator {
   void emitInterfaceDecl(const Interface &interface);
 
   /// The set of interface records to emit.
-  std::vector<const llvm::Record *> defs;
+  std::vector<const Record *> defs;
   // The stream to emit to.
   raw_ostream &os;
   /// The C++ value type of the interface, e.g. Operation*.
@@ -118,7 +119,7 @@ class InterfaceGenerator {
 
 /// A specialized generator for attribute interfaces.
 struct AttrInterfaceGenerator : public InterfaceGenerator {
-  AttrInterfaceGenerator(const llvm::RecordKeeper &records, raw_ostream &os)
+  AttrInterfaceGenerator(const RecordKeeper &records, raw_ostream &os)
       : InterfaceGenerator(getAllInterfaceDefinitions(records, "Attr"), os) {
     valueType = "::mlir::Attribute";
     interfaceBaseType = "AttributeInterface";
@@ -133,7 +134,7 @@ struct AttrInterfaceGenerator : public InterfaceGenerator {
 };
 /// A specialized generator for operation interfaces.
 struct OpInterfaceGenerator : public InterfaceGenerator {
-  OpInterfaceGenerator(const llvm::RecordKeeper &records, raw_ostream &os)
+  OpInterfaceGenerator(const RecordKeeper &records, raw_ostream &os)
       : InterfaceGenerator(getAllInterfaceDefinitions(records, "Op"), os) {
     valueType = "::mlir::Operation *";
     interfaceBaseType = "OpInterface";
@@ -149,7 +150,7 @@ struct OpInterfaceGenerator : public InterfaceGenerator {
 };
 /// A specialized generator for type interfaces.
 struct TypeInterfaceGenerator : public InterfaceGenerator {
-  TypeInterfaceGenerator(const llvm::RecordKeeper &records, raw_ostream &os)
+  TypeInterfaceGenerator(const RecordKeeper &records, raw_ostream &os)
       : InterfaceGenerator(getAllInterfaceDefinitions(records, "Type"), os) {
     valueType = "::mlir::Type";
     interfaceBaseType = "TypeInterface";
@@ -607,13 +608,13 @@ bool InterfaceGenerator::emitInterfaceDecls() {
   llvm::emitSourceFileHeader("Interface Declarations", os);
   // Sort according to ID, so defs are emitted in the order in which they appear
   // in the Tablegen file.
-  std::vector<const llvm::Record *> sortedDefs(defs);
-  llvm::sort(sortedDefs, [](const llvm::Record *lhs, const llvm::Record *rhs) {
+  std::vector<const Record *> sortedDefs(defs);
+  llvm::sort(sortedDefs, [](const Record *lhs, const Record *rhs) {
     return lhs->getID() < rhs->getID();
   });
-  for (const llvm::Record *def : sortedDefs)
+  for (const Record *def : sortedDefs)
     emitInterfaceDecl(Interface(def));
-  for (const llvm::Record *def : sortedDefs)
+  for (const Record *def : sortedDefs)
     emitModelMethodsDef(Interface(def));
   return false;
 }
@@ -622,8 +623,7 @@ bool InterfaceGenerator::emitInterfaceDecls() {
 // GEN: Interface documentation
 //===----------------------------------------------------------------------===//
 
-static void emitInterfaceDoc(const llvm::Record &interfaceDef,
-                             raw_ostream &os) {
+static void emitInterfaceDoc(const Record &interfaceDef, raw_ostream &os) {
   Interface interface(&interfaceDef);
 
   // Emit the interface name followed by the description.
@@ -684,15 +684,15 @@ struct InterfaceGenRegistration {
         genDefDesc(("Generate " + genDesc + " interface definitions").str()),
         genDocDesc(("Generate " + genDesc + " interface documentation").str()),
         genDecls(genDeclArg, genDeclDesc,
-                 [](const llvm::RecordKeeper &records, raw_ostream &os) {
+                 [](const RecordKeeper &records, raw_ostream &os) {
                    return GeneratorT(records, os).emitInterfaceDecls();
                  }),
         genDefs(genDefArg, genDefDesc,
-                [](const llvm::RecordKeeper &records, raw_ostream &os) {
+                [](const RecordKeeper &records, raw_ostream &os) {
                   return GeneratorT(records, os).emitInterfaceDefs();
                 }),
         genDocs(genDocArg, genDocDesc,
-                [](const llvm::RecordKeeper &records, raw_ostream &os) {
+                [](const RecordKeeper &records, raw_ostream &os) {
                   return GeneratorT(records, os).emitInterfaceDocs();
                 }) {}
 
diff --git a/mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp b/mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp
index 553ab6adc65b0..0c5c936f5adde 100644
--- a/mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp
@@ -23,6 +23,9 @@
 
 using namespace mlir;
 using namespace mlir::tblgen;
+using llvm::formatv;
+using llvm::Record;
+using llvm::RecordKeeper;
 
 /// File header and includes.
 ///   {0} is the dialect namespace.
@@ -315,9 +318,9 @@ static std::string sanitizeName(StringRef name) {
 }
 
 static std::string attrSizedTraitForKind(const char *kind) {
-  return llvm::formatv("::mlir::OpTrait::AttrSized{0}{1}Segments",
-                       llvm::StringRef(kind).take_front().upper(),
-                       llvm::StringRef(kind).drop_front());
+  return formatv("::mlir::OpTrait::AttrSized{0}{1}Segments",
+                 StringRef(kind).take_front().upper(),
+                 StringRef(kind).drop_front());
 }
 
 /// Emits accessors to "elements" of an Op definition. Currently, the supported
@@ -328,15 +331,14 @@ static void emitElementAccessors(
     unsigned numVariadicGroups, unsigned numElements,
     llvm::function_ref<const NamedTypeConstraint &(const Operator &, int)>
         getElement) {
-  assert(llvm::is_contained(
-             llvm::SmallVector<StringRef, 2>{"operand", "result"}, kind) &&
+  assert(llvm::is_contained(SmallVector<StringRef, 2>{"operand", "result"},
+                            kind) &&
          "unsupported kind");
 
   // Traits indicating how to process variadic elements.
-  std::string sameSizeTrait =
-      llvm::formatv("::mlir::OpTrait::SameVariadic{0}{1}Size",
-                    llvm::StringRef(kind).take_front().upper(),
-                    llvm::StringRef(kind).drop_front());
+  std::string sameSizeTrait = formatv("::mlir::OpTrait::SameVariadic{0}{1}Size",
+                                      StringRef(kind).take_front().upper(),
+                                      StringRef(kind).drop_front());
   std::string attrSizedTrait = attrSizedTraitForKind(kind);
 
   // If there is only one variable-length element group, its size can be
@@ -351,15 +353,14 @@ static void emitElementAccessors(
       if (element.name.empty())
         continue;
       if (element.isVariableLength()) {
-        os << llvm::formatv(element.isOptional() ? opOneOptionalTemplate
-                                                 : opOneVariadicTemplate,
-                            sanitizeName(element.name), kind, numElements, i);
+        os << formatv(element.isOptional() ? opOneOptionalTemplate
+                                           : opOneVariadicTemplate,
+                      sanitizeName(element.name), kind, numElements, i);
       } else if (seenVariableLength) {
-        os << llvm::formatv(opSingleAfterVariableTemplate,
-                            sanitizeName(element.name), kind, numElements, i);
+        os << formatv(opSingleAfterVariableTemplate, sanitizeName(element.name),
+                      kind, numElements, i);
       } else {
-        os << llvm::formatv(opSingleTemplate, sanitizeName(element.name), kind,
-                            i);
+        os << formatv(opSingleTemplate, sanitizeName(element.name), kind, i);
       }
     }
     return;
@@ -382,14 +383,13 @@ static void emitElementAccessors(
     for (unsigned i = 0; i < numElements; ++i) {
       const NamedTypeConstraint &element = getElement(op, i);
       if (!element.name.empty()) {
-        os << llvm::formatv(opVariadicEqualPrefixTemplate,
-                            sanitizeName(element.name), kind, numSimpleLength,
-                            numVariadicGroups, numPrecedingSimple,
-                            numPrecedingVariadic);
-        os << llvm::formatv(element.isVariableLength()
-                                ? opVariadicEqualVariadicTemplate
-                                : opVariadicEqualSimpleTemplate,
-                            kind);
+        os << formatv(opVariadicEqualPrefixTemplate, sanitizeName(element.name),
+                      kind, numSimpleLength, numVariadicGroups,
+                      numPrecedingSimple, numPrecedingVariadic);
+        os << formatv(element.isVariableLength()
+                          ? opVariadicEqualVariadicTemplate
+                          : opVariadicEqualSimpleTemplate,
+                      kind);
       }
       if (element.isVariableLength())
         ++numPrecedingVariadic;
@@ -412,9 +412,9 @@ static void emitElementAccessors(
         trailing = "[0]";
       else if (element.isOptional())
         trailing = std::string(
-            llvm::formatv(opVariadicSegmentOptionalTrailingTemplate, kind));
-      os << llvm::formatv(opVariadicSegmentTemplate, sanitizeName(element.name),
-                          kind, i, trailing);
+            formatv(opVariadicSegmentOptionalTrailingTemplate, kind));
+      os << formatv(opVariadicSegmentTemplate, sanitizeName(element.name), kind,
+                    i, trailing);
     }
     return;
   }
@@ -459,27 +459,21 @@ static void emitAttributeAccessors(const Operator &op, raw_ostream &os) {
 
     // Unit attributes are handled specially.
     if (namedAttr.attr.getStorageType().trim() == "::mlir::UnitAttr") {
-      os << llvm::formatv(unitAttributeGetterTemplate, sanitizedName,
-                          namedAttr.name);
-      os << llvm::formatv(unitAttributeSetterTemplate, sanitizedName,
-                          namedAttr.name);
-      os << llvm::formatv(attributeDeleterTemplate, sanitizedName,
-                          namedAttr.name);
+      os << formatv(unitAttributeGetterTemplate, sanitizedName, namedAttr.name);
+      os << formatv(unitAttributeSetterTemplate, sanitizedName, namedAttr.name);
+      os << formatv(attributeDeleterTemplate, sanitizedName, namedAttr.name);
       continue;
     }
 
     if (namedAttr.attr.isOptional()) {
-      os << llvm::formatv(optionalAttributeGetterTemplate, sanitizedName,
-                          namedAttr.name);
-      os << llvm::formatv(optionalAttributeSetterTemplate, sanitizedName,
-                          namedAttr.name);
-      os << llvm::formatv(attributeDeleterTemplate, sanitizedName,
-                          namedAttr.name);
+      os << formatv(optionalAttributeGetterTemplate, sanitizedName,
+                    namedAttr.name);
+      os << formatv(optionalAttributeSetterTemplate, sanitizedName,
+                    namedAttr.name);
+      os << formatv(attributeDeleterTemplate, sanitizedName, namedAttr.name);
     } else {
-      os << llvm::formatv(attributeGetterTemplate, sanitizedName,
-                          namedAttr.name);
-      os << llvm::formatv(attributeSetterTemplate, sanitizedName,
-                          namedAttr.name);
+      os << formatv(attributeGetterTemplate, sanitizedName, namedAttr.name);
+      os << formatv(attributeSetterTemplate, sanitizedName, namedAttr.name);
       // Non-optional attributes cannot be deleted.
     }
   }
@@ -595,7 +589,7 @@ static bool canInferType(const Operator &op) {
 /// accept them as arguments.
 static void
 populateBuilderArgsResults(const Operator &op,
-                           llvm::SmallVectorImpl<std::string> &builderArgs) {
+                           SmallVectorImpl<std::string> &builderArgs) {
   if (canInferType(op))
     return;
 
@@ -607,7 +601,7 @@ populateBuilderArgsResults(const Operator &op,
         // to properly match the built-in result accessor.
         name = "result";
       } else {
-        name = llvm::formatv("_gen_res_{0}", i);
+        name = formatv("_gen_res_{0}", i);
       }
     }
     name = sanitizeName(name);
@@ -620,14 +614,13 @@ populateBuilderArgsResults(const Operator &op,
 /// appear in the `arguments` field of the op definition. Additionally,
 /// `operandNames` is populated with names of operands in their order of
 /// appearance.
-static void
-populateBuilderArgs(const Operator &op,
-                    llvm::SmallVectorImpl<std::string> &builderArgs,
-                    llvm::SmallVectorImpl<std::string> &operandNames) {
+static void populateBuilderArgs(const Operator &op,
+                                SmallVectorImpl<std::string> &builderArgs,
+                                SmallVectorImpl<std::string> &operandNames) {
   for (int i = 0, e = op.getNumArgs(); i < e; ++i) {
     std::string name = op.getArgName(i).str();
     if (name.empty())
-      name = llvm::formatv("_gen_arg_{0}", i);
+      name = formatv("_gen_arg_{0}", i);
     name = sanitizeName(name);
     builderArgs.push_back(name);
     if (!op.getArg(i).is<NamedAttribute *>())
@@ -637,15 +630,16 @@ populateBuilderArgs(const Operator &op,
 
 /// Populates `builderArgs` with the Python-compatible names of builder function
 /// successor arguments. Additionally, `successorArgNames` is also populated.
-static void populateBuilderArgsSuccessors(
-    const Operator &op, llvm::SmallVectorImpl<std::string> &builderArgs,
-    llvm::SmallVectorImpl<std::string> &successorArgNames) {
+static void
+populateBuilderArgsSuccessors(const Operator &op,
+                              SmallVectorImpl<std::string> &builderArgs,
+                              SmallVectorImpl<std::string> &successorArgNames) {
 
   for (int i = 0, e = op.getNumSuccessors(); i < e; ++i) {
     NamedSuccessor successor = op.getSuccessor(i);
     std::string name = std::string(successor.name);
     if (name.empty())
-      name = llvm::formatv("_gen_successor_{0}", i);
+      name = formatv("_gen_successor_{0}", i);
     name = sanitizeName(name);
     builderArgs.push_back(name);
     successorArgNames.push_back(name);
@@ -658,9 +652,8 @@ static void populateBuilderArgsSuccessors(
 /// operands and attributes in the same order as they appear in the `arguments`
 /// field.
 static void
-populateBuilderLinesAttr(const Operator &op,
-                         llvm::ArrayRef<std::string> argNames,
-                         llvm::SmallVectorImpl<std::string> &builderLines) {
+populateBuilderLinesAttr(const Operator &op, ArrayRef<std::string> argNames,
+                         SmallVectorImpl<std::string> &builderLines) {
   builderLines.push_back("_ods_context = _ods_get_default_loc_context(loc)");
   for (int i = 0, e = op.getNumArgs(); i < e; ++i) {
     Argument arg = op.getArg(i);
@@ -670,12 +663,12 @@ populateBuilderLinesAttr(const Operator &op,
 
     // Unit attributes are handled specially.
     if (attribute->attr.getStorageType().trim() == "::mlir::UnitAttr") {
-      builderLines.push_back(llvm::formatv(initUnitAttributeTemplate,
-                                           attribute->name, argNames[i]));
+      builderLines.push_back(
+          formatv(initUnitAttributeTemplate, attribute->name, argNames[i]));
       continue;
     }
 
-    builderLines.push_back(llvm::formatv(
+    builderLines.push_back(formatv(
         attribute->attr.isOptional() || attribute->attr.hasDefaultValue()
             ? initOptionalAttributeWithBuilderTemplate
             : initAttributeWithBuilderTemplate,
@@ -686,30 +679,30 @@ populateBuilderLinesAttr(const Operator &op,
 /// Populates `builderLines` with additional lines that are required in the
 /// builder to set up successors. successorArgNames is expected to correspond
 /// to the Python argument name for each successor on the op.
-static void populateBuilderLinesSuccessors(
-    const Operator &op, llvm::ArrayRef<std::string> successorArgNames,
-    llvm::SmallVectorImpl<std::string> &builderLines) {
+static void
+populateBuilderLinesSuccessors(const Operator &op,
+                               ArrayRef<std::string> successorArgNames,
+                               SmallVectorImpl<std::string> &builderLines) {
   if (successorArgNames.empty()) {
-    builderLines.push_back(llvm::formatv(initSuccessorsTemplate, "None"));
+    builderLines.push_back(formatv(initSuccessorsTemplate, "None"));
     return;
   }
 
-  builderLines.push_back(llvm::formatv(initSuccessorsTemplate, "[]"));
+  builderLines.push_back(formatv(initSuccessorsTemplate, "[]"));
   for (int i = 0, e = successorArgNames.size(); i < e; ++i) {
     auto &argName = successorArgNames[i];
     const NamedSuccessor &successor = op.getSuccessor(i);
-    builderLines.push_back(
-        llvm::formatv(addSuccessorTemplate,
-                      successor.isVariadic() ? "extend" : "append", argName));
+    builderLines.push_back(formatv(addSuccessorTemplate,
+                                   successor.isVariadic() ? "extend" : "append",
+                                   argName));
   }
 }
 
 /// Populates `builderLines` with additional lines that are required in the
 /// builder to set up op operands.
 static void
-populateBuilderLinesOperand(const Operator &op,
-                            llvm::ArrayRef<std::string> names,
-                            llvm::SmallVectorImpl<std::string> &builderLines) {
+populateBuilderLinesOperand(const Operator &op, ArrayRef<std::string> names,
+                            SmallVectorImpl<std::string> &builderLines) {
   bool sizedSegments = op.getTrait(attrSizedTraitForKind("operand")) != nullptr;
 
   // For each element, find or generate a name.
@@ -718,7 +711,7 @@ populateBuilderLinesOperand(const Operator &op,
     std::string name = names[i];
 
     // Choose the formatting string based on the element kind.
-    llvm::StringRef formatString;
+    StringRef formatString;
     if (!element.isVariableLength()) {
       formatString = singleOperandAppendTemplate;
     } else if (element.isOptional()) {
@@ -738,7 +731,7 @@ populateBuilderLinesOperand(const Operator &op,
       }
     }
 
-    builderLines.push_back(llvm::formatv(formatString.data(), name));
+    builderLines.push_back(formatv(formatString.data(), name));
   }
 }
 
@@ -758,7 +751,7 @@ constexpr const char *appendSameResultsTemplate = "results.extend([{0}] * {1})";
 /// Appends the given multiline string as individual strings into
 /// `builderLines`.
 static void appendLineByLine(StringRef string,
-                             llvm::SmallVectorImpl<std::string> &builderLines) {
+                             SmallVectorImpl<std::string> &builderLines) {
 
   std::pair<StringRef, StringRef> split = std::make_pair(string, string);
   do {
@@ -770,14 +763,13 @@ static void appendLineByLine(StringRef string,
 /// Populates `builderLines` with additional lines that are required in the
 /// builder to set up op results.
 static void
-populateBuilderLinesResult(const Operator &op,
-                           llvm::ArrayRef<std::string> names,
-                           llvm::SmallVectorImpl<std::string> &builderLines) {
+populateBuilderLinesResult(const Operator &op, ArrayRef<std::string> names,
+                           SmallVectorImpl<std::string> &builderLines) {
   bool sizedSegments = op.getTrait(attrSizedTraitForKind("result")) != nullptr;
 
   if (hasSameArgumentAndResultTypes(op)) {
-    builderLines.push_back(llvm::formatv(
-        appendSameResultsTemplate, "operands[0].type", op.getNumResults()));
+    builderLines.push_back(formatv(appendSameResultsTemplate,
+                                   "operands[0].type", op.getNumResults()));
     return;
   }
 
@@ -785,12 +777,11 @@ populateBuilderLinesResult(const Operator &op,
     const NamedAttribute &firstAttr = op.getAttribute(0);
     assert(!firstAttr.name.empty() && "unexpected empty name for the attribute "
                                       "from which the type is derived");
-    appendLineByLine(
-        llvm::formatv(deriveTypeFromAttrTemplate, firstAttr.name).str(),
-        builderLines);
-    builderLines.push_back(llvm::formatv(appendSameResultsTemplate,
-                                         "_ods_derived_result_type",
-                                         op.getNumResults()));
+    appendLineByLine(formatv(deriveTypeFromAttrTemplate, firstAttr.name).str(),
+                     builderLines);
+    builderLines.push_back(formatv(appendSameResultsTemplate,
+                                   "_ods_derived_result_type",
+                                   op.getNumResults()));
     return;
   }
 
@@ -803,7 +794,7 @@ populateBuilderLinesResult(const Operator &op,
     std::string name = names[i];
 
     // Choose the formatting string based on the element kind.
-    llvm::StringRef formatString;
+    StringRef formatString;
     if (!element.isVariableLength()) {
       formatString = singleResultAppendTemplate;
     } else if (element.isOptional()) {
@@ -819,17 +810,16 @@ populateBuilderLinesResult(const Operator &op,
       }
     }
 
-    builderLines.push_back(llvm::formatv(formatString.data(), name));
+    builderLines.push_back(formatv(formatString.data(), name));
   }
 }
 
 /// If the operation has variadic regions, adds a builder argument to specify
 /// the number of those regions and builder lines to forward it to the generic
 /// constructor.
-static void
-populateBuilderRegions(const Operator &op,
-                       llvm::SmallVectorImpl<std::string> &builderArgs,
-                       llvm::SmallVectorImpl<std::string> &builderLines) {
+static void populateBuilderRegions(const Operator &op,
+                                   SmallVectorImpl<std::string> &builderArgs,
+                                   SmallVectorImpl<std::string> &builderLines) {
   if (op.hasNoVariadicRegions())
     return;
 
@@ -844,19 +834,19 @@ populateBuilderRegions(const Operator &op,
           .str();
   builderArgs.push_back(name);
   builderLines.push_back(
-      llvm::formatv("regions = {0} + {1}", op.getNumRegions() - 1, name));
+      formatv("regions = {0} + {1}", op.getNumRegions() - 1, name));
 }
 
 /// Emits a default builder constructing an operation from the list of its
 /// result types, followed by a list of its operands. Returns vector
 /// of fully built functionArgs for downstream users (to save having to
 /// rebuild anew).
-static llvm::SmallVector<std::string> emitDefaultOpBuilder(const Operator &op,
-                                                           raw_ostream &os) {
-  llvm::SmallVector<std::string> builderArgs;
-  llvm::SmallVector<std::string> builderLines;
-  llvm::SmallVector<std::string> operandArgNames;
-  llvm::SmallVector<std::string> successorArgNames;
+static SmallVector<std::string> emitDefaultOpBuilder(const Operator &op,
+                                                     raw_ostream &os) {
+  SmallVector<std::string> builderArgs;
+  SmallVector<std::string> builderLines;
+  SmallVector<std::string> operandArgNames;
+  SmallVector<std::string> successorArgNames;
   builderArgs.reserve(op.getNumOperands() + op.getNumResults() +
                       op.getNumNativeAttributes() + op.getNumSuccessors());
   populateBuilderArgsResults(op, builderArgs);
@@ -866,10 +856,10 @@ static llvm::SmallVector<std::string> emitDefaultOpBuilder(const Operator &op,
   populateBuilderArgsSuccessors(op, builderArgs, successorArgNames);
 
   populateBuilderLinesOperand(op, operandArgNames, builderLines);
-  populateBuilderLinesAttr(
-      op, llvm::ArrayRef(builderArgs).drop_front(numResultArgs), builderLines);
+  populateBuilderLinesAttr(op, ArrayRef(builderArgs).drop_front(numResultArgs),
+                           builderLines);
   populateBuilderLinesResult(
-      op, llvm::ArrayRef(builderArgs).take_front(numResultArgs), builderLines);
+      op, ArrayRef(builderArgs).take_front(numResultArgs), builderLines);
   populateBuilderLinesSuccessors(op, successorArgNames, builderLines);
   populateBuilderRegions(op, builderArgs, builderLines);
 
@@ -896,7 +886,7 @@ static llvm::SmallVector<std::string> emitDefaultOpBuilder(const Operator &op,
   };
 
   // StringRefs in functionArgs refer to strings allocated by builderArgs.
-  llvm::SmallVector<llvm::StringRef> functionArgs;
+  SmallVector<StringRef> functionArgs;
 
   // Add positional arguments.
   for (size_t i = 0, cnt = builderArgs.size(); i < cnt; ++i) {
@@ -929,11 +919,10 @@ static llvm::SmallVector<std::string> emitDefaultOpBuilder(const Operator &op,
   initArgs.push_back("loc=loc");
   initArgs.push_back("ip=ip");
 
-  os << llvm::formatv(initTemplate, llvm::join(functionArgs, ", "),
-                      llvm::join(builderLines, "\n    "),
-                      llvm::join(initArgs, ", "));
+  os << formatv(initTemplate, llvm::join(functionArgs, ", "),
+                llvm::join(builderLines, "\n    "), llvm::join(initArgs, ", "));
   return llvm::to_vector<8>(
-      llvm::map_range(functionArgs, [](llvm::StringRef s) { return s.str(); }));
+      llvm::map_range(functionArgs, [](StringRef s) { return s.str(); }));
 }
 
 static void emitSegmentSpec(
@@ -955,15 +944,15 @@ static void emitSegmentSpec(
   }
   segmentSpec.append("]");
 
-  os << llvm::formatv(opClassSizedSegmentsTemplate, kind, segmentSpec);
+  os << formatv(opClassSizedSegmentsTemplate, kind, segmentSpec);
 }
 
 static void emitRegionAttributes(const Operator &op, raw_ostream &os) {
   // Emit _ODS_REGIONS = (min_region_count, has_no_variadic_regions).
   // Note that the base OpView class defines this as (0, True).
   unsigned minRegionCount = op.getNumRegions() - op.getNumVariadicRegions();
-  os << llvm::formatv(opClassRegionSpecTemplate, minRegionCount,
-                      op.hasNoVariadicRegions() ? "True" : "False");
+  os << formatv(opClassRegionSpecTemplate, minRegionCount,
+                op.hasNoVariadicRegions() ? "True" : "False");
 }
 
 /// Emits named accessors to regions.
@@ -975,20 +964,20 @@ static void emitRegionAccessors(const Operator &op, raw_ostream &os) {
 
     assert((!region.isVariadic() || en.index() == op.getNumRegions() - 1) &&
            "expected only the last region to be variadic");
-    os << llvm::formatv(regionAccessorTemplate, sanitizeName(region.name),
-                        std::to_string(en.index()) +
-                            (region.isVariadic() ? ":" : ""));
+    os << formatv(regionAccessorTemplate, sanitizeName(region.name),
+                  std::to_string(en.index()) +
+                      (region.isVariadic() ? ":" : ""));
   }
 }
 
 /// Emits builder that extracts results from op
 static void emitValueBuilder(const Operator &op,
-                             llvm::SmallVector<std::string> functionArgs,
+                             SmallVector<std::string> functionArgs,
                              raw_ostream &os) {
   // Params with (possibly) default args.
   auto valueBuilderParams =
       llvm::map_range(functionArgs, [](const std::string &argAndMaybeDefault) {
-        llvm::SmallVector<llvm::StringRef> argMaybeDefault =
+        SmallVector<StringRef> argMaybeDefault =
             llvm::to_vector<2>(llvm::split(argAndMaybeDefault, "="));
         auto arg = llvm::convertToSnakeFromCamelCase(argMaybeDefault[0]);
         if (argMaybeDefault.size() == 2)
@@ -1005,7 +994,7 @@ static void emitValueBuilder(const Operator &op,
       });
   std::string nameWithoutDialect =
       op.getOperationName().substr(op.getOperationName().find('.') + 1);
-  os << llvm::formatv(
+  os << formatv(
       valueBuilderTemplate, sanitizeName(nameWithoutDialect),
       op.getCppClassName(), llvm::join(valueBuilderParams, ", "),
       llvm::join(opBuilderArgs, ", "),
@@ -1016,8 +1005,7 @@ static void emitValueBuilder(const Operator &op,
 
 /// Emits bindings for a specific Op to the given output stream.
 static void emitOpBindings(const Operator &op, raw_ostream &os) {
-  os << llvm::formatv(opClassTemplate, op.getCppClassName(),
-                      op.getOperationName());
+  os << formatv(opClassTemplate, op.getCppClassName(), op.getOperationName());
 
   // Sized segments.
   if (op.getTrait(attrSizedTraitForKind("operand")) != nullptr) {
@@ -1028,7 +1016,7 @@ static void emitOpBindings(const Operator &op, raw_ostream &os) {
   }
 
   emitRegionAttributes(op, os);
-  llvm::SmallVector<std::string> functionArgs = emitDefaultOpBuilder(op, os);
+  SmallVector<std::string> functionArgs = emitDefaultOpBuilder(op, os);
   emitOperandAccessors(op, os);
   emitAttributeAccessors(op, os);
   emitResultAccessors(op, os);
@@ -1039,17 +1027,17 @@ static void emitOpBindings(const Operator &op, raw_ostream &os) {
 /// Emits bindings for the dialect specified in the command line, including file
 /// headers and utilities. Returns `false` on success to comply with Tablegen
 /// registration requirements.
-static bool emitAllOps(const llvm::RecordKeeper &records, raw_ostream &os) {
+static bool emitAllOps(const RecordKeeper &records, raw_ostream &os) {
   if (clDialectName.empty())
     llvm::PrintFatalError("dialect name not provided");
 
   os << fileHeader;
   if (!clDialectExtensionName.empty())
-    os << llvm::formatv(dialectExtensionTemplate, clDialectName.getValue());
+    os << formatv(dialectExtensionTemplate, clDialectName.getValue());
   else
-    os << llvm::formatv(dialectClassTemplate, clDialectName.getValue());
+    os << formatv(dialectClassTemplate, clDialectName.getValue());
 
-  for (const llvm::Record *rec : records.getAllDerivedDefinitions("Op")) {
+  for (const Record *rec : records.getAllDerivedDefinitions("Op")) {
     Operator op(rec);
     if (op.getDialectName() == clDialectName.getValue())
       emitOpBindings(op, os);
diff --git a/mlir/tools/mlir-tblgen/PassCAPIGen.cpp b/mlir/tools/mlir-tblgen/PassCAPIGen.cpp
index 9f33a4129aacc..8c13c9b031335 100644
--- a/mlir/tools/mlir-tblgen/PassCAPIGen.cpp
+++ b/mlir/tools/mlir-tblgen/PassCAPIGen.cpp
@@ -20,6 +20,8 @@
 
 using namespace mlir;
 using namespace mlir::tblgen;
+using llvm::formatv;
+using llvm::RecordKeeper;
 
 static llvm::cl::OptionCategory
     passGenCat("Options for -gen-pass-capi-header and -gen-pass-capi-impl");
@@ -56,7 +58,7 @@ const char *const fileFooter = R"(
 )";
 
 /// Emit TODO
-static bool emitCAPIHeader(const llvm::RecordKeeper &records, raw_ostream &os) {
+static bool emitCAPIHeader(const RecordKeeper &records, raw_ostream &os) {
   os << fileHeader;
   os << "// Registration for the entire group\n";
   os << "MLIR_CAPI_EXPORTED void mlirRegister" << groupName
@@ -64,7 +66,7 @@ static bool emitCAPIHeader(const llvm::RecordKeeper &records, raw_ostream &os) {
   for (const auto *def : records.getAllDerivedDefinitions("PassBase")) {
     Pass pass(def);
     StringRef defName = pass.getDef()->getName();
-    os << llvm::formatv(passDecl, groupName, defName);
+    os << formatv(passDecl, groupName, defName);
   }
   os << fileFooter;
   return false;
@@ -91,9 +93,9 @@ void mlirRegister{0}Passes(void) {{
 }
 )";
 
-static bool emitCAPIImpl(const llvm::RecordKeeper &records, raw_ostream &os) {
+static bool emitCAPIImpl(const RecordKeeper &records, raw_ostream &os) {
   os << "/* Autogenerated by mlir-tblgen; don't manually edit. */";
-  os << llvm::formatv(passGroupRegistrationCode, groupName);
+  os << formatv(passGroupRegistrationCode, groupName);
 
   for (const auto *def : records.getAllDerivedDefinitions("PassBase")) {
     Pass pass(def);
@@ -103,10 +105,9 @@ static bool emitCAPIImpl(const llvm::RecordKeeper &records, raw_ostream &os) {
     if (StringRef constructor = pass.getConstructor(); !constructor.empty())
       constructorCall = constructor.str();
     else
-      constructorCall =
-          llvm::formatv("create{0}()", pass.getDef()->getName()).str();
+      constructorCall = formatv("create{0}()", pass.getDef()->getName()).str();
 
-    os << llvm::formatv(passCreateDef, groupName, defName, constructorCall);
+    os << formatv(passCreateDef, groupName, defName, constructorCall);
   }
   return false;
 }
diff --git a/mlir/tools/mlir-tblgen/PassDocGen.cpp b/mlir/tools/mlir-tblgen/PassDocGen.cpp
index 8febba1915625..a2cb514ece3eb 100644
--- a/mlir/tools/mlir-tblgen/PassDocGen.cpp
+++ b/mlir/tools/mlir-tblgen/PassDocGen.cpp
@@ -18,6 +18,7 @@
 
 using namespace mlir;
 using namespace mlir::tblgen;
+using llvm::RecordKeeper;
 
 /// Emit the documentation for the given pass.
 static void emitDoc(const Pass &pass, raw_ostream &os) {
@@ -56,9 +57,9 @@ static void emitDoc(const Pass &pass, raw_ostream &os) {
   }
 }
 
-static void emitDocs(const llvm::RecordKeeper &recordKeeper, raw_ostream &os) {
+static void emitDocs(const RecordKeeper &records, raw_ostream &os) {
   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
-  auto passDefs = recordKeeper.getAllDerivedDefinitions("PassBase");
+  auto passDefs = records.getAllDerivedDefinitions("PassBase");
 
   // Collect the registered passes, sorted by argument name.
   SmallVector<Pass, 16> passes(passDefs.begin(), passDefs.end());
@@ -74,7 +75,7 @@ static void emitDocs(const llvm::RecordKeeper &recordKeeper, raw_ostream &os) {
 
 static mlir::GenRegistration
     genRegister("gen-pass-doc", "Generate pass documentation",
-                [](const llvm::RecordKeeper &records, raw_ostream &os) {
+                [](const RecordKeeper &records, raw_ostream &os) {
                   emitDocs(records, os);
                   return false;
                 });
diff --git a/mlir/tools/mlir-tblgen/PassGen.cpp b/mlir/tools/mlir-tblgen/PassGen.cpp
index 655843f26201a..4b4ac41b9effb 100644
--- a/mlir/tools/mlir-tblgen/PassGen.cpp
+++ b/mlir/tools/mlir-tblgen/PassGen.cpp
@@ -21,6 +21,8 @@
 
 using namespace mlir;
 using namespace mlir::tblgen;
+using llvm::formatv;
+using llvm::RecordKeeper;
 
 static llvm::cl::OptionCategory passGenCat("Options for -gen-pass-decls");
 static llvm::cl::opt<std::string>
@@ -28,10 +30,10 @@ static llvm::cl::opt<std::string>
               llvm::cl::cat(passGenCat));
 
 /// Extract the list of passes from the TableGen records.
-static std::vector<Pass> getPasses(const llvm::RecordKeeper &recordKeeper) {
+static std::vector<Pass> getPasses(const RecordKeeper &records) {
   std::vector<Pass> passes;
 
-  for (const auto *def : recordKeeper.getAllDerivedDefinitions("PassBase"))
+  for (const auto *def : records.getAllDerivedDefinitions("PassBase"))
     passes.emplace_back(def);
 
   return passes;
@@ -91,7 +93,7 @@ static void emitPassOptionsStruct(const Pass &pass, raw_ostream &os) {
   if (options.empty())
     return;
 
-  os << llvm::formatv("struct {0}Options {{\n", passName);
+  os << formatv("struct {0}Options {{\n", passName);
 
   for (const PassOption &opt : options) {
     std::string type = opt.getType().str();
@@ -99,7 +101,7 @@ static void emitPassOptionsStruct(const Pass &pass, raw_ostream &os) {
     if (opt.isListOption())
       type = "::llvm::SmallVector<" + type + ">";
 
-    os.indent(2) << llvm::formatv("{0} {1}", type, opt.getCppVariableName());
+    os.indent(2) << formatv("{0} {1}", type, opt.getCppVariableName());
 
     if (std::optional<StringRef> defaultVal = opt.getDefaultValue())
       os << " = " << defaultVal;
@@ -128,9 +130,9 @@ static void emitPassDecls(const Pass &pass, raw_ostream &os) {
 
     // Declaration of the constructor with options.
     if (ArrayRef<PassOption> options = pass.getOptions(); !options.empty())
-      os << llvm::formatv("std::unique_ptr<::mlir::Pass> create{0}("
-                          "{0}Options options);\n",
-                          passName);
+      os << formatv("std::unique_ptr<::mlir::Pass> create{0}("
+                    "{0}Options options);\n",
+                    passName);
   }
 
   os << "#undef " << enableVarName << "\n";
@@ -147,14 +149,13 @@ static void emitRegistrations(llvm::ArrayRef<Pass> passes, raw_ostream &os) {
     if (StringRef constructor = pass.getConstructor(); !constructor.empty())
       constructorCall = constructor.str();
     else
-      constructorCall =
-          llvm::formatv("create{0}()", pass.getDef()->getName()).str();
+      constructorCall = formatv("create{0}()", pass.getDef()->getName()).str();
 
-    os << llvm::formatv(passRegistrationCode, pass.getDef()->getName(),
-                        constructorCall);
+    os << formatv(passRegistrationCode, pass.getDef()->getName(),
+                  constructorCall);
   }
 
-  os << llvm::formatv(passGroupRegistrationCode, groupName);
+  os << formatv(passGroupRegistrationCode, groupName);
 
   for (const Pass &pass : passes)
     os << "  register" << pass.getDef()->getName() << "();\n";
@@ -270,9 +271,9 @@ static void emitPassOptionDecls(const Pass &pass, raw_ostream &os) {
     os.indent(2) << "::mlir::Pass::"
                  << (opt.isListOption() ? "ListOption" : "Option");
 
-    os << llvm::formatv(R"(<{0}> {1}{{*this, "{2}", ::llvm::cl::desc("{3}"))",
-                        opt.getType(), opt.getCppVariableName(),
-                        opt.getArgument(), opt.getDescription());
+    os << formatv(R"(<{0}> {1}{{*this, "{2}", ::llvm::cl::desc("{3}"))",
+                  opt.getType(), opt.getCppVariableName(), opt.getArgument(),
+                  opt.getDescription());
     if (std::optional<StringRef> defaultVal = opt.getDefaultValue())
       os << ", ::llvm::cl::init(" << defaultVal << ")";
     if (std::optional<StringRef> additionalFlags = opt.getAdditionalFlags())
@@ -284,9 +285,9 @@ static void emitPassOptionDecls(const Pass &pass, raw_ostream &os) {
 /// Emit the declarations for each of the pass statistics.
 static void emitPassStatisticDecls(const Pass &pass, raw_ostream &os) {
   for (const PassStatistic &stat : pass.getStatistics()) {
-    os << llvm::formatv(
-        "  ::mlir::Pass::Statistic {0}{{this, \"{1}\", \"{2}\"};\n",
-        stat.getCppVariableName(), stat.getName(), stat.getDescription());
+    os << formatv("  ::mlir::Pass::Statistic {0}{{this, \"{1}\", \"{2}\"};\n",
+                  stat.getCppVariableName(), stat.getName(),
+                  stat.getDescription());
   }
 }
 
@@ -300,11 +301,10 @@ static void emitPassDefs(const Pass &pass, raw_ostream &os) {
   os << "#ifdef " << enableVarName << "\n";
 
   if (emitDefaultConstructors) {
-    os << llvm::formatv(friendDefaultConstructorDeclTemplate, passName);
+    os << formatv(friendDefaultConstructorDeclTemplate, passName);
 
     if (emitDefaultConstructorWithOptions)
-      os << llvm::formatv(friendDefaultConstructorWithOptionsDeclTemplate,
-                          passName);
+      os << formatv(friendDefaultConstructorWithOptionsDeclTemplate, passName);
   }
 
   std::string dependentDialectRegistrations;
@@ -313,24 +313,23 @@ static void emitPassDefs(const Pass &pass, raw_ostream &os) {
     llvm::interleave(
         pass.getDependentDialects(), dialectsOs,
         [&](StringRef dependentDialect) {
-          dialectsOs << llvm::formatv(dialectRegistrationTemplate,
-                                      dependentDialect);
+          dialectsOs << formatv(dialectRegistrationTemplate, dependentDialect);
         },
         "\n    ");
   }
 
   os << "namespace impl {\n";
-  os << llvm::formatv(baseClassBegin, passName, pass.getBaseClass(),
-                      pass.getArgument(), pass.getSummary(),
-                      dependentDialectRegistrations);
+  os << formatv(baseClassBegin, passName, pass.getBaseClass(),
+                pass.getArgument(), pass.getSummary(),
+                dependentDialectRegistrations);
 
   if (ArrayRef<PassOption> options = pass.getOptions(); !options.empty()) {
-    os.indent(2) << llvm::formatv(
-        "{0}Base({0}Options options) : {0}Base() {{\n", passName);
+    os.indent(2) << formatv("{0}Base({0}Options options) : {0}Base() {{\n",
+                            passName);
 
     for (const PassOption &opt : pass.getOptions())
-      os.indent(4) << llvm::formatv("{0} = std::move(options.{0});\n",
-                                    opt.getCppVariableName());
+      os.indent(4) << formatv("{0} = std::move(options.{0});\n",
+                              opt.getCppVariableName());
 
     os.indent(2) << "}\n";
   }
@@ -344,21 +343,20 @@ static void emitPassDefs(const Pass &pass, raw_ostream &os) {
   os << "private:\n";
 
   if (emitDefaultConstructors) {
-    os << llvm::formatv(friendDefaultConstructorDefTemplate, passName);
+    os << formatv(friendDefaultConstructorDefTemplate, passName);
 
     if (!pass.getOptions().empty())
-      os << llvm::formatv(friendDefaultConstructorWithOptionsDefTemplate,
-                          passName);
+      os << formatv(friendDefaultConstructorWithOptionsDefTemplate, passName);
   }
 
   os << "};\n";
   os << "} // namespace impl\n";
 
   if (emitDefaultConstructors) {
-    os << llvm::formatv(defaultConstructorDefTemplate, passName);
+    os << formatv(defaultConstructorDefTemplate, passName);
 
     if (emitDefaultConstructorWithOptions)
-      os << llvm::formatv(defaultConstructorWithOptionsDefTemplate, passName);
+      os << formatv(defaultConstructorWithOptionsDefTemplate, passName);
   }
 
   os << "#undef " << enableVarName << "\n";
@@ -367,7 +365,7 @@ static void emitPassDefs(const Pass &pass, raw_ostream &os) {
 
 static void emitPass(const Pass &pass, raw_ostream &os) {
   StringRef passName = pass.getDef()->getName();
-  os << llvm::formatv(passHeader, passName);
+  os << formatv(passHeader, passName);
 
   emitPassDecls(pass, os);
   emitPassDefs(pass, os);
@@ -436,22 +434,20 @@ static void emitOldPassDecl(const Pass &pass, raw_ostream &os) {
     llvm::interleave(
         pass.getDependentDialects(), dialectsOs,
         [&](StringRef dependentDialect) {
-          dialectsOs << llvm::formatv(dialectRegistrationTemplate,
-                                      dependentDialect);
+          dialectsOs << formatv(dialectRegistrationTemplate, dependentDialect);
         },
         "\n    ");
   }
-  os << llvm::formatv(oldPassDeclBegin, defName, pass.getBaseClass(),
-                      pass.getArgument(), pass.getSummary(),
-                      dependentDialectRegistrations);
+  os << formatv(oldPassDeclBegin, defName, pass.getBaseClass(),
+                pass.getArgument(), pass.getSummary(),
+                dependentDialectRegistrations);
   emitPassOptionDecls(pass, os);
   emitPassStatisticDecls(pass, os);
   os << "};\n";
 }
 
-static void emitPasses(const llvm::RecordKeeper &recordKeeper,
-                       raw_ostream &os) {
-  std::vector<Pass> passes = getPasses(recordKeeper);
+static void emitPasses(const RecordKeeper &records, raw_ostream &os) {
+  std::vector<Pass> passes = getPasses(records);
   os << "/* Autogenerated by mlir-tblgen; don't manually edit */\n";
 
   os << "\n";
@@ -479,7 +475,7 @@ static void emitPasses(const llvm::RecordKeeper &recordKeeper,
 
 static mlir::GenRegistration
     genPassDecls("gen-pass-decls", "Generate pass declarations",
-                 [](const llvm::RecordKeeper &records, raw_ostream &os) {
+                 [](const RecordKeeper &records, raw_ostream &os) {
                    emitPasses(records, os);
                    return false;
                  });
diff --git a/mlir/tools/mlir-tblgen/RewriterGen.cpp b/mlir/tools/mlir-tblgen/RewriterGen.cpp
index 598eb8ea12fcc..e2764f8493cd8 100644
--- a/mlir/tools/mlir-tblgen/RewriterGen.cpp
+++ b/mlir/tools/mlir-tblgen/RewriterGen.cpp
@@ -268,7 +268,7 @@ class PatternEmitter {
 // inlining them.
 class StaticMatcherHelper {
 public:
-  StaticMatcherHelper(raw_ostream &os, const RecordKeeper &recordKeeper,
+  StaticMatcherHelper(raw_ostream &os, const RecordKeeper &records,
                       RecordOperatorMap &mapper);
 
   // Determine if we should inline the match logic or delegate to a static
@@ -1886,9 +1886,9 @@ void PatternEmitter::createAggregateLocalVarsForOpArgs(
 }
 
 StaticMatcherHelper::StaticMatcherHelper(raw_ostream &os,
-                                         const RecordKeeper &recordKeeper,
+                                         const RecordKeeper &records,
                                          RecordOperatorMap &mapper)
-    : opMap(mapper), staticVerifierEmitter(os, recordKeeper) {}
+    : opMap(mapper), staticVerifierEmitter(os, records) {}
 
 void StaticMatcherHelper::populateStaticMatchers(raw_ostream &os) {
   // PatternEmitter will use the static matcher if there's one generated. To
@@ -1951,17 +1951,17 @@ StringRef StaticMatcherHelper::getVerifierName(DagLeaf leaf) {
   return staticVerifierEmitter.getTypeConstraintFn(leaf.getAsConstraint());
 }
 
-static void emitRewriters(const RecordKeeper &recordKeeper, raw_ostream &os) {
-  emitSourceFileHeader("Rewriters", os, recordKeeper);
+static void emitRewriters(const RecordKeeper &records, raw_ostream &os) {
+  emitSourceFileHeader("Rewriters", os, records);
 
-  auto patterns = recordKeeper.getAllDerivedDefinitions("Pattern");
+  auto patterns = records.getAllDerivedDefinitions("Pattern");
 
   // We put the map here because it can be shared among multiple patterns.
   RecordOperatorMap recordOpMap;
 
   // Exam all the patterns and generate static matcher for the duplicated
   // DagNode.
-  StaticMatcherHelper staticMatcher(os, recordKeeper, recordOpMap);
+  StaticMatcherHelper staticMatcher(os, records, recordOpMap);
   for (const Record *p : patterns)
     staticMatcher.addPattern(p);
   staticMatcher.populateStaticConstraintFunctions(os);
diff --git a/mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp b/mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp
index 8f830cdf513fb..3480c81ff7d4b 100644
--- a/mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp
+++ b/mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp
@@ -98,15 +98,15 @@ class Availability {
   StringRef getMergeInstance() const;
 
   // Returns the underlying LLVM TableGen Record.
-  const llvm::Record *getDef() const { return def; }
+  const Record *getDef() const { return def; }
 
 private:
   // The TableGen definition of this availability.
-  const llvm::Record *def;
+  const Record *def;
 };
 } // namespace
 
-Availability::Availability(const llvm::Record *def) : def(def) {
+Availability::Availability(const Record *def) : def(def) {
   assert(def->isSubClassOf("Availability") &&
          "must be subclass of TableGen 'Availability' class");
 }
@@ -166,7 +166,8 @@ std::vector<Availability> getAvailabilities(const Record &def) {
   std::vector<Availability> availabilities;
 
   if (def.getValue("availability")) {
-    std::vector<Record *> availDefs = def.getValueAsListOfDefs("availability");
+    std::vector<const Record *> availDefs =
+        def.getValueAsListOfDefs("availability");
     availabilities.reserve(availDefs.size());
     for (const Record *avail : availDefs)
       availabilities.emplace_back(avail);
@@ -195,12 +196,10 @@ static void emitInterfaceDef(const Availability &availability,
      << "}\n";
 }
 
-static bool emitInterfaceDefs(const RecordKeeper &recordKeeper,
-                              raw_ostream &os) {
-  llvm::emitSourceFileHeader("Availability Interface Definitions", os,
-                             recordKeeper);
+static bool emitInterfaceDefs(const RecordKeeper &records, raw_ostream &os) {
+  llvm::emitSourceFileHeader("Availability Interface Definitions", os, records);
 
-  auto defs = recordKeeper.getAllDerivedDefinitions("Availability");
+  auto defs = records.getAllDerivedDefinitions("Availability");
   SmallVector<const Record *, 1> handledClasses;
   for (const Record *def : defs) {
     SmallVector<const Record *, 1> parent;
@@ -285,12 +284,11 @@ static void emitInterfaceDecl(const Availability &availability,
   os << "};\n\n";
 }
 
-static bool emitInterfaceDecls(const RecordKeeper &recordKeeper,
-                               raw_ostream &os) {
+static bool emitInterfaceDecls(const RecordKeeper &records, raw_ostream &os) {
   llvm::emitSourceFileHeader("Availability Interface Declarations", os,
-                             recordKeeper);
+                             records);
 
-  auto defs = recordKeeper.getAllDerivedDefinitions("Availability");
+  auto defs = records.getAllDerivedDefinitions("Availability");
   SmallVector<const Record *, 4> handledClasses;
   for (const Record *def : defs) {
     SmallVector<const Record *, 1> parent;
@@ -450,11 +448,11 @@ static void emitEnumDecl(const Record &enumDef, raw_ostream &os) {
     os << "} // namespace " << ns << "\n";
 }
 
-static bool emitEnumDecls(const RecordKeeper &recordKeeper, raw_ostream &os) {
+static bool emitEnumDecls(const RecordKeeper &records, raw_ostream &os) {
   llvm::emitSourceFileHeader("SPIR-V Enum Availability Declarations", os,
-                             recordKeeper);
+                             records);
 
-  auto defs = recordKeeper.getAllDerivedDefinitions("EnumAttrInfo");
+  auto defs = records.getAllDerivedDefinitions("EnumAttrInfo");
   for (const auto *def : defs)
     emitEnumDecl(*def, os);
 
@@ -482,11 +480,11 @@ static void emitEnumDef(const Record &enumDef, raw_ostream &os) {
   os << "\n";
 }
 
-static bool emitEnumDefs(const RecordKeeper &recordKeeper, raw_ostream &os) {
+static bool emitEnumDefs(const RecordKeeper &records, raw_ostream &os) {
   llvm::emitSourceFileHeader("SPIR-V Enum Availability Definitions", os,
-                             recordKeeper);
+                             records);
 
-  auto defs = recordKeeper.getAllDerivedDefinitions("EnumAttrInfo");
+  auto defs = records.getAllDerivedDefinitions("EnumAttrInfo");
   for (const auto *def : defs)
     emitEnumDef(*def, os);
 
@@ -1120,9 +1118,8 @@ static void initExtendedSetDeserializationDispatch(StringRef extensionSetName,
                 extensionSetName, instructionID, words);
 }
 
-static void
-emitExtendedSetDeserializationDispatch(const RecordKeeper &recordKeeper,
-                                       raw_ostream &os) {
+static void emitExtendedSetDeserializationDispatch(const RecordKeeper &records,
+                                                   raw_ostream &os) {
   StringRef extensionSetName("extensionSetName"),
       instructionID("instructionID"), words("words");
 
@@ -1138,7 +1135,7 @@ emitExtendedSetDeserializationDispatch(const RecordKeeper &recordKeeper,
 
   initExtendedSetDeserializationDispatch(extensionSetName, instructionID, words,
                                          os);
-  auto defs = recordKeeper.getAllDerivedDefinitions("SPIRV_ExtInstOp");
+  auto defs = records.getAllDerivedDefinitions("SPIRV_ExtInstOp");
   for (const auto *def : defs) {
     if (!def->getValueAsBit("autogenSerialization")) {
       continue;
@@ -1179,16 +1176,15 @@ emitExtendedSetDeserializationDispatch(const RecordKeeper &recordKeeper,
 
 /// Emits all the autogenerated serialization/deserializations functions for the
 /// SPIRV_Ops.
-static bool emitSerializationFns(const RecordKeeper &recordKeeper,
-                                 raw_ostream &os) {
+static bool emitSerializationFns(const RecordKeeper &records, raw_ostream &os) {
   llvm::emitSourceFileHeader("SPIR-V Serialization Utilities/Functions", os,
-                             recordKeeper);
+                             records);
 
   std::string dSerFnString, dDesFnString, serFnString, deserFnString,
       utilsString;
   raw_string_ostream dSerFn(dSerFnString), dDesFn(dDesFnString),
       serFn(serFnString), deserFn(deserFnString);
-  Record *attrClass = recordKeeper.getClass("Attr");
+  Record *attrClass = records.getClass("Attr");
 
   // Emit the serialization and deserialization functions simultaneously.
   StringRef opVar("op");
@@ -1197,7 +1193,7 @@ static bool emitSerializationFns(const RecordKeeper &recordKeeper,
   // Handle the SPIR-V ops.
   initDispatchSerializationFn(opVar, dSerFn);
   initDispatchDeserializationFn(opcode, words, dDesFn);
-  auto defs = recordKeeper.getAllDerivedDefinitions("SPIRV_Op");
+  auto defs = records.getAllDerivedDefinitions("SPIRV_Op");
   for (const auto *def : defs) {
     Operator op(def);
     emitSerializationFunction(attrClass, def, op, serFn);
@@ -1213,7 +1209,7 @@ static bool emitSerializationFns(const RecordKeeper &recordKeeper,
   finalizeDispatchSerializationFn(opVar, dSerFn);
   finalizeDispatchDeserializationFn(opcode, dDesFn);
 
-  emitExtendedSetDeserializationDispatch(recordKeeper, dDesFn);
+  emitExtendedSetDeserializationDispatch(records, dDesFn);
 
   os << "#ifdef GET_SERIALIZATION_FNS\n\n";
   os << serFn.str();
@@ -1260,10 +1256,10 @@ static void emitEnumGetAttrNameFnDefn(const EnumAttr &enumAttr,
   os << "}\n";
 }
 
-static bool emitAttrUtils(const RecordKeeper &recordKeeper, raw_ostream &os) {
-  llvm::emitSourceFileHeader("SPIR-V Attribute Utilities", os, recordKeeper);
+static bool emitAttrUtils(const RecordKeeper &records, raw_ostream &os) {
+  llvm::emitSourceFileHeader("SPIR-V Attribute Utilities", os, records);
 
-  auto defs = recordKeeper.getAllDerivedDefinitions("EnumAttrInfo");
+  auto defs = records.getAllDerivedDefinitions("EnumAttrInfo");
   os << "#ifndef MLIR_DIALECT_SPIRV_IR_ATTR_UTILS_H_\n";
   os << "#define MLIR_DIALECT_SPIRV_IR_ATTR_UTILS_H_\n";
   emitEnumGetAttrNameFnDecl(os);
@@ -1404,12 +1400,11 @@ static void emitAvailabilityImpl(const Operator &srcOp, raw_ostream &os) {
   }
 }
 
-static bool emitAvailabilityImpl(const RecordKeeper &recordKeeper,
-                                 raw_ostream &os) {
+static bool emitAvailabilityImpl(const RecordKeeper &records, raw_ostream &os) {
   llvm::emitSourceFileHeader("SPIR-V Op Availability Implementations", os,
-                             recordKeeper);
+                             records);
 
-  auto defs = recordKeeper.getAllDerivedDefinitions("SPIRV_Op");
+  auto defs = records.getAllDerivedDefinitions("SPIRV_Op");
   for (const auto *def : defs) {
     Operator op(def);
     if (def->getValueAsBit("autogenAvailability"))
@@ -1433,12 +1428,12 @@ static mlir::GenRegistration
 // SPIR-V Capability Implication AutoGen
 //===----------------------------------------------------------------------===//
 
-static bool emitCapabilityImplication(const RecordKeeper &recordKeeper,
+static bool emitCapabilityImplication(const RecordKeeper &records,
                                       raw_ostream &os) {
-  llvm::emitSourceFileHeader("SPIR-V Capability Implication", os, recordKeeper);
+  llvm::emitSourceFileHeader("SPIR-V Capability Implication", os, records);
 
   EnumAttr enumAttr(
-      recordKeeper.getDef("SPIRV_CapabilityAttr")->getValueAsDef("enum"));
+      records.getDef("SPIRV_CapabilityAttr")->getValueAsDef("enum"));
 
   os << "ArrayRef<spirv::Capability> "
         "spirv::getDirectImpliedCapabilities(spirv::Capability cap) {\n"
@@ -1449,7 +1444,8 @@ static bool emitCapabilityImplication(const RecordKeeper &recordKeeper,
     if (!def.getValue("implies"))
       continue;
 
-    std::vector<Record *> impliedCapsDefs = def.getValueAsListOfDefs("implies");
+    std::vector<const Record *> impliedCapsDefs =
+        def.getValueAsListOfDefs("implies");
     os << "  case spirv::Capability::" << enumerant.getSymbol()
        << ": {static const spirv::Capability implies[" << impliedCapsDefs.size()
        << "] = {";
diff --git a/mlir/tools/tblgen-to-irdl/OpDefinitionsGen.cpp b/mlir/tools/tblgen-to-irdl/OpDefinitionsGen.cpp
index d0a3552fb123d..a763105fa0fd6 100644
--- a/mlir/tools/tblgen-to-irdl/OpDefinitionsGen.cpp
+++ b/mlir/tools/tblgen-to-irdl/OpDefinitionsGen.cpp
@@ -249,7 +249,7 @@ Value createTypeConstraint(OpBuilder &builder, tblgen::Constraint constraint) {
     std::vector<Value> constraints;
     constraints.push_back(createTypeConstraint(
         builder, tblgen::Constraint(predRec.getValueAsDef("baseType"))));
-    for (Record *child : predRec.getValueAsListOfDefs("predicateList")) {
+    for (const Record *child : predRec.getValueAsListOfDefs("predicateList")) {
       constraints.push_back(createPredicate(builder, tblgen::Pred(child)));
     }
     auto op = builder.create<irdl::AllOfOp>(UnknownLoc::get(ctx), constraints);
@@ -273,7 +273,8 @@ Value createAttrConstraint(OpBuilder &builder, tblgen::Constraint constraint) {
     std::vector<Value> constraints;
     constraints.push_back(createAttrConstraint(
         builder, tblgen::Constraint(predRec.getValueAsDef("baseAttr"))));
-    for (Record *child : predRec.getValueAsListOfDefs("attrConstraints")) {
+    for (const Record *child :
+         predRec.getValueAsListOfDefs("attrConstraints")) {
       constraints.push_back(createPredicate(
           builder, tblgen::Pred(child->getValueAsDef("predicate"))));
     }
@@ -283,7 +284,8 @@ Value createAttrConstraint(OpBuilder &builder, tblgen::Constraint constraint) {
 
   if (predRec.isSubClassOf("AnyAttrOf")) {
     std::vector<Value> constraints;
-    for (Record *child : predRec.getValueAsListOfDefs("allowedAttributes")) {
+    for (const Record *child :
+         predRec.getValueAsListOfDefs("allowedAttributes")) {
       constraints.push_back(
           createAttrConstraint(builder, tblgen::Constraint(child)));
     }
@@ -338,6 +340,29 @@ Value createAttrConstraint(OpBuilder &builder, tblgen::Constraint constraint) {
   return createPredicate(builder, constraint.getPredicate());
 }
 
+Value createRegionConstraint(OpBuilder &builder, tblgen::Region constraint) {
+  MLIRContext *ctx = builder.getContext();
+  const Record &predRec = constraint.getDef();
+
+  if (predRec.getName() == "AnyRegion") {
+    ValueRange entryBlockArgs = {};
+    auto op =
+        builder.create<irdl::RegionOp>(UnknownLoc::get(ctx), entryBlockArgs);
+    return op.getResult();
+  }
+
+  if (predRec.isSubClassOf("SizedRegion")) {
+    ValueRange entryBlockArgs = {};
+    auto ty = IntegerType::get(ctx, 32);
+    auto op = builder.create<irdl::RegionOp>(
+        UnknownLoc::get(ctx), entryBlockArgs,
+        IntegerAttr::get(ty, predRec.getValueAsInt("blocks")));
+    return op.getResult();
+  }
+
+  return createPredicate(builder, constraint.getPredicate());
+}
+
 /// Returns the name of the operation without the dialect prefix.
 static StringRef getOperatorName(tblgen::Operator &tblgenOp) {
   StringRef opName = tblgenOp.getDef().getValueAsString("opName");
@@ -404,6 +429,12 @@ irdl::OperationOp createIRDLOperation(OpBuilder &builder,
     attrNames.push_back(StringAttr::get(ctx, namedAttr.name));
   }
 
+  SmallVector<Value> regions;
+  for (auto namedRegion : tblgenOp.getRegions()) {
+    regions.push_back(
+        createRegionConstraint(consBuilder, namedRegion.constraint));
+  }
+
   // Create the operands and results operations.
   if (!operands.empty())
     consBuilder.create<irdl::OperandsOp>(UnknownLoc::get(ctx), operands,
@@ -414,6 +445,8 @@ irdl::OperationOp createIRDLOperation(OpBuilder &builder,
   if (!attributes.empty())
     consBuilder.create<irdl::AttributesOp>(UnknownLoc::get(ctx), attributes,
                                            ArrayAttr::get(ctx, attrNames));
+  if (!regions.empty())
+    consBuilder.create<irdl::RegionsOp>(UnknownLoc::get(ctx), regions);
 
   return op;
 }
@@ -451,8 +484,7 @@ static irdl::DialectOp createIRDLDialect(OpBuilder &builder) {
                                          StringAttr::get(ctx, selectedDialect));
 }
 
-static bool emitDialectIRDLDefs(const RecordKeeper &recordKeeper,
-                                raw_ostream &os) {
+static bool emitDialectIRDLDefs(const RecordKeeper &records, raw_ostream &os) {
   // Initialize.
   MLIRContext ctx;
   ctx.getOrLoadDialect<irdl::IRDLDialect>();
@@ -468,7 +500,7 @@ static bool emitDialectIRDLDefs(const RecordKeeper &recordKeeper,
   builder = builder.atBlockBegin(&dialect.getBody().emplaceBlock());
 
   for (const Record *type :
-       recordKeeper.getAllDerivedDefinitionsIfDefined("TypeDef")) {
+       records.getAllDerivedDefinitionsIfDefined("TypeDef")) {
     tblgen::TypeDef tblgenType(type);
     if (tblgenType.getDialect().getName() != selectedDialect)
       continue;
@@ -476,15 +508,14 @@ static bool emitDialectIRDLDefs(const RecordKeeper &recordKeeper,
   }
 
   for (const Record *attr :
-       recordKeeper.getAllDerivedDefinitionsIfDefined("AttrDef")) {
+       records.getAllDerivedDefinitionsIfDefined("AttrDef")) {
     tblgen::AttrDef tblgenAttr(attr);
     if (tblgenAttr.getDialect().getName() != selectedDialect)
       continue;
     createIRDLAttr(builder, tblgenAttr);
   }
 
-  for (const Record *def :
-       recordKeeper.getAllDerivedDefinitionsIfDefined("Op")) {
+  for (const Record *def : records.getAllDerivedDefinitionsIfDefined("Op")) {
     tblgen::Operator tblgenOp(def);
     if (tblgenOp.getDialectName() != selectedDialect)
       continue;
diff --git a/mlir/unittests/IR/CMakeLists.txt b/mlir/unittests/IR/CMakeLists.txt
index 05cb36e190316..547e536dd9cbb 100644
--- a/mlir/unittests/IR/CMakeLists.txt
+++ b/mlir/unittests/IR/CMakeLists.txt
@@ -8,6 +8,7 @@ add_mlir_unittest(MLIRIRTests
   InterfaceTest.cpp
   IRMapping.cpp
   InterfaceAttachmentTest.cpp
+  LocationTest.cpp
   OperationSupportTest.cpp
   PatternMatchTest.cpp
   ShapedTypeTest.cpp
diff --git a/mlir/unittests/IR/LocationTest.cpp b/mlir/unittests/IR/LocationTest.cpp
new file mode 100644
index 0000000000000..03374ee0b8467
--- /dev/null
+++ b/mlir/unittests/IR/LocationTest.cpp
@@ -0,0 +1,53 @@
+//===- LocationTest.cpp - unit tests for affine map API -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/IR/Location.h"
+#include "mlir/IR/Builders.h"
+#include "gtest/gtest.h"
+
+using namespace mlir;
+
+// Check that we only walk *locations* and not non-location attributes.
+TEST(LocationTest, Walk) {
+  MLIRContext ctx;
+  Builder builder(&ctx);
+  BoolAttr trueAttr = builder.getBoolAttr(true);
+
+  Location loc1 = FileLineColLoc::get(builder.getStringAttr("foo"), 1, 2);
+  Location loc2 = FileLineColLoc::get(builder.getStringAttr("foo"), 3, 4);
+  Location fused = builder.getFusedLoc({loc1, loc2}, trueAttr);
+
+  SmallVector<Attribute> visited;
+  fused->walk([&](Location l) {
+    visited.push_back(LocationAttr(l));
+    return WalkResult::advance();
+  });
+
+  EXPECT_EQ(llvm::ArrayRef(visited), ArrayRef<Attribute>({fused, loc1, loc2}));
+}
+
+// Check that we skip location attrs nested under a non-location attr.
+TEST(LocationTest, SkipNested) {
+  MLIRContext ctx;
+  Builder builder(&ctx);
+
+  Location loc1 = FileLineColLoc::get(builder.getStringAttr("foo"), 1, 2);
+  Location loc2 = FileLineColLoc::get(builder.getStringAttr("foo"), 3, 4);
+  Location loc3 = FileLineColLoc::get(builder.getStringAttr("bar"), 1, 2);
+  Location loc4 = FileLineColLoc::get(builder.getStringAttr("bar"), 3, 4);
+  ArrayAttr arr = builder.getArrayAttr({loc3, loc4});
+  Location fused = builder.getFusedLoc({loc1, loc2}, arr);
+
+  SmallVector<Attribute> visited;
+  fused->walk([&](Location l) {
+    visited.push_back(LocationAttr(l));
+    return WalkResult::advance();
+  });
+
+  EXPECT_EQ(llvm::ArrayRef(visited), ArrayRef<Attribute>({fused, loc1, loc2}));
+}
diff --git a/mlir/utils/lldb-scripts/mlirDataFormatters.py b/mlir/utils/lldb-scripts/mlirDataFormatters.py
index 54d3d70364040..38e8278eefbbd 100644
--- a/mlir/utils/lldb-scripts/mlirDataFormatters.py
+++ b/mlir/utils/lldb-scripts/mlirDataFormatters.py
@@ -60,6 +60,7 @@ def build_ptr_str_from_addr(addrValue: lldb.SBValue, type: lldb.SBType):
     "mlir::Float8E4M3FNUZType": '"f8E4M3FNUZ"',
     "mlir::Float8E4M3B11FNUZType": '"f8E4M3B11FNUZ"',
     "mlir::Float8E3M4Type": '"f8E3M4"',
+    "mlir::Float8E8M0FNUType": '"f8E8M0FNU"',
     "mlir::BFloat16Type": '"bf16"',
     "mlir::Float16Type": '"f16"',
     "mlir::FloatTF32Type": '"tf32"',
diff --git a/mlir/utils/tree-sitter-mlir/grammar.js b/mlir/utils/tree-sitter-mlir/grammar.js
index f7d916dfb57e2..2dadd46c4760c 100644
--- a/mlir/utils/tree-sitter-mlir/grammar.js
+++ b/mlir/utils/tree-sitter-mlir/grammar.js
@@ -231,7 +231,8 @@ const common = {
       token(seq(choice('si', 'ui', 'i'), /[1-9]/, repeat(/[0-9]/))),
   float_type : $ => token(
       choice('f16', 'f32', 'f64', 'f80', 'f128', 'bf16', 'f8E3M4', 'f8E4M3FN',
-             'f8E4M3', 'f8E5M2', 'f4E2M1FN', 'f6E2M3FN', 'f6E3M2FN')),
+             'f8E4M3', 'f8E5M2', 'f4E2M1FN', 'f6E2M3FN', 'f6E3M2FN',
+             'f8E8M0FNU')),
   index_type : $ => token('index'),
   none_type : $ => token('none'),
   complex_type : $ => seq(token('complex'), '<', $._prim_type, '>'),
diff --git a/openmp/docs/SupportAndFAQ.rst b/openmp/docs/SupportAndFAQ.rst
index cd2d6a4703221..dee707cf50f91 100644
--- a/openmp/docs/SupportAndFAQ.rst
+++ b/openmp/docs/SupportAndFAQ.rst
@@ -52,7 +52,7 @@ All patches go through the regular `LLVM review process
 Q: How to build an OpenMP GPU offload capable compiler?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 To build an *effective* OpenMP offload capable compiler, only one extra CMake
-option, ``LLVM_ENABLE_RUNTIMES="openmp"``, is needed when building LLVM (Generic
+option, ``LLVM_ENABLE_RUNTIMES="openmp;offload"``, is needed when building LLVM (Generic
 information about building LLVM is available `here
 <https://llvm.org/docs/GettingStarted.html>`__.). Make sure all backends that
 are targeted by OpenMP are enabled. That can be done by adjusting the CMake 
diff --git a/runtimes/cmake/Modules/WarningFlags.cmake b/runtimes/cmake/Modules/WarningFlags.cmake
index 068d22150d387..90edf3a957438 100644
--- a/runtimes/cmake/Modules/WarningFlags.cmake
+++ b/runtimes/cmake/Modules/WarningFlags.cmake
@@ -23,6 +23,7 @@ function(cxx_add_warning_flags target enable_werror enable_pedantic)
       -Wundef
       -Wunused-template
       -Wformat-nonliteral
+      -Wzero-length-array
       )
 
   if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
diff --git a/utils/bazel/llvm-project-overlay/clang-tools-extra/clang-tidy/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang-tools-extra/clang-tidy/BUILD.bazel
index 57c46c91402d9..db53df08fecc8 100644
--- a/utils/bazel/llvm-project-overlay/clang-tools-extra/clang-tidy/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang-tools-extra/clang-tidy/BUILD.bazel
@@ -79,6 +79,7 @@ clang_tidy_library(
         "//clang:rewrite",
         "//clang:sema",
         "//clang:serialization",
+        "//clang:static_analyzer_checkers_gen",
         "//clang:tooling",
         "//clang:tooling_core",
     ] + select({
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 140d48c8f9684..8f3bbe68648fe 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -148,6 +148,9 @@ libc_support_library(
 libc_support_library(
     name = "hdr_stdio_macros",
     hdrs = ["hdr/stdio_macros.h"],
+    deps = [
+        ":hdr_stdio_overlay",
+    ],
 )
 
 libc_support_library(
@@ -155,6 +158,11 @@ libc_support_library(
     hdrs = ["hdr/limits_macros.h"],
 )
 
+libc_support_library(
+    name = "hdr_stdio_overlay",
+    hdrs = ["hdr/stdio_overlay.h"],
+)
+
 ############################ Type Proxy Header Files ###########################
 
 libc_support_library(
@@ -210,11 +218,17 @@ libc_support_library(
 libc_support_library(
     name = "types_off_t",
     hdrs = ["hdr/types/off_t.h"],
+    deps = [
+        ":hdr_stdio_overlay",
+    ],
 )
 
 libc_support_library(
     name = "types_FILE",
     hdrs = ["hdr/types/FILE.h"],
+    deps = [
+        ":hdr_stdio_overlay",
+    ],
 )
 
 libc_support_library(
@@ -749,6 +763,7 @@ libc_support_library(
         ":__support_threads_mutex",
         ":errno",
         ":hdr_stdio_macros",
+        ":hdr_stdio_overlay",
         ":types_off_t",
     ],
 )
@@ -761,6 +776,7 @@ libc_support_library(
         ":__support_error_or",
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_stdio_overlay",
         ":types_off_t",
     ],
 )
@@ -3893,9 +3909,9 @@ libc_support_library(
         ":__support_ctype_utils",
         ":__support_fputil_fp_bits",
         ":__support_str_to_integer",
+        ":errno",
         ":printf_config",
         ":printf_core_structs",
-        ":errno"
     ],
 )
 
@@ -4104,6 +4120,7 @@ libc_function(
         ":__support_common",
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_stdio_overlay",
         ":types_FILE",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl b/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
index 1da0a24b7791a..7cb30739f1e9c 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
+++ b/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
@@ -36,7 +36,6 @@ def libc_test(name, srcs, libc_function_deps = [], copts = [], deps = [], local_
             "//libc/test/UnitTest:LibcUnitTest",
             "//libc:__support_macros_config",
         ] + deps,
-        features = ["-link_llvmlibc"],  # Do not link libllvmlibc.a
         copts = copts + libc_common_copts(),
         linkstatic = 1,
         **kwargs
diff --git a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
index a35e6c8361506..96202bf47b848 100644
--- a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
@@ -1091,6 +1091,7 @@ cc_library(
         "//lldb:Utility",
         "//lldb:UtilityPrivateHeaders",
         "//llvm:Core",
+        "//llvm:Support",
         "//llvm:TargetParser",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index 727b19e47ec82..4e195949a6d78 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -1598,6 +1598,18 @@ gentbl(
     td_srcs = [":omp_td_files"],
 )
 
+cc_library(
+    name = "FrontendAtomic",
+    hdrs = glob([
+        "include/llvm/Frontend/Atomic/*.h",
+    ]),
+    copts = llvm_copts,
+    deps = [
+        ":Support",
+        ":ir_headers",
+    ],
+)
+
 cc_library(
     name = "FrontendOpenMP",
     srcs = glob([
@@ -1618,6 +1630,7 @@ cc_library(
         ":BitReader",
         ":Core",
         ":Demangle",
+        ":FrontendAtomic",
         ":FrontendOffloading",
         ":MC",
         ":Scalar",
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index a2f70f765eec3..1c91ede63d0e9 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -3745,11 +3745,14 @@ cc_library(
     includes = ["include"],
     deps = [
         ":ArithDialect",
+        ":ArithUtils",
         ":BytecodeOpInterface",
         ":DialectUtils",
         ":IR",
+        ":MaskableOpInterface",
         ":ShapedOpInterfaces",
         ":SideEffectInterfaces",
+        ":VectorDialect",
         ":ViewLikeInterface",
         ":XeGPUEnumsIncGen",
         ":XeGPUIncGen",
@@ -6299,11 +6302,17 @@ gentbl_cc_library(
 
 cc_library(
     name = "NVVMDialect",
-    srcs = ["lib/Dialect/LLVMIR/IR/NVVMDialect.cpp"],
-    hdrs = ["include/mlir/Dialect/LLVMIR/NVVMDialect.h"],
+    srcs = [
+        "lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp",
+        "lib/Dialect/LLVMIR/IR/NVVMDialect.cpp",
+    ],
+    hdrs = [
+        "include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.h",
+        "include/mlir/Dialect/LLVMIR/NVVMDialect.h",
+    ],
     includes = ["include"],
     deps = [
-        ":BasicPtxBuilderInterface",
+        ":BasicPtxBuilderIntGen",
         ":BytecodeOpInterface",
         ":ConvertToLLVMInterface",
         ":DialectUtils",
@@ -6504,21 +6513,6 @@ gentbl_cc_library(
     deps = [":NVVMOpsTdFiles"],
 )
 
-cc_library(
-    name = "BasicPtxBuilderInterface",
-    srcs = ["lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp"],
-    hdrs = [
-        "include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.h",
-    ],
-    includes = ["include"],
-    deps = [
-        ":BasicPtxBuilderIntGen",
-        ":IR",
-        ":LLVMDialect",
-        ":Support",
-    ],
-)
-
 cc_library(
     name = "NVVMToLLVM",
     srcs = ["lib/Conversion/NVVMToLLVM/NVVMToLLVM.cpp"],
@@ -7694,6 +7688,7 @@ cc_library(
         ":DialectUtils",
         ":TensorDialect",
         ":ValueBoundsOpInterface",
+        ":VectorDialect",
     ],
 )
 
@@ -13320,6 +13315,7 @@ cc_library(
         ":ArithUtils",
         ":MemRefDialect",
         ":VectorDialect",
+        ":ViewLikeInterface",
         "//llvm:Support",
     ],
 )
@@ -13901,6 +13897,7 @@ cc_library(
         ":BufferViewFlowOpInterfaceIncGen",
         ":BufferizableOpInterfaceIncGen",
         ":BufferizationEnumsIncGen",
+        ":FunctionInterfaces",
         ":IR",
         ":Support",
         "//llvm:Support",