diff --git a/clang/test/Sema/constant-builtins-fminimum-num.cpp b/clang/test/Sema/constant-builtins-fminimum-num.cpp
@@ -0,0 +1,69 @@
+// RUN: %clang_cc1 -std=c++17 -fsyntax-only -verify %s
+// FIXME: %clang_cc1 -std=c++17 -fsyntax-only -verify -fexperimental-new-constant-interpreter %s
+// expected-no-diagnostics
+
+constexpr double NaN = __builtin_nan("");
+constexpr double SNaN = __builtin_nans("");
+constexpr double Inf = __builtin_inf();
+constexpr double NegInf = -__builtin_inf();
+
+#define FMINIMUMNUM_TEST_SIMPLE(T, FUNC)                           \
+    static_assert(T(1.2345) == FUNC(T(1.2345), T(6.7890))); \
+    static_assert(T(1.2345) == FUNC(T(6.7890), T(1.2345)));
+
+#define FMINIMUMNUM_TEST_NAN(T, FUNC)                          \
+    static_assert(Inf == FUNC(NaN, Inf));               \
+    static_assert(NegInf == FUNC(NegInf, NaN));         \
+    static_assert(0.0 == FUNC(NaN, 0.0));               \
+    static_assert(-0.0 == FUNC(-0.0, NaN));             \
+    static_assert(T(-1.2345) == FUNC(NaN, T(-1.2345))); \
+    static_assert(T(1.2345) == FUNC(T(1.2345), NaN));   \
+    static_assert(__builtin_isnan(FUNC(NaN, NaN)));
+
+#define FMINIMUMNUM_TEST_SNAN(T, FUNC)                          \
+    static_assert(Inf == FUNC(SNaN, Inf));               \
+    static_assert(NegInf == FUNC(NegInf, SNaN));         \
+    static_assert(0.0 == FUNC(SNaN, 0.0));               \
+    static_assert(-0.0 == FUNC(-0.0, SNaN));             \
+    static_assert(T(-1.2345) == FUNC(SNaN, T(-1.2345))); \
+    static_assert(T(1.2345) == FUNC(T(1.2345), SNaN));   \
+    static_assert(__builtin_isnan(FUNC(SNaN, SNaN)));    \
+    static_assert(__builtin_isnan(FUNC(NaN, SNaN)));    \
+    static_assert(!__builtin_issignaling(FUNC(SNaN, SNaN)));  \
+    static_assert(!__builtin_issignaling(FUNC(NaN, SNaN)));
+
+#define FMINIMUMNUM_TEST_INF(T, FUNC)                        \
+    static_assert(NegInf == FUNC(NegInf, Inf));       \
+    static_assert(0.0 == FUNC(Inf, 0.0));             \
+    static_assert(-0.0 == FUNC(-0.0, Inf));           \
+    static_assert(T(1.2345) == FUNC(Inf, T(1.2345))); \
+    static_assert(T(-1.2345) == FUNC(T(-1.2345), Inf));
+
+#define FMINIMUMNUM_TEST_NEG_INF(T, FUNC)                     \
+    static_assert(NegInf == FUNC(Inf, NegInf));        \
+    static_assert(NegInf == FUNC(NegInf, 0.0));        \
+    static_assert(NegInf == FUNC(-0.0, NegInf));       \
+    static_assert(NegInf == FUNC(NegInf, T(-1.2345))); \
+    static_assert(NegInf == FUNC(T(1.2345), NegInf));
+
+#define FMINIMUMNUM_TEST_BOTH_ZERO(T, FUNC)                                 \
+    static_assert(__builtin_copysign(1.0, FUNC(0.0, 0.0)) == 1.0);   \
+    static_assert(__builtin_copysign(1.0, FUNC(-0.0, 0.0)) == -1.0); \
+    static_assert(__builtin_copysign(1.0, FUNC(0.0, -0.0)) == -1.0); \
+    static_assert(__builtin_copysign(1.0, FUNC(-0.0, -0.0)) == -1.0);
+
+#define LIST_FMINIMUMNUM_TESTS(T, FUNC) \
+    FMINIMUMNUM_TEST_SIMPLE(T, FUNC)    \
+    FMINIMUMNUM_TEST_NAN(T, FUNC)       \
+    FMINIMUMNUM_TEST_SNAN(T, FUNC)      \
+    FMINIMUMNUM_TEST_INF(T, FUNC)       \
+    FMINIMUMNUM_TEST_NEG_INF(T, FUNC)   \
+    FMINIMUMNUM_TEST_BOTH_ZERO(T, FUNC)
+
+LIST_FMINIMUMNUM_TESTS(double, __builtin_fminimum_num)
+LIST_FMINIMUMNUM_TESTS(float, __builtin_fminimum_numf)
+LIST_FMINIMUMNUM_TESTS((long double), __builtin_fminimum_numl)
+LIST_FMINIMUMNUM_TESTS(__fp16, __builtin_fminimum_numf16)
+#ifdef __FLOAT128__
+LIST_FMINIMUMNUM_TESTS(__float128, __builtin_fminimum_numf128)
+#endif
diff --git a/compiler-rt/lib/builtins/cpu_model/riscv.c b/compiler-rt/lib/builtins/cpu_model/riscv.c
@@ -22,8 +22,8 @@ struct {
 
 struct {
   unsigned mvendorid;
-  unsigned marchid;
-  unsigned mimpid;
+  unsigned long long marchid;
+  unsigned long long mimpid;
 } __riscv_cpu_model __attribute__((visibility("hidden"), nocommon));
 
 // NOTE: Should sync-up with RISCVFeatures.td

diff --git a/flang/include/flang/Optimizer/CodeGen/Target.h b/flang/include/flang/Optimizer/CodeGen/Target.h
@@ -126,6 +126,11 @@ class CodeGenSpecifics {
   structArgumentType(mlir::Location loc, fir::RecordType recTy,
                      const Marshalling &previousArguments) const = 0;
 
+  /// Type representation of a `fir.type<T>` type argument when returned by
+  /// value. Such value may need to be converted to a hidden reference argument.
+  virtual Marshalling structReturnType(mlir::Location loc,
+                                       fir::RecordType eleTy) const = 0;
+
   /// Type representation of a `boxchar<n>` type argument when passed by value.
   /// An argument value may need to be passed as a (safe) reference argument.
   ///

diff --git a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
@@ -177,6 +177,27 @@ inline mlir::NamedAttribute getAdaptToByRefAttr(Builder &builder) {
 }
 
 bool isDummyArgument(mlir::Value v);
+
+template <fir::FortranProcedureFlagsEnum Flag>
+inline bool hasProcedureAttr(fir::FortranProcedureFlagsEnumAttr flags) {
+  return flags && bitEnumContainsAny(flags.getValue(), Flag);
+}
+
+template <fir::FortranProcedureFlagsEnum Flag>
+inline bool hasProcedureAttr(mlir::Operation *op) {
+  if (auto firCallOp = mlir::dyn_cast<fir::CallOp>(op))
+    return hasProcedureAttr<Flag>(firCallOp.getProcedureAttrsAttr());
+  if (auto firCallOp = mlir::dyn_cast<fir::DispatchOp>(op))
+    return hasProcedureAttr<Flag>(firCallOp.getProcedureAttrsAttr());
+  return hasProcedureAttr<Flag>(
+      op->getAttrOfType<fir::FortranProcedureFlagsEnumAttr>(
+          getFortranProcedureFlagsAttrName()));
+}
+
+inline bool hasBindcAttr(mlir::Operation *op) {
+  return hasProcedureAttr<fir::FortranProcedureFlagsEnum::bind_c>(op);
+}
+
 } // namespace fir
 
 #endif // FORTRAN_OPTIMIZER_DIALECT_FIROPSSUPPORT_H
diff --git a/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp b/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
@@ -257,6 +257,10 @@ AliasAnalysis::Source AliasAnalysis::getSource(mlir::Value v,
   while (defOp && !breakFromLoop) {
     ty = defOp->getResultTypes()[0];
     llvm::TypeSwitch<Operation *>(defOp)
+        .Case<hlfir::AsExprOp>([&](auto op) {
+          v = op.getVar();
+          defOp = v.getDefiningOp();
+        })
         .Case<fir::AllocaOp, fir::AllocMemOp>([&](auto op) {
           // Unique memory allocation.
           type = SourceKind::Allocate;

diff --git a/flang/lib/Optimizer/CodeGen/Target.cpp b/flang/lib/Optimizer/CodeGen/Target.cpp
@@ -100,6 +100,11 @@ struct GenericTarget : public CodeGenSpecifics {
     TODO(loc, "passing VALUE BIND(C) derived type for this target");
   }
 
+  CodeGenSpecifics::Marshalling
+  structReturnType(mlir::Location loc, fir::RecordType ty) const override {
+    TODO(loc, "returning BIND(C) derived type for this target");
+  }
+
   CodeGenSpecifics::Marshalling
   integerArgumentType(mlir::Location loc,
                       mlir::IntegerType argTy) const override {
@@ -533,14 +538,17 @@ struct TargetX86_64 : public GenericTarget<TargetX86_64> {
   /// When \p recTy is a one field record type that can be passed
   /// like the field on its own, returns the field type. Returns
   /// a null type otherwise.
-  mlir::Type passAsFieldIfOneFieldStruct(fir::RecordType recTy) const {
+  mlir::Type passAsFieldIfOneFieldStruct(fir::RecordType recTy,
+                                         bool allowComplex = false) const {
     auto typeList = recTy.getTypeList();
     if (typeList.size() != 1)
       return {};
     mlir::Type fieldType = typeList[0].second;
     if (mlir::isa<mlir::FloatType, mlir::IntegerType, fir::LogicalType>(
             fieldType))
       return fieldType;
+    if (allowComplex && mlir::isa<mlir::ComplexType>(fieldType))
+      return fieldType;
     if (mlir::isa<fir::CharacterType>(fieldType)) {
       // Only CHARACTER(1) are expected in BIND(C) contexts, which is the only
       // contexts where derived type may be passed in registers.
@@ -593,7 +601,7 @@ struct TargetX86_64 : public GenericTarget<TargetX86_64> {
     postMerge(byteOffset, Lo, Hi);
     if (Lo == ArgClass::Memory || Lo == ArgClass::X87 ||
         Lo == ArgClass::ComplexX87)
-      return passOnTheStack(loc, recTy);
+      return passOnTheStack(loc, recTy, /*isResult=*/false);
     int neededIntRegisters = 0;
     int neededSSERegisters = 0;
     if (Lo == ArgClass::SSE)
@@ -609,7 +617,7 @@ struct TargetX86_64 : public GenericTarget<TargetX86_64> {
     // all in registers or all on the stack).
     if (!hasEnoughRegisters(loc, neededIntRegisters, neededSSERegisters,
                             previousArguments))
-      return passOnTheStack(loc, recTy);
+      return passOnTheStack(loc, recTy, /*isResult=*/false);
 
     if (auto fieldType = passAsFieldIfOneFieldStruct(recTy)) {
       CodeGenSpecifics::Marshalling marshal;
@@ -641,17 +649,65 @@ struct TargetX86_64 : public GenericTarget<TargetX86_64> {
     return marshal;
   }
 
+  CodeGenSpecifics::Marshalling
+  structReturnType(mlir::Location loc, fir::RecordType recTy) const override {
+    std::uint64_t byteOffset = 0;
+    ArgClass Lo, Hi;
+    Lo = Hi = ArgClass::NoClass;
+    byteOffset = classifyStruct(loc, recTy, byteOffset, Lo, Hi);
+    mlir::MLIRContext *context = recTy.getContext();
+    postMerge(byteOffset, Lo, Hi);
+    if (Lo == ArgClass::Memory)
+      return passOnTheStack(loc, recTy, /*isResult=*/true);
+
+    // Note that X87/ComplexX87 are passed in memory, but returned via %st0
+    // %st1 registers. Here, they are returned as fp80 or {fp80, fp80} by
+    // passAsFieldIfOneFieldStruct, and LLVM will use the expected registers.
+
+    // Note that {_Complex long double} is not 100% clear from an ABI
+    // perspective because the aggregate post merger rules say it should be
+    // passed in memory because it is bigger than 2 eight bytes. This has the
+    // funny effect of
+    // {_Complex long double} return to be dealt with differently than
+    // _Complex long double.
+
+    if (auto fieldType =
+            passAsFieldIfOneFieldStruct(recTy, /*allowComplex=*/true)) {
+      if (auto complexType = mlir::dyn_cast<mlir::ComplexType>(fieldType))
+        return complexReturnType(loc, complexType.getElementType());
+      CodeGenSpecifics::Marshalling marshal;
+      marshal.emplace_back(fieldType, AT{});
+      return marshal;
+    }
+
+    if (Hi == ArgClass::NoClass || Hi == ArgClass::SSEUp) {
+      // Return a single integer or floating point argument.
+      mlir::Type lowType = pickLLVMArgType(loc, context, Lo, byteOffset);
+      CodeGenSpecifics::Marshalling marshal;
+      marshal.emplace_back(lowType, AT{});
+      return marshal;
+    }
+    // Will be returned in two different registers. Generate {lowTy, HiTy} for
+    // the LLVM IR result type.
+    CodeGenSpecifics::Marshalling marshal;
+    mlir::Type lowType = pickLLVMArgType(loc, context, Lo, 8u);
+    mlir::Type hiType = pickLLVMArgType(loc, context, Hi, byteOffset - 8u);
+    marshal.emplace_back(mlir::TupleType::get(context, {lowType, hiType}),
+                         AT{});
+    return marshal;
+  }
+
   /// Marshal an argument that must be passed on the stack.
-  CodeGenSpecifics::Marshalling passOnTheStack(mlir::Location loc,
-                                               mlir::Type ty) const {
+  CodeGenSpecifics::Marshalling
+  passOnTheStack(mlir::Location loc, mlir::Type ty, bool isResult) const {
     CodeGenSpecifics::Marshalling marshal;
     auto sizeAndAlign =
         fir::getTypeSizeAndAlignmentOrCrash(loc, ty, getDataLayout(), kindMap);
     // The stack is always 8 byte aligned (note 14 in 3.2.3).
     unsigned short align =
         std::max(sizeAndAlign.second, static_cast<unsigned short>(8));
     marshal.emplace_back(fir::ReferenceType::get(ty),
-                         AT{align, /*byval=*/true, /*sret=*/false});
+                         AT{align, /*byval=*/!isResult, /*sret=*/isResult});
     return marshal;
   }
 };

diff --git a/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp b/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp
@@ -142,20 +142,16 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
 
   mlir::ModuleOp getModule() { return getOperation(); }
 
-  template <typename A, typename B, typename C>
+  template <typename Ty, typename Callback>
   std::optional<std::function<mlir::Value(mlir::Operation *)>>
-  rewriteCallComplexResultType(
-      mlir::Location loc, A ty, B &newResTys,
-      fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs, C &newOpers,
-      mlir::Value &savedStackPtr) {
-    if (noComplexConversion) {
-      newResTys.push_back(ty);
-      return std::nullopt;
-    }
-    auto m = specifics->complexReturnType(loc, ty.getElementType());
-    // Currently targets mandate COMPLEX is a single aggregate or packed
-    // scalar, including the sret case.
-    assert(m.size() == 1 && "target of complex return not supported");
+  rewriteCallResultType(mlir::Location loc, mlir::Type originalResTy,
+                        Ty &newResTys,
+                        fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs,
+                        Callback &newOpers, mlir::Value &savedStackPtr,
+                        fir::CodeGenSpecifics::Marshalling &m) {
+    // Currently, targets mandate COMPLEX or STRUCT is a single aggregate or
+    // packed scalar, including the sret case.
+    assert(m.size() == 1 && "return type not supported on this target");
     auto resTy = std::get<mlir::Type>(m[0]);
     auto attr = std::get<fir::CodeGenSpecifics::Attributes>(m[0]);
     if (attr.isSRet()) {
@@ -170,7 +166,7 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
       newInTyAndAttrs.push_back(m[0]);
       newOpers.push_back(stack);
       return [=](mlir::Operation *) -> mlir::Value {
-        auto memTy = fir::ReferenceType::get(ty);
+        auto memTy = fir::ReferenceType::get(originalResTy);
         auto cast = rewriter->create<fir::ConvertOp>(loc, memTy, stack);
         return rewriter->create<fir::LoadOp>(loc, cast);
       };
@@ -180,11 +176,41 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
       // We are going to generate an alloca, so save the stack pointer.
       if (!savedStackPtr)
         savedStackPtr = genStackSave(loc);
-      return this->convertValueInMemory(loc, call->getResult(0), ty,
+      return this->convertValueInMemory(loc, call->getResult(0), originalResTy,
                                         /*inputMayBeBigger=*/true);
     };
   }
 
+  template <typename Ty, typename Callback>
+  std::optional<std::function<mlir::Value(mlir::Operation *)>>
+  rewriteCallComplexResultType(
+      mlir::Location loc, mlir::ComplexType ty, Ty &newResTys,
+      fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs, Callback &newOpers,
+      mlir::Value &savedStackPtr) {
+    if (noComplexConversion) {
+      newResTys.push_back(ty);
+      return std::nullopt;
+    }
+    auto m = specifics->complexReturnType(loc, ty.getElementType());
+    return rewriteCallResultType(loc, ty, newResTys, newInTyAndAttrs, newOpers,
+                                 savedStackPtr, m);
+  }
+
+  template <typename Ty, typename Callback>
+  std::optional<std::function<mlir::Value(mlir::Operation *)>>
+  rewriteCallStructResultType(
+      mlir::Location loc, fir::RecordType recTy, Ty &newResTys,
+      fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs, Callback &newOpers,
+      mlir::Value &savedStackPtr) {
+    if (noStructConversion) {
+      newResTys.push_back(recTy);
+      return std::nullopt;
+    }
+    auto m = specifics->structReturnType(loc, recTy);
+    return rewriteCallResultType(loc, recTy, newResTys, newInTyAndAttrs,
+                                 newOpers, savedStackPtr, m);
+  }
+
   void passArgumentOnStackOrWithNewType(
       mlir::Location loc, fir::CodeGenSpecifics::TypeAndAttr newTypeAndAttr,
       mlir::Type oldType, mlir::Value oper,
@@ -356,6 +382,11 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
                                                 newInTyAndAttrs, newOpers,
                                                 savedStackPtr);
           })
+          .template Case<fir::RecordType>([&](fir::RecordType recTy) {
+            wrap = rewriteCallStructResultType(loc, recTy, newResTys,
+                                               newInTyAndAttrs, newOpers,
+                                               savedStackPtr);
+          })
           .Default([&](mlir::Type ty) { newResTys.push_back(ty); });
     } else if (fnTy.getResults().size() > 1) {
       TODO(loc, "multiple results not supported yet");
@@ -562,6 +593,24 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
     }
   }
 
+  template <typename Ty>
+  void
+  lowerStructSignatureRes(mlir::Location loc, fir::RecordType recTy,
+                          Ty &newResTys,
+                          fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs) {
+    if (noComplexConversion) {
+      newResTys.push_back(recTy);
+      return;
+    } else {
+      for (auto &tup : specifics->structReturnType(loc, recTy)) {
+        if (std::get<fir::CodeGenSpecifics::Attributes>(tup).isSRet())
+          newInTyAndAttrs.push_back(tup);
+        else
+          newResTys.push_back(std::get<mlir::Type>(tup));
+      }
+    }
+  }
+
   void
   lowerStructSignatureArg(mlir::Location loc, fir::RecordType recTy,
                           fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs) {
@@ -595,6 +644,9 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
           .Case<mlir::ComplexType>([&](mlir::ComplexType ty) {
             lowerComplexSignatureRes(loc, ty, newResTys, newInTyAndAttrs);
           })
+          .Case<fir::RecordType>([&](fir::RecordType ty) {
+            lowerStructSignatureRes(loc, ty, newResTys, newInTyAndAttrs);
+          })
           .Default([&](mlir::Type ty) { newResTys.push_back(ty); });
     }
     llvm::SmallVector<mlir::Type> trailingInTys;
@@ -696,7 +748,8 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
     for (auto ty : func.getResults())
       if ((mlir::isa<fir::BoxCharType>(ty) && !noCharacterConversion) ||
           (fir::isa_complex(ty) && !noComplexConversion) ||
-          (mlir::isa<mlir::IntegerType>(ty) && hasCCallingConv)) {
+          (mlir::isa<mlir::IntegerType>(ty) && hasCCallingConv) ||
+          (mlir::isa<fir::RecordType>(ty) && !noStructConversion)) {
         LLVM_DEBUG(llvm::dbgs() << "rewrite " << signature << " for target\n");
         return false;
       }
@@ -770,6 +823,9 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
                                                 rewriter->getUnitAttr()));
             newResTys.push_back(retTy);
           })
+          .Case<fir::RecordType>([&](fir::RecordType recTy) {
+            doStructReturn(func, recTy, newResTys, newInTyAndAttrs, fixups);
+          })
           .Default([&](mlir::Type ty) { newResTys.push_back(ty); });
 
     // Saved potential shift in argument. Handling of result can add arguments
@@ -1062,21 +1118,12 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
     return false;
   }
 
-  /// Convert a complex return value. This can involve converting the return
-  /// value to a "hidden" first argument or packing the complex into a wide
-  /// GPR.
   template <typename Ty, typename FIXUPS>
-  void doComplexReturn(mlir::func::FuncOp func, mlir::ComplexType cmplx,
-                       Ty &newResTys,
-                       fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs,
-                       FIXUPS &fixups) {
-    if (noComplexConversion) {
-      newResTys.push_back(cmplx);
-      return;
-    }
-    auto m =
-        specifics->complexReturnType(func.getLoc(), cmplx.getElementType());
-    assert(m.size() == 1);
+  void doReturn(mlir::func::FuncOp func, Ty &newResTys,
+                fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs,
+                FIXUPS &fixups, fir::CodeGenSpecifics::Marshalling &m) {
+    assert(m.size() == 1 &&
+           "expect result to be turned into single argument or result so far");
     auto &tup = m[0];
     auto attr = std::get<fir::CodeGenSpecifics::Attributes>(tup);
     auto argTy = std::get<mlir::Type>(tup);
@@ -1117,6 +1164,36 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
     newResTys.push_back(argTy);
   }
 
+  /// Convert a complex return value. This can involve converting the return
+  /// value to a "hidden" first argument or packing the complex into a wide
+  /// GPR.
+  template <typename Ty, typename FIXUPS>
+  void doComplexReturn(mlir::func::FuncOp func, mlir::ComplexType cmplx,
+                       Ty &newResTys,
+                       fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs,
+                       FIXUPS &fixups) {
+    if (noComplexConversion) {
+      newResTys.push_back(cmplx);
+      return;
+    }
+    auto m =
+        specifics->complexReturnType(func.getLoc(), cmplx.getElementType());
+    doReturn(func, newResTys, newInTyAndAttrs, fixups, m);
+  }
+
+  template <typename Ty, typename FIXUPS>
+  void doStructReturn(mlir::func::FuncOp func, fir::RecordType recTy,
+                      Ty &newResTys,
+                      fir::CodeGenSpecifics::Marshalling &newInTyAndAttrs,
+                      FIXUPS &fixups) {
+    if (noStructConversion) {
+      newResTys.push_back(recTy);
+      return;
+    }
+    auto m = specifics->structReturnType(func.getLoc(), recTy);
+    doReturn(func, newResTys, newInTyAndAttrs, fixups, m);
+  }
+
   template <typename FIXUPS>
   void
   createFuncOpArgFixups(mlir::func::FuncOp func,

diff --git a/flang/lib/Optimizer/Transforms/AbstractResult.cpp b/flang/lib/Optimizer/Transforms/AbstractResult.cpp
@@ -32,6 +32,36 @@ using namespace mlir;
 namespace fir {
 namespace {
 
+// Helper to only build the symbol table if needed because its build time is
+// linear on the number of symbols in the module.
+struct LazySymbolTable {
+  LazySymbolTable(mlir::Operation *op)
+      : module{op->getParentOfType<mlir::ModuleOp>()} {}
+  void build() {
+    if (table)
+      return;
+    table = std::make_unique<mlir::SymbolTable>(module);
+  }
+
+  template <typename T>
+  T lookup(llvm::StringRef name) {
+    build();
+    return table->lookup<T>(name);
+  }
+
+private:
+  std::unique_ptr<mlir::SymbolTable> table;
+  mlir::ModuleOp module;
+};
+
+bool hasScalarDerivedResult(mlir::FunctionType funTy) {
+  // C_PTR/C_FUNPTR are results to void* in this pass, do not consider
+  // them as normal derived types.
+  return funTy.getNumResults() == 1 &&
+         mlir::isa<fir::RecordType>(funTy.getResult(0)) &&
+         !fir::isa_builtin_cptr_type(funTy.getResult(0));
+}
+
 static mlir::Type getResultArgumentType(mlir::Type resultType,
                                         bool shouldBoxResult) {
   return llvm::TypeSwitch<mlir::Type, mlir::Type>(resultType)
@@ -190,7 +220,15 @@ class SaveResultOpConversion
   llvm::LogicalResult
   matchAndRewrite(fir::SaveResultOp op,
                   mlir::PatternRewriter &rewriter) const override {
-    rewriter.eraseOp(op);
+    mlir::Operation *call = op.getValue().getDefiningOp();
+    mlir::Type type = op.getValue().getType();
+    if (mlir::isa<fir::RecordType>(type) && call && fir::hasBindcAttr(call) &&
+        !fir::isa_builtin_cptr_type(type)) {
+      rewriter.replaceOpWithNewOp<fir::StoreOp>(op, op.getValue(),
+                                                op.getMemref());
+    } else {
+      rewriter.eraseOp(op);
+    }
     return mlir::success();
   }
 };
@@ -300,6 +338,12 @@ class AbstractResultOpt
     auto *context = &getContext();
     // Convert function type itself if it has an abstract result.
     auto funcTy = mlir::cast<mlir::FunctionType>(func.getFunctionType());
+    // Scalar derived result of BIND(C) function must be returned according
+    // to the C struct return ABI which is target dependent and implemented in
+    // the target-rewrite pass.
+    if (hasScalarDerivedResult(funcTy) &&
+        fir::hasBindcAttr(func.getOperation()))
+      return;
     if (hasAbstractResult(funcTy)) {
       if (fir::isa_builtin_cptr_type(funcTy.getResult(0))) {
         func.setType(getCPtrFunctionType(funcTy));
@@ -395,6 +439,8 @@ class AbstractResultOpt
       return;
     }
 
+    LazySymbolTable symbolTable(op);
+
     mlir::RewritePatternSet patterns(context);
     mlir::ConversionTarget target = *context;
     const bool shouldBoxResult = this->passResultAsBox.getValue();
@@ -409,14 +455,29 @@ class AbstractResultOpt
                            mlir::func::FuncDialect>();
     target.addIllegalOp<fir::SaveResultOp>();
     target.addDynamicallyLegalOp<fir::CallOp>([](fir::CallOp call) {
-      return !hasAbstractResult(call.getFunctionType());
+      mlir::FunctionType funTy = call.getFunctionType();
+      if (hasScalarDerivedResult(funTy) &&
+          fir::hasBindcAttr(call.getOperation()))
+        return true;
+      return !hasAbstractResult(funTy);
     });
-    target.addDynamicallyLegalOp<fir::AddrOfOp>([](fir::AddrOfOp addrOf) {
-      if (auto funTy = mlir::dyn_cast<mlir::FunctionType>(addrOf.getType()))
+    target.addDynamicallyLegalOp<fir::AddrOfOp>([&symbolTable](
+                                                    fir::AddrOfOp addrOf) {
+      if (auto funTy = mlir::dyn_cast<mlir::FunctionType>(addrOf.getType())) {
+        if (hasScalarDerivedResult(funTy)) {
+          auto func = symbolTable.lookup<mlir::func::FuncOp>(
+              addrOf.getSymbol().getRootReference().getValue());
+          return func && fir::hasBindcAttr(func.getOperation());
+        }
         return !hasAbstractResult(funTy);
+      }
       return true;
     });
     target.addDynamicallyLegalOp<fir::DispatchOp>([](fir::DispatchOp dispatch) {
+      mlir::FunctionType funTy = dispatch.getFunctionType();
+      if (hasScalarDerivedResult(funTy) &&
+          fir::hasBindcAttr(dispatch.getOperation()))
+        return true;
       return !hasAbstractResult(dispatch.getFunctionType());
     });
 

diff --git a/flang/test/Analysis/AliasAnalysis/alias-analysis-9.fir b/flang/test/Analysis/AliasAnalysis/alias-analysis-9.fir
@@ -0,0 +1,48 @@
+// Use --mlir-disable-threading so that the AA queries are serialized
+// as well as its diagnostic output.
+// RUN: fir-opt %s -pass-pipeline='builtin.module(func.func(test-fir-alias-analysis))' -split-input-file --mlir-disable-threading 2>&1 | FileCheck %s
+
+// Fortran source code:
+// program TestTmpArrayAssignment
+//   integer :: a, b, c
+//   integer :: arrayD(3)
+//
+//   arrayD = [ a, b, c ]
+// end program TestTmpArrayAssignment
+
+// CHECK-LABEL: Testing : "_QPTestTmpArrayAssignment"
+// CHECK-DAG: ArrayD#0 <-> tmp_array#0: NoAlias
+func.func @_QPTestTmpArrayAssignment() attributes {fir.bindc_name = "testtmparrayassignment"} {
+  %0 = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
+  %1:2 = hlfir.declare %0 {uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %c3 = arith.constant 3 : index
+  %2 = fir.alloca !fir.array<3xi32> {bindc_name = "arrayd", uniq_name = "_QFEarrayd", test.ptr = "ArrayD" }
+  %3 = fir.shape %c3 : (index) -> !fir.shape<1>
+  %4:2 = hlfir.declare %2(%3) {uniq_name = "_QFEarrayd"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
+  %5 = fir.alloca i32 {bindc_name = "b", uniq_name = "_QFEb"}
+  %6:2 = hlfir.declare %5 {uniq_name = "_QFEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %7 = fir.alloca i32 {bindc_name = "c", uniq_name = "_QFEc"}
+  %8:2 = hlfir.declare %7 {uniq_name = "_QFEc"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %c3_0 = arith.constant 3 : index
+  %c1 = arith.constant 1 : index
+  %c1_1 = arith.constant 1 : index
+  %9 = fir.allocmem !fir.array<3xi32> {bindc_name = ".tmp.arrayctor", uniq_name = ""}
+  %10 = fir.shape %c3_0 : (index) -> !fir.shape<1>
+  %11:2 = hlfir.declare %9(%10) {uniq_name = ".tmp.arrayctor"} : (!fir.heap<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.heap<!fir.array<3xi32>>, !fir.heap<!fir.array<3xi32>>)
+  %12 = fir.load %1#0 : !fir.ref<i32>
+  %13 = arith.addi %c1, %c1_1 : index
+  %14 = hlfir.designate %11#0 (%c1)  : (!fir.heap<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+  hlfir.assign %12 to %14 : i32, !fir.ref<i32>
+  %15 = fir.load %6#0 : !fir.ref<i32>
+  %16 = arith.addi %13, %c1_1 : index
+  %17 = hlfir.designate %11#0 (%13)  : (!fir.heap<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+  hlfir.assign %15 to %17 : i32, !fir.ref<i32>
+  %18 = fir.load %8#0 : !fir.ref<i32>
+  %19 = hlfir.designate %11#0 (%16)  : (!fir.heap<!fir.array<3xi32>>, index) -> !fir.ref<i32>
+  hlfir.assign %18 to %19 : i32, !fir.ref<i32>
+  %true = arith.constant true
+  %20 = hlfir.as_expr %11#0 move %true {test.ptr = "tmp_array"}: (!fir.heap<!fir.array<3xi32>>, i1) -> !hlfir.expr<3xi32>
+  hlfir.assign %20 to %4#0 : !hlfir.expr<3xi32>, !fir.ref<!fir.array<3xi32>>
+  hlfir.destroy %20 : !hlfir.expr<3xi32>
+  return
+}
diff --git a/flang/test/Fir/abstract-results-bindc.fir b/flang/test/Fir/abstract-results-bindc.fir
@@ -0,0 +1,60 @@
+// Test that bind_c derived type results are not moved to a hidden argument
+// by the abstract-result pass. They will be dealt with according to the C
+// struct returning ABI for the target in the target-rewrite pass.
+// RUN: fir-opt %s --abstract-result | FileCheck %s
+
+!t = !fir.type<t{i:f32, j: i32, k: f32}>
+!cptr = !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>
+
+func.func private @foo() -> !t attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+
+func.func @test_call(%x: !fir.ref<!t>) {
+  %0 = fir.call @foo() proc_attrs<bind_c> : () -> !t
+  fir.save_result %0 to %x : !t, !fir.ref<!t>
+  return
+}
+
+func.func @test_addr_of() -> (() -> !t) {
+  %0 = fir.address_of(@foo) : () -> !t
+  return %0 : () -> !t
+}
+
+func.func @test_dispatch(%x: !fir.ref<!t>, %y : !fir.class<!fir.type<somet>>) {
+  %0 = fir.dispatch "bar"(%y : !fir.class<!fir.type<somet>>) (%y : !fir.class<!fir.type<somet>>) -> !t proc_attrs<bind_c> {pass_arg_pos = 0 : i32}
+  fir.save_result %0 to %x : !t, !fir.ref<!t>
+  return
+}
+
+
+func.func private @return_cptr() -> !cptr attributes {fir.proc_attrs = #fir.proc_attrs<bind_c>}
+func.func @test_return_cptr(%x: !fir.ref<!cptr>) {
+  %0 = fir.call @return_cptr() proc_attrs<bind_c> : () -> !cptr
+  fir.save_result %0 to %x : !cptr, !fir.ref<!cptr>
+  return
+}
+
+
+// CHECK-LABEL:   func.func @test_call(
+// CHECK-SAME:                         %[[VAL_0:.*]]: !fir.ref<!fir.type<t{i:f32,j:i32,k:f32}>>) {
+// CHECK:           %[[VAL_1:.*]] = fir.call @foo() proc_attrs<bind_c> : () -> !fir.type<t{i:f32,j:i32,k:f32}>
+// CHECK:           fir.store %[[VAL_1]] to %[[VAL_0]] : !fir.ref<!fir.type<t{i:f32,j:i32,k:f32}>>
+// CHECK:           return
+// CHECK:         }
+// CHECK-LABEL:   func.func @test_addr_of() -> (() -> !fir.type<t{i:f32,j:i32,k:f32}>) {
+// CHECK:           %[[VAL_0:.*]] = fir.address_of(@foo) : () -> !fir.type<t{i:f32,j:i32,k:f32}>
+// CHECK:           return %[[VAL_0]] : () -> !fir.type<t{i:f32,j:i32,k:f32}>
+// CHECK:         }
+// CHECK-LABEL:   func.func @test_dispatch(
+// CHECK-SAME:                             %[[VAL_0:.*]]: !fir.ref<!fir.type<t{i:f32,j:i32,k:f32}>>,
+// CHECK-SAME:                             %[[VAL_1:.*]]: !fir.class<!fir.type<somet>>) {
+// CHECK:           %[[VAL_2:.*]] = fir.dispatch "bar"(%[[VAL_1]] : !fir.class<!fir.type<somet>>) (%[[VAL_1]] : !fir.class<!fir.type<somet>>) -> !fir.type<t{i:f32,j:i32,k:f32}> proc_attrs <bind_c> {pass_arg_pos = 0 : i32}
+// CHECK:           fir.store %[[VAL_2]] to %[[VAL_0]] : !fir.ref<!fir.type<t{i:f32,j:i32,k:f32}>>
+// CHECK:           return
+// CHECK:         }
+// CHECK-LABEL:   func.func @test_return_cptr(
+// CHECK-SAME:                                %[[VAL_0:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) {
+// CHECK:           %[[VAL_1:.*]] = fir.call @return_cptr() : () -> !fir.ref<none>
+// CHECK:           %[[VAL_2:.*]] = fir.field_index __address, !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>
+// CHECK:           %[[VAL_3:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_2]] : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.field) -> !fir.ref<i64>
+// CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<none>) -> i64
+// CHECK:           fir.store %[[VAL_4]] to %[[VAL_3]] : !fir.ref<i64>
diff --git a/flang/test/Fir/struct-return-x86-64.fir b/flang/test/Fir/struct-return-x86-64.fir
@@ -0,0 +1,120 @@
+// Test X86-64 ABI rewrite of struct returned by value (BIND(C), VALUE derived types).
+// REQUIRES: x86-registered-target
+// RUN: fir-opt --target-rewrite %s | FileCheck %s
+
+!fits_in_reg = !fir.type<t1{i:f32,j:i32,k:f32}>
+!too_big = !fir.type<t2{i:!fir.array<5xf32>}>
+
+module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
+
+  func.func private @test_inreg() -> !fits_in_reg
+  func.func @test_call_inreg(%arg0: !fir.ref<!fits_in_reg>) {
+    %0 = fir.call @test_inreg() : () -> !fits_in_reg
+    fir.store %0 to %arg0 : !fir.ref<!fits_in_reg>
+    return
+  }
+  func.func @test_addr_of_inreg() -> (() -> ()) {
+    %0 = fir.address_of(@test_inreg) : () -> !fits_in_reg
+    %1 = fir.convert %0 : (() -> !fits_in_reg) -> (() -> ())
+    return %1 : () -> ()
+  }
+  func.func @test_dispatch_inreg(%arg0: !fir.ref<!fits_in_reg>, %arg1: !fir.class<!fir.type<somet>>) {
+    %0 = fir.dispatch "bar"(%arg1 : !fir.class<!fir.type<somet>>) (%arg1 : !fir.class<!fir.type<somet>>) -> !fits_in_reg {pass_arg_pos = 0 : i32}
+    fir.store %0 to %arg0 : !fir.ref<!fits_in_reg>
+    return
+  }
+
+  func.func private @test_sret() -> !too_big
+  func.func @test_call_sret(%arg0: !fir.ref<!too_big>) {
+    %0 = fir.call @test_sret() : () -> !too_big
+    fir.store %0 to %arg0 : !fir.ref<!too_big>
+    return
+  }
+  func.func @test_addr_of_sret() -> (() -> ()) {
+    %0 = fir.address_of(@test_sret) : () -> !too_big
+    %1 = fir.convert %0 : (() -> !too_big) -> (() -> ())
+    return %1 : () -> ()
+  }
+  func.func @test_dispatch_sret(%arg0: !fir.ref<!too_big>, %arg1: !fir.class<!fir.type<somet>>) {
+    %0 = fir.dispatch "bar"(%arg1 : !fir.class<!fir.type<somet>>) (%arg1 : !fir.class<!fir.type<somet>>) -> !too_big {pass_arg_pos = 0 : i32}
+    fir.store %0 to %arg0 : !fir.ref<!too_big>
+    return
+  }
+  func.func private @test_fp_80() -> !fir.type<t3{i:f80}>
+  func.func private @test_complex_80() -> !fir.type<t4{i:complex<f80>}>
+  func.func private @test_two_fp_80() -> !fir.type<t5{i:f80,j:f80}>
+  func.func private @test_fp128() -> !fir.type<t6{i:f128}>
+}
+
+// CHECK-LABEL:   func.func private @test_inreg() -> tuple<i64, f32>
+
+// CHECK-LABEL:   func.func @test_call_inreg(
+// CHECK-SAME:                               %[[VAL_0:.*]]: !fir.ref<!fir.type<t1{i:f32,j:i32,k:f32}>>) {
+// CHECK:           %[[VAL_1:.*]] = fir.call @test_inreg() : () -> tuple<i64, f32>
+// CHECK:           %[[VAL_2:.*]] = llvm.intr.stacksave : !llvm.ptr
+// CHECK:           %[[VAL_3:.*]] = fir.alloca tuple<i64, f32>
+// CHECK:           fir.store %[[VAL_1]] to %[[VAL_3]] : !fir.ref<tuple<i64, f32>>
+// CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.ref<tuple<i64, f32>>) -> !fir.ref<!fir.type<t1{i:f32,j:i32,k:f32}>>
+// CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_4]] : !fir.ref<!fir.type<t1{i:f32,j:i32,k:f32}>>
+// CHECK:           llvm.intr.stackrestore %[[VAL_2]] : !llvm.ptr
+// CHECK:           fir.store %[[VAL_5]] to %[[VAL_0]] : !fir.ref<!fir.type<t1{i:f32,j:i32,k:f32}>>
+// CHECK:           return
+// CHECK:         }
+
+// CHECK-LABEL:   func.func @test_addr_of_inreg() -> (() -> ()) {
+// CHECK:           %[[VAL_0:.*]] = fir.address_of(@test_inreg) : () -> tuple<i64, f32>
+// CHECK:           %[[VAL_1:.*]] = fir.convert %[[VAL_0]] : (() -> tuple<i64, f32>) -> (() -> ())
+// CHECK:           return %[[VAL_1]] : () -> ()
+// CHECK:         }
+
+// CHECK-LABEL:   func.func @test_dispatch_inreg(
+// CHECK-SAME:                                   %[[VAL_0:.*]]: !fir.ref<!fir.type<t1{i:f32,j:i32,k:f32}>>,
+// CHECK-SAME:                                   %[[VAL_1:.*]]: !fir.class<!fir.type<somet>>) {
+// CHECK:           %[[VAL_2:.*]] = fir.dispatch "bar"(%[[VAL_1]] : !fir.class<!fir.type<somet>>) (%[[VAL_1]] : !fir.class<!fir.type<somet>>) -> tuple<i64, f32> {pass_arg_pos = 0 : i32}
+// CHECK:           %[[VAL_3:.*]] = llvm.intr.stacksave : !llvm.ptr
+// CHECK:           %[[VAL_4:.*]] = fir.alloca tuple<i64, f32>
+// CHECK:           fir.store %[[VAL_2]] to %[[VAL_4]] : !fir.ref<tuple<i64, f32>>
+// CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (!fir.ref<tuple<i64, f32>>) -> !fir.ref<!fir.type<t1{i:f32,j:i32,k:f32}>>
+// CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_5]] : !fir.ref<!fir.type<t1{i:f32,j:i32,k:f32}>>
+// CHECK:           llvm.intr.stackrestore %[[VAL_3]] : !llvm.ptr
+// CHECK:           fir.store %[[VAL_6]] to %[[VAL_0]] : !fir.ref<!fir.type<t1{i:f32,j:i32,k:f32}>>
+// CHECK:           return
+// CHECK:         }
+// CHECK:         func.func private @test_sret(!fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>> {llvm.align = 8 : i32, llvm.sret = !fir.type<t2{i:!fir.array<5xf32>}>})
+
+// CHECK-LABEL:   func.func @test_call_sret(
+// CHECK-SAME:                              %[[VAL_0:.*]]: !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>) {
+// CHECK:           %[[VAL_1:.*]] = llvm.intr.stacksave : !llvm.ptr
+// CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.type<t2{i:!fir.array<5xf32>}>
+// CHECK:           fir.call @test_sret(%[[VAL_2]]) : (!fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>) -> ()
+// CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>) -> !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>
+// CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]] : !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>
+// CHECK:           llvm.intr.stackrestore %[[VAL_1]] : !llvm.ptr
+// CHECK:           fir.store %[[VAL_4]] to %[[VAL_0]] : !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>
+// CHECK:           return
+// CHECK:         }
+
+// CHECK-LABEL:   func.func @test_addr_of_sret() -> (() -> ()) {
+// CHECK:           %[[VAL_0:.*]] = fir.address_of(@test_sret) : (!fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>) -> ()
+// CHECK:           %[[VAL_1:.*]] = fir.convert %[[VAL_0]] : ((!fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>) -> ()) -> (() -> ())
+// CHECK:           return %[[VAL_1]] : () -> ()
+// CHECK:         }
+
+// CHECK-LABEL:   func.func @test_dispatch_sret(
+// CHECK-SAME:                                  %[[VAL_0:.*]]: !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>,
+// CHECK-SAME:                                  %[[VAL_1:.*]]: !fir.class<!fir.type<somet>>) {
+// CHECK:           %[[VAL_2:.*]] = llvm.intr.stacksave : !llvm.ptr
+// CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.type<t2{i:!fir.array<5xf32>}>
+// CHECK:           fir.dispatch "bar"(%[[VAL_1]] : !fir.class<!fir.type<somet>>) (%[[VAL_3]], %[[VAL_1]] : !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>, !fir.class<!fir.type<somet>>) {pass_arg_pos = 1 : i32}
+// CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>) -> !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>
+// CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_4]] : !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>
+// CHECK:           llvm.intr.stackrestore %[[VAL_2]] : !llvm.ptr
+// CHECK:           fir.store %[[VAL_5]] to %[[VAL_0]] : !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>
+// CHECK:           return
+// CHECK:         }
+
+
+// CHECK: func.func private @test_fp_80() -> f80
+// CHECK: func.func private @test_complex_80(!fir.ref<!fir.type<t4{i:complex<f80>}>> {llvm.align = 16 : i32, llvm.sret = !fir.type<t4{i:complex<f80>}>})
+// CHECK: func.func private @test_two_fp_80(!fir.ref<!fir.type<t5{i:f80,j:f80}>> {llvm.align = 16 : i32, llvm.sret = !fir.type<t5{i:f80,j:f80}>})
+// CHECK: func.func private @test_fp128() -> f128
diff --git a/lldb/CodeOwners.rst → lldb/Maintainers.rst b/lldb/CodeOwners.rst → lldb/Maintainers.rst
@@ -1,27 +1,31 @@
 ================
-LLDB Code Owners
+LLDB Maintainers
 ================
 
-This file is a list of the `code owners <https://llvm.org/docs/DeveloperPolicy.html#code-owners>`_ for LLDB.
+This file is a list of the `maintainers <https://llvm.org/docs/DeveloperPolicy.html#maintainers>`_ for LLDB.
 
 .. contents::
    :depth: 2
    :local:
 
-Current Code Owners
+Current Maintainers
 ===================
-The following people are the active code owners for the project. Please reach
+The following people are the active maintainers for the project. Please reach
 out to them for code reviews, questions about their area of expertise, or other
 assistance.
 
-All parts of LLDB not covered by someone else
-----------------------------------------------
+Lead Maintainer
+---------------
+
+Responsible for project as a whole, and for any areas not covered by a specific
+maintainer.
+
 | Jonas Devlieghere
 | jonas\@devlieghere.com (email), jdevlieghere (GitHub), jdevlieghere (Discourse), jdevlieghere (Discord)
 
 Components
 ----------
-These code owners are responsible for particular high-level components within
+These maintainers are responsible for particular high-level components within
 LLDB.
 
 ABI
@@ -242,10 +246,10 @@ lldb-dap
 | Walter Erquinigo
 | a20012251\@gmail.com (email), walter-erquinigo (GitHub), wallace (Discourse), werquinigo (Discord)
 
-Former Code Owners
+Former Maintainers
 ==================
-The following people have graciously spent time performing code ownership
-responsibilities but are no longer active in that role. Thank you for all your
+The following people have graciously spent time performing maintainership
+duties but are no longer active in that role. Thank you for all your
 help with the success of the project!
 
 | Kamil Rytarowski (kamil\@netbsd.org)

diff --git a/llvm/docs/HowToAddABuilder.rst b/llvm/docs/HowToAddABuilder.rst
@@ -276,4 +276,33 @@ Leave it on the staging buildmaster
   impacting the broader community.  The sponsoring organization simply
   has to take on the responsibility of all bisection and triage.
 
-
+Managing a Worker From The Web Interface
+========================================
+
+Tasks such as clearing pending building requests can be done using
+the Buildbot web interface. To do this you must be recognised as an admin
+of the worker:
+
+* Set your public GitHub profile email to one that was included in the
+  ``admin`` information you set up on the worker. It does not matter if this
+  is your primary account email or a "verified email". To confirm this has been
+  done correctly, go to ``github.com/<your GitHub username>`` and you should
+  see the email address listed there.
+
+  A worker can have many admins, if they are listed in the form
+  ``First Last <first.last@example.com>, First2 Last2 <first2.last2@example.com>``.
+  You only need to have one of those addresses in your profile to be recognised
+  as an admin.
+
+  If you need to add an email address, you can edit the ``admin`` file and
+  restart the worker. You should see the new admin details in the web interface
+  shortly afterwards.
+
+* Connect GitHub to Buildbot by clicking on the "Anonymous" button on the
+  top right of the page, then "Login with GitHub" and authorise the app.
+
+Some tasks don't give immediate feedback, so if nothing happens within a short
+time, try again with the browser's web console open. Sometimes you will see
+403 errors and other messages that might indicate you don't have the correct
+details set up.
+
diff --git a/llvm/include/llvm/CodeGen/MIRParser/MIParser.h b/llvm/include/llvm/CodeGen/MIRParser/MIParser.h
@@ -37,16 +37,15 @@ class TargetRegisterClass;
 class TargetSubtargetInfo;
 
 struct VRegInfo {
-  enum uint8_t {
-    UNKNOWN, NORMAL, GENERIC, REGBANK
-  } Kind = UNKNOWN;
+  enum : uint8_t { UNKNOWN, NORMAL, GENERIC, REGBANK } Kind = UNKNOWN;
   bool Explicit = false; ///< VReg was explicitly specified in the .mir file.
   union {
     const TargetRegisterClass *RC;
     const RegisterBank *RegBank;
   } D;
   Register VReg;
   Register PreferredReg;
+  std::vector<uint8_t> Flags;
 };
 
 using Name2RegClassMap = StringMap<const TargetRegisterClass *>;
@@ -150,6 +149,8 @@ struct PerTargetMIParsingState {
   /// Return null if the name isn't a register bank.
   const RegisterBank *getRegBank(StringRef Name);
 
+  bool getVRegFlagValue(StringRef FlagName, uint8_t &FlagValue) const;
+
   PerTargetMIParsingState(const TargetSubtargetInfo &STI)
     : Subtarget(STI) {
     initNames2RegClasses();

diff --git a/llvm/include/llvm/CodeGen/MIRYamlMapping.h b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
@@ -191,6 +191,7 @@ struct VirtualRegisterDefinition {
   UnsignedValue ID;
   StringValue Class;
   StringValue PreferredRegister;
+  std::vector<FlowStringValue> RegisterFlags;
 
   // TODO: Serialize the target specific register hints.
 
@@ -206,6 +207,8 @@ template <> struct MappingTraits<VirtualRegisterDefinition> {
     YamlIO.mapRequired("class", Reg.Class);
     YamlIO.mapOptional("preferred-register", Reg.PreferredRegister,
                        StringValue()); // Don't print out when it's empty.
+    YamlIO.mapOptional("flags", Reg.RegisterFlags,
+                       std::vector<FlowStringValue>());
   }
 
   static const bool flow = true;

diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -1213,6 +1213,15 @@ class TargetRegisterInfo : public MCRegisterInfo {
   virtual bool isNonallocatableRegisterCalleeSave(MCRegister Reg) const {
     return false;
   }
+
+  virtual std::optional<uint8_t> getVRegFlagValue(StringRef Name) const {
+    return {};
+  }
+
+  virtual SmallVector<StringLiteral>
+  getVRegFlagsOfReg(Register Reg, const MachineFunction &MF) const {
+    return {};
+  }
 };
 
 //===----------------------------------------------------------------------===//

diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -127,6 +127,16 @@ bool PerTargetMIParsingState::getRegisterByName(StringRef RegName,
   return false;
 }
 
+bool PerTargetMIParsingState::getVRegFlagValue(StringRef FlagName,
+                                               uint8_t &FlagValue) const {
+  const auto *TRI = Subtarget.getRegisterInfo();
+  std::optional<uint8_t> FV = TRI->getVRegFlagValue(FlagName);
+  if (!FV)
+    return true;
+  FlagValue = *FV;
+  return false;
+}
+
 void PerTargetMIParsingState::initNames2InstrOpCodes() {
   if (!Names2InstrOpCodes.empty())
     return;
@@ -1776,6 +1786,7 @@ bool MIParser::parseRegisterOperand(MachineOperand &Dest,
 
         MRI.setRegClassOrRegBank(Reg, static_cast<RegisterBank *>(nullptr));
         MRI.setType(Reg, Ty);
+        MRI.noteNewVirtualRegister(Reg);
       }
     }
   } else if (consumeIfPresent(MIToken::lparen)) {

diff --git a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
@@ -696,6 +696,16 @@ bool MIRParserImpl::parseRegisterInfo(PerFunctionMIParsingState &PFS,
                                  VReg.PreferredRegister.Value, Error))
         return error(Error, VReg.PreferredRegister.SourceRange);
     }
+
+    for (const auto &FlagStringValue : VReg.RegisterFlags) {
+      uint8_t FlagValue;
+      if (Target->getVRegFlagValue(FlagStringValue.Value, FlagValue))
+        return error(FlagStringValue.SourceRange.Start,
+                     Twine("use of undefined register flag '") +
+                         FlagStringValue.Value + "'");
+      Info.Flags.push_back(FlagValue);
+    }
+    RegInfo.noteNewVirtualRegister(Info.VReg);
   }
 
   // Parse the liveins.

diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -113,7 +113,8 @@ class MIRPrinter {
 
   void print(const MachineFunction &MF);
 
-  void convert(yaml::MachineFunction &MF, const MachineRegisterInfo &RegInfo,
+  void convert(yaml::MachineFunction &YamlMF, const MachineFunction &MF,
+               const MachineRegisterInfo &RegInfo,
                const TargetRegisterInfo *TRI);
   void convert(ModuleSlotTracker &MST, yaml::MachineFrameInfo &YamlMFI,
                const MachineFrameInfo &MFI);
@@ -231,7 +232,7 @@ void MIRPrinter::print(const MachineFunction &MF) {
   YamlMF.NoVRegs = MF.getProperties().hasProperty(
       MachineFunctionProperties::Property::NoVRegs);
 
-  convert(YamlMF, MF.getRegInfo(), MF.getSubtarget().getRegisterInfo());
+  convert(YamlMF, MF, MF.getRegInfo(), MF.getSubtarget().getRegisterInfo());
   MachineModuleSlotTracker MST(MMI, &MF);
   MST.incorporateFunction(MF.getFunction());
   convert(MST, YamlMF.FrameInfo, MF.getFrameInfo());
@@ -316,10 +317,21 @@ printStackObjectDbgInfo(const MachineFunction::VariableDbgInfo &DebugVar,
   }
 }
 
-void MIRPrinter::convert(yaml::MachineFunction &MF,
+static void printRegFlags(Register Reg,
+                          std::vector<yaml::FlowStringValue> &RegisterFlags,
+                          const MachineFunction &MF,
+                          const TargetRegisterInfo *TRI) {
+  auto FlagValues = TRI->getVRegFlagsOfReg(Reg, MF);
+  for (auto &Flag : FlagValues) {
+    RegisterFlags.push_back(yaml::FlowStringValue(Flag.str()));
+  }
+}
+
+void MIRPrinter::convert(yaml::MachineFunction &YamlMF,
+                         const MachineFunction &MF,
                          const MachineRegisterInfo &RegInfo,
                          const TargetRegisterInfo *TRI) {
-  MF.TracksRegLiveness = RegInfo.tracksLiveness();
+  YamlMF.TracksRegLiveness = RegInfo.tracksLiveness();
 
   // Print the virtual register definitions.
   for (unsigned I = 0, E = RegInfo.getNumVirtRegs(); I < E; ++I) {
@@ -332,7 +344,8 @@ void MIRPrinter::convert(yaml::MachineFunction &MF,
     Register PreferredReg = RegInfo.getSimpleHint(Reg);
     if (PreferredReg)
       printRegMIR(PreferredReg, VReg.PreferredRegister, TRI);
-    MF.VirtualRegisters.push_back(VReg);
+    printRegFlags(Reg, VReg.RegisterFlags, MF, TRI);
+    YamlMF.VirtualRegisters.push_back(VReg);
   }
 
   // Print the live ins.
@@ -341,7 +354,7 @@ void MIRPrinter::convert(yaml::MachineFunction &MF,
     printRegMIR(LI.first, LiveIn.Register, TRI);
     if (LI.second)
       printRegMIR(LI.second, LiveIn.VirtualRegister, TRI);
-    MF.LiveIns.push_back(LiveIn);
+    YamlMF.LiveIns.push_back(LiveIn);
   }
 
   // Prints the callee saved registers.
@@ -353,7 +366,7 @@ void MIRPrinter::convert(yaml::MachineFunction &MF,
       printRegMIR(*I, Reg, TRI);
       CalleeSavedRegisters.push_back(Reg);
     }
-    MF.CalleeSavedRegisters = CalleeSavedRegisters;
+    YamlMF.CalleeSavedRegisters = CalleeSavedRegisters;
   }
 }
 

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1146,7 +1146,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       Subtarget->requiresStrictAlign() ? MaxStoresPerMemcpyOptSize : 16;
 
   MaxStoresPerMemmoveOptSize = 4;
-  MaxStoresPerMemmove = 4;
+  MaxStoresPerMemmove =
+      Subtarget->requiresStrictAlign() ? MaxStoresPerMemmoveOptSize : 16;
 
   MaxLoadsPerMemcmpOptSize = 4;
   MaxLoadsPerMemcmp =

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp
@@ -482,6 +482,29 @@ void SplitGraph::Node::visitAllDependencies(
   }
 }
 
+/// Checks if \p I has MD_callees and if it does, parse it and put the function
+/// in \p Callees.
+///
+/// \returns true if there was metadata and it was parsed correctly. false if
+/// there was no MD or if it contained unknown entries and parsing failed.
+/// If this returns false, \p Callees will contain incomplete information
+/// and must not be used.
+static bool handleCalleesMD(const Instruction &I,
+                            SetVector<Function *> &Callees) {
+  auto *MD = I.getMetadata(LLVMContext::MD_callees);
+  if (!MD)
+    return false;
+
+  for (const auto &Op : MD->operands()) {
+    Function *Callee = mdconst::extract_or_null<Function>(Op);
+    if (!Callee)
+      return false;
+    Callees.insert(Callee);
+  }
+
+  return true;
+}
+
 void SplitGraph::buildGraph(CallGraph &CG) {
   SplitModuleTimer SMT("buildGraph", "graph construction");
   LLVM_DEBUG(
@@ -519,28 +542,38 @@ void SplitGraph::buildGraph(CallGraph &CG) {
                  Fn.printAsOperand(dbgs());
                  dbgs() << " - analyzing function\n");
 
-      bool HasIndirectCall = false;
+      SetVector<Function *> KnownCallees;
+      bool HasUnknownIndirectCall = false;
       for (const auto &Inst : instructions(Fn)) {
         // look at all calls without a direct callee.
-        if (const auto *CB = dyn_cast<CallBase>(&Inst);
-            CB && !CB->getCalledFunction()) {
-          // inline assembly can be ignored, unless InlineAsmIsIndirectCall is
-          // true.
-          if (CB->isInlineAsm()) {
-            LLVM_DEBUG(dbgs() << "    found inline assembly\n");
-            continue;
-          }
-
-          // everything else is handled conservatively.
-          HasIndirectCall = true;
-          break;
+        const auto *CB = dyn_cast<CallBase>(&Inst);
+        if (!CB || CB->getCalledFunction())
+          continue;
+
+        // inline assembly can be ignored, unless InlineAsmIsIndirectCall is
+        // true.
+        if (CB->isInlineAsm()) {
+          LLVM_DEBUG(dbgs() << "    found inline assembly\n");
+          continue;
         }
+
+        if (handleCalleesMD(Inst, KnownCallees))
+          continue;
+        // If we failed to parse any !callees MD, or some was missing,
+        // the entire KnownCallees list is now unreliable.
+        KnownCallees.clear();
+
+        // Everything else is handled conservatively. If we fall into the
+        // conservative case don't bother analyzing further.
+        HasUnknownIndirectCall = true;
+        break;
       }
 
-      if (HasIndirectCall) {
+      if (HasUnknownIndirectCall) {
         LLVM_DEBUG(dbgs() << "    indirect call found\n");
         FnsWithIndirectCalls.push_back(&Fn);
-      }
+      } else if (!KnownCallees.empty())
+        DirectCallees.insert(KnownCallees.begin(), KnownCallees.end());
     }
 
     Node &N = getNode(Cache, Fn);

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1718,6 +1718,17 @@ bool GCNTargetMachine::parseMachineFunctionInfo(
     MFI->reserveWWMRegister(ParsedReg);
   }
 
+  for (const auto &[_, Info] : PFS.VRegInfosNamed) {
+    for (uint8_t Flag : Info->Flags) {
+      MFI->setFlag(Info->VReg, Flag);
+    }
+  }
+  for (const auto &[_, Info] : PFS.VRegInfos) {
+    for (uint8_t Flag : Info->Flags) {
+      MFI->setFlag(Info->VReg, Flag);
+    }
+  }
+
   auto parseAndCheckArgument = [&](const std::optional<yaml::SIArgument> &A,
                                    const TargetRegisterClass &RC,
                                    ArgDescriptor &Arg, unsigned UserSGPRs,

diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -3851,3 +3851,13 @@ SIRegisterInfo::getSubRegAlignmentNumBits(const TargetRegisterClass *RC,
   }
   return 0;
 }
+
+SmallVector<StringLiteral>
+SIRegisterInfo::getVRegFlagsOfReg(Register Reg,
+                                  const MachineFunction &MF) const {
+  SmallVector<StringLiteral> RegFlags;
+  const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
+  if (FuncInfo->checkFlag(Reg, AMDGPU::VirtRegFlag::WWM_REG))
+    RegFlags.push_back("WWM_REG");
+  return RegFlags;
+}
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -457,6 +457,14 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
   // No check if the subreg is supported by the current RC is made.
   unsigned getSubRegAlignmentNumBits(const TargetRegisterClass *RC,
                                      unsigned SubReg) const;
+
+  std::optional<uint8_t> getVRegFlagValue(StringRef Name) const override {
+    return Name == "WWM_REG" ? AMDGPU::VirtRegFlag::WWM_REG
+                             : std::optional<uint8_t>{};
+  }
+
+  SmallVector<StringLiteral>
+  getVRegFlagsOfReg(Register Reg, const MachineFunction &MF) const override;
 };
 
 namespace AMDGPU {

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -411,13 +411,13 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::SELECT, XLenVT, Custom);
 
   static const unsigned FPLegalNodeTypes[] = {
-      ISD::FMINNUM,       ISD::FMAXNUM,       ISD::FMINIMUMNUM,
-      ISD::FMAXIMUMNUM,   ISD::LRINT,         ISD::LLRINT,
-      ISD::LROUND,        ISD::LLROUND,       ISD::STRICT_LRINT,
-      ISD::STRICT_LLRINT, ISD::STRICT_LROUND, ISD::STRICT_LLROUND,
-      ISD::STRICT_FMA,    ISD::STRICT_FADD,   ISD::STRICT_FSUB,
-      ISD::STRICT_FMUL,   ISD::STRICT_FDIV,   ISD::STRICT_FSQRT,
-      ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS};
+      ISD::FMINNUM,       ISD::FMAXNUM,        ISD::FMINIMUMNUM,
+      ISD::FMAXIMUMNUM,   ISD::LRINT,          ISD::LLRINT,
+      ISD::LROUND,        ISD::LLROUND,        ISD::STRICT_LRINT,
+      ISD::STRICT_LLRINT, ISD::STRICT_LROUND,  ISD::STRICT_LLROUND,
+      ISD::STRICT_FMA,    ISD::STRICT_FADD,    ISD::STRICT_FSUB,
+      ISD::STRICT_FMUL,   ISD::STRICT_FDIV,    ISD::STRICT_FSQRT,
+      ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::FCANONICALIZE};
 
   static const ISD::CondCode FPCCToExpand[] = {
       ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
@@ -441,7 +441,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
       ISD::STRICT_FSQRT, ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS,
       ISD::SETCC,        ISD::FCEIL,         ISD::FFLOOR,
       ISD::FTRUNC,       ISD::FRINT,         ISD::FROUND,
-      ISD::FROUNDEVEN};
+      ISD::FROUNDEVEN,   ISD::FCANONICALIZE};
 
   if (Subtarget.hasStdExtZfbfmin()) {
     setOperationAction(ISD::BITCAST, MVT::i16, Custom);

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td
@@ -398,6 +398,7 @@ foreach Ext = DExts in {
   defm : PatFprFpr_m<fmaximumnum, FMAX_D, Ext>;
   defm : PatFprFpr_m<riscv_fmin, FMIN_D, Ext>;
   defm : PatFprFpr_m<riscv_fmax, FMAX_D, Ext>;
+  def : Pat<(f64 (fcanonicalize FPR64:$rs1)), (FMIN_D $rs1, $rs1)>;
 }
 
 /// Setcc

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
@@ -623,6 +623,7 @@ foreach Ext = FExts in {
   defm : PatFprFpr_m<fmaximumnum, FMAX_S, Ext>;
   defm : PatFprFpr_m<riscv_fmin, FMIN_S, Ext>;
   defm : PatFprFpr_m<riscv_fmax, FMAX_S, Ext>;
+  def : Pat<(f32 (fcanonicalize FPR32:$rs1)), (FMIN_S $rs1, $rs1)>;
 }
 
 /// Setcc

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td
@@ -369,6 +369,7 @@ foreach Ext = ZfhExts in {
   defm : PatFprFpr_m<fmaximumnum, FMAX_H, Ext>;
   defm : PatFprFpr_m<riscv_fmin, FMIN_H, Ext>;
   defm : PatFprFpr_m<riscv_fmax, FMAX_H, Ext>;
+  def : Pat<(f16 (fcanonicalize FPR16:$rs1)), (FMIN_H $rs1, $rs1)>;
 }
 
 /// Setcc

diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memmove.mir b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memmove.mir
@@ -61,11 +61,12 @@ body:             |
 
     ; CHECK-LABEL: name: test_memmove1
     ; CHECK: liveins: $x0, $x1, $x2
-    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
-    ; CHECK: [[COPY2:%[0-9]+]]:_(s64) = COPY $x2
-    ; CHECK: G_MEMMOVE [[COPY]](p0), [[COPY1]](p0), [[COPY2]](s64), 1 :: (store (s8) into %ir.0, align 4), (load (s8) from %ir.1, align 4)
-    ; CHECK: RET_ReallyLR
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x2
+    ; CHECK-NEXT: G_MEMMOVE [[COPY]](p0), [[COPY1]](p0), [[COPY2]](s64), 1 :: (store (s8) into %ir.0, align 4), (load (s8) from %ir.1, align 4)
+    ; CHECK-NEXT: RET_ReallyLR
     %0:_(p0) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s64) = COPY $x2
@@ -83,23 +84,24 @@ body:             |
 
     ; CHECK-LABEL: name: test_memmove2_const
     ; CHECK: liveins: $x0, $x1
-    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY1]](p0) :: (load (s128) from %ir.1, align 4)
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C]](s64)
-    ; CHECK: [[LOAD1:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD]](p0) :: (load (s128) from %ir.1 + 16, align 4)
-    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-    ; CHECK: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s64)
-    ; CHECK: [[LOAD2:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD1]](p0) :: (load (s128) from %ir.1 + 32, align 4)
-    ; CHECK: G_STORE [[LOAD]](s128), [[COPY]](p0) :: (store (s128) into %ir.0, align 4)
-    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; CHECK: G_STORE [[LOAD1]](s128), [[PTR_ADD2]](p0) :: (store (s128) into %ir.0 + 16, align 4)
-    ; CHECK: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-    ; CHECK: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; CHECK: G_STORE [[LOAD2]](s128), [[PTR_ADD3]](p0) :: (store (s128) into %ir.0 + 32, align 4)
-    ; CHECK: RET_ReallyLR
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY1]](p0) :: (load (s128) from %ir.1, align 4)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[LOAD1:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD]](p0) :: (load (s128) from %ir.1 + 16, align 4)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s64)
+    ; CHECK-NEXT: [[LOAD2:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD1]](p0) :: (load (s128) from %ir.1 + 32, align 4)
+    ; CHECK-NEXT: G_STORE [[LOAD]](s128), [[COPY]](p0) :: (store (s128) into %ir.0, align 4)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C2]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD1]](s128), [[PTR_ADD2]](p0) :: (store (s128) into %ir.0 + 16, align 4)
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C3]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD2]](s128), [[PTR_ADD3]](p0) :: (store (s128) into %ir.0 + 32, align 4)
+    ; CHECK-NEXT: RET_ReallyLR
     %0:_(p0) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s64) = G_CONSTANT i64 48
@@ -117,11 +119,42 @@ body:             |
 
     ; CHECK-LABEL: name: test_memmove3_const_toolarge
     ; CHECK: liveins: $x0, $x1
-    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 96
-    ; CHECK: G_MEMMOVE [[COPY]](p0), [[COPY1]](p0), [[C]](s64), 1 :: (store (s8) into %ir.0, align 4), (load (s8) from %ir.1, align 4)
-    ; CHECK: RET_ReallyLR
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY1]](p0) :: (load (s128) from %ir.1, align 4)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[LOAD1:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD]](p0) :: (load (s128) from %ir.1 + 16, align 4)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s64)
+    ; CHECK-NEXT: [[LOAD2:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD1]](p0) :: (load (s128) from %ir.1 + 32, align 4)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C2]](s64)
+    ; CHECK-NEXT: [[LOAD3:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD2]](p0) :: (load (s128) from %ir.1 + 48, align 4)
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
+    ; CHECK-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C3]](s64)
+    ; CHECK-NEXT: [[LOAD4:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD3]](p0) :: (load (s128) from %ir.1 + 64, align 4)
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 80
+    ; CHECK-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C4]](s64)
+    ; CHECK-NEXT: [[LOAD5:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD4]](p0) :: (load (s128) from %ir.1 + 80, align 4)
+    ; CHECK-NEXT: G_STORE [[LOAD]](s128), [[COPY]](p0) :: (store (s128) into %ir.0, align 4)
+    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C5]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD1]](s128), [[PTR_ADD5]](p0) :: (store (s128) into %ir.0 + 16, align 4)
+    ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C6]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD2]](s128), [[PTR_ADD6]](p0) :: (store (s128) into %ir.0 + 32, align 4)
+    ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; CHECK-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C7]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD3]](s128), [[PTR_ADD7]](p0) :: (store (s128) into %ir.0 + 48, align 4)
+    ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
+    ; CHECK-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C8]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD4]](s128), [[PTR_ADD8]](p0) :: (store (s128) into %ir.0 + 64, align 4)
+    ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 80
+    ; CHECK-NEXT: [[PTR_ADD9:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C9]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD5]](s128), [[PTR_ADD9]](p0) :: (store (s128) into %ir.0 + 80, align 4)
+    ; CHECK-NEXT: RET_ReallyLR
     %0:_(p0) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s64) = G_CONSTANT i64 96
@@ -139,29 +172,30 @@ body:             |
 
     ; CHECK-LABEL: name: test_memmove4_const_unaligned
     ; CHECK: liveins: $x0, $x1
-    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY1]](p0) :: (load (s128) from %ir.1, align 4)
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C]](s64)
-    ; CHECK: [[LOAD1:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD]](p0) :: (load (s128) from %ir.1 + 16, align 4)
-    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-    ; CHECK: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s64)
-    ; CHECK: [[LOAD2:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD1]](p0) :: (load (s128) from %ir.1 + 32, align 4)
-    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; CHECK: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C2]](s64)
-    ; CHECK: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s32) from %ir.1 + 48)
-    ; CHECK: G_STORE [[LOAD]](s128), [[COPY]](p0) :: (store (s128) into %ir.0, align 4)
-    ; CHECK: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; CHECK: G_STORE [[LOAD1]](s128), [[PTR_ADD3]](p0) :: (store (s128) into %ir.0 + 16, align 4)
-    ; CHECK: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-    ; CHECK: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-    ; CHECK: G_STORE [[LOAD2]](s128), [[PTR_ADD4]](p0) :: (store (s128) into %ir.0 + 32, align 4)
-    ; CHECK: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-    ; CHECK: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C5]](s64)
-    ; CHECK: G_STORE [[LOAD3]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %ir.0 + 48)
-    ; CHECK: RET_ReallyLR
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY1]](p0) :: (load (s128) from %ir.1, align 4)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[LOAD1:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD]](p0) :: (load (s128) from %ir.1 + 16, align 4)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s64)
+    ; CHECK-NEXT: [[LOAD2:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD1]](p0) :: (load (s128) from %ir.1 + 32, align 4)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C2]](s64)
+    ; CHECK-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s32) from %ir.1 + 48)
+    ; CHECK-NEXT: G_STORE [[LOAD]](s128), [[COPY]](p0) :: (store (s128) into %ir.0, align 4)
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C3]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD1]](s128), [[PTR_ADD3]](p0) :: (store (s128) into %ir.0 + 16, align 4)
+    ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD2]](s128), [[PTR_ADD4]](p0) :: (store (s128) into %ir.0 + 32, align 4)
+    ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; CHECK-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C5]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD3]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %ir.0 + 48)
+    ; CHECK-NEXT: RET_ReallyLR
     %0:_(p0) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s64) = G_CONSTANT i64 52
@@ -179,23 +213,24 @@ body:             |
 
     ; CHECK-LABEL: name: test_memmove_addrspace
     ; CHECK: liveins: $x0, $x1
-    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $x0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(p2) = COPY $x1
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY1]](p2) :: (load (s128) from %ir.1, align 4, addrspace 2)
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(p2) = G_PTR_ADD [[COPY1]], [[C]](s64)
-    ; CHECK: [[LOAD1:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD]](p2) :: (load (s128) from %ir.1 + 16, align 4, addrspace 2)
-    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-    ; CHECK: [[PTR_ADD1:%[0-9]+]]:_(p2) = G_PTR_ADD [[COPY1]], [[C1]](s64)
-    ; CHECK: [[LOAD2:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD1]](p2) :: (load (s128) from %ir.1 + 32, align 4, addrspace 2)
-    ; CHECK: G_STORE [[LOAD]](s128), [[COPY]](p1) :: (store (s128) into %ir.0, align 4, addrspace 1)
-    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-    ; CHECK: [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
-    ; CHECK: G_STORE [[LOAD1]](s128), [[PTR_ADD2]](p1) :: (store (s128) into %ir.0 + 16, align 4, addrspace 1)
-    ; CHECK: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-    ; CHECK: [[PTR_ADD3:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; CHECK: G_STORE [[LOAD2]](s128), [[PTR_ADD3]](p1) :: (store (s128) into %ir.0 + 32, align 4, addrspace 1)
-    ; CHECK: RET_ReallyLR
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p2) = COPY $x1
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY1]](p2) :: (load (s128) from %ir.1, align 4, addrspace 2)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p2) = G_PTR_ADD [[COPY1]], [[C]](s64)
+    ; CHECK-NEXT: [[LOAD1:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD]](p2) :: (load (s128) from %ir.1 + 16, align 4, addrspace 2)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p2) = G_PTR_ADD [[COPY1]], [[C1]](s64)
+    ; CHECK-NEXT: [[LOAD2:%[0-9]+]]:_(s128) = G_LOAD [[PTR_ADD1]](p2) :: (load (s128) from %ir.1 + 32, align 4, addrspace 2)
+    ; CHECK-NEXT: G_STORE [[LOAD]](s128), [[COPY]](p1) :: (store (s128) into %ir.0, align 4, addrspace 1)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C2]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD1]](s128), [[PTR_ADD2]](p1) :: (store (s128) into %ir.0 + 16, align 4, addrspace 1)
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = G_PTR_ADD [[COPY]], [[C3]](s64)
+    ; CHECK-NEXT: G_STORE [[LOAD2]](s128), [[PTR_ADD3]](p1) :: (store (s128) into %ir.0 + 32, align 4, addrspace 1)
+    ; CHECK-NEXT: RET_ReallyLR
     %0:_(p1) = COPY $x0
     %1:_(p2) = COPY $x1
     %2:_(s64) = G_CONSTANT i64 48

diff --git a/llvm/test/CodeGen/AArch64/memmove-inline.ll b/llvm/test/CodeGen/AArch64/memmove-inline.ll
@@ -0,0 +1,122 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-ALIGNED
+; RUN: llc -mtriple=aarch64 -mattr=+strict-align < %s | FileCheck %s --check-prefixes=CHECK,CHECK-UNALIGNED
+
+; Small (16 bytes here) unaligned memmove() should be a function call if
+; strict-alignment is turned on.
+define void @t16(ptr %out, ptr %in) {
+; CHECK-ALIGNED-LABEL: t16:
+; CHECK-ALIGNED:       // %bb.0: // %entry
+; CHECK-ALIGNED-NEXT:    ldr q0, [x1]
+; CHECK-ALIGNED-NEXT:    str q0, [x0]
+; CHECK-ALIGNED-NEXT:    ret
+;
+; CHECK-UNALIGNED-LABEL: t16:
+; CHECK-UNALIGNED:       // %bb.0: // %entry
+; CHECK-UNALIGNED-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-UNALIGNED-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-UNALIGNED-NEXT:    .cfi_offset w30, -16
+; CHECK-UNALIGNED-NEXT:    mov w2, #16 // =0x10
+; CHECK-UNALIGNED-NEXT:    bl memmove
+; CHECK-UNALIGNED-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-UNALIGNED-NEXT:    ret
+entry:
+  call void @llvm.memmove.p0.p0.i64(ptr %out, ptr %in, i64 16, i1 false)
+  ret void
+}
+
+; Small (16 bytes here) aligned memmove() should be inlined even if
+; strict-alignment is turned on.
+define void @t16_aligned(ptr align 8 %out, ptr align 8 %in) {
+; CHECK-ALIGNED-LABEL: t16_aligned:
+; CHECK-ALIGNED:       // %bb.0: // %entry
+; CHECK-ALIGNED-NEXT:    ldr q0, [x1]
+; CHECK-ALIGNED-NEXT:    str q0, [x0]
+; CHECK-ALIGNED-NEXT:    ret
+;
+; CHECK-UNALIGNED-LABEL: t16_aligned:
+; CHECK-UNALIGNED:       // %bb.0: // %entry
+; CHECK-UNALIGNED-NEXT:    ldp x9, x8, [x1]
+; CHECK-UNALIGNED-NEXT:    stp x9, x8, [x0]
+; CHECK-UNALIGNED-NEXT:    ret
+entry:
+  call void @llvm.memmove.p0.p0.i64(ptr align 8 %out, ptr align 8 %in, i64 16, i1 false)
+  ret void
+}
+
+; Tiny (4 bytes here) unaligned memmove() should be inlined with byte sized
+; loads and stores if strict-alignment is turned on.
+define void @t4(ptr %out, ptr %in) {
+; CHECK-ALIGNED-LABEL: t4:
+; CHECK-ALIGNED:       // %bb.0: // %entry
+; CHECK-ALIGNED-NEXT:    ldr w8, [x1]
+; CHECK-ALIGNED-NEXT:    str w8, [x0]
+; CHECK-ALIGNED-NEXT:    ret
+;
+; CHECK-UNALIGNED-LABEL: t4:
+; CHECK-UNALIGNED:       // %bb.0: // %entry
+; CHECK-UNALIGNED-NEXT:    ldrb w8, [x1, #3]
+; CHECK-UNALIGNED-NEXT:    ldrb w9, [x1, #2]
+; CHECK-UNALIGNED-NEXT:    ldrb w10, [x1]
+; CHECK-UNALIGNED-NEXT:    ldrb w11, [x1, #1]
+; CHECK-UNALIGNED-NEXT:    strb w8, [x0, #3]
+; CHECK-UNALIGNED-NEXT:    strb w9, [x0, #2]
+; CHECK-UNALIGNED-NEXT:    strb w11, [x0, #1]
+; CHECK-UNALIGNED-NEXT:    strb w10, [x0]
+; CHECK-UNALIGNED-NEXT:    ret
+entry:
+  call void @llvm.memmove.p0.p0.i64(ptr %out, ptr %in, i64 4, i1 false)
+  ret void
+}
+
+define void @t256(ptr %out, ptr %in) {
+; CHECK-ALIGNED-LABEL: t256:
+; CHECK-ALIGNED:       // %bb.0: // %entry
+; CHECK-ALIGNED-NEXT:    ldp q0, q1, [x1]
+; CHECK-ALIGNED-NEXT:    ldp q2, q3, [x1, #32]
+; CHECK-ALIGNED-NEXT:    ldp q4, q5, [x1, #64]
+; CHECK-ALIGNED-NEXT:    ldp q6, q7, [x1, #96]
+; CHECK-ALIGNED-NEXT:    ldp q16, q17, [x1, #224]
+; CHECK-ALIGNED-NEXT:    ldp q18, q19, [x1, #128]
+; CHECK-ALIGNED-NEXT:    ldp q20, q21, [x1, #160]
+; CHECK-ALIGNED-NEXT:    ldp q22, q23, [x1, #192]
+; CHECK-ALIGNED-NEXT:    stp q0, q1, [x0]
+; CHECK-ALIGNED-NEXT:    stp q2, q3, [x0, #32]
+; CHECK-ALIGNED-NEXT:    stp q4, q5, [x0, #64]
+; CHECK-ALIGNED-NEXT:    stp q6, q7, [x0, #96]
+; CHECK-ALIGNED-NEXT:    stp q18, q19, [x0, #128]
+; CHECK-ALIGNED-NEXT:    stp q20, q21, [x0, #160]
+; CHECK-ALIGNED-NEXT:    stp q22, q23, [x0, #192]
+; CHECK-ALIGNED-NEXT:    stp q16, q17, [x0, #224]
+; CHECK-ALIGNED-NEXT:    ret
+;
+; CHECK-UNALIGNED-LABEL: t256:
+; CHECK-UNALIGNED:       // %bb.0: // %entry
+; CHECK-UNALIGNED-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-UNALIGNED-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-UNALIGNED-NEXT:    .cfi_offset w30, -16
+; CHECK-UNALIGNED-NEXT:    mov w2, #256 // =0x100
+; CHECK-UNALIGNED-NEXT:    bl memmove
+; CHECK-UNALIGNED-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-UNALIGNED-NEXT:    ret
+entry:
+  call void @llvm.memmove.p0.p0.i64(ptr %out, ptr %in, i64 256, i1 false)
+  ret void
+}
+
+define void @t257(ptr %out, ptr %in) {
+; CHECK-LABEL: t257:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    mov w2, #257 // =0x101
+; CHECK-NEXT:    bl memmove
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  call void @llvm.memmove.p0.p0.i64(ptr %out, ptr %in, i64 257, i1 false)
+  ret void
+}
+
+declare void @llvm.memmove.p0.p0.i64(ptr nocapture, ptr nocapture readonly, i64, i1)
diff --git a/llvm/test/CodeGen/AMDGPU/limit-coalesce.mir b/llvm/test/CodeGen/AMDGPU/limit-coalesce.mir
@@ -2,13 +2,13 @@
 
 # Check that coalescer does not create wider register tuple than in source
 
-# CHECK:  - { id: 2, class: vreg_64, preferred-register: '' }
-# CHECK:  - { id: 3, class: vreg_64, preferred-register: '' }
-# CHECK:  - { id: 4, class: vreg_64, preferred-register: '' }
-# CHECK:  - { id: 5, class: vreg_96, preferred-register: '' }
-# CHECK:  - { id: 6, class: vreg_96, preferred-register: '' }
-# CHECK:  - { id: 7, class: vreg_128, preferred-register: '' }
-# CHECK:  - { id: 8, class: vreg_128, preferred-register: '' }
+# CHECK:  - { id: 2, class: vreg_64, preferred-register: '', flags: [  ] }
+# CHECK:  - { id: 3, class: vreg_64, preferred-register: '', flags: [  ] }
+# CHECK:  - { id: 4, class: vreg_64, preferred-register: '', flags: [  ] }
+# CHECK:  - { id: 5, class: vreg_96, preferred-register: '', flags: [  ] }
+# CHECK:  - { id: 6, class: vreg_96, preferred-register: '', flags: [  ] }
+# CHECK:  - { id: 7, class: vreg_128, preferred-register: '', flags: [  ] }
+# CHECK:  - { id: 8, class: vreg_128, preferred-register: '', flags: [  ] }
 # No more registers shall be defined
 # CHECK-NEXT: liveins:
 # CHECK:    FLAT_STORE_DWORDX2 $vgpr0_vgpr1, %4,

diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir
@@ -578,3 +578,18 @@ body:             |
     SI_RETURN
 
 ...
+---
+name: vregs
+# FULL: registers:
+# FULL-NEXT:   - { id: 0, class: vgpr_32, preferred-register: '$vgpr1', flags: [ WWM_REG ] }
+# FULL-NEXT:   - { id: 1, class: sgpr_64, preferred-register: '$sgpr0_sgpr1', flags: [  ] }
+# FULL-NEXT:   - { id: 2, class: sgpr_64, preferred-register: '', flags: [  ] }
+registers:
+  - { id: 0, class: vgpr_32, preferred-register: $vgpr1, flags: [ WWM_REG ]}
+  - { id: 1, class: sgpr_64, preferred-register: $sgpr0_sgpr1 }
+  - { id: 2, class: sgpr_64, flags: [ ] }
+body: |
+  bb.0:
+    %2:sgpr_64 = COPY %1
+    %1:sgpr_64 = COPY %0
+...
diff --git a/llvm/test/CodeGen/MIR/Generic/register-flag-error.mir b/llvm/test/CodeGen/MIR/Generic/register-flag-error.mir
@@ -0,0 +1,13 @@
+# RUN: not llc -run-pass=none -filetype=null %s 2>&1 | FileCheck %s --check-prefix=ERR
+
+---
+name:  flags
+registers:
+  - { id: 0, class: _, flags: [ 'VFLAG_ERR' ] }
+body: |
+  bb.0:
+    liveins: $w0
+    %0 = G_ADD $w0, $w0
+...
+# ERR: use of undefined register flag
+# ERR: VFLAG_ERR
diff --git a/llvm/test/CodeGen/MIR/X86/expected-named-register-in-allocation-hint.mir b/llvm/test/CodeGen/MIR/X86/expected-named-register-in-allocation-hint.mir
@@ -14,8 +14,8 @@ name:            test
 tracksRegLiveness: true
 registers:
   - { id: 0, class: gr32 }
-  # CHECK: - { id: 1, class: gr32, preferred-register: '%0' }
-  # CHECK: - { id: 2, class: gr32, preferred-register: '$edi' }
+  # CHECK: - { id: 1, class: gr32, preferred-register: '%0', flags: [  ] }
+  # CHECK: - { id: 2, class: gr32, preferred-register: '$edi', flags: [  ] }
   - { id: 1, class: gr32, preferred-register: '%0' }
   - { id: 2, class: gr32, preferred-register: '$edi' }
 body: |

diff --git a/llvm/test/CodeGen/MIR/X86/generic-instr-type.mir b/llvm/test/CodeGen/MIR/X86/generic-instr-type.mir
@@ -18,11 +18,11 @@
 ---
 name:            test_vregs
 # CHECK:      registers:
-# CHECK-NEXT:   - { id: 0, class: _, preferred-register: '' }
-# CHECK-NEXT:   - { id: 1, class: _, preferred-register: '' }
-# CHECK-NEXT:   - { id: 2, class: _, preferred-register: '' }
-# CHECK-NEXT:   - { id: 3, class: _, preferred-register: '' }
-# CHECK-NEXT:   - { id: 4, class: _, preferred-register: '' }
+# CHECK-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 1, class: _, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 2, class: _, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 3, class: _, preferred-register: '', flags: [  ] }
+# CHECK-NEXT:   - { id: 4, class: _, preferred-register: '', flags: [  ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }

diff --git a/llvm/test/CodeGen/MIR/X86/register-operand-class.mir b/llvm/test/CodeGen/MIR/X86/register-operand-class.mir
@@ -6,11 +6,11 @@
 ---
 # CHECK-LABEL: name: func
 # CHECK: registers:
-# CHECK:   - { id: 0, class: gr32, preferred-register: '' }
-# CHECK:   - { id: 1, class: gr64, preferred-register: '' }
-# CHECK:   - { id: 2, class: gr32, preferred-register: '' }
-# CHECK:   - { id: 3, class: gr16, preferred-register: '' }
-# CHECK:   - { id: 4, class: _, preferred-register: '' }
+# CHECK:   - { id: 0, class: gr32, preferred-register: '', flags: [   ] }
+# CHECK:   - { id: 1, class: gr64, preferred-register: '', flags: [   ] }
+# CHECK:   - { id: 2, class: gr32, preferred-register: '', flags: [   ] }
+# CHECK:   - { id: 3, class: gr16, preferred-register: '', flags: [   ] }
+# CHECK:   - { id: 4, class: _, preferred-register: '', flags: [   ] }
 name: func
 body: |
   bb.0:

diff --git a/llvm/test/CodeGen/MIR/X86/roundtrip.mir b/llvm/test/CodeGen/MIR/X86/roundtrip.mir
@@ -2,8 +2,8 @@
 ---
 # CHECK-LABEL: name: func0
 # CHECK: registers:
-# CHECK:   - { id: 0, class: gr32, preferred-register: '' }
-# CHECK:   - { id: 1, class: gr32, preferred-register: '' }
+# CHECK:   - { id: 0, class: gr32, preferred-register: '', flags: [   ] }
+# CHECK:   - { id: 1, class: gr32, preferred-register: '', flags: [   ] }
 # CHECK: body: |
 # CHECK:   bb.0:
 # CHECK:     %0:gr32 = MOV32r0 implicit-def $eflags

diff --git a/llvm/test/CodeGen/MIR/X86/simple-register-allocation-hints.mir b/llvm/test/CodeGen/MIR/X86/simple-register-allocation-hints.mir
@@ -15,9 +15,9 @@
 name:            test
 tracksRegLiveness: true
 # CHECK: registers:
-# CHECK-NEXT:  - { id: 0, class: gr32, preferred-register: '' }
-# CHECK-NEXT:  - { id: 1, class: gr32, preferred-register: '$esi' }
-# CHECK-NEXT:  - { id: 2, class: gr32, preferred-register: '$edi' }
+# CHECK-NEXT:  - { id: 0, class: gr32, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:  - { id: 1, class: gr32, preferred-register: '$esi', flags: [   ] }
+# CHECK-NEXT:  - { id: 2, class: gr32, preferred-register: '$edi', flags: [   ] }
 registers:
   - { id: 0, class: gr32 }
   - { id: 1, class: gr32, preferred-register: '$esi' }

diff --git a/llvm/test/CodeGen/MIR/X86/virtual-registers.mir b/llvm/test/CodeGen/MIR/X86/virtual-registers.mir
@@ -33,9 +33,9 @@
 name:            bar
 tracksRegLiveness: true
 # CHECK:      registers:
-# CHECK-NEXT:   - { id: 0, class: gr32, preferred-register: '' }
-# CHECK-NEXT:   - { id: 1, class: gr32, preferred-register: '' }
-# CHECK-NEXT:   - { id: 2, class: gr32, preferred-register: '' }
+# CHECK-NEXT:   - { id: 0, class: gr32, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:   - { id: 1, class: gr32, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:   - { id: 2, class: gr32, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: gr32 }
   - { id: 1, class: gr32 }
@@ -67,9 +67,9 @@ name:            foo
 tracksRegLiveness: true
 # CHECK: name: foo
 # CHECK:      registers:
-# CHECK-NEXT:   - { id: 0, class: gr32, preferred-register: '' }
-# CHECK-NEXT:   - { id: 1, class: gr32, preferred-register: '' }
-# CHECK-NEXT:   - { id: 2, class: gr32, preferred-register: '' }
+# CHECK-NEXT:   - { id: 0, class: gr32, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:   - { id: 1, class: gr32, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:   - { id: 2, class: gr32, preferred-register: '', flags: [   ] }
 registers:
   - { id: 2, class: gr32 }
   - { id: 0, class: gr32 }

diff --git a/llvm/test/CodeGen/RISCV/fp-fcanonicalize.ll b/llvm/test/CodeGen/RISCV/fp-fcanonicalize.ll
diff --git a/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-v128.mir b/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-v128.mir
@@ -26,9 +26,9 @@ alignment:       16
 legalized:       false
 regBankSelected: false
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: _, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: _, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -56,9 +56,9 @@ alignment:       16
 legalized:       false
 regBankSelected: false
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: _, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: _, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -86,9 +86,9 @@ alignment:       16
 legalized:       false
 regBankSelected: false
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: _, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: _, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }

diff --git a/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-v256.mir b/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-v256.mir
@@ -26,9 +26,9 @@ alignment:       16
 legalized:       false
 regBankSelected: false
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: _, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: _, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -56,9 +56,9 @@ alignment:       16
 legalized:       false
 regBankSelected: false
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: _, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: _, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -86,9 +86,9 @@ alignment:       16
 legalized:       false
 regBankSelected: false
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: _, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: _, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }

diff --git a/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-v512.mir b/llvm/test/CodeGen/X86/GlobalISel/legalize-mul-v512.mir
@@ -28,9 +28,9 @@ alignment:       16
 legalized:       false
 regBankSelected: false
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: _, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: _, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -58,9 +58,9 @@ alignment:       16
 legalized:       false
 regBankSelected: false
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: _, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: _, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -88,9 +88,9 @@ alignment:       16
 legalized:       false
 regBankSelected: false
 # ALL:      registers:
-# ALL-NEXT:   - { id: 0, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 1, class: _, preferred-register: '' }
-# ALL-NEXT:   - { id: 2, class: _, preferred-register: '' }
+# ALL-NEXT:   - { id: 0, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 1, class: _, preferred-register: '', flags: [   ] }
+# ALL-NEXT:   - { id: 2, class: _, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }

diff --git a/llvm/test/CodeGen/X86/GlobalISel/regbankselect-AVX2.mir b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-AVX2.mir
@@ -33,8 +33,8 @@ selected:        false
 tracksRegLiveness: true
 # CHECK-LABEL: name:            test_mul_vec256
 # CHECK: registers:
-# CHECK:  - { id: 0, class: vecr, preferred-register: '' }
-# CHECK:  - { id: 1, class: vecr, preferred-register: '' }
+# CHECK:  - { id: 0, class: vecr, preferred-register: '', flags: [   ] }
+# CHECK:  - { id: 1, class: vecr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -56,8 +56,8 @@ selected:        false
 tracksRegLiveness: true
 # CHECK-LABEL: name:            test_add_vec256
 # CHECK: registers:
-# CHECK:  - { id: 0, class: vecr, preferred-register: '' }
-# CHECK:  - { id: 1, class: vecr, preferred-register: '' }
+# CHECK:  - { id: 0, class: vecr, preferred-register: '', flags: [   ] }
+# CHECK:  - { id: 1, class: vecr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -79,8 +79,8 @@ selected:        false
 tracksRegLiveness: true
 # CHECK-LABEL: name:            test_sub_vec256
 # CHECK: registers:
-# CHECK:  - { id: 0, class: vecr, preferred-register: '' }
-# CHECK:  - { id: 1, class: vecr, preferred-register: '' }
+# CHECK:  - { id: 0, class: vecr, preferred-register: '', flags: [   ] }
+# CHECK:  - { id: 1, class: vecr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -100,8 +100,8 @@ alignment:       16
 legalized:       true
 regBankSelected: false
 # CHECK:       registers:
-# CHECK-NEXT:    - { id: 0, class: gpr, preferred-register: '' }
-# CHECK-NEXT:    - { id: 1, class: vecr, preferred-register: '' }
+# CHECK-NEXT:    - { id: 0, class: gpr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:    - { id: 1, class: vecr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -122,8 +122,8 @@ alignment:       16
 legalized:       true
 regBankSelected: false
 # CHECK:       registers:
-# CHECK-NEXT:    - { id: 0, class: vecr, preferred-register: '' }
-# CHECK-NEXT:    - { id: 1, class: gpr, preferred-register: '' }
+# CHECK-NEXT:    - { id: 0, class: vecr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:    - { id: 1, class: gpr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }

diff --git a/llvm/test/CodeGen/X86/GlobalISel/regbankselect-AVX512.mir b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-AVX512.mir
@@ -33,8 +33,8 @@ alignment:       16
 legalized:       true
 regBankSelected: false
 # CHECK:       registers:
-# CHECK-NEXT:    - { id: 0, class: vecr, preferred-register: '' }
-# CHECK-NEXT:    - { id: 1, class: vecr, preferred-register: '' }
+# CHECK-NEXT:    - { id: 0, class: vecr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:    - { id: 1, class: vecr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -53,8 +53,8 @@ alignment:       16
 legalized:       true
 regBankSelected: false
 # CHECK:       registers:
-# CHECK-NEXT:    - { id: 0, class: vecr, preferred-register: '' }
-# CHECK-NEXT:    - { id: 1, class: vecr, preferred-register: '' }
+# CHECK-NEXT:    - { id: 0, class: vecr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:    - { id: 1, class: vecr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -73,8 +73,8 @@ alignment:       16
 legalized:       true
 regBankSelected: false
 # CHECK:       registers:
-# CHECK-NEXT:    - { id: 0, class: vecr, preferred-register: '' }
-# CHECK-NEXT:    - { id: 1, class: vecr, preferred-register: '' }
+# CHECK-NEXT:    - { id: 0, class: vecr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:    - { id: 1, class: vecr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -93,8 +93,8 @@ alignment:       16
 legalized:       true
 regBankSelected: false
 # CHECK:       registers:
-# CHECK-NEXT:    - { id: 0, class: gpr, preferred-register: '' }
-# CHECK-NEXT:    - { id: 1, class: vecr, preferred-register: '' }
+# CHECK-NEXT:    - { id: 0, class: gpr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:    - { id: 1, class: vecr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
@@ -115,8 +115,8 @@ alignment:       16
 legalized:       true
 regBankSelected: false
 # CHECK:       registers:
-# CHECK-NEXT:    - { id: 0, class: vecr, preferred-register: '' }
-# CHECK-NEXT:    - { id: 1, class: gpr, preferred-register: '' }
+# CHECK-NEXT:    - { id: 0, class: vecr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:    - { id: 1, class: gpr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }

diff --git a/llvm/test/CodeGen/X86/GlobalISel/regbankselect-X32.mir b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-X32.mir
@@ -14,11 +14,11 @@ alignment:       16
 legalized:       true
 regBankSelected: false
 # CHECK:      registers:
-# CHECK-NEXT:   - { id: 0, class: gpr, preferred-register: '' }
-# CHECK-NEXT:   - { id: 1, class: gpr, preferred-register: '' }
-# CHECK-NEXT:   - { id: 2, class: gpr, preferred-register: '' }
-# CHECK-NEXT:   - { id: 3, class: gpr, preferred-register: '' }
-# CHECK-NEXT:   - { id: 4, class: gpr, preferred-register: '' }
+# CHECK-NEXT:   - { id: 0, class: gpr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:   - { id: 1, class: gpr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:   - { id: 2, class: gpr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:   - { id: 3, class: gpr, preferred-register: '', flags: [   ] }
+# CHECK-NEXT:   - { id: 4, class: gpr, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }

diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-GV-32.mir b/llvm/test/CodeGen/X86/GlobalISel/select-GV-32.mir
@@ -25,12 +25,12 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # X32:                 registers:
-# X32-NEXT:              - { id: 0, class: gr32, preferred-register: '' }
-# X32-NEXT:              - { id: 1, class: gr32, preferred-register: '' }
+# X32-NEXT:              - { id: 0, class: gr32, preferred-register: '', flags: [   ] }
+# X32-NEXT:              - { id: 1, class: gr32, preferred-register: '', flags: [   ] }
 #
 # X32ABI:              registers:
-# X32ABI-NEXT:           - { id: 0, class: low32_addr_access, preferred-register: '' }
-# X32ABI-NEXT:           - { id: 1, class: gr32, preferred-register: '' }
+# X32ABI-NEXT:           - { id: 0, class: low32_addr_access, preferred-register: '', flags: [   ] }
+# X32ABI-NEXT:           - { id: 1, class: gr32, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
@@ -60,8 +60,8 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # X32ALL:              registers:
-# X32ALL-NEXT:           - { id: 0, class: gr32, preferred-register: '' }
-# X32ALL-NEXT:           - { id: 1, class: gr32, preferred-register: '' }
+# X32ALL-NEXT:           - { id: 0, class: gr32, preferred-register: '', flags: [   ] }
+# X32ALL-NEXT:           - { id: 1, class: gr32, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }

diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-GV-64.mir b/llvm/test/CodeGen/X86/GlobalISel/select-GV-64.mir
@@ -25,8 +25,8 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # X64ALL:              registers:
-# X64ALL-NEXT:           - { id: 0, class: gr64, preferred-register: '' }
-# X64ALL-NEXT:           - { id: 1, class: gr64, preferred-register: '' }
+# X64ALL-NEXT:           - { id: 0, class: gr64, preferred-register: '', flags: [   ] }
+# X64ALL-NEXT:           - { id: 1, class: gr64, preferred-register: '', flags: [   ] }
 #
 registers:
   - { id: 0, class: gpr, preferred-register: '' }
@@ -58,8 +58,8 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # X64ALL:              registers:
-# X64ALL-NEXT:           - { id: 0, class: gr32, preferred-register: '' }
-# X64ALL-NEXT:           - { id: 1, class: gr64, preferred-register: '' }
+# X64ALL-NEXT:           - { id: 0, class: gr32, preferred-register: '', flags: [   ] }
+# X64ALL-NEXT:           - { id: 1, class: gr64, preferred-register: '', flags: [   ] }
 #
 registers:
   - { id: 0, class: gpr, preferred-register: '' }

diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-add-v128.mir b/llvm/test/CodeGen/X86/GlobalISel/select-add-v128.mir
@@ -32,19 +32,19 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # NOVL:            registers:
-# NOVL-NEXT:         - { id: 0, class: vr128, preferred-register: '' }
-# NOVL-NEXT:         - { id: 1, class: vr128, preferred-register: '' }
-# NOVL-NEXT:         - { id: 2, class: vr128, preferred-register: '' }
+# NOVL-NEXT:         - { id: 0, class: vr128, preferred-register: '', flags: [   ] }
+# NOVL-NEXT:         - { id: 1, class: vr128, preferred-register: '', flags: [   ] }
+# NOVL-NEXT:         - { id: 2, class: vr128, preferred-register: '', flags: [   ] }
 #
 # AVX512VL:        registers:
-# AVX512VL-NEXT:     - { id: 0, class: vr128, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 1, class: vr128, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 2, class: vr128, preferred-register: '' }
+# AVX512VL-NEXT:     - { id: 0, class: vr128, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 1, class: vr128, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 2, class: vr128, preferred-register: '', flags: [   ] }
 #
 # AVX512BWVL:      registers:
-# AVX512BWVL-NEXT:   - { id: 0, class: vr128x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 1, class: vr128x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 2, class: vr128x, preferred-register: '' }
+# AVX512BWVL-NEXT:   - { id: 0, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 1, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 2, class: vr128x, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
   - { id: 1, class: vecr }
@@ -74,19 +74,19 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # NOVL:            registers:
-# NOVL-NEXT:         - { id: 0, class: vr128, preferred-register: '' }
-# NOVL-NEXT:         - { id: 1, class: vr128, preferred-register: '' }
-# NOVL-NEXT:         - { id: 2, class: vr128, preferred-register: '' }
+# NOVL-NEXT:         - { id: 0, class: vr128, preferred-register: '', flags: [   ] }
+# NOVL-NEXT:         - { id: 1, class: vr128, preferred-register: '', flags: [   ] }
+# NOVL-NEXT:         - { id: 2, class: vr128, preferred-register: '', flags: [   ] }
 #
 # AVX512VL:        registers:
-# AVX512VL-NEXT:     - { id: 0, class: vr128, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 1, class: vr128, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 2, class: vr128, preferred-register: '' }
+# AVX512VL-NEXT:     - { id: 0, class: vr128, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 1, class: vr128, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 2, class: vr128, preferred-register: '', flags: [   ] }
 #
 # AVX512BWVL:      registers:
-# AVX512BWVL-NEXT:   - { id: 0, class: vr128x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 1, class: vr128x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 2, class: vr128x, preferred-register: '' }
+# AVX512BWVL-NEXT:   - { id: 0, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 1, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 2, class: vr128x, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
   - { id: 1, class: vecr }
@@ -116,19 +116,19 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # NOVL:            registers:
-# NOVL-NEXT:         - { id: 0, class: vr128, preferred-register: '' }
-# NOVL-NEXT:         - { id: 1, class: vr128, preferred-register: '' }
-# NOVL-NEXT:         - { id: 2, class: vr128, preferred-register: '' }
+# NOVL-NEXT:         - { id: 0, class: vr128, preferred-register: '', flags: [   ] }
+# NOVL-NEXT:         - { id: 1, class: vr128, preferred-register: '', flags: [   ] }
+# NOVL-NEXT:         - { id: 2, class: vr128, preferred-register: '', flags: [   ] }
 #
 # AVX512VL:        registers:
-# AVX512VL-NEXT:     - { id: 0, class: vr128x, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 1, class: vr128x, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 2, class: vr128x, preferred-register: '' }
+# AVX512VL-NEXT:     - { id: 0, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 1, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 2, class: vr128x, preferred-register: '', flags: [   ] }
 #
 # AVX512BWVL:      registers:
-# AVX512BWVL-NEXT:   - { id: 0, class: vr128x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 1, class: vr128x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 2, class: vr128x, preferred-register: '' }
+# AVX512BWVL-NEXT:   - { id: 0, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 1, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 2, class: vr128x, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
   - { id: 1, class: vecr }
@@ -158,19 +158,19 @@ alignment:       16
 legalized:       true
 regBankSelected: true
 # NOVL:            registers:
-# NOVL-NEXT:         - { id: 0, class: vr128, preferred-register: '' }
-# NOVL-NEXT:         - { id: 1, class: vr128, preferred-register: '' }
-# NOVL-NEXT:         - { id: 2, class: vr128, preferred-register: '' }
+# NOVL-NEXT:         - { id: 0, class: vr128, preferred-register: '', flags: [   ] }
+# NOVL-NEXT:         - { id: 1, class: vr128, preferred-register: '', flags: [   ] }
+# NOVL-NEXT:         - { id: 2, class: vr128, preferred-register: '', flags: [   ] }
 #
 # AVX512VL:        registers:
-# AVX512VL-NEXT:     - { id: 0, class: vr128x, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 1, class: vr128x, preferred-register: '' }
-# AVX512VL-NEXT:     - { id: 2, class: vr128x, preferred-register: '' }
+# AVX512VL-NEXT:     - { id: 0, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 1, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512VL-NEXT:     - { id: 2, class: vr128x, preferred-register: '', flags: [   ] }
 #
 # AVX512BWVL:      registers:
-# AVX512BWVL-NEXT:   - { id: 0, class: vr128x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 1, class: vr128x, preferred-register: '' }
-# AVX512BWVL-NEXT:   - { id: 2, class: vr128x, preferred-register: '' }
+# AVX512BWVL-NEXT:   - { id: 0, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 1, class: vr128x, preferred-register: '', flags: [   ] }
+# AVX512BWVL-NEXT:   - { id: 2, class: vr128x, preferred-register: '', flags: [   ] }
 registers:
   - { id: 0, class: vecr }
   - { id: 1, class: vecr }