[clang] Mark trivial_abi types as "trivially relocatable".

This change enables library code to skip paired move-construction and destruction for `trivial_abi` types, as if they were trivially-movable and trivially-destructible. This offers an extension to the performance fix offered by `trivial_abi`: rather than only offering trivial-type-like performance for pass-by-value, it also offers it for library code that moves values but not as arguments.

For example, if we use `memcpy` for trivially relocatable types inside of vector reallocation, and mark `unique_ptr` as `trivial_abi` (via `_LIBCPP_ABI_ENABLE_UNIQUE_PTR_TRIVIAL_ABI` / `_LIBCPP_ABI_UNSTABLE` / etc.), this would speed up `vector<unique_ptr>::push_back` by 40% on my benchmarks. (Though note that in this case, the compiler could have done this anyway, but happens not to due to the inlining horizon.)

If accepted, I intend to follow up with exactly such changes to library code, including and especially `std::vector`, making them use a trivial relocation operation on trivially relocatable types.

**D50119 and P1144:**

This change is very similar to D50119, which was rejected from Clang. (That change was an implementation of P1144, which is not yet part of the C++ standard.)

The intent of this change, rather than trying to pick a winning proposal for trivial relocation operations, is to extend the behavior of `trivial_abi` in a way that could be made compatible with any such proposal. If P1144 or any similar proposal were accepted, then `trivial_abi`, `__is_trivially_relocatable`, and everything else in this change would be redefined in terms of that.

**Safety:**

It's worth pointing out, specifically, that `trivial_abi` already implies trivial relocatability in a narrow sense: a `trivial_abi` type, when passed by value, has its constructor run in one location, and its destructor run in another, after the type has been trivially relocated (through registers).

Trivial relocatability optimizations could change the number of paired constructor/destructor calls, but this seems unlikely to matter for `trivial_abi` types.

Reviewed By: rsmith

Differential Revision: https://reviews.llvm.org/D114732

clang/docs/LanguageExtensions.rst

@@ -1365,6 +1365,11 @@ The following type trait primitives are supported by Clang. Those traits marked
 * ``__is_trivially_constructible`` (C++, GNU, Microsoft)
 * ``__is_trivially_copyable`` (C++, GNU, Microsoft)
 * ``__is_trivially_destructible`` (C++, MSVC 2013)
+* ``__is_trivially_relocatable`` (Clang): Returns true if moving an object
+  of the given type, and then destroying the source object, is known to be
+  functionally equivalent to copying the underlying bytes and then dropping the
+  source object on the floor. This is true of trivial types and types which
+  were made trivially relocatable via the ``clang::trivial_abi`` attribute.
 * ``__is_union`` (C++, GNU, Microsoft, Embarcadero)
 * ``__is_unsigned`` (C++, Embarcadero):
   Returns false for enumeration types. Note, before Clang 13, returned true for

clang/include/clang/AST/Type.h

@@ -829,6 +829,8 @@ class QualType {
   /// Return true if this is a trivially copyable type (C++0x [basic.types]p9)
   bool isTriviallyCopyableType(const ASTContext &Context) const;
 
+  /// Return true if this is a trivially relocatable type.
+  bool isTriviallyRelocatableType(const ASTContext &Context) const;
 
   /// Returns true if it is a class and it might be dynamic.
   bool mayBeDynamicClass() const;

clang/include/clang/Basic/AttrDocs.td

@@ -3295,6 +3295,9 @@ If a type is trivial for the purposes of calls, has a non-trivial destructor,
 and is passed as an argument by value, the convention is that the callee will
 destroy the object before returning.
 
+If a type is trivial for the purpose of calls, it is assumed to be trivially
+relocatable for the purpose of ``__is_trivially_relocatable``.
+
 Attribute ``trivial_abi`` has no effect in the following cases:
 
 - The class directly declares a virtual base or virtual methods.

clang/include/clang/Basic/TokenKinds.def

@@ -510,6 +510,7 @@ TYPE_TRAIT_1(__has_unique_object_representations,
 KEYWORD(__underlying_type           , KEYCXX)
 
 // Clang-only C++ Type Traits
+TYPE_TRAIT_1(__is_trivially_relocatable, IsTriviallyRelocatable, KEYCXX)
 TYPE_TRAIT_2(__reference_binds_to_temporary, ReferenceBindsToTemporary, KEYCXX)
 
 // Embarcadero Expression Traits

clang/lib/AST/Type.cpp

@@ -2495,6 +2495,25 @@ bool QualType::isTriviallyCopyableType(const ASTContext &Context) const {
   return false;
 }
 
+bool QualType::isTriviallyRelocatableType(const ASTContext &Context) const {
+  QualType BaseElementType = Context.getBaseElementType(*this);
+
+  if (BaseElementType->isIncompleteType()) {
+    return false;
+  } else if (const auto *RD = BaseElementType->getAsRecordDecl()) {
+    return RD->canPassInRegisters();
+  } else {
+    switch (isNonTrivialToPrimitiveDestructiveMove()) {
+    case PCK_Trivial:
+      return !isDestructedType();
+    case PCK_ARCStrong:
+      return true;
+    default:
+      return false;
+    }
+  }
+}
+
 bool QualType::isNonWeakInMRRWithObjCWeak(const ASTContext &Context) const {
   return !Context.getLangOpts().ObjCAutoRefCount &&
          Context.getLangOpts().ObjCWeak &&

clang/lib/Sema/SemaExprCXX.cpp

@@ -11,8 +11,6 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#include "clang/Sema/Template.h"
-#include "clang/Sema/SemaInternal.h"
 #include "TreeTransform.h"
 #include "TypeLocBuilder.h"
 #include "clang/AST/ASTContext.h"
@@ -27,14 +25,17 @@
 #include "clang/Basic/AlignedAllocation.h"
 #include "clang/Basic/PartialDiagnostic.h"
 #include "clang/Basic/TargetInfo.h"
+#include "clang/Basic/TypeTraits.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/SemaLambda.h"
+#include "clang/Sema/Template.h"
 #include "clang/Sema/TemplateDeduction.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/STLExtras.h"
@@ -4746,6 +4747,8 @@ static bool CheckUnaryTypeTraitTypeCompleteness(Sema &S, TypeTrait UTT,
   case UTT_IsStandardLayout:
   case UTT_IsPOD:
   case UTT_IsLiteral:
+  // By analogy, is_trivially_relocatable imposes the same constraints.
+  case UTT_IsTriviallyRelocatable:
   // Per the GCC type traits documentation, T shall be a complete type, cv void,
   // or an array of unknown bound. But GCC actually imposes the same constraints
   // as above.
@@ -5210,6 +5213,8 @@ static bool EvaluateUnaryTypeTrait(Sema &Self, TypeTrait UTT,
     return !T->isIncompleteType();
   case UTT_HasUniqueObjectRepresentations:
     return C.hasUniqueObjectRepresentations(T);
+  case UTT_IsTriviallyRelocatable:
+    return T.isTriviallyRelocatableType(C);
   }
 }
 

clang/test/SemaCXX/attr-trivial-abi.cpp

@@ -5,44 +5,62 @@ void __attribute__((trivial_abi)) foo(); // expected-warning {{'trivial_abi' att
 // Should not crash.
 template <class>
 class __attribute__((trivial_abi)) a { a(a &&); };
+#ifdef _WIN32
+// On Windows, to be trivial-for-calls, an object must be trivially copyable.
+// (And it is only trivially relocatable, currently, if it is trivial for calls.)
+// In this case, it is suppressed by an explicitly defined move constructor.
+// Similar concerns apply to later tests that have #ifdef _WIN32.
+static_assert(!__is_trivially_relocatable(a<int>), "");
+#else
+static_assert(__is_trivially_relocatable(a<int>), "");
+#endif
 
 struct [[clang::trivial_abi]] S0 {
   int a;
 };
+static_assert(__is_trivially_relocatable(S0), "");
 
 struct __attribute__((trivial_abi)) S1 {
   int a;
 };
+static_assert(__is_trivially_relocatable(S1), "");
 
 struct __attribute__((trivial_abi)) S3 { // expected-warning {{'trivial_abi' cannot be applied to 'S3'}} expected-note {{is polymorphic}}
   virtual void m();
 };
+static_assert(!__is_trivially_relocatable(S3), "");
 
 struct S3_2 {
   virtual void m();
 } __attribute__((trivial_abi)); // expected-warning {{'trivial_abi' cannot be applied to 'S3_2'}} expected-note {{is polymorphic}}
+static_assert(!__is_trivially_relocatable(S3_2), "");
 
 struct __attribute__((trivial_abi)) S3_3 { // expected-warning {{'trivial_abi' cannot be applied to 'S3_3'}} expected-note {{has a field of a non-trivial class type}}
   S3_3(S3_3 &&);
   S3_2 s32;
 };
+static_assert(!__is_trivially_relocatable(S3_3), "");
 
 // Diagnose invalid trivial_abi even when the type is templated because it has a non-trivial field.
 template <class T>
 struct __attribute__((trivial_abi)) S3_4 { // expected-warning {{'trivial_abi' cannot be applied to 'S3_4'}} expected-note {{has a field of a non-trivial class type}}
   S3_4(S3_4 &&);
   S3_2 s32;
 };
+static_assert(!__is_trivially_relocatable(S3_4<int>), "");
 
 struct S4 {
   int a;
 };
+static_assert(__is_trivially_relocatable(S4), "");
 
 struct __attribute__((trivial_abi)) S5 : public virtual S4 { // expected-warning {{'trivial_abi' cannot be applied to 'S5'}} expected-note {{has a virtual base}}
 };
+static_assert(!__is_trivially_relocatable(S5), "");
 
 struct __attribute__((trivial_abi)) S9 : public S4 {
 };
+static_assert(__is_trivially_relocatable(S9), "");
 
 struct __attribute__((trivial_abi(1))) S8 { // expected-error {{'trivial_abi' attribute takes no arguments}}
   int a;
@@ -55,6 +73,8 @@ struct __attribute__((trivial_abi)) S10 {
 };
 
 S10<int *> p1;
+static_assert(__is_trivially_relocatable(S10<int>), "");
+static_assert(!__is_trivially_relocatable(S10<S3>), "");
 
 template <class T>
 struct S14 {
@@ -66,11 +86,15 @@ struct __attribute__((trivial_abi)) S15 : S14<T> {
 };
 
 S15<int> s15;
+static_assert(__is_trivially_relocatable(S15<int>), "");
+static_assert(!__is_trivially_relocatable(S15<S3>), "");
 
 template <class T>
 struct __attribute__((trivial_abi)) S16 {
   S14<T> a;
 };
+static_assert(__is_trivially_relocatable(S16<int>), "");
+static_assert(!__is_trivially_relocatable(S16<S3>), "");
 
 S16<int> s16;
 
@@ -79,34 +103,50 @@ struct __attribute__((trivial_abi)) S17 {
 };
 
 S17<int> s17;
+static_assert(__is_trivially_relocatable(S17<int>), "");
+static_assert(__is_trivially_relocatable(S17<S3>), "");
 
 namespace deletedCopyMoveConstructor {
 struct __attribute__((trivial_abi)) CopyMoveDeleted { // expected-warning {{'trivial_abi' cannot be applied to 'CopyMoveDeleted'}} expected-note {{copy constructors and move constructors are all deleted}}
   CopyMoveDeleted(const CopyMoveDeleted &) = delete;
   CopyMoveDeleted(CopyMoveDeleted &&) = delete;
 };
+static_assert(!__is_trivially_relocatable(CopyMoveDeleted), "");
 
 struct __attribute__((trivial_abi)) S18 { // expected-warning {{'trivial_abi' cannot be applied to 'S18'}} expected-note {{copy constructors and move constructors are all deleted}}
   CopyMoveDeleted a;
 };
+static_assert(!__is_trivially_relocatable(S18), "");
 
 struct __attribute__((trivial_abi)) CopyDeleted {
   CopyDeleted(const CopyDeleted &) = delete;
   CopyDeleted(CopyDeleted &&) = default;
 };
+#ifdef _WIN32
+static_assert(!__is_trivially_relocatable(CopyDeleted), "");
+#else
+static_assert(__is_trivially_relocatable(CopyDeleted), "");
+#endif
 
 struct __attribute__((trivial_abi)) MoveDeleted {
   MoveDeleted(const MoveDeleted &) = default;
   MoveDeleted(MoveDeleted &&) = delete;
 };
+static_assert(__is_trivially_relocatable(MoveDeleted), "");
 
 struct __attribute__((trivial_abi)) S19 { // expected-warning {{'trivial_abi' cannot be applied to 'S19'}} expected-note {{copy constructors and move constructors are all deleted}}
   CopyDeleted a;
   MoveDeleted b;
 };
+static_assert(!__is_trivially_relocatable(S19), "");
 
 // This is fine since the move constructor isn't deleted.
 struct __attribute__((trivial_abi)) S20 {
   int &&a; // a member of rvalue reference type deletes the copy constructor.
 };
+#ifdef _WIN32
+static_assert(!__is_trivially_relocatable(S20), "");
+#else
+static_assert(__is_trivially_relocatable(S20), "");
+#endif
 } // namespace deletedCopyMoveConstructor

clang/test/SemaCXX/type-traits.cpp

@@ -2854,3 +2854,64 @@ void test() { (void) __is_constructible(int, T32768(int)); }
 #undef T16384
 #undef T32768
 } // namespace type_trait_expr_numargs_overflow
+
+namespace is_trivially_relocatable {
+
+static_assert(!__is_trivially_relocatable(void), "");
+static_assert(__is_trivially_relocatable(int), "");
+static_assert(__is_trivially_relocatable(int[]), "");
+
+enum Enum {};
+static_assert(__is_trivially_relocatable(Enum), "");
+static_assert(__is_trivially_relocatable(Enum[]), "");
+
+union Union {int x;};
+static_assert(__is_trivially_relocatable(Union), "");
+static_assert(__is_trivially_relocatable(Union[]), "");
+
+struct Trivial {};
+static_assert(__is_trivially_relocatable(Trivial), "");
+static_assert(__is_trivially_relocatable(Trivial[]), "");
+
+struct Incomplete; // expected-note {{forward declaration of 'is_trivially_relocatable::Incomplete'}}
+bool unused = __is_trivially_relocatable(Incomplete); // expected-error {{incomplete type}}
+
+struct NontrivialDtor {
+  ~NontrivialDtor() {}
+};
+static_assert(!__is_trivially_relocatable(NontrivialDtor), "");
+static_assert(!__is_trivially_relocatable(NontrivialDtor[]), "");
+
+struct NontrivialCopyCtor {
+  NontrivialCopyCtor(const NontrivialCopyCtor&) {}
+};
+static_assert(!__is_trivially_relocatable(NontrivialCopyCtor), "");
+static_assert(!__is_trivially_relocatable(NontrivialCopyCtor[]), "");
+
+struct NontrivialMoveCtor {
+  NontrivialMoveCtor(NontrivialMoveCtor&&) {}
+};
+static_assert(!__is_trivially_relocatable(NontrivialMoveCtor), "");
+static_assert(!__is_trivially_relocatable(NontrivialMoveCtor[]), "");
+
+struct [[clang::trivial_abi]] TrivialAbiNontrivialDtor {
+  ~TrivialAbiNontrivialDtor() {}
+};
+static_assert(__is_trivially_relocatable(TrivialAbiNontrivialDtor), "");
+static_assert(__is_trivially_relocatable(TrivialAbiNontrivialDtor[]), "");
+
+struct [[clang::trivial_abi]] TrivialAbiNontrivialCopyCtor {
+  TrivialAbiNontrivialCopyCtor(const TrivialAbiNontrivialCopyCtor&) {}
+};
+static_assert(__is_trivially_relocatable(TrivialAbiNontrivialCopyCtor), "");
+static_assert(__is_trivially_relocatable(TrivialAbiNontrivialCopyCtor[]), "");
+
+// A more complete set of tests for the behavior of trivial_abi can be found in
+// clang/test/SemaCXX/attr-trivial-abi.cpp
+struct [[clang::trivial_abi]] TrivialAbiNontrivialMoveCtor {
+  TrivialAbiNontrivialMoveCtor(TrivialAbiNontrivialMoveCtor&&) {}
+};
+static_assert(__is_trivially_relocatable(TrivialAbiNontrivialMoveCtor), "");
+static_assert(__is_trivially_relocatable(TrivialAbiNontrivialMoveCtor[]), "");
+
+} // namespace is_trivially_relocatable

clang/test/SemaObjCXX/arc-type-traits.mm

@@ -12,7 +12,7 @@
 #define TRAIT_IS_FALSE(Trait, Type) char JOIN2(Trait,__LINE__)[Trait(Type)? -1 : 1]
 #define TRAIT_IS_TRUE_2(Trait, Type1, Type2) char JOIN2(Trait,__LINE__)[Trait(Type1, Type2)? 1 : -1]
 #define TRAIT_IS_FALSE_2(Trait, Type1, Type2) char JOIN2(Trait,__LINE__)[Trait(Type1, Type2)? -1 : 1]
-  
+
 struct HasStrong { id obj; };
 struct HasWeak { __weak id obj; };
 struct HasUnsafeUnretained { __unsafe_unretained id obj; };
@@ -213,3 +213,11 @@
 TRAIT_IS_TRUE_2(__is_trivially_constructible, HasUnsafeUnretained, HasUnsafeUnretained);
 TRAIT_IS_TRUE_2(__is_trivially_constructible, HasUnsafeUnretained, HasUnsafeUnretained&&);
 
+// __is_trivially_relocatable
+TRAIT_IS_TRUE(__is_trivially_relocatable, __strong id);
+TRAIT_IS_FALSE(__is_trivially_relocatable, __weak id);
+TRAIT_IS_TRUE(__is_trivially_relocatable, __autoreleasing id);
+TRAIT_IS_TRUE(__is_trivially_relocatable, __unsafe_unretained id);
+TRAIT_IS_TRUE(__is_trivially_relocatable, HasStrong);
+TRAIT_IS_FALSE(__is_trivially_relocatable, HasWeak);
+TRAIT_IS_TRUE(__is_trivially_relocatable, HasUnsafeUnretained);

clang/test/SemaObjCXX/objc-weak-type-traits.mm

@@ -8,7 +8,7 @@
 #define TRAIT_IS_FALSE(Trait, Type) static_assert(!Trait(Type), "")
 #define TRAIT_IS_TRUE_2(Trait, Type1, Type2) static_assert(Trait(Type1, Type2), "")
 #define TRAIT_IS_FALSE_2(Trait, Type1, Type2) static_assert(!Trait(Type1, Type2), "")
-  
+
 struct HasStrong { id obj; };
 struct HasWeak { __weak id obj; };
 struct HasUnsafeUnretained { __unsafe_unretained id obj; };
@@ -208,3 +208,12 @@
 TRAIT_IS_FALSE_2(__is_trivially_constructible, HasWeak, HasWeak&&);
 TRAIT_IS_TRUE_2(__is_trivially_constructible, HasUnsafeUnretained, HasUnsafeUnretained);
 TRAIT_IS_TRUE_2(__is_trivially_constructible, HasUnsafeUnretained, HasUnsafeUnretained&&);
+
+// __is_trivially_relocatable
+TRAIT_IS_TRUE(__is_trivially_relocatable, __strong id);
+TRAIT_IS_FALSE(__is_trivially_relocatable, __weak id);
+TRAIT_IS_TRUE(__is_trivially_relocatable, __autoreleasing id);
+TRAIT_IS_TRUE(__is_trivially_relocatable, __unsafe_unretained id);
+TRAIT_IS_TRUE(__is_trivially_relocatable, HasStrong);
+TRAIT_IS_FALSE(__is_trivially_relocatable, HasWeak);
+TRAIT_IS_TRUE(__is_trivially_relocatable, HasUnsafeUnretained);