[clang][bytecode] Allocate InitMaps via Program/InterpState allocators

clang/lib/AST/ByteCode/Descriptor.cpp

@@ -52,10 +52,8 @@ static void dtorTy(Block *, std::byte *Ptr, const Descriptor *) {
 template <typename T>
 static void ctorArrayTy(Block *, std::byte *Ptr, bool, bool, bool, bool, bool,
                         const Descriptor *D) {
-  new (Ptr) InitMapPtr(std::nullopt);
-
   if constexpr (needsCtor<T>()) {
-    Ptr += sizeof(InitMapPtr);
+    Ptr += sizeof(InitMap *);
     for (unsigned I = 0, NE = D->getNumElems(); I < NE; ++I) {
       new (&reinterpret_cast<T *>(Ptr)[I]) T();
     }
@@ -64,13 +62,8 @@ static void ctorArrayTy(Block *, std::byte *Ptr, bool, bool, bool, bool, bool,
 
 template <typename T>
 static void dtorArrayTy(Block *, std::byte *Ptr, const Descriptor *D) {
-  InitMapPtr &IMP = *reinterpret_cast<InitMapPtr *>(Ptr);
-
-  if (IMP)
-    IMP = std::nullopt;
-
   if constexpr (needsCtor<T>()) {
-    Ptr += sizeof(InitMapPtr);
+    Ptr += sizeof(InitMap *);
     for (unsigned I = 0, NE = D->getNumElems(); I < NE; ++I) {
       reinterpret_cast<T *>(Ptr)[I].~T();
     }
@@ -239,12 +232,6 @@ static bool needsRecordDtor(const Record *R) {
 
 static BlockCtorFn getCtorPrim(PrimType T) {
   switch (T) {
-  case PT_Float:
-    return ctorTy<PrimConv<PT_Float>::T>;
-  case PT_IntAP:
-    return ctorTy<PrimConv<PT_IntAP>::T>;
-  case PT_IntAPS:
-    return ctorTy<PrimConv<PT_IntAPS>::T>;
   case PT_Ptr:
     return ctorTy<PrimConv<PT_Ptr>::T>;
   case PT_MemberPtr:
@@ -257,12 +244,6 @@ static BlockCtorFn getCtorPrim(PrimType T) {
 
 static BlockDtorFn getDtorPrim(PrimType T) {
   switch (T) {
-  case PT_Float:
-    return dtorTy<PrimConv<PT_Float>::T>;
-  case PT_IntAP:
-    return dtorTy<PrimConv<PT_IntAP>::T>;
-  case PT_IntAPS:
-    return dtorTy<PrimConv<PT_IntAPS>::T>;
   case PT_Ptr:
     return dtorTy<PrimConv<PT_Ptr>::T>;
   case PT_MemberPtr:
@@ -273,14 +254,16 @@ static BlockDtorFn getDtorPrim(PrimType T) {
   llvm_unreachable("Unhandled PrimType");
 }
 
-static BlockCtorFn getCtorArrayPrim(PrimType Type) {
-  TYPE_SWITCH(Type, return ctorArrayTy<T>);
-  llvm_unreachable("unknown Expr");
-}
-
-static BlockDtorFn getDtorArrayPrim(PrimType Type) {
-  TYPE_SWITCH(Type, return dtorArrayTy<T>);
-  llvm_unreachable("unknown Expr");
+static BlockDtorFn getDtorArrayPrim(PrimType T) {
+  switch (T) {
+  case PT_Ptr:
+    return dtorArrayTy<PrimConv<PT_Ptr>::T>;
+  case PT_MemberPtr:
+    return dtorArrayTy<PrimConv<PT_MemberPtr>::T>;
+  default:
+    return nullptr;
+  }
+  llvm_unreachable("Unhandled PrimType");
 }
 
 /// Primitives.
@@ -302,10 +285,9 @@ Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD,
                        bool IsMutable)
     : Source(D), ElemSize(primSize(Type)), Size(ElemSize * NumElems),
       MDSize(MD.value_or(0)),
-      AllocSize(align(MDSize) + align(Size) + sizeof(InitMapPtr)), PrimT(Type),
+      AllocSize(align(MDSize) + align(Size) + sizeof(InitMap *)), PrimT(Type),
       IsConst(IsConst), IsMutable(IsMutable), IsTemporary(IsTemporary),
-      IsArray(true), CtorFn(getCtorArrayPrim(Type)),
-      DtorFn(getDtorArrayPrim(Type)) {
+      IsArray(true), DtorFn(getDtorArrayPrim(Type)) {
   assert(Source && "Missing source");
   assert(NumElems <= (MaxArrayElemBytes / ElemSize));
 }
@@ -315,10 +297,9 @@ Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD,
                        bool IsTemporary, bool IsConst, UnknownSize)
     : Source(D), ElemSize(primSize(Type)), Size(UnknownSizeMark),
       MDSize(MD.value_or(0)),
-      AllocSize(MDSize + sizeof(InitMapPtr) + alignof(void *)), PrimT(Type),
+      AllocSize(MDSize + sizeof(InitMap *) + alignof(void *)), PrimT(Type),
       IsConst(IsConst), IsMutable(false), IsTemporary(IsTemporary),
-      IsArray(true), CtorFn(getCtorArrayPrim(Type)),
-      DtorFn(getDtorArrayPrim(Type)) {
+      IsArray(true), DtorFn(getDtorArrayPrim(Type)) {
   assert(Source && "Missing source");
 }
 
@@ -468,15 +449,16 @@ bool Descriptor::hasTrivialDtor() const {
 
 bool Descriptor::isUnion() const { return isRecord() && ElemRecord->isUnion(); }
 
-InitMap::InitMap(unsigned N)
-    : UninitFields(N), Data(std::make_unique<T[]>(numFields(N))) {}
+InitMap::InitMap(unsigned N) : UninitFields(N) {
+  std::memset(data(), 0, numFields(N) * sizeof(T));
+}
 
 bool InitMap::initializeElement(unsigned I) {
   unsigned Bucket = I / PER_FIELD;
   T Mask = T(1) << (I % PER_FIELD);
   if (!(data()[Bucket] & Mask)) {
     data()[Bucket] |= Mask;
-    UninitFields -= 1;
+    --UninitFields;
   }
   return UninitFields == 0;
 }

clang/lib/AST/ByteCode/Descriptor.h

@@ -16,6 +16,7 @@
 #include "PrimType.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/Expr.h"
+#include <limits>
 
 namespace clang {
 namespace interp {
@@ -27,7 +28,6 @@ struct Descriptor;
 enum PrimType : uint8_t;
 
 using DeclTy = llvm::PointerUnion<const Decl *, const Expr *>;
-using InitMapPtr = std::optional<std::pair<bool, std::shared_ptr<InitMap>>>;
 
 /// Invoked whenever a block is created. The constructor method fills in the
 /// inline descriptors of all fields and array elements. It also initializes
@@ -146,7 +146,7 @@ struct Descriptor final {
 
   /// Maximum number of bytes to be used for array elements.
   static constexpr unsigned MaxArrayElemBytes =
-      std::numeric_limits<decltype(AllocSize)>::max() - sizeof(InitMapPtr) -
+      std::numeric_limits<decltype(AllocSize)>::max() - sizeof(InitMap *) -
       align(std::max(*InlineDescMD, *GlobalMD));
 
   /// Pointer to the record, if block contains records.
@@ -278,7 +278,7 @@ struct Descriptor final {
 };
 
 /// Bitfield tracking the initialisation status of elements of primitive arrays.
-struct InitMap final {
+struct alignas(alignof(uint64_t)) InitMap final {
 private:
   /// Type packing bits.
   using T = uint64_t;
@@ -289,12 +289,33 @@ struct InitMap final {
   /// Initializes the map with no fields set.
   explicit InitMap(unsigned N);
 
+  /// Checks if all elements have been initialized.
+  static bool allInitialized(const InitMap *IM) {
+    return reinterpret_cast<uintptr_t>(IM) ==
+           std::numeric_limits<uintptr_t>::max();
+  }
+
+  /// Marks all elements as initialized.
+  static void markAllInitialized(InitMap *&IMPtr) {
+    std::memset(&IMPtr, static_cast<int>(std::numeric_limits<uintptr_t>::max()),
+                sizeof(void *));
+  }
+
+  /// Returns the number of bytes needed to allocate the InitMap for
+  /// \param N elements.
+  static unsigned allocBytes(unsigned N) {
+    return align(sizeof(InitMap)) + (numFields(N) * sizeof(T));
+  }
+
 private:
   friend class Pointer;
 
   /// Returns a pointer to storage.
-  T *data() { return Data.get(); }
-  const T *data() const { return Data.get(); }
+  T *data() {
+    return reinterpret_cast<T *>(reinterpret_cast<std::byte *>(this) +
+                                 align(sizeof(InitMap)));
+  }
+  const T *data() const { return const_cast<InitMap *>(this)->data(); }
 
   /// Initializes an element. Returns true when object if fully initialized.
   bool initializeElement(unsigned I);
@@ -307,7 +328,6 @@ struct InitMap final {
   }
   /// Number of fields not initialized.
   unsigned UninitFields;
-  std::unique_ptr<T[]> Data;
 };
 
 } // namespace interp

clang/lib/AST/ByteCode/Disasm.cpp

@@ -440,7 +440,7 @@ LLVM_DUMP_METHOD void Descriptor::dumpFull(unsigned Offset,
     OS.indent(Spaces) << "Elements: " << getNumElems() << '\n';
     OS.indent(Spaces) << "Element type: " << primTypeToString(getPrimType())
                       << '\n';
-    unsigned FO = Offset + sizeof(InitMapPtr);
+    unsigned FO = Offset + sizeof(InitMap *);
     for (unsigned I = 0; I != getNumElems(); ++I) {
       OS.indent(Spaces) << "Element " << I << " offset: " << FO << '\n';
       FO += getElemSize();

clang/lib/AST/ByteCode/EvaluationResult.cpp

@@ -54,12 +54,11 @@ static bool CheckArrayInitialized(InterpState &S, SourceLocation Loc,
   } else {
     // Primitive arrays.
     if (S.getContext().canClassify(ElemType)) {
-      if (BasePtr.allElementsInitialized()) {
+      if (BasePtr.allElementsInitialized())
         return true;
-      } else {
-        DiagnoseUninitializedSubobject(S, Loc, BasePtr.getField());
-        return false;
-      }
+
+      DiagnoseUninitializedSubobject(S, Loc, BasePtr.getField());
+      return false;
     }
 
     for (size_t I = 0; I != NumElems; ++I) {

clang/lib/AST/ByteCode/Interp.cpp

@@ -2318,7 +2318,7 @@ bool FinishInitGlobal(InterpState &S, CodePtr OpPC) {
 
   finishGlobalRecurse(S, Ptr);
   if (Ptr.canBeInitialized()) {
-    Ptr.initialize();
+    Ptr.initialize(S);
     Ptr.activate();
   }