From 0db670a49b72eab7fbcae056c17345f2bc76f283 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Mon, 29 Sep 2025 16:12:15 +0200 Subject: [PATCH] [clang][bytecode] Add support for (toplevel) array fillers. This adds support for array fillers, for global, toplevel descriptors. Toplevel meaning that this gets an array filler: ```c++ constexpr int foo[200] = {1}; ``` But this does not: ```c++ struct S { int i[10] = {3}; }; constexpr S s{}; ``` For multidimensional arrays, only the topmost dimension gets an array filler: ```c++ constexpr float arr_f[3][5] = { {1, 2, 3, 4, 5}, }; ``` This means that this patch fixes e.g. `test/SemaCXX/large-array-init`, but does not fix `test/CodeGenCXX/cxx11-initializer-aggregate.cpp`, as that contains arrays such as: ```c++ struct B { int n; int arr[1024 * 1024 * 1024 * 2u]; } b = {1, {2}}; // ... unsigned char data_3[1024][1024][1024] = {{{0}}}; // ... unsigned char data_12[1024][1024][1024] = {{{1}}}; ``` additionally, adding array fillers regresses performance in the common case of no array fillers: https://llvm-compile-time-tracker.com/compare.php?from=02052caa09b27b422c452a2e1be2e3bfed710156&to=5b654212c1442d814aa8ed1c8de960432e479cdd&stat=instructions:u --- clang/lib/AST/ByteCode/Compiler.cpp | 65 +++- clang/lib/AST/ByteCode/Descriptor.cpp | 309 +++++++++++++++++- clang/lib/AST/ByteCode/Descriptor.h | 49 ++- clang/lib/AST/ByteCode/Disasm.cpp | 38 ++- clang/lib/AST/ByteCode/EvaluationResult.cpp | 16 +- clang/lib/AST/ByteCode/Interp.cpp | 2 + clang/lib/AST/ByteCode/Interp.h | 49 ++- clang/lib/AST/ByteCode/InterpBlock.h | 3 + .../lib/AST/ByteCode/InterpBuiltinBitCast.cpp | 5 + clang/lib/AST/ByteCode/InterpFrame.cpp | 29 +- clang/lib/AST/ByteCode/InterpFrame.h | 14 +- clang/lib/AST/ByteCode/InterpHelpers.cpp | 58 ++++ clang/lib/AST/ByteCode/InterpHelpers.h | 14 + clang/lib/AST/ByteCode/Opcodes.td | 2 + clang/lib/AST/ByteCode/Pointer.cpp | 17 +- clang/lib/AST/ByteCode/Pointer.h | 42 ++- clang/lib/AST/ByteCode/Program.cpp | 67 +++- clang/lib/AST/ByteCode/Program.h | 15 +- clang/lib/AST/CMakeLists.txt | 1 + clang/test/AST/ByteCode/array-fillers.cpp | 50 +++ 20 files changed, 778 insertions(+), 67 deletions(-) create mode 100644 clang/lib/AST/ByteCode/InterpHelpers.cpp create mode 100644 clang/lib/AST/ByteCode/InterpHelpers.h create mode 100644 clang/test/AST/ByteCode/array-fillers.cpp diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp index 74cae030bb9bb..5b5bfd5981ea3 100644 --- a/clang/lib/AST/ByteCode/Compiler.cpp +++ b/clang/lib/AST/ByteCode/Compiler.cpp @@ -16,6 +16,11 @@ #include "PrimType.h" #include "Program.h" #include "clang/AST/Attr.h" +#include "clang/Basic/UnsignedOrNone.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/Support/Casting.h" + +#define DEBUG_TYPE "exprconstant" using namespace clang; using namespace clang::interp; @@ -1942,9 +1947,9 @@ bool Compiler::visitInitList(ArrayRef Inits, const ConstantArrayType *CAT = Ctx.getASTContext().getAsConstantArrayType(QT); - uint64_t NumElems = CAT->getZExtSize(); + uint64_t Capacity = CAT->getZExtSize(); - if (!this->emitCheckArraySize(NumElems, E)) + if (!this->emitCheckArraySize(Capacity, E)) return false; OptPrimType InitT = classify(CAT->getElementType()); @@ -1977,15 +1982,59 @@ bool Compiler::visitInitList(ArrayRef Inits, } } - // Expand the filler expression. - // FIXME: This should go away. - if (ArrayFiller) { - for (; ElementIndex != NumElems; ++ElementIndex) { - if (!this->visitArrayElemInit(ElementIndex, ArrayFiller, InitT)) + auto isInit = [&](const Expr *E) -> bool { + if (const auto *VD = dyn_cast_if_present(this->InitializingDecl)) + return E == VD->getInit(); + return false; + }; + + LLVM_DEBUG(llvm::dbgs() << "The number of elements to initialize: " + << ElementIndex << ".\n"); + + if (isInit(E) && ArraySize::shouldUseFiller(ElementIndex, Capacity) && + Context::shouldBeGloballyIndexed(InitializingDecl)) { + assert(ArrayFiller); + + unsigned TempOffset = 0; + if (InitT) { + TempOffset = this->allocateLocalPrimitive(ArrayFiller, *InitT, + /*IsConst=*/false); + if (!this->visit(ArrayFiller)) return false; + if (!this->emitSetLocal(*InitT, TempOffset, ArrayFiller)) + return false; + } else { + if (UnsignedOrNone LocalOffset = this->allocateLocal(ArrayFiller)) { + TempOffset = *LocalOffset; + if (!this->emitGetPtrLocal(*LocalOffset, E)) + return false; + + if (!this->visitInitializer(ArrayFiller)) + return false; + + if (!this->emitFinishInitPop(ArrayFiller)) + return false; + } else { + return false; + } + } + + // Now copy the array filler from the local to the array. + if (!this->emitGetPtrLocal(TempOffset, E)) + return false; + if (!this->emitSetArrayFillerPtr(TempOffset, E)) + return false; + } else { + // Expand the filler expression. + if (ArrayFiller) { + for (; ElementIndex != Capacity; ++ElementIndex) { + if (!this->visitArrayElemInit(ElementIndex, ArrayFiller, InitT)) + return false; + } } } + // assert(false); return this->emitFinishInit(E); } @@ -4884,6 +4933,8 @@ Compiler::visitVarDecl(const VarDecl *VD, const Expr *Init, } // Local variables. InitLinkScope ILS(this, InitLink::Decl(VD)); + this->InitializingDecl = VD; + // DeclScope LocalScope(this, VD); if (VarT) { unsigned Offset = this->allocateLocalPrimitive( diff --git a/clang/lib/AST/ByteCode/Descriptor.cpp b/clang/lib/AST/ByteCode/Descriptor.cpp index 0a819599287ee..2398d93cfbf08 100644 --- a/clang/lib/AST/ByteCode/Descriptor.cpp +++ b/clang/lib/AST/ByteCode/Descriptor.cpp @@ -17,6 +17,7 @@ #include "Record.h" #include "Source.h" #include "clang/AST/ExprCXX.h" +#include "llvm/Support/raw_ostream.h" using namespace clang; using namespace clang::interp; @@ -288,10 +289,10 @@ Descriptor::Descriptor(const DeclTy &D, const Type *SourceTy, PrimType Type, MetadataSize MD, bool IsConst, bool IsTemporary, bool IsMutable, bool IsVolatile) : Source(D), SourceType(SourceTy), ElemSize(primSize(Type)), Size(ElemSize), - MDSize(MD.value_or(0)), AllocSize(align(Size + MDSize)), PrimT(Type), - IsConst(IsConst), IsMutable(IsMutable), IsTemporary(IsTemporary), - IsVolatile(IsVolatile), CtorFn(getCtorPrim(Type)), - DtorFn(getDtorPrim(Type)) { + Capacity(Size), MDSize(MD.value_or(0)), AllocSize(align(Size + MDSize)), + PrimT(Type), IsConst(IsConst), IsMutable(IsMutable), + IsTemporary(IsTemporary), IsVolatile(IsVolatile), + CtorFn(getCtorPrim(Type)), DtorFn(getDtorPrim(Type)) { assert(AllocSize >= Size); assert(Source && "Missing source"); } @@ -301,7 +302,7 @@ Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD, size_t NumElems, bool IsConst, bool IsTemporary, bool IsMutable) : Source(D), ElemSize(primSize(Type)), Size(ElemSize * NumElems), - MDSize(MD.value_or(0)), + Capacity(NumElems), MDSize(MD.value_or(0)), AllocSize(align(MDSize) + align(Size) + sizeof(InitMapPtr)), PrimT(Type), IsConst(IsConst), IsMutable(IsMutable), IsTemporary(IsTemporary), IsArray(true), CtorFn(getCtorArrayPrim(Type)), @@ -310,11 +311,26 @@ Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD, assert(NumElems <= (MaxArrayElemBytes / ElemSize)); } +Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD, + ArraySize ArrSize, bool IsConst, bool IsTemporary, + bool IsMutable) + : Source(D), ElemSize(primSize(Type)), + Size(ElemSize * (ArrSize.Size + ArrSize.hasFiller())), + // Size(ElemSize * (ArrSize.Size)), + Capacity(ArrSize.Capacity), MDSize(MD.value_or(0)), + AllocSize(align(MDSize) + align(Size) + sizeof(InitMapPtr)), PrimT(Type), + IsConst(IsConst), IsMutable(IsMutable), IsTemporary(IsTemporary), + IsArray(true), CtorFn(getCtorArrayPrim(Type)), + DtorFn(getDtorArrayPrim(Type)) { + assert(Source && "Missing source"); + assert(ArrSize.Size <= (MaxArrayElemBytes / ElemSize)); +} + /// Primitive unknown-size arrays. Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD, bool IsTemporary, bool IsConst, UnknownSize) : Source(D), ElemSize(primSize(Type)), Size(UnknownSizeMark), - MDSize(MD.value_or(0)), + Capacity(Size), MDSize(MD.value_or(0)), AllocSize(MDSize + sizeof(InitMapPtr) + alignof(void *)), PrimT(Type), IsConst(IsConst), IsMutable(false), IsTemporary(IsTemporary), IsArray(true), CtorFn(getCtorArrayPrim(Type)), @@ -329,7 +345,24 @@ Descriptor::Descriptor(const DeclTy &D, const Type *SourceTy, bool IsMutable) : Source(D), SourceType(SourceTy), ElemSize(Elem->getAllocSize() + sizeof(InlineDescriptor)), - Size(ElemSize * NumElems), MDSize(MD.value_or(0)), + Size(ElemSize * NumElems), Capacity(NumElems), MDSize(MD.value_or(0)), + AllocSize(std::max(alignof(void *), Size) + MDSize), + ElemDesc(Elem), IsConst(IsConst), IsMutable(IsMutable), + IsTemporary(IsTemporary), IsArray(true), CtorFn(ctorArrayDesc), + DtorFn(Elem->DtorFn ? dtorArrayDesc : nullptr) { + assert(Source && "Missing source"); +} + +/// Arrays of composite elements. +Descriptor::Descriptor(const DeclTy &D, const Type *SourceTy, + const Descriptor *Elem, MetadataSize MD, + ArraySize ArrSize, bool IsConst, bool IsTemporary, + bool IsMutable) + : Source(D), SourceType(SourceTy), + ElemSize(Elem->getAllocSize() + sizeof(InlineDescriptor)), + Size(ElemSize * (ArrSize.Size + ArrSize.hasFiller())), + // Size(ElemSize * (ArrSize.Size)), + Capacity(ArrSize.Capacity), MDSize(MD.value_or(0)), AllocSize(std::max(alignof(void *), Size) + MDSize), ElemDesc(Elem), IsConst(IsConst), IsMutable(IsMutable), IsTemporary(IsTemporary), IsArray(true), CtorFn(ctorArrayDesc), @@ -341,7 +374,7 @@ Descriptor::Descriptor(const DeclTy &D, const Type *SourceTy, Descriptor::Descriptor(const DeclTy &D, const Descriptor *Elem, MetadataSize MD, bool IsTemporary, UnknownSize) : Source(D), ElemSize(Elem->getAllocSize() + sizeof(InlineDescriptor)), - Size(UnknownSizeMark), MDSize(MD.value_or(0)), + Size(UnknownSizeMark), Capacity(Size), MDSize(MD.value_or(0)), AllocSize(MDSize + alignof(void *)), ElemDesc(Elem), IsConst(true), IsMutable(false), IsTemporary(IsTemporary), IsArray(true), CtorFn(ctorArrayDesc), DtorFn(Elem->DtorFn ? dtorArrayDesc : nullptr) { @@ -353,16 +386,16 @@ Descriptor::Descriptor(const DeclTy &D, const Record *R, MetadataSize MD, bool IsConst, bool IsTemporary, bool IsMutable, bool IsVolatile) : Source(D), ElemSize(std::max(alignof(void *), R->getFullSize())), - Size(ElemSize), MDSize(MD.value_or(0)), AllocSize(Size + MDSize), - ElemRecord(R), IsConst(IsConst), IsMutable(IsMutable), - IsTemporary(IsTemporary), IsVolatile(IsVolatile), CtorFn(ctorRecord), - DtorFn(needsRecordDtor(R) ? dtorRecord : nullptr) { + Size(ElemSize), Capacity(Size), MDSize(MD.value_or(0)), + AllocSize(Size + MDSize), ElemRecord(R), IsConst(IsConst), + IsMutable(IsMutable), IsTemporary(IsTemporary), IsVolatile(IsVolatile), + CtorFn(ctorRecord), DtorFn(needsRecordDtor(R) ? dtorRecord : nullptr) { assert(Source && "Missing source"); } /// Dummy. Descriptor::Descriptor(const DeclTy &D, MetadataSize MD) - : Source(D), ElemSize(1), Size(1), MDSize(MD.value_or(0)), + : Source(D), ElemSize(1), Size(1), Capacity(Size), MDSize(MD.value_or(0)), AllocSize(MDSize), ElemRecord(nullptr), IsConst(true), IsMutable(false), IsTemporary(false) { assert(Source && "Missing source"); @@ -468,10 +501,255 @@ bool Descriptor::hasTrivialDtor() const { bool Descriptor::isUnion() const { return isRecord() && ElemRecord->isUnion(); } +using BlockMoveFn = void (*)(Block *Storage, std::byte *SrcFieldPtr, + std::byte *DstFieldPtr, + const Descriptor *FieldDesc, + const Descriptor *NewDesc); + +template +static void moveTy(Block *, std::byte *Src, std::byte *Dst, const Descriptor *, + const Descriptor *) { + + auto *SrcPtr = reinterpret_cast(Src); + if constexpr (!std::is_same_v && + !std::is_same_v && + !std::is_same_v) { + } + auto *DstPtr = reinterpret_cast(Dst); + new (DstPtr) T(std::move(*SrcPtr)); +} +static BlockMoveFn getMovePrim(PrimType Type) { + TYPE_SWITCH(Type, return moveTy); + llvm_unreachable("unknown PrimType"); +} + +/// Moving a primitive array. +template +static void moveArrayTy(Block *, std::byte *Src, std::byte *Dst, + const Descriptor *D, const Descriptor *NewDesc) { + InitMapPtr &SrcIMP = *reinterpret_cast(Src); + SrcIMP = std::nullopt; + + auto *NewInitMap = new (Dst) InitMapPtr( + std::make_pair(false, std::make_shared(NewDesc->getNumElems()))); + + Src += sizeof(InitMapPtr); + Dst += sizeof(InitMapPtr); + + for (unsigned I = 0, NE = D->getNumElemsWithoutFiller(); I < NE; ++I) { + auto *SrcPtr = &reinterpret_cast(Src)[I]; + auto *DstPtr = &reinterpret_cast(Dst)[I]; + new (DstPtr) T(std::move(*SrcPtr)); + NewInitMap->value().second->initializeElement(I); + } +} + +static BlockMoveFn getMoveArrayPrim(PrimType Type) { + TYPE_SWITCH(Type, return moveArrayTy); + llvm_unreachable("unknown Expr"); +} + +static void moveRecord(Block *B, std::byte *Src, std::byte *Dst, + const Descriptor *D, const Descriptor *NewDesc); +static void moveArrayDesc(Block *B, std::byte *Src, std::byte *Dst, + const Descriptor *D, const Descriptor *NewDesc); +static BlockMoveFn getMoveFn(const Descriptor *D) { + if (D->isPrimitive()) + return getMovePrim(D->getPrimType()); + if (D->isPrimitiveArray()) + return getMoveArrayPrim(D->getPrimType()); + if (D->isCompositeArray()) + return moveArrayDesc; + if (D->isRecord()) + return moveRecord; + + return nullptr; +} + +static BlockMoveFn getElemMoveFn(const Descriptor *D) { + assert(D->isArray()); + if (D->isPrimitiveArray()) + return getMovePrim(D->getPrimType()); + assert(D->ElemDesc); + return getMoveFn(D->ElemDesc); +} + +static void moveRecord(Block *B, std::byte *Src, std::byte *Dst, + const Descriptor *D, const Descriptor *) { + assert(D); + assert(D->ElemRecord); + + for (const auto &F : D->ElemRecord->fields()) { + auto FieldOffset = F.Offset; + const auto *SrcDesc = + reinterpret_cast(Src + FieldOffset) - 1; + auto *DestDesc = + reinterpret_cast(Dst + FieldOffset) - 1; + std::memcpy(DestDesc, SrcDesc, sizeof(InlineDescriptor)); + + if (auto Fn = getMoveFn(F.Desc)) + Fn(B, Src + FieldOffset, Dst + FieldOffset, F.Desc, nullptr); + } + + for (const auto &Base : D->ElemRecord->bases()) { + auto BaseOffset = Base.Offset; + const auto *SrcDesc = + reinterpret_cast(Src + BaseOffset) - 1; + auto *DestDesc = reinterpret_cast(Dst + BaseOffset) - 1; + std::memcpy(DestDesc, SrcDesc, sizeof(InlineDescriptor)); + + if (auto Fn = getMoveFn(Base.Desc)) + Fn(B, Src + BaseOffset, Dst + BaseOffset, Base.Desc, nullptr); + } + + for (const auto &VBase : D->ElemRecord->virtual_bases()) { + auto VBaseOffset = VBase.Offset; + const auto *SrcDesc = + reinterpret_cast(Src + VBaseOffset) - 1; + auto *DestDesc = + reinterpret_cast(Dst + VBaseOffset) - 1; + std::memcpy(DestDesc, SrcDesc, sizeof(InlineDescriptor)); + } +} + +static void moveArrayDesc(Block *B, std::byte *Src, std::byte *Dst, + const Descriptor *D, const Descriptor *NewDesc) { + const unsigned NumElems = D->getNumElemsWithoutFiller(); + const unsigned ElemSize = + D->ElemDesc->getAllocSize() + sizeof(InlineDescriptor); + + unsigned ElemOffset = 0; + for (unsigned I = 0; I != NumElems; ++I, ElemOffset += ElemSize) { + auto *SrcPtr = Src + ElemOffset; + auto *DstPtr = Dst + ElemOffset; + + auto *SrcDesc = reinterpret_cast(SrcPtr); + auto *SrcElemLoc = reinterpret_cast(SrcDesc + 1); + auto *DstDesc = reinterpret_cast(DstPtr); + auto *DstElemLoc = reinterpret_cast(DstDesc + 1); + + *DstDesc = *SrcDesc; + + if (auto Fn = getMoveFn(D->ElemDesc)) { + Fn(B, SrcElemLoc, DstElemLoc, D->ElemDesc, NewDesc); + } + } +} + +void Descriptor::moveArrayData(const Block *From, Block *To) const { + assert(From->getDescriptor() == this); + + // First, copy block-level metadata. + assert(From->getDescriptor()->getMetadataSize() == + To->getDescriptor()->getMetadataSize()); + std::memcpy(To->rawData(), From->rawData(), MDSize); + unsigned OldNumElems = this->getNumElemsWithoutFiller(); + unsigned NewNumElems = To->getDescriptor()->getNumElemsWithoutFiller(); + assert(NewNumElems > OldNumElems); + assert(this->hasArrayFiller()); + + unsigned ElemSize = getElemSize(); + // Copy all the old data over. + + if (this->isPrimitiveArray()) { + auto MoveFn = getMoveArrayPrim(this->getPrimType()); + MoveFn(To, const_cast(From->data()), To->data(), this, + To->getDescriptor()); + } else { + assert(isCompositeArray()); + moveArrayDesc(To, const_cast(From->data()), To->data(), this, + To->getDescriptor()); + } + + // Now fill the rest by copying over the array filler. + unsigned ElemIndex = this->getNumElemsWithoutFiller(); + assert(ElemSize == To->getDescriptor()->getElemSize()); + unsigned ArrayFillerIndex = this->getNumElemsWithoutFiller(); + unsigned ArrayFillerOffset = + From->getDescriptor()->getMetadataSize() + (ElemSize * ArrayFillerIndex); + unsigned DstOffset = To->getDescriptor()->getMetadataSize(); + DstOffset += ElemIndex * ElemSize; + if (this->isPrimitiveArray()) { + DstOffset += sizeof(InitMapPtr); + ArrayFillerOffset += sizeof(InitMapPtr); + } + + DstOffset = ArrayFillerOffset; + + bool ElemsHaveInlineDesc = this->isCompositeArray(); + + for (; ElemIndex != NewNumElems; ++ElemIndex) { + if (ElemsHaveInlineDesc) { + auto *SrcDesc = + reinterpret_cast(To->data() + ArrayFillerOffset); + auto *DstDesc = + reinterpret_cast(To->data() + DstOffset); + *DstDesc = *SrcDesc; + DstOffset += sizeof(InlineDescriptor); + } else { + InitMapPtr &DstIMP = *reinterpret_cast(To->data()); + DstIMP->second->initializeElement(ElemIndex); + } + + auto MoveFn = getElemMoveFn(this); + if (ElemsHaveInlineDesc) + MoveFn( + To, + const_cast( + From->rawData() + (ArrayFillerOffset + sizeof(InlineDescriptor))), + To->rawData() + DstOffset, ElemDesc, To->getDescriptor()); + else + MoveFn(To, const_cast(From->rawData() + (ArrayFillerOffset)), + To->rawData() + DstOffset, ElemDesc, To->getDescriptor()); + + if (ElemsHaveInlineDesc) + DstOffset += (ElemSize - sizeof(InlineDescriptor)); + else + DstOffset += (ElemSize); + } + + // At last, if we didn't expand to the full capacity, we need to carry the + // evaluated array filler around with us. + if (NewNumElems < To->getDescriptor()->Capacity) { + if (ElemsHaveInlineDesc) { + auto *SrcDesc = + reinterpret_cast(To->data() + ArrayFillerOffset); + auto *DstDesc = + reinterpret_cast(To->data() + DstOffset); + + *DstDesc = *SrcDesc; + DstOffset += sizeof(InlineDescriptor); + } else { + InitMapPtr &DstIMP = *reinterpret_cast(To->data()); + DstIMP->second->initializeElement(ElemIndex); + } + + // Note that DstOffset gets advanced in the loop above and now points to the + // element after the filled-in elements. + auto MoveFn = getElemMoveFn(this); + if (ElemsHaveInlineDesc) + MoveFn( + To, + const_cast( + From->rawData() + (ArrayFillerOffset + sizeof(InlineDescriptor))), + To->rawData() + DstOffset, ElemDesc, To->getDescriptor()); + else + MoveFn(To, const_cast(From->rawData() + (ArrayFillerOffset)), + To->rawData() + DstOffset, ElemDesc, To->getDescriptor()); + } +} + InitMap::InitMap(unsigned N) - : UninitFields(N), Data(std::make_unique(numFields(N))) {} + : UninitFields(N), Data(std::make_unique(numFields(N))) { +#ifndef NDEBUG + NumFields = N; +#endif +} bool InitMap::initializeElement(unsigned I) { +#ifndef NDEBUG + assert(I < NumFields); +#endif unsigned Bucket = I / PER_FIELD; T Mask = T(1) << (I % PER_FIELD); if (!(data()[Bucket] & Mask)) { @@ -482,6 +760,9 @@ bool InitMap::initializeElement(unsigned I) { } bool InitMap::isElementInitialized(unsigned I) const { +#ifndef NDEBUG + assert(I < NumFields); +#endif unsigned Bucket = I / PER_FIELD; return data()[Bucket] & (T(1) << (I % PER_FIELD)); } diff --git a/clang/lib/AST/ByteCode/Descriptor.h b/clang/lib/AST/ByteCode/Descriptor.h index 90dc2b4aa3111..989247a763fbd 100644 --- a/clang/lib/AST/ByteCode/Descriptor.h +++ b/clang/lib/AST/ByteCode/Descriptor.h @@ -29,6 +29,29 @@ enum PrimType : uint8_t; using DeclTy = llvm::PointerUnion; using InitMapPtr = std::optional>>; +struct ArraySize { + unsigned Size; + unsigned Capacity; + ArraySize(unsigned S, unsigned C) : Size(S), Capacity(C) { + assert(Size <= Capacity); + } + bool hasFiller() const { return Capacity != Size; } + + static bool shouldUseFiller(unsigned S, unsigned C) { + return (C - S) >= 2; + // return C >= 32 && (C - S) >= 16; + // return (C - S) >= 5; + } + + static ArraySize getNextSize(unsigned S, unsigned C) { + // llvm::errs() << S << " / " << C << '\n'; + + unsigned S2 = std::min(S * 2, C); + + return ArraySize(S2, C); + } +}; + /// Invoked whenever a block is created. The constructor method fills in the /// inline descriptors of all fields and array elements. It also initializes /// all the fields which contain non-trivial types. @@ -120,7 +143,7 @@ static_assert(sizeof(GlobalInlineDescriptor) != sizeof(InlineDescriptor), ""); /// Describes a memory block created by an allocation site. struct Descriptor final { -private: +public: /// Original declaration, used to emit the error message. const DeclTy Source; const Type *SourceType = nullptr; @@ -128,6 +151,7 @@ struct Descriptor final { const unsigned ElemSize; /// Size of the storage, in host bytes. const unsigned Size; + const unsigned Capacity; /// Size of the metadata. const unsigned MDSize; /// Size of the allocation (storage + metadata), in host bytes. @@ -181,6 +205,9 @@ struct Descriptor final { Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD, size_t NumElems, bool IsConst, bool IsTemporary, bool IsMutable); + Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD, ArraySize ArrSize, + bool IsConst, bool IsTemporary, bool IsMutable); + /// Allocates a descriptor for an array of primitives of unknown size. Descriptor(const DeclTy &D, PrimType Type, MetadataSize MDSize, bool IsConst, bool IsTemporary, UnknownSize); @@ -190,6 +217,10 @@ struct Descriptor final { MetadataSize MD, unsigned NumElems, bool IsConst, bool IsTemporary, bool IsMutable); + Descriptor(const DeclTy &D, const Type *SourceTy, const Descriptor *Elem, + MetadataSize MD, ArraySize ArrSize, bool IsConst, bool IsTemporary, + bool IsMutable); + /// Allocates a descriptor for an array of composites of unknown size. Descriptor(const DeclTy &D, const Descriptor *Elem, MetadataSize MD, bool IsTemporary, UnknownSize); @@ -240,7 +271,7 @@ struct Descriptor final { /// Returns the allocated size, including metadata. unsigned getAllocSize() const { return AllocSize; } - /// returns the size of an element when the structure is viewed as an array. + /// Returns the size of an element when the structure is viewed as an array. unsigned getElemSize() const { return ElemSize; } /// Returns the size of the metadata. unsigned getMetadataSize() const { return MDSize; } @@ -250,6 +281,15 @@ struct Descriptor final { return Size == UnknownSizeMark ? 0 : (getSize() / getElemSize()); } + unsigned getNumElemsWithoutFiller() const { + assert(!isUnknownSizeArray()); + unsigned N = getNumElems(); + return N - (N != Capacity); + } + bool hasArrayFiller() const { return getNumElems() != Capacity; } + + void moveArrayData(const Block *From, Block *To) const; + /// Checks if the descriptor is of an array of primitives. bool isPrimitiveArray() const { return IsArray && !ElemDesc; } /// Checks if the descriptor is of an array of composites. @@ -284,12 +324,15 @@ struct InitMap final { using T = uint64_t; /// Bits stored in a single field. static constexpr uint64_t PER_FIELD = sizeof(T) * CHAR_BIT; +#ifndef NDEBUG + unsigned NumFields; +#endif public: /// Initializes the map with no fields set. explicit InitMap(unsigned N); -private: +public: friend class Pointer; /// Returns a pointer to storage. diff --git a/clang/lib/AST/ByteCode/Disasm.cpp b/clang/lib/AST/ByteCode/Disasm.cpp index fd0903f2e652c..f14e056526dab 100644 --- a/clang/lib/AST/ByteCode/Disasm.cpp +++ b/clang/lib/AST/ByteCode/Disasm.cpp @@ -393,11 +393,17 @@ LLVM_DUMP_METHOD void Descriptor::dump(llvm::raw_ostream &OS) const { } // Print a few interesting bits about the descriptor. - if (isPrimitiveArray()) - OS << " primitive-array"; - else if (isCompositeArray()) - OS << " composite-array"; - else if (isUnion()) + if (isPrimitiveArray()) { + OS << " primitive-array(" << getNumElemsWithoutFiller() << "/" << Capacity + << ")"; + if (hasArrayFiller()) + OS << " has-filler"; + } else if (isCompositeArray()) { + OS << " composite-array(" << getNumElemsWithoutFiller() << "/" << Capacity + << ")"; + if (hasArrayFiller()) + OS << " has-filler"; + } else if (isUnion()) OS << " union"; else if (isRecord()) OS << " record"; @@ -484,8 +490,6 @@ LLVM_DUMP_METHOD void InterpFrame::dump(llvm::raw_ostream &OS, OS << " (" << F->getName() << ")"; } OS << "\n"; - OS.indent(Spaces) << "This: " << getThis() << "\n"; - OS.indent(Spaces) << "RVO: " << getRVOPtr() << "\n"; OS.indent(Spaces) << "Depth: " << Depth << "\n"; OS.indent(Spaces) << "ArgSize: " << ArgSize << "\n"; OS.indent(Spaces) << "Args: " << (void *)Args << "\n"; @@ -563,6 +567,26 @@ LLVM_DUMP_METHOD void Block::dump(llvm::raw_ostream &OS) const { OS << " Dynamic: " << isDynamic() << "\n"; } +LLVM_DUMP_METHOD void Block::dumpContents() const { + llvm::raw_ostream &OS = llvm::errs(); + const Descriptor *Desc = getDescriptor(); + assert(Desc); + + Desc->dump(OS); + OS << ' '; + if (Desc->isPrimitiveArray()) { + PrimType ElemT = Desc->getPrimType(); + Pointer BasePtr = Pointer(const_cast(this)); + for (unsigned I = 0; I != Desc->getNumElems(); ++I) { + TYPE_SWITCH(ElemT, { OS << BasePtr.elem(I); }); + OS << ' '; + } + OS << '\n'; + } else { + assert(false && "Unimplemented content type in Block::dumpContents()"); + } +} + LLVM_DUMP_METHOD void EvaluationResult::dump() const { auto &OS = llvm::errs(); diff --git a/clang/lib/AST/ByteCode/EvaluationResult.cpp b/clang/lib/AST/ByteCode/EvaluationResult.cpp index 7c3c21cf28251..00360a6ea7ffc 100644 --- a/clang/lib/AST/ByteCode/EvaluationResult.cpp +++ b/clang/lib/AST/ByteCode/EvaluationResult.cpp @@ -42,27 +42,27 @@ static bool CheckArrayInitialized(InterpState &S, SourceLocation Loc, if (ElemType->isRecordType()) { const Record *R = BasePtr.getElemRecord(); - for (size_t I = 0; I != NumElems; ++I) { + for (size_t I = 0; I != BasePtr.getNumAllocatedElems(); ++I) { Pointer ElemPtr = BasePtr.atIndex(I).narrow(); Result &= CheckFieldsInitialized(S, Loc, ElemPtr, R); } } else if (const auto *ElemCAT = dyn_cast(ElemType)) { - for (size_t I = 0; I != NumElems; ++I) { + + for (size_t I = 0; I != BasePtr.getNumAllocatedElems(); ++I) { Pointer ElemPtr = BasePtr.atIndex(I).narrow(); Result &= CheckArrayInitialized(S, Loc, ElemPtr, ElemCAT); } } else { // Primitive arrays. if (S.getContext().canClassify(ElemType)) { - if (BasePtr.allElementsInitialized()) { + if (BasePtr.allElementsInitialized()) return true; - } else { - DiagnoseUninitializedSubobject(S, Loc, BasePtr.getField()); - return false; - } + + DiagnoseUninitializedSubobject(S, Loc, BasePtr.getField()); + return false; } - for (size_t I = 0; I != NumElems; ++I) { + for (size_t I = 0; I != BasePtr.getNumAllocatedElems(); ++I) { if (!BasePtr.isElementInitialized(I)) { DiagnoseUninitializedSubobject(S, Loc, BasePtr.getField()); Result = false; diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp index a72282caf5e73..83b784f5b5152 100644 --- a/clang/lib/AST/ByteCode/Interp.cpp +++ b/clang/lib/AST/ByteCode/Interp.cpp @@ -1597,6 +1597,8 @@ bool Call(InterpState &S, CodePtr OpPC, const Function *Func, if (!Func->isFullyCompiled()) compileFunction(S, Func); + // Func->dump(); + if (!CheckCallable(S, OpPC, Func)) return cleanup(); diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h index 2f7e2d98f3576..fbed3806ea735 100644 --- a/clang/lib/AST/ByteCode/Interp.h +++ b/clang/lib/AST/ByteCode/Interp.h @@ -22,6 +22,7 @@ #include "Function.h" #include "InterpBuiltinBitCast.h" #include "InterpFrame.h" +#include "InterpHelpers.h" #include "InterpStack.h" #include "InterpState.h" #include "MemberPointer.h" @@ -1960,6 +1961,7 @@ template ::T> bool Store(InterpState &S, CodePtr OpPC) { const T &Value = S.Stk.pop(); const Pointer &Ptr = S.Stk.peek(); + if (!CheckStore(S, OpPC, Ptr)) return false; if (Ptr.canBeInitialized()) @@ -1972,6 +1974,7 @@ template ::T> bool StorePop(InterpState &S, CodePtr OpPC) { const T &Value = S.Stk.pop(); const Pointer &Ptr = S.Stk.pop(); + if (!CheckStore(S, OpPC, Ptr)) return false; if (Ptr.canBeInitialized()) @@ -2139,7 +2142,10 @@ bool InitElem(InterpState &S, CodePtr OpPC, uint32_t Idx) { if (!CheckLive(S, OpPC, Ptr, AK_Assign)) return false; - if (Idx >= Desc->getNumElems()) { + + ensureArraySize(S.P, Ptr, Idx); + + if (Idx >= (Desc->getNumElems())) { // CheckRange. if (S.getLangOpts().CPlusPlus) { const SourceInfo &Loc = S.Current->getSource(OpPC); @@ -2171,9 +2177,11 @@ bool InitElemPop(InterpState &S, CodePtr OpPC, uint32_t Idx) { return true; } + ensureArraySize(S.P, Ptr, Idx); + if (!CheckLive(S, OpPC, Ptr, AK_Assign)) return false; - if (Idx >= Desc->getNumElems()) { + if (Idx >= Desc->getNumElemsWithoutFiller()) { // CheckRange. if (S.getLangOpts().CPlusPlus) { const SourceInfo &Loc = S.Current->getSource(OpPC); @@ -2264,7 +2272,7 @@ std::optional OffsetHelper(InterpState &S, CodePtr OpPC, assert(Ptr.isBlockPointer()); - uint64_t MaxIndex = static_cast(Ptr.getNumElems()); + uint64_t MaxIndex = static_cast(Ptr.getCapacity()); uint64_t Index; if (Ptr.isOnePastEnd()) Index = MaxIndex; @@ -2308,8 +2316,9 @@ std::optional OffsetHelper(InterpState &S, CodePtr OpPC, } } - if (Invalid && S.getLangOpts().CPlusPlus) + if (Invalid && S.getLangOpts().CPlusPlus) { return std::nullopt; + } // Offset is valid - compute it on unsigned. int64_t WideIndex = static_cast(Index); @@ -2320,6 +2329,8 @@ std::optional OffsetHelper(InterpState &S, CodePtr OpPC, else Result = WideIndex - WideOffset; + ensureArraySize(S.P, Ptr, Result); + // When the pointer is one-past-end, going back to index 0 is the only // useful thing we can do. Any other index has been diagnosed before and // we don't get here. @@ -3098,6 +3109,7 @@ inline bool ArrayElemPtr(InterpState &S, CodePtr OpPC) { } if (Offset.isZero()) { + ensureArraySize(S.P, Ptr, 0); if (const Descriptor *Desc = Ptr.getFieldDesc(); Desc && Desc->isArray() && Ptr.getIndex() == 0) { S.Stk.push(Ptr.atIndex(0).narrow()); @@ -3129,6 +3141,7 @@ inline bool ArrayElemPtrPop(InterpState &S, CodePtr OpPC) { } if (Offset.isZero()) { + ensureArraySize(S.P, Ptr, 0); if (const Descriptor *Desc = Ptr.getFieldDesc(); Desc && Desc->isArray() && Ptr.getIndex() == 0) { S.Stk.push(Ptr.atIndex(0).narrow()); @@ -3164,6 +3177,13 @@ template ::T> inline bool ArrayElemPop(InterpState &S, CodePtr OpPC, uint32_t Index) { const Pointer &Ptr = S.Stk.pop(); + // if (Index == Ptr.getFieldDesc()->getNumElemsWithoutFiller() && + // Index < Ptr.getFieldDesc()->Capacity && + // Ptr.getFieldDesc()->hasArrayFiller()) { + // S.Stk.push(Ptr.elem(Ptr.getNumElems())); + // return true; + // } + if (!CheckLoad(S, OpPC, Ptr)) return false; @@ -3193,6 +3213,27 @@ inline bool CopyArray(InterpState &S, CodePtr OpPC, uint32_t SrcIndex, return true; } +inline bool SetArrayFillerPtr(InterpState &S, CodePtr OpPC, + uint32_t LocalIndex) { + const auto &FillerValue = S.Stk.pop(); + const auto &Arr = S.Stk.peek(); + + assert(Arr.getFieldDesc()->hasArrayFiller()); + assert(Arr.isArrayRoot()); + assert(FillerValue.isRoot()); + assert(Arr.getNumAllocatedElems() < Arr.getNumAllocatedElemsWithFiller()); + + Pointer ArrayFillerDest = Arr.atIndex(Arr.getNumAllocatedElems()).narrow(); + if (Arr.getFieldDesc()->isPrimitiveArray()) { + TYPE_SWITCH(ArrayFillerDest.getFieldDesc()->getPrimType(), { + new (&ArrayFillerDest.deref()) T(FillerValue.deref()); + }); + return true; + } + + return DoMemcpy(S, OpPC, FillerValue, ArrayFillerDest); +} + /// Just takes a pointer and checks if it's an incomplete /// array type. inline bool ArrayDecay(InterpState &S, CodePtr OpPC) { diff --git a/clang/lib/AST/ByteCode/InterpBlock.h b/clang/lib/AST/ByteCode/InterpBlock.h index 73fdc8d85da11..a549ee9c81c47 100644 --- a/clang/lib/AST/ByteCode/InterpBlock.h +++ b/clang/lib/AST/ByteCode/InterpBlock.h @@ -142,8 +142,11 @@ class Block final { IsInitialized = false; } + void moveArrayData(Block *To) const { Desc->moveArrayData(this, To); } + void dump() const { dump(llvm::errs()); } void dump(llvm::raw_ostream &OS) const; + void dumpContents() const; bool isAccessible() const { return AccessFlags == 0; } diff --git a/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp b/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp index 4bd9c66fc9974..f5d51a8147123 100644 --- a/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp +++ b/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp @@ -11,6 +11,7 @@ #include "Context.h" #include "Floating.h" #include "Integral.h" +#include "InterpHelpers.h" #include "InterpState.h" #include "MemberPointer.h" #include "Pointer.h" @@ -265,6 +266,8 @@ bool clang::interp::readPointerToBuffer(const Context &Ctx, Endian TargetEndianness = ASTCtx.getTargetInfo().isLittleEndian() ? Endian::Little : Endian::Big; + ensureArraySize(Ctx.getProgram(), FromPtr); + return enumeratePointerFields( FromPtr, Ctx, Buffer.size(), [&](const Pointer &P, PrimType T, Bits BitOffset, Bits FullBitWidth, @@ -384,6 +387,8 @@ bool clang::interp::DoBitCastPtr(InterpState &S, CodePtr OpPC, readPointerToBuffer(S.getContext(), FromPtr, Buffer, /*ReturnOnUninit=*/false); + ensureArraySize(S.P, ToPtr); + // Now read the values out of the buffer again and into ToPtr. Endian TargetEndianness = ASTCtx.getTargetInfo().isLittleEndian() ? Endian::Little : Endian::Big; diff --git a/clang/lib/AST/ByteCode/InterpFrame.cpp b/clang/lib/AST/ByteCode/InterpFrame.cpp index 039acb5d72b2c..cbde63eedea67 100644 --- a/clang/lib/AST/ByteCode/InterpFrame.cpp +++ b/clang/lib/AST/ByteCode/InterpFrame.cpp @@ -81,7 +81,7 @@ void InterpFrame::destroyScopes() { return; for (auto &Scope : Func->scopes()) { for (auto &Local : Scope.locals()) { - S.deallocate(localBlock(Local.Offset)); + S.deallocate(localBlock(Local.Offset, false)); } } } @@ -192,6 +192,33 @@ void InterpFrame::describe(llvm::raw_ostream &OS) const { OS << ")"; } +void InterpFrame::reallocLocal(Block *Prev, const Descriptor *NewDesc) { + // llvm::errs() << __PRETTY_FUNCTION__ << '\n'; + // llvm::errs() << "Prev: " << Prev << '\n'; + // this->dump(); + + unsigned Offset = ((char *)Prev) - Locals.get() + sizeof(Block); + + // First, we need a new block for the new descriptor. + Block *B = new (S.allocate(sizeof(Block) + NewDesc->getAllocSize())) + Block(Prev->getEvalID(), Prev->getDeclID(), NewDesc, false, false, false); + B->invokeCtor(); + + // llvm::errs() << "Reallocated LOCAL in InterpFrame: "<< Prev << " -> " << B + // << '\n'; + + // Pointer(B).initializeAllElements(); + + Prev->moveArrayData(B); + Prev->movePointersTo(B); + + assert(!Prev->hasPointers()); + + ReallocatedLocals[Offset] = B; + // llvm::errs() << " !!!!!!!!!!!! " << Offset << " should now point to " << B + // << '\n'; +} + SourceRange InterpFrame::getCallRange() const { if (!Caller->Func) { if (SourceRange NullRange = S.getRange(nullptr, {}); NullRange.isValid()) diff --git a/clang/lib/AST/ByteCode/InterpFrame.h b/clang/lib/AST/ByteCode/InterpFrame.h index fa9de2e1e7c6d..c79a80587b080 100644 --- a/clang/lib/AST/ByteCode/InterpFrame.h +++ b/clang/lib/AST/ByteCode/InterpFrame.h @@ -142,6 +142,7 @@ class InterpFrame final : public Frame { void dump() const { dump(llvm::errs(), 0); } void dump(llvm::raw_ostream &OS, unsigned Indent = 0) const; + void reallocLocal(Block *Prev, const Descriptor *NewDesc); private: /// Returns an original argument from the stack. @@ -156,7 +157,16 @@ class InterpFrame final : public Frame { } /// Returns a pointer to a local's block. - Block *localBlock(unsigned Offset) const { + Block *localBlock(unsigned Offset, bool CheckReallocs = true) const { + if (CheckReallocs) { + // llvm::errs() << __PRETTY_FUNCTION__ << ": " << Offset << ". Reallocated + // locals: " << ReallocatedLocals.size() << '\n'; + if (auto It = ReallocatedLocals.find(Offset); + It != ReallocatedLocals.end()) { + // llvm::errs() << "AAAAAAAha! localblock() on a reallocated local!\n"; + return It->second; + } + } return reinterpret_cast(Locals.get() + Offset - sizeof(Block)); } @@ -186,6 +196,8 @@ class InterpFrame final : public Frame { const size_t FrameOffset; /// Mapping from arg offsets to their argument blocks. llvm::DenseMap> Params; + + llvm::DenseMap ReallocatedLocals; }; } // namespace interp diff --git a/clang/lib/AST/ByteCode/InterpHelpers.cpp b/clang/lib/AST/ByteCode/InterpHelpers.cpp new file mode 100644 index 0000000000000..fd458b224e0a2 --- /dev/null +++ b/clang/lib/AST/ByteCode/InterpHelpers.cpp @@ -0,0 +1,58 @@ + + +#include "InterpHelpers.h" +#include "Descriptor.h" +#include "InterpBlock.h" +#include "Program.h" + +namespace clang { +namespace interp { + +void ensureArraySize(Program &P, const Pointer &Ptr, unsigned RequestedIndex) { + if (!Ptr.isBlockPointer()) + return; + + assert(Ptr.getFieldDesc()); + assert(Ptr.getDeclDesc()); + if (!Ptr.getDeclDesc()->isArray()) + return; + + // No fillers for these. + if (!Ptr.isStatic() || Ptr.isUnknownSizeArray() || Ptr.block()->isDynamic()) + return; + + assert(Ptr.getFieldDesc()->isArray()); + + bool NeedsRealloc = RequestedIndex >= Ptr.getNumAllocatedElems() && + RequestedIndex < Ptr.getCapacity(); + + // llvm::errs() << "NeedsRealloc: " << NeedsRealloc << '\n'; + if (!NeedsRealloc) + return; + + assert(Ptr.getFieldDesc()->hasArrayFiller()); + unsigned RequestedSize = RequestedIndex + 1; + assert(RequestedSize <= Ptr.getNumElems()); + + const Descriptor *D = Ptr.getFieldDesc(); + ArraySize NewArraySize = ArraySize::getNextSize(RequestedSize, D->Capacity); + + const Descriptor *NewDesc = nullptr; + if (D->isPrimitiveArray()) { + NewDesc = P.allocateDescriptor(D->Source, D->getPrimType(), + Descriptor::GlobalMD, NewArraySize, + D->IsConst, D->IsTemporary, D->IsMutable); + } else if (D->isCompositeArray()) { + NewDesc = P.allocateDescriptor(D->Source, D->SourceType, D->ElemDesc, + Descriptor::GlobalMD, NewArraySize, + D->IsConst, D->IsTemporary, D->IsMutable); + } else { + llvm_unreachable("Should be either a primitive or composite array"); + } + + assert(NewDesc); + P.reallocGlobal(const_cast(Ptr.block()), NewDesc); +} + +} // namespace interp +} // namespace clang diff --git a/clang/lib/AST/ByteCode/InterpHelpers.h b/clang/lib/AST/ByteCode/InterpHelpers.h new file mode 100644 index 0000000000000..1eaa54900788f --- /dev/null +++ b/clang/lib/AST/ByteCode/InterpHelpers.h @@ -0,0 +1,14 @@ + +#include "Pointer.h" + +namespace clang { +namespace interp { +void ensureArraySize(Program &P, const Pointer &Ptr, unsigned RequestedIndex); + +inline void ensureArraySize(Program &P, const Pointer &Ptr) { + if (!Ptr.getFieldDesc()->isArray()) + return; + ensureArraySize(P, Ptr, Ptr.getNumElems() - 1); +} +} // namespace interp +} // namespace clang diff --git a/clang/lib/AST/ByteCode/Opcodes.td b/clang/lib/AST/ByteCode/Opcodes.td index 532c4448e6f40..63a065cfe2ce8 100644 --- a/clang/lib/AST/ByteCode/Opcodes.td +++ b/clang/lib/AST/ByteCode/Opcodes.td @@ -381,6 +381,8 @@ def CopyArray : Opcode { let HasGroup = 1; } +def SetArrayFillerPtr : Opcode { let Args = [ArgUint32]; } + //===----------------------------------------------------------------------===// // Direct field accessors //===----------------------------------------------------------------------===// diff --git a/clang/lib/AST/ByteCode/Pointer.cpp b/clang/lib/AST/ByteCode/Pointer.cpp index e417bdfb81b8f..a785e342fa457 100644 --- a/clang/lib/AST/ByteCode/Pointer.cpp +++ b/clang/lib/AST/ByteCode/Pointer.cpp @@ -812,9 +812,12 @@ std::optional Pointer::toRValue(const Context &Ctx, } if (const auto *AT = Ty->getAsArrayTypeUnsafe()) { - const size_t NumElems = Ptr.getNumElems(); + const Descriptor *Desc = Ptr.getFieldDesc(); + + size_t Capacity = Desc->Capacity; + const size_t NumElems = Ptr.getNumAllocatedElems(); QualType ElemTy = AT->getElementType(); - R = APValue(APValue::UninitArray{}, NumElems, NumElems); + R = APValue(APValue::UninitArray{}, NumElems, Desc->Capacity); bool Ok = true; OptPrimType ElemT = Ctx.classify(ElemTy); @@ -826,6 +829,16 @@ std::optional Pointer::toRValue(const Context &Ctx, Ok &= Composite(ElemTy, Ptr.atIndex(I).narrow(), Slot); } } + + if (NumElems != Capacity) { + APValue &Slot = R.getArrayFiller(); + if (ElemT) { + TYPE_SWITCH(*ElemT, Slot = Ptr.elem(NumElems).toAPValue(ASTCtx)); + } else { + Ok &= Composite(ElemTy, Ptr.atIndex(NumElems).narrow(), Slot); + } + } + return Ok; } diff --git a/clang/lib/AST/ByteCode/Pointer.h b/clang/lib/AST/ByteCode/Pointer.h index cd738ce8b2a3e..958d3467ef24c 100644 --- a/clang/lib/AST/ByteCode/Pointer.h +++ b/clang/lib/AST/ByteCode/Pointer.h @@ -596,6 +596,33 @@ class Pointer { unsigned getNumElems() const { if (!isBlockPointer()) return ~0u; + // return getFieldDesc()->getNumElemsWithoutFiller(); + // return getSize() / elemSize(); + return getCapacity(); + } + + unsigned getNumAllocatedElems() const { + if (!isBlockPointer()) + return ~0u; + + assert(getFieldDesc()->isArray()); + // return getSize() / elemSize(); + return getFieldDesc()->getNumElemsWithoutFiller(); + } + + unsigned getNumAllocatedElemsWithFiller() const { + if (!isBlockPointer()) + return ~0u; + + assert(getFieldDesc()->isArray()); + return getFieldDesc()->getNumElems(); // XXX WITH FILLER + } + + unsigned getCapacity() const { + if (!isBlockPointer()) + return ~0u; + if (getFieldDesc()->IsArray) + return getFieldDesc()->Capacity; return getSize() / elemSize(); } @@ -636,6 +663,15 @@ class Pointer { if (isUnknownSizeArray()) return false; + if (isPastEnd()) + return true; + + return getIndex() == getNumElems(); + + // if (getFieldDesc()->IsArray && getIndex() >= + // getFieldDesc()->getNumElems() && getIndex() < getFieldDesc()->Capacity) + // return false; + return isPastEnd() || (getSize() == getOffset()); } @@ -644,6 +680,10 @@ class Pointer { if (isIntegralPointer()) return false; + // if (getFieldDesc()->IsArray && getIndex() >= + // getFieldDesc()->getNumElems() && getIndex() < getFieldDesc()->Capacity) + // return false; + return !isZero() && Offset > BS.Pointee->getSize(); } @@ -682,7 +722,7 @@ class Pointer { assert(BS.Pointee); assert(isDereferencable()); assert(getFieldDesc()->isPrimitiveArray()); - assert(I < getFieldDesc()->getNumElems()); + // assert(I < getFieldDesc()->getNumElems()); unsigned ElemByteOffset = I * getFieldDesc()->getElemSize(); unsigned ReadOffset = BS.Base + sizeof(InitMapPtr) + ElemByteOffset; diff --git a/clang/lib/AST/ByteCode/Program.cpp b/clang/lib/AST/ByteCode/Program.cpp index e0b2852f0e906..ac6b7145634f0 100644 --- a/clang/lib/AST/ByteCode/Program.cpp +++ b/clang/lib/AST/ByteCode/Program.cpp @@ -14,6 +14,7 @@ #include "clang/AST/Decl.h" #include "clang/AST/DeclCXX.h" #include "clang/AST/DeclTemplate.h" +#include "clang/AST/Expr.h" using namespace clang; using namespace clang::interp; @@ -262,7 +263,7 @@ UnsignedOrNone Program::createGlobal(const DeclTy &D, QualType Ty, IsTemporary, /*IsMutable=*/false, IsVolatile); else Desc = createDescriptor(D, Ty.getTypePtr(), Descriptor::GlobalMD, IsConst, - IsTemporary, /*IsMutable=*/false, IsVolatile); + IsTemporary, /*IsMutable=*/false, IsVolatile, Init); if (!Desc) return std::nullopt; @@ -283,6 +284,23 @@ UnsignedOrNone Program::createGlobal(const DeclTy &D, QualType Ty, return I; } +void Program::reallocGlobal(Block *Prev, const Descriptor *NewDesc) { + assert(Prev->getDescriptor()->IsArray); + assert(NewDesc->IsArray); + + auto *G = new (Allocator, NewDesc->getAllocSize()) + Global(Ctx.getEvalID(), Prev->getDeclID(), NewDesc, true, false, false); + G->block()->invokeCtor(); + + auto [_, PrevIndex] = + *GlobalIndices.find(Prev->getDescriptor()->getSource().getOpaqueValue()); + + Prev->moveArrayData(G->block()); + + Globals[PrevIndex] = G; + Prev->movePointersTo(G->block()); +} + Function *Program::getFunction(const FunctionDecl *F) { F = F->getCanonicalDecl(); assert(F); @@ -394,6 +412,17 @@ Record *Program::getOrCreateRecord(const RecordDecl *RD) { return R; } +static unsigned getNumInits(const Expr *E, unsigned Capacity) { + unsigned N = Capacity; + if (const auto *ILE = dyn_cast_if_present(E)) + N = ILE->getNumInitsWithEmbedExpanded(); + if (const auto *PE = dyn_cast_if_present(E)) + N = PE->getNumExprs(); + if (ArraySize::shouldUseFiller(N, Capacity)) + return N; + return Capacity; +} + Descriptor *Program::createDescriptor(const DeclTy &D, const Type *Ty, Descriptor::MetadataSize MDSize, bool IsConst, bool IsTemporary, @@ -413,27 +442,37 @@ Descriptor *Program::createDescriptor(const DeclTy &D, const Type *Ty, QualType ElemTy = ArrayType->getElementType(); // Array of well-known bounds. if (const auto *CAT = dyn_cast(ArrayType)) { + size_t Capacity = CAT->getZExtSize(); + size_t Size = getNumInits(Init, Capacity); size_t NumElems = CAT->getZExtSize(); + + if (const auto *VD = + dyn_cast_if_present(D.dyn_cast())) { + if (!Context::shouldBeGloballyIndexed(VD)) + Size = Capacity; + } + if (OptPrimType T = Ctx.classify(ElemTy)) { // Arrays of primitives. unsigned ElemSize = primSize(*T); if (std::numeric_limits::max() / ElemSize <= NumElems) { return {}; } - return allocateDescriptor(D, *T, MDSize, NumElems, IsConst, IsTemporary, - IsMutable); + return allocateDescriptor(D, *T, MDSize, ArraySize(Size, Capacity), + IsConst, IsTemporary, IsMutable); } - // Arrays of composites. In this case, the array is a list of pointers, - // followed by the actual elements. - const Descriptor *ElemDesc = createDescriptor( - D, ElemTy.getTypePtr(), std::nullopt, IsConst, IsTemporary); - if (!ElemDesc) - return nullptr; - unsigned ElemSize = ElemDesc->getAllocSize() + sizeof(InlineDescriptor); - if (std::numeric_limits::max() / ElemSize <= NumElems) - return {}; - return allocateDescriptor(D, Ty, ElemDesc, MDSize, NumElems, IsConst, - IsTemporary, IsMutable); + + // Arrays of composites. + const Descriptor *ElemDesc = createDescriptor( + D, ElemTy.getTypePtr(), std::nullopt, IsConst, IsTemporary); + if (!ElemDesc) + return nullptr; + unsigned ElemSize = ElemDesc->getAllocSize() + sizeof(InlineDescriptor); + if (std::numeric_limits::max() / ElemSize <= NumElems) + return {}; + return allocateDescriptor(D, Ty, ElemDesc, MDSize, + ArraySize(Size, Capacity), IsConst, IsTemporary, + IsMutable); } // Array of unknown bounds - cannot be accessed and pointer arithmetic diff --git a/clang/lib/AST/ByteCode/Program.h b/clang/lib/AST/ByteCode/Program.h index 28fcc97f5339d..19154541e0b9e 100644 --- a/clang/lib/AST/ByteCode/Program.h +++ b/clang/lib/AST/ByteCode/Program.h @@ -161,6 +161,12 @@ class Program final { return std::nullopt; return CurrentDeclaration; } + /// Creates a new descriptor. + template Descriptor *allocateDescriptor(Ts &&...Args) { + return new (Allocator) Descriptor(std::forward(Args)...); + } + + void reallocGlobal(Block *Prev, const Descriptor *NewDesc); private: friend class DeclScope; @@ -204,6 +210,10 @@ class Program final { Block *block() { return &B; } const Block *block() const { return &B; } + GlobalInlineDescriptor getInlineDesc() const { + return *reinterpret_cast(B.rawData()); + } + private: /// Required metadata - does not actually track pointers. Block B; @@ -223,11 +233,6 @@ class Program final { /// Dummy parameter to generate pointers from. llvm::DenseMap DummyVariables; - /// Creates a new descriptor. - template Descriptor *allocateDescriptor(Ts &&...Args) { - return new (Allocator) Descriptor(std::forward(Args)...); - } - /// No declaration ID. static constexpr unsigned NoDeclaration = ~0u; /// Last declaration ID. diff --git a/clang/lib/AST/CMakeLists.txt b/clang/lib/AST/CMakeLists.txt index d4fd7a7f16d53..b1fe7211f5ce1 100644 --- a/clang/lib/AST/CMakeLists.txt +++ b/clang/lib/AST/CMakeLists.txt @@ -82,6 +82,7 @@ add_clang_library(clangAST ByteCode/EvaluationResult.cpp ByteCode/DynamicAllocator.cpp ByteCode/Interp.cpp + ByteCode/InterpHelpers.cpp ByteCode/InterpBlock.cpp ByteCode/InterpFrame.cpp ByteCode/InterpStack.cpp diff --git a/clang/test/AST/ByteCode/array-fillers.cpp b/clang/test/AST/ByteCode/array-fillers.cpp new file mode 100644 index 0000000000000..395c93ff61d29 --- /dev/null +++ b/clang/test/AST/ByteCode/array-fillers.cpp @@ -0,0 +1,50 @@ +// RUN: %clang_cc1 %s -std=c++20 -verify=both,expected -fexperimental-new-constant-interpreter + +// both-no-diagnostics + + +constexpr int F[100] = {1,2}; +static_assert(F[98] == 0); +static_assert(F[99] == 0); +static_assert(F[0] == 1); +static_assert(F[1] == 2); +static_assert(F[2] == 0); + +constexpr _Complex double Doubles[4] = {{1.0, 2.0}}; +static_assert(__real(Doubles[0]) == 1.0, ""); +static_assert(__imag(Doubles[0]) == 2.0, ""); + +static_assert(__real(Doubles[1]) == 0.0, ""); +static_assert(__imag(Doubles[1]) == 0.0, ""); + +static_assert(__real(Doubles[2]) == 0.0, ""); +static_assert(__imag(Doubles[2]) == 0.0, ""); +static_assert(__real(Doubles[3]) == 0.0, ""); +static_assert(__imag(Doubles[3]) == 0.0, ""); + +static_assert(__real(Doubles[0]) == 1.0, ""); +static_assert(__imag(Doubles[0]) == 2.0, ""); + +struct S { + int x = 20; +}; +constexpr S s[20] = {}; +static_assert(s[0].x == 20); +static_assert(s[1].x == 20); +static_assert(s[2].x == 20); +static_assert(s[3].x == 20); +static_assert(s[4].x == 20); + +constexpr int test() { + int a[4] = {}; + int r = a[2]; + return r; +} +static_assert(test() == 0); + +constexpr int test2() { + char buff[2] = {}; + buff[0] = 'B'; + return buff[1] == '\0' && buff[0] == 'B'; +} +static_assert(test2());