[codeview] Add support for splitting field list records over 64KB

The basic structure is that once a list record goes over 64K, the last
subrecord of the list is an LF_INDEX record that refers to the next
record. Because the type record graph must be toplogically sorted, this
means we have to emit them in reverse order. We build the type record in
order of declaration, so this means that if we don't want extra copies,
we need to detect when we were about to split a record, and leave space
for a continuation subrecord that will point to the eventual split
top-level record.

Also adds dumping support for these records.

Next we should make sure that large method overload lists work properly.

llvm-svn: 273294

llvm/include/llvm/DebugInfo/CodeView/FieldListRecordBuilder.h

@@ -47,7 +47,7 @@ class FieldListRecordBuilder : public ListRecordBuilder {
 public:
   FieldListRecordBuilder();
 
-  void reset() { ListRecordBuilder::reset(TypeRecordKind::FieldList); }
+  void reset() { ListRecordBuilder::reset(); }
 
   void writeBaseClass(const BaseClassRecord &Record);
   void writeEnumerator(const EnumeratorRecord &Record);

llvm/include/llvm/DebugInfo/CodeView/ListRecordBuilder.h

@@ -14,6 +14,7 @@
 
 namespace llvm {
 namespace codeview {
+class TypeTableBuilder;
 
 class ListRecordBuilder {
 private:
@@ -28,23 +29,35 @@ class ListRecordBuilder {
 public:
   llvm::StringRef str() { return Builder.str(); }
 
-  void reset(TypeRecordKind K) {
-    Builder.reset(K);
+  void reset() {
+    Builder.reset(Kind);
     ContinuationOffsets.clear();
-    SubrecordCount = 0;
+    SubrecordStart = 0;
   }
 
-  unsigned getSubrecordCount() { return SubrecordCount; }
+  void writeListContinuation(const ListContinuationRecord &R);
+
+  /// Writes this list record as a possible sequence of records.
+  TypeIndex writeListRecord(TypeTableBuilder &Table);
 
 protected:
   void finishSubRecord();
 
   TypeRecordBuilder &getBuilder() { return Builder; }
 
 private:
+  size_t getLastContinuationStart() const {
+    return ContinuationOffsets.empty() ? 0 : ContinuationOffsets.back();
+  }
+  size_t getLastContinuationEnd() const { return Builder.size(); }
+  unsigned getLastContinuationSize() const {
+    return getLastContinuationEnd() - getLastContinuationStart();
+  }
+
+  TypeRecordKind Kind;
   TypeRecordBuilder Builder;
   SmallVector<size_t, 4> ContinuationOffsets;
-  unsigned SubrecordCount = 0;
+  size_t SubrecordStart = 0;
 };
 }
 }

llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h

@@ -1169,6 +1169,29 @@ class VirtualBaseClassRecord : public TypeRecord {
   uint64_t VTableIndex;
 };
 
+/// LF_INDEX - Used to chain two large LF_FIELDLIST or LF_METHODLIST records
+/// together. The first will end in an LF_INDEX record that points to the next.
+class ListContinuationRecord : public TypeRecord {
+public:
+  ListContinuationRecord(TypeIndex ContinuationIndex)
+      : TypeRecord(TypeRecordKind::ListContinuation),
+        ContinuationIndex(ContinuationIndex) {}
+
+  TypeIndex getContinuationIndex() const { return ContinuationIndex; }
+
+  bool remapTypeIndices(ArrayRef<TypeIndex> IndexMap);
+
+  static ErrorOr<ListContinuationRecord> deserialize(TypeRecordKind Kind,
+                                                     ArrayRef<uint8_t> &Data);
+
+private:
+  struct Layout {
+    ulittle16_t Pad0;
+    TypeIndex ContinuationIndex;
+  };
+  TypeIndex ContinuationIndex;
+};
+
 typedef CVRecord<TypeLeafKind> CVType;
 typedef VarStreamArray<CVType> CVTypeArray;
 }

llvm/include/llvm/DebugInfo/CodeView/TypeRecordBuilder.h

@@ -43,11 +43,18 @@ class TypeRecordBuilder {
   void writeNullTerminatedString(const char *Value);
   void writeNullTerminatedString(StringRef Value);
   void writeGuid(StringRef Guid);
+  void writeBytes(StringRef Value) { Stream << Value; }
 
   llvm::StringRef str();
 
   uint64_t size() const { return Stream.tell(); }
 
+  void truncate(uint64_t Size) {
+    // This works because raw_svector_ostream is not buffered.
+    assert(Size < Buffer.size());
+    Buffer.resize(Size);
+  }
+
   void reset(TypeRecordKind K) {
     Buffer.clear();
     writeTypeRecordKind(K);

llvm/include/llvm/DebugInfo/CodeView/TypeRecords.def

@@ -71,6 +71,7 @@ MEMBER_RECORD(LF_MEMBER, 0x150d, DataMember)
 MEMBER_RECORD(LF_NESTTYPE, 0x1510, NestedType)
 MEMBER_RECORD(LF_ONEMETHOD, 0x1511, OneMethod)
 MEMBER_RECORD(LF_ENUMERATE, 0x1502, Enumerator)
+MEMBER_RECORD(LF_INDEX, 0x1404, ListContinuation)
 
 // ID leaf records. Subsequent leaf types may be referenced from .debug$S.
 TYPE_RECORD(LF_FUNC_ID, 0x1601, FuncId)
@@ -168,7 +169,6 @@ CV_TYPE(LF_DIMVARLU, 0x120a)
 // Member type records. These are generally not length prefixed, and appear
 // inside of a field list record.
 CV_TYPE(LF_FRIENDFCN_ST, 0x1403)
-CV_TYPE(LF_INDEX, 0x1404)
 CV_TYPE(LF_MEMBER_ST, 0x1405)
 CV_TYPE(LF_STMEMBER_ST, 0x1406)
 CV_TYPE(LF_METHOD_ST, 0x1407)

llvm/lib/DebugInfo/CodeView/ListRecordBuilder.cpp

@@ -7,27 +7,92 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/ADT/SmallString.h"
 #include "llvm/DebugInfo/CodeView/ListRecordBuilder.h"
+#include "llvm/DebugInfo/CodeView/TypeTableBuilder.h"
 
 using namespace llvm;
 using namespace codeview;
 
-ListRecordBuilder::ListRecordBuilder(TypeRecordKind Kind) : Builder(Kind) {}
+ListRecordBuilder::ListRecordBuilder(TypeRecordKind Kind)
+    : Kind(Kind), Builder(Kind) {}
 
-void ListRecordBuilder::finishSubRecord() {
-  SubrecordCount++;
+void ListRecordBuilder::writeListContinuation(const ListContinuationRecord &R) {
+  TypeRecordBuilder &Builder = getBuilder();
+
+  assert(getLastContinuationSize() < 65535 - 8 && "continuation won't fit");
+
+  Builder.writeTypeRecordKind(TypeRecordKind::ListContinuation);
+  Builder.writeUInt16(0);
+  Builder.writeTypeIndex(R.getContinuationIndex());
+
+  // End the current segment manually so that nothing comes after the
+  // continuation.
+  ContinuationOffsets.push_back(Builder.size());
+  SubrecordStart = Builder.size();
+}
 
+void ListRecordBuilder::finishSubRecord() {
   // The builder starts at offset 2 in the actual CodeView buffer, so add an
   // additional offset of 2 before computing the alignment.
   uint32_t Remainder = (Builder.size() + 2) % 4;
   if (Remainder != 0) {
     for (int32_t PaddingBytesLeft = 4 - Remainder; PaddingBytesLeft > 0;
          --PaddingBytesLeft) {
-      Builder.writeUInt8(0xf0 + PaddingBytesLeft);
+      Builder.writeUInt8(LF_PAD0 + PaddingBytesLeft);
     }
   }
 
-  // TODO: Split the list into multiple records if it's longer than 64KB, using
-  // a subrecord of TypeRecordKind::Index to chain the records together.
-  assert(Builder.size() < 65536);
+  // Check if this subrecord makes the current segment not fit in 64K minus the
+  // space for a continuation record (8 bytes). If the segment does not fit,
+  // back up and insert a continuation record, sliding the current subrecord
+  // down.
+  if (getLastContinuationSize() > 65535 - 8) {
+    SmallString<128> SubrecordCopy(Builder.str().slice(SubrecordStart, Builder.size()));
+    Builder.truncate(SubrecordStart);
+
+    // Write a placeholder continuation record.
+    Builder.writeTypeRecordKind(TypeRecordKind::ListContinuation);
+    Builder.writeUInt16(0);
+    Builder.writeUInt32(0);
+    ContinuationOffsets.push_back(Builder.size());
+    assert(Builder.size() == SubrecordStart + 8 && "wrong continuation size");
+    assert(getLastContinuationSize() < 65535 && "segment too big");
+
+    // Start a new list record of the appropriate kind, and copy the previous
+    // subrecord into place.
+    Builder.writeTypeRecordKind(Kind);
+    Builder.writeBytes(SubrecordCopy);
+  }
+
+  SubrecordStart = Builder.size();
+}
+
+TypeIndex ListRecordBuilder::writeListRecord(TypeTableBuilder &Table) {
+  // Get the continuation segments as a reversed vector of StringRefs for
+  // convenience.
+  SmallVector<StringRef, 1> Segments;
+  StringRef Data = str();
+  size_t LastEnd = 0;
+  for (size_t SegEnd : ContinuationOffsets) {
+    Segments.push_back(Data.slice(LastEnd, SegEnd));
+    LastEnd = SegEnd;
+  }
+  Segments.push_back(Data.slice(LastEnd, Builder.size()));
+
+  // Pop the last record off and emit it directly.
+  StringRef LastRec = Segments.pop_back_val();
+  TypeIndex ContinuationIndex = Table.writeRecord(LastRec);
+
+  // Emit each record with a continuation in reverse order, so that each one
+  // references the previous record.
+  for (StringRef Rec : reverse(Segments)) {
+    assert(*reinterpret_cast<const ulittle16_t *>(Rec.data()) ==
+           unsigned(Kind));
+    ulittle32_t *ContinuationPtr =
+        reinterpret_cast<ulittle32_t *>(const_cast<char *>(Rec.end())) - 1;
+    *ContinuationPtr = ContinuationIndex.getIndex();
+    ContinuationIndex = Table.writeRecord(Rec);
+  }
+  return ContinuationIndex;
 }

llvm/lib/DebugInfo/CodeView/TypeDumper.cpp

@@ -195,6 +195,8 @@ static StringRef getLeafTypeName(TypeLeafKind LT) {
   case ename:                                                                  \
     return #name;
 #include "llvm/DebugInfo/CodeView/TypeRecords.def"
+  case LF_FIELDLIST:
+    return "FieldList";
   default:
     break;
   }
@@ -214,6 +216,9 @@ Error CVTypeDumper::visitTypeBegin(const CVRecord<TypeLeafKind> &Record) {
 }
 
 Error CVTypeDumper::visitTypeEnd(const CVRecord<TypeLeafKind> &Record) {
+  if (Record.Type == LF_FIELDLIST)
+    Name = "<field list>";
+
   // Always record some name for every type, even if Name is empty. CVUDTNames
   // is indexed by type index, and must have one entry for every type.
   recordType(Name);
@@ -612,6 +617,12 @@ Error CVTypeDumper::visitVirtualBaseClass(VirtualBaseClassRecord &Base) {
   return Error::success();
 }
 
+Error CVTypeDumper::visitListContinuation(ListContinuationRecord &Cont) {
+  DictScope S(*W, "ListContinuation");
+  printTypeIndex("ContinuationIndex", Cont.getContinuationIndex());
+  return Error::success();
+}
+
 StringRef CVTypeDumper::getTypeName(TypeIndex TI) {
   if (TI.isNoneType())
     return "<no type>";

llvm/lib/DebugInfo/CodeView/TypeRecord.cpp

@@ -365,6 +365,14 @@ VirtualBaseClassRecord::deserialize(TypeRecordKind Kind,
                                 Offset, Index);
 }
 
+ErrorOr<ListContinuationRecord>
+ListContinuationRecord::deserialize(TypeRecordKind Kind,
+                                    ArrayRef<uint8_t> &Data) {
+  const Layout *L = nullptr;
+  CV_DESERIALIZE(Data, L);
+  return ListContinuationRecord(L->ContinuationIndex);
+}
+
 //===----------------------------------------------------------------------===//
 // Type index remapping
 //===----------------------------------------------------------------------===//
@@ -556,3 +564,7 @@ bool VirtualBaseClassRecord::remapTypeIndices(ArrayRef<TypeIndex> IndexMap) {
   Success &= remapIndex(IndexMap, VBPtrType);
   return Success;
 }
+
+bool ListContinuationRecord::remapTypeIndices(ArrayRef<TypeIndex> IndexMap) {
+  return remapIndex(IndexMap, ContinuationIndex);
+}

llvm/lib/DebugInfo/CodeView/TypeTableBuilder.cpp

@@ -267,9 +267,7 @@ TypeIndex TypeTableBuilder::writeRecord(TypeRecordBuilder &Builder) {
 }
 
 TypeIndex TypeTableBuilder::writeFieldList(FieldListRecordBuilder &FieldList) {
-  // TODO: Split the list into multiple records if it's longer than 64KB, using
-  // a subrecord of TypeRecordKind::Index to chain the records together.
-  return writeRecord(FieldList.str());
+  return FieldList.writeListRecord(*this);
 }
 
 TypeIndex TypeTableBuilder::writeMethodOverloadList(

llvm/test/DebugInfo/COFF/enum.ll

@@ -5,7 +5,7 @@
 ; E e;
 
 ; CHECK:     CodeViewTypes [
-; CHECK:       UnknownLeaf (0x1000) {
+; CHECK:       FieldList (0x1000) {
 ; CHECK-NEXT:    TypeLeafKind: LF_FIELDLIST (0x1203)
 ; CHECK-NEXT:    Enumerator {
 ; CHECK-NEXT:      AccessSpecifier: Public (0x3)
@@ -19,7 +19,7 @@
 ; CHECK-NEXT:    Properties [ (0x0)
 ; CHECK-NEXT:    ]
 ; CHECK-NEXT:    UnderlyingType: int (0x74)
-; CHECK-NEXT:    FieldListType: BLAH (0x1000)
+; CHECK-NEXT:    FieldListType: <field list> (0x1000)
 ; CHECK-NEXT:    Name: E
 ; CHECK-NEXT:  }
 

llvm/test/DebugInfo/COFF/types-data-members.ll

@@ -81,7 +81,7 @@
 ; CHECK:       Const (0x1)
 ; CHECK:     ]
 ; CHECK:   }
-; CHECK:   UnknownLeaf (0x1005) {
+; CHECK:   FieldList (0x1005) {
 ; CHECK:     TypeLeafKind: LF_FIELDLIST (0x1203)
 ; CHECK:     DataMember {
 ; CHECK:       AccessSpecifier: Public (0x3)
@@ -113,7 +113,7 @@
 ; CHECK:     Properties [ (0x200)
 ; CHECK:       HasUniqueName (0x200)
 ; CHECK:     ]
-; CHECK:     FieldList: sdm (0x1005)
+; CHECK:     FieldList: <field list> (0x1005)
 ; CHECK:     DerivedFrom: 0x0
 ; CHECK:     VShape: 0x0
 ; CHECK:     SizeOf: 12
@@ -132,7 +132,7 @@
 ; CHECK:     Name: Union
 ; CHECK:     LinkageName: .?ATUnion@@
 ; CHECK:   }
-; CHECK:   UnknownLeaf (0x1008) {
+; CHECK:   FieldList (0x1008) {
 ; CHECK:     TypeLeafKind: LF_FIELDLIST (0x1203)
 ; CHECK:     DataMember {
 ; CHECK:       AccessSpecifier: Public (0x3)
@@ -153,7 +153,7 @@
 ; CHECK:     Properties [ (0x200)
 ; CHECK:       HasUniqueName (0x200)
 ; CHECK:     ]
-; CHECK:     FieldList: b (0x1008)
+; CHECK:     FieldList: <field list> (0x1008)
 ; CHECK:     SizeOf: 4
 ; CHECK:     Name: Union
 ; CHECK:     LinkageName: .?ATUnion@@
@@ -171,7 +171,7 @@
 ; CHECK:     SizeOf: 0
 ; CHECK:     Name: Class
 ; CHECK:   }
-; CHECK:   UnknownLeaf (0x100B) {
+; CHECK:   FieldList (0x100B) {
 ; CHECK:     TypeLeafKind: LF_FIELDLIST (0x1203)
 ; CHECK:     DataMember {
 ; CHECK:       AccessSpecifier: Public (0x3)
@@ -198,7 +198,7 @@
 ; CHECK:     Properties [ (0x200)
 ; CHECK:       HasUniqueName (0x200)
 ; CHECK:     ]
-; CHECK:     FieldList: prot (0x100B)
+; CHECK:     FieldList: <field list> (0x100B)
 ; CHECK:     DerivedFrom: 0x0
 ; CHECK:     VShape: 0x0
 ; CHECK:     SizeOf: 12
@@ -249,7 +249,7 @@
 ; CHECK:     IsVolatile: 0
 ; CHECK:     IsUnaligned: 0
 ; CHECK:   }
-; CHECK:   UnknownLeaf (0x1011) {
+; CHECK:   FieldList (0x1011) {
 ; CHECK:     TypeLeafKind: LF_FIELDLIST (0x1203)
 ; CHECK:     DataMember {
 ; CHECK:       AccessSpecifier: Public (0x3)
@@ -270,7 +270,7 @@
 ; CHECK:     Properties [ (0x200)
 ; CHECK:       HasUniqueName (0x200)
 ; CHECK:     ]
-; CHECK:     FieldList: d (0x1011)
+; CHECK:     FieldList: <field list> (0x1011)
 ; CHECK:     DerivedFrom: 0x0
 ; CHECK:     VShape: 0x0
 ; CHECK:     SizeOf: 48
@@ -291,7 +291,7 @@
 ; CHECK:     Name: Nested
 ; CHECK:     LinkageName: .?AUNested@Class@@
 ; CHECK:   }
-; CHECK:   UnknownLeaf (0x1014) {
+; CHECK:   FieldList (0x1014) {
 ; CHECK:     TypeLeafKind: LF_FIELDLIST (0x1203)
 ; CHECK:     DataMember {
 ; CHECK:       AccessSpecifier: Public (0x3)
@@ -306,7 +306,7 @@
 ; CHECK:     Properties [ (0x200)
 ; CHECK:       HasUniqueName (0x200)
 ; CHECK:     ]
-; CHECK:     FieldList: n (0x1014)
+; CHECK:     FieldList: <field list> (0x1014)
 ; CHECK:     DerivedFrom: 0x0
 ; CHECK:     VShape: 0x0
 ; CHECK:     SizeOf: 4