[pdb] Write PDB TPI Stream from Yaml.

This writes the full sequence of type records described in
Yaml to the TPI stream of the PDB file.

Reviewed By: rnk
Differential Revision: https://reviews.llvm.org/D24316

llvm-svn: 281063

llvm/include/llvm/DebugInfo/CodeView/MemoryTypeTableBuilder.h

@@ -20,7 +20,8 @@ namespace codeview {
 
 class MemoryTypeTableBuilder : public TypeTableBuilder {
 public:
-  MemoryTypeTableBuilder() {}
+  explicit MemoryTypeTableBuilder(BumpPtrAllocator &Allocator)
+      : RecordStorage(Allocator) {}
 
   bool empty() const { return Records.empty(); }
 
@@ -33,12 +34,13 @@ class MemoryTypeTableBuilder : public TypeTableBuilder {
     }
   }
 
-protected:
   TypeIndex writeRecord(llvm::StringRef Data) override;
 
+  ArrayRef<StringRef> getRecords() const { return Records; }
+
 private:
   std::vector<StringRef> Records;
-  BumpPtrAllocator RecordStorage;
+  BumpPtrAllocator &RecordStorage;
   DenseMap<StringRef, TypeIndex> HashedRecords;
 };
 

llvm/include/llvm/DebugInfo/CodeView/TypeRecordBuilder.h

@@ -47,6 +47,7 @@ class TypeRecordBuilder {
   llvm::StringRef str();
 
   uint64_t size() const { return Stream.tell(); }
+  TypeRecordKind kind() const { return Kind; }
 
   void truncate(uint64_t Size) {
     // This works because raw_svector_ostream is not buffered.
@@ -56,10 +57,12 @@ class TypeRecordBuilder {
 
   void reset(TypeRecordKind K) {
     Buffer.clear();
+    Kind = K;
     writeTypeRecordKind(K);
   }
 
 private:
+  TypeRecordKind Kind;
   llvm::SmallVector<char, 256> Buffer;
   llvm::raw_svector_ostream Stream;
   llvm::support::endian::Writer<llvm::support::endianness::little> Writer;

llvm/include/llvm/DebugInfo/CodeView/TypeSerializationVisitor.h

@@ -0,0 +1,74 @@
+//===- TypeSerializationVisitor.h -------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_CODEVIEW_TYPESERIALIZATIONVISITOR_H
+#define LLVM_DEBUGINFO_CODEVIEW_TYPESERIALIZATIONVISITOR_H
+
+#include "llvm/DebugInfo/CodeView/FieldListRecordBuilder.h"
+#include "llvm/DebugInfo/CodeView/MemoryTypeTableBuilder.h"
+#include "llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h"
+
+#include "llvm/ADT/StringRef.h"
+
+namespace llvm {
+namespace codeview {
+
+class TypeSerializationVisitor : public TypeVisitorCallbacks {
+public:
+  TypeSerializationVisitor(FieldListRecordBuilder &FieldListBuilder,
+                           MemoryTypeTableBuilder &TypeTableBuilder)
+      : FieldListBuilder(FieldListBuilder), TypeTableBuilder(TypeTableBuilder) {
+  }
+
+  virtual Expected<TypeLeafKind> visitTypeBegin(const CVType &Record) override {
+    if (Record.Type == TypeLeafKind::LF_FIELDLIST)
+      FieldListBuilder.reset();
+    return Record.Type;
+  }
+
+  virtual Error visitTypeEnd(const CVRecord<TypeLeafKind> &Record) override {
+    if (Record.Type == TypeLeafKind::LF_FIELDLIST)
+      TypeTableBuilder.writeFieldList(FieldListBuilder);
+    return Error::success();
+  }
+
+#define TYPE_RECORD(EnumName, EnumVal, Name)                                   \
+  virtual Error visitKnownRecord(const CVRecord<TypeLeafKind> &CVR,            \
+                                 Name##Record &Record) override {              \
+    visitKnownRecordImpl(Record);                                              \
+    return Error::success();                                                   \
+  }
+#define TYPE_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName)
+#define MEMBER_RECORD(EnumName, EnumVal, Name)                                 \
+  virtual Error visitKnownRecord(const CVRecord<TypeLeafKind> &CVR,            \
+                                 Name##Record &Record) override {              \
+    visitMemberRecordImpl(Record);                                             \
+    return Error::success();                                                   \
+  }
+#define MEMBER_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName)
+#include "llvm/DebugInfo/CodeView/TypeRecords.def"
+
+private:
+  template <typename RecordKind> void visitKnownRecordImpl(RecordKind &Record) {
+    TypeTableBuilder.writeKnownType(Record);
+  }
+  template <typename RecordKind>
+  void visitMemberRecordImpl(RecordKind &Record) {
+    FieldListBuilder.writeMemberType(Record);
+  }
+
+  void visitKnownRecordImpl(FieldListRecord &FieldList) {}
+
+  FieldListRecordBuilder &FieldListBuilder;
+  MemoryTypeTableBuilder &TypeTableBuilder;
+};
+}
+}
+
+#endif

llvm/include/llvm/DebugInfo/CodeView/TypeTableBuilder.h

@@ -63,6 +63,11 @@ class TypeTableBuilder {
   TypeIndex writeRecord(TypeRecordBuilder &builder);
 
   virtual TypeIndex writeRecord(llvm::StringRef record) = 0;
+
+  ArrayRef<TypeRecordKind> getRecordKinds() const { return RecordKinds; }
+
+private:
+  std::vector<TypeRecordKind> RecordKinds;
 };
 }
 }

llvm/include/llvm/DebugInfo/MSF/SequencedItemStream.h

@@ -0,0 +1,91 @@
+//===- SequencedItemStream.h ------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_MSF_SEQUENCEDITEMSTREAM_H
+#define LLVM_DEBUGINFO_MSF_SEQUENCEDITEMSTREAM_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/DebugInfo/MSF/MSFError.h"
+#include "llvm/DebugInfo/MSF/StreamInterface.h"
+#include "llvm/Support/Error.h"
+#include <cstdint>
+#include <memory>
+#include <type_traits>
+
+namespace llvm {
+namespace msf {
+template <typename T> struct SequencedItemTraits {
+  static size_t length(const T &Item) = delete;
+  static ArrayRef<uint8_t> bytes(const T &Item) = delete;
+};
+
+/// SequencedItemStream represents a sequence of objects stored in a
+/// standard container but for which it is useful to view as a stream of
+/// contiguous bytes.  An example of this might be if you have a std::vector
+/// of TPI records, where each record contains a byte sequence that
+/// represents that one record serialized, but where each consecutive item
+/// might not be allocated immediately after the previous item.  Using a
+/// SequencedItemStream, we can adapt the VarStreamArray class to trivially
+/// extract one item at a time, allowing the data to be used anywhere a
+/// VarStreamArray could be used.
+template <typename T, typename Traits = SequencedItemTraits<T>>
+class SequencedItemStream : public ReadableStream {
+public:
+  SequencedItemStream() {}
+
+  Error readBytes(uint32_t Offset, uint32_t Size,
+                  ArrayRef<uint8_t> &Buffer) const override {
+    auto ExpectedIndex = translateOffsetIndex(Offset);
+    if (!ExpectedIndex)
+      return ExpectedIndex.takeError();
+    const auto &Item = Items[*ExpectedIndex];
+    if (Size > Traits::length(Item))
+      return make_error<MSFError>(msf_error_code::insufficient_buffer);
+    Buffer = Traits::bytes(Item).take_front(Size);
+    return Error::success();
+  }
+
+  Error readLongestContiguousChunk(uint32_t Offset,
+                                   ArrayRef<uint8_t> &Buffer) const override {
+    auto ExpectedIndex = translateOffsetIndex(Offset);
+    if (!ExpectedIndex)
+      return ExpectedIndex.takeError();
+    Buffer = Traits::bytes(Items[*ExpectedIndex]);
+    return Error::success();
+  }
+
+  void setItems(ArrayRef<T> ItemArray) { Items = ItemArray; }
+
+  uint32_t getLength() const override {
+    uint32_t Size = 0;
+    for (const auto &Item : Items)
+      Size += Traits::length(Item);
+    return Size;
+  }
+
+private:
+  Expected<uint32_t> translateOffsetIndex(uint32_t Offset) const {
+    uint32_t CurrentOffset = 0;
+    uint32_t CurrentIndex = 0;
+    for (const auto &Item : Items) {
+      if (CurrentOffset >= Offset)
+        break;
+      CurrentOffset += Traits::length(Item);
+      ++CurrentIndex;
+    }
+    if (CurrentOffset != Offset)
+      return make_error<MSFError>(msf_error_code::insufficient_buffer);
+    return CurrentIndex;
+  }
+  ArrayRef<T> Items;
+};
+} // end namespace msf
+} // end namespace llvm
+
+#endif // LLVM_DEBUGINFO_MSF_SEQUENCEDITEMSTREAM_H

llvm/include/llvm/DebugInfo/MSF/StreamArray.h

@@ -10,6 +10,7 @@
 #ifndef LLVM_DEBUGINFO_MSF_STREAMARRAY_H
 #define LLVM_DEBUGINFO_MSF_STREAMARRAY_H
 
+#include "llvm/DebugInfo/MSF/SequencedItemStream.h"
 #include "llvm/DebugInfo/MSF/StreamRef.h"
 #include "llvm/Support/Error.h"
 

llvm/include/llvm/DebugInfo/PDB/Raw/PDBFile.h

@@ -92,6 +92,8 @@ class PDBFile : public msf::IMSFFile {
   Expected<SymbolStream &> getPDBSymbolStream();
   Expected<NameHashTable &> getStringTable();
 
+  BumpPtrAllocator &getAllocator() { return Allocator; }
+
 private:
   BumpPtrAllocator &Allocator;
 

llvm/include/llvm/DebugInfo/PDB/Raw/PDBFileBuilder.h

@@ -28,6 +28,7 @@ class MSFBuilder;
 namespace pdb {
 class DbiStreamBuilder;
 class InfoStreamBuilder;
+class TpiStreamBuilder;
 
 class PDBFileBuilder {
 public:
@@ -40,6 +41,7 @@ class PDBFileBuilder {
   msf::MSFBuilder &getMsfBuilder();
   InfoStreamBuilder &getInfoBuilder();
   DbiStreamBuilder &getDbiBuilder();
+  TpiStreamBuilder &getTpiBuilder();
 
   Expected<std::unique_ptr<PDBFile>>
   build(std::unique_ptr<msf::WritableStream> PdbFileBuffer);
@@ -54,6 +56,7 @@ class PDBFileBuilder {
   std::unique_ptr<msf::MSFBuilder> Msf;
   std::unique_ptr<InfoStreamBuilder> Info;
   std::unique_ptr<DbiStreamBuilder> Dbi;
+  std::unique_ptr<TpiStreamBuilder> Tpi;
 };
 }
 }

llvm/include/llvm/DebugInfo/PDB/Raw/RawTypes.h

@@ -266,6 +266,34 @@ struct PDB_UniqueId {
   char Guid[16];
 };
 
+// The header preceeding the global TPI stream.
+// This corresponds to `HDR` in PDB/dbi/tpi.h.
+struct TpiStreamHeader {
+  struct EmbeddedBuf {
+    support::little32_t Off;
+    support::ulittle32_t Length;
+  };
+
+  support::ulittle32_t Version;
+  support::ulittle32_t HeaderSize;
+  support::ulittle32_t TypeIndexBegin;
+  support::ulittle32_t TypeIndexEnd;
+  support::ulittle32_t TypeRecordBytes;
+
+  // The following members correspond to `TpiHash` in PDB/dbi/tpi.h.
+  support::ulittle16_t HashStreamIndex;
+  support::ulittle16_t HashAuxStreamIndex;
+  support::ulittle32_t HashKeySize;
+  support::ulittle32_t NumHashBuckets;
+
+  EmbeddedBuf HashValueBuffer;
+  EmbeddedBuf IndexOffsetBuffer;
+  EmbeddedBuf HashAdjBuffer;
+};
+
+const uint32_t MinTpiHashBuckets = 0x1000;
+const uint32_t MaxTpiHashBuckets = 0x40000;
+
 /// The header preceeding the global PDB Stream (Stream 1)
 struct InfoStreamHeader {
   support::ulittle32_t Version;

llvm/include/llvm/DebugInfo/PDB/Raw/TpiStream.h

@@ -27,7 +27,7 @@ namespace pdb {
 class PDBFile;
 
 class TpiStream {
-  struct HeaderInfo;
+  friend class TpiStreamBuilder;
 
 public:
   TpiStream(const PDBFile &File,
@@ -66,7 +66,7 @@ class TpiStream {
   msf::FixedStreamArray<TypeIndexOffset> TypeIndexOffsets;
   msf::FixedStreamArray<TypeIndexOffset> HashAdjustments;
 
-  const HeaderInfo *Header;
+  const TpiStreamHeader *Header;
 };
 }
 }

llvm/include/llvm/DebugInfo/PDB/Raw/TpiStreamBuilder.h

@@ -0,0 +1,76 @@
+//===- TpiStreamBuilder.h - PDB Tpi Stream Creation -------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_PDB_RAW_PDBTPISTREAMBUILDER_H
+#define LLVM_DEBUGINFO_PDB_RAW_PDBTPISTREAMBUILDER_H
+
+#include "llvm/ADT/Optional.h"
+#include "llvm/DebugInfo/CodeView/TypeRecord.h"
+#include "llvm/DebugInfo/MSF/ByteStream.h"
+#include "llvm/DebugInfo/MSF/SequencedItemStream.h"
+#include "llvm/DebugInfo/PDB/Raw/RawConstants.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/Error.h"
+
+#include <vector>
+
+namespace llvm {
+namespace codeview {
+class TypeRecord;
+}
+namespace msf {
+struct MSFLayout;
+class ReadableStreamRef;
+class WritableStream;
+
+template <> struct SequencedItemTraits<llvm::codeview::CVType> {
+  static size_t length(const codeview::CVType &Item) { return Item.Length; }
+  static ArrayRef<uint8_t> bytes(const codeview::CVType &Item) {
+    return Item.RawData;
+  }
+};
+}
+namespace pdb {
+class PDBFile;
+class TpiStream;
+struct TpiStreamHeader;
+
+class TpiStreamBuilder {
+public:
+  explicit TpiStreamBuilder(BumpPtrAllocator &Allocator);
+  ~TpiStreamBuilder();
+
+  TpiStreamBuilder(const TpiStreamBuilder &) = delete;
+  TpiStreamBuilder &operator=(const TpiStreamBuilder &) = delete;
+
+  void setVersionHeader(PdbRaw_TpiVer Version);
+  void addTypeRecord(const codeview::CVType &Record);
+
+  Expected<std::unique_ptr<TpiStream>> build(PDBFile &File,
+                                             const msf::WritableStream &Buffer);
+
+  Error commit(const msf::MSFLayout &Layout, const msf::WritableStream &Buffer);
+
+  uint32_t calculateSerializedLength() const;
+
+private:
+  Error finalize();
+
+  BumpPtrAllocator &Allocator;
+
+  Optional<PdbRaw_TpiVer> VerHeader;
+  std::vector<codeview::CVType> TypeRecords;
+  msf::SequencedItemStream<codeview::CVType> TypeRecordStream;
+
+  const TpiStreamHeader *Header;
+};
+}
+}
+
+#endif