[pdb] Use MappedBlockStream to parse the PDB directory.

In order to efficiently write PDBs, we need to be able to make a
StreamWriter class similar to a StreamReader, which can transparently deal
with writing to discontiguous streams, and we need to use this for all
writing, similar to how we use StreamReader for all reading.

Most discontiguous streams are the typical numbered streams that appear in
a PDB file and are described by the directory, but the exception to this,
that until now has been parsed by hand, is the directory itself.
MappedBlockStream works by querying the directory to find out which blocks
a stream occupies and various other things, so naturally the same logic
could not possibly work to describe the blocks that the directory itself
resided on.

To solve this, I've introduced an abstraction IPDBStreamData, which allows
the client to query for the list of blocks occupied by the stream, as well
as the stream length. I provide two implementations of this: one which
queries the directory (for indexed streams), and one which queries the
super block (for the directory stream).

This has the side benefit of vastly simplifying the code to parse the
directory. Whereas before a mini state machine was rolled by hand, now we
simply use FixedStreamArray to read out the stream sizes, then build a
vector of FixedStreamArrays for the stream map, all in just a few lines of
code.

Reviewed By: ruiu
Differential Revision: http://reviews.llvm.org/D21046

llvm-svn: 271982

llvm/include/llvm/DebugInfo/CodeView/StreamArray.h

@@ -224,6 +224,8 @@ template <typename T> class FixedStreamArray {
     return FixedStreamArrayIterator<T>(*this, size());
   }
 
+  StreamRef getUnderlyingStream() const { return Stream; }
+
 private:
   StreamRef Stream;
 };

llvm/include/llvm/DebugInfo/CodeView/StreamReader.h

@@ -52,6 +52,20 @@ class StreamReader {
     return Error::success();
   }
 
+  template <typename T>
+  Error readArray(ArrayRef<T> &Array, uint32_t NumElements) {
+    ArrayRef<uint8_t> Bytes;
+    if (NumElements == 0) {
+      Array = ArrayRef<T>();
+      return Error::success();
+    }
+
+    if (auto EC = readBytes(Bytes, NumElements * sizeof(T)))
+      return EC;
+    Array = ArrayRef<T>(reinterpret_cast<const T *>(Bytes.data()), NumElements);
+    return Error::success();
+  }
+
   template <typename T, typename U>
   Error readArray(VarStreamArray<T, U> &Array, uint32_t Size) {
     StreamRef S;

llvm/include/llvm/DebugInfo/PDB/Raw/IPDBFile.h

@@ -12,6 +12,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/DebugInfo/CodeView/StreamArray.h"
 #include "llvm/Support/Endian.h"
 
 #include <stdint.h>
@@ -28,7 +29,8 @@ class IPDBFile {
 
   virtual uint32_t getNumStreams() const = 0;
   virtual uint32_t getStreamByteSize(uint32_t StreamIndex) const = 0;
-  virtual ArrayRef<uint32_t> getStreamBlockList(uint32_t StreamIndex) const = 0;
+  virtual ArrayRef<support::ulittle32_t>
+  getStreamBlockList(uint32_t StreamIndex) const = 0;
 
   virtual StringRef getBlockData(uint32_t BlockIndex,
                                  uint32_t NumBytes) const = 0;

llvm/include/llvm/DebugInfo/PDB/Raw/IPDBStreamData.h

@@ -0,0 +1,38 @@
+//===- IPDBStreamData.h - Base interface for PDB Stream Data ----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_PDB_RAW_IPDBSTREAMDATA_H
+#define LLVM_DEBUGINFO_PDB_RAW_IPDBSTREAMDATA_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/Endian.h"
+
+namespace llvm {
+namespace pdb {
+/// IPDBStream abstracts the notion of PDB stream data.  Although we already
+/// have another stream abstraction (namely in the form of StreamInterface
+/// and MappedBlockStream), they assume that the stream data is referenced
+/// the same way.  Namely, by looking in the directory to get the list of
+/// stream blocks, and by looking in the array of stream lengths to get the
+/// length.  This breaks down for the directory itself, however, since its
+/// length and list of blocks are stored elsewhere.  By abstracting the
+/// notion of stream data further, we can use a MappedBlockStream to read
+/// from the directory itself, or from an indexed stream which references
+/// the directory.
+class IPDBStreamData {
+public:
+  virtual ~IPDBStreamData() {}
+
+  virtual uint32_t getLength() = 0;
+  virtual ArrayRef<support::ulittle32_t> getStreamBlocks() = 0;
+};
+}
+}
+
+#endif

llvm/include/llvm/DebugInfo/PDB/Raw/IndexedStreamData.h

@@ -0,0 +1,34 @@
+//===- IndexedStreamData.h - Standard PDB Stream Data -----------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_PDB_RAW_INDEXEDSTREAMDATA_H
+#define LLVM_DEBUGINFO_PDB_RAW_INDEXEDSTREAMDATA_H
+
+#include "llvm/DebugInfo/PDB/Raw/IPDBStreamData.h"
+
+namespace llvm {
+namespace pdb {
+class IPDBFile;
+
+class IndexedStreamData : public IPDBStreamData {
+public:
+  IndexedStreamData(uint32_t StreamIdx, const IPDBFile &File);
+  virtual ~IndexedStreamData() {}
+
+  uint32_t getLength() override;
+  ArrayRef<support::ulittle32_t> getStreamBlocks() override;
+
+private:
+  uint32_t StreamIdx;
+  const IPDBFile &File;
+};
+}
+}
+
+#endif

llvm/include/llvm/DebugInfo/PDB/Raw/InfoStream.h

@@ -24,7 +24,7 @@ namespace pdb {
 class PDBFile;
 class InfoStream {
 public:
-  InfoStream(PDBFile &File);
+  InfoStream(const PDBFile &File);
 
   Error reload();
 
@@ -36,10 +36,7 @@ class InfoStream {
   uint32_t getNamedStreamIndex(llvm::StringRef Name) const;
   iterator_range<StringMapConstIterator<uint32_t>> named_streams() const;
 
-  PDBFile &getFile() { return Pdb; }
-
 private:
-  PDBFile &Pdb;
   MappedBlockStream Stream;
 
   // PDB file format version.  We only support VC70.  See the enumeration

llvm/include/llvm/DebugInfo/PDB/Raw/MappedBlockStream.h

@@ -14,6 +14,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/DebugInfo/CodeView/StreamInterface.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
 #include <vector>
@@ -22,15 +23,16 @@ namespace llvm {
 namespace pdb {
 
 class IPDBFile;
+class IPDBStreamData;
 
 class MappedBlockStream : public codeview::StreamInterface {
 public:
-  MappedBlockStream(uint32_t StreamIdx, const IPDBFile &File);
+  MappedBlockStream(std::unique_ptr<IPDBStreamData> Data, const IPDBFile &File);
 
   Error readBytes(uint32_t Offset, uint32_t Size,
                   ArrayRef<uint8_t> &Buffer) const override;
 
-  uint32_t getLength() const override { return StreamLength; }
+  uint32_t getLength() const override;
 
   uint32_t getNumBytesCopied() const;
 
@@ -39,11 +41,11 @@ class MappedBlockStream : public codeview::StreamInterface {
   bool tryReadContiguously(uint32_t Offset, uint32_t Size,
                            ArrayRef<uint8_t> &Buffer) const;
 
-  uint32_t StreamLength;
-  std::vector<uint32_t> BlockList;
+  const IPDBFile &Pdb;
+  std::unique_ptr<IPDBStreamData> Data;
+
   mutable llvm::BumpPtrAllocator Pool;
   mutable DenseMap<uint32_t, uint8_t *> CacheMap;
-  const IPDBFile &Pdb;
 };
 
 } // end namespace pdb

llvm/include/llvm/DebugInfo/PDB/Raw/ModStream.h

@@ -26,7 +26,7 @@ class ModInfo;
 
 class ModStream {
 public:
-  ModStream(PDBFile &File, const ModInfo &Module);
+  ModStream(const PDBFile &File, const ModInfo &Module);
   ~ModStream();
 
   Error reload();

llvm/include/llvm/DebugInfo/PDB/Raw/PDBFile.h

@@ -11,6 +11,7 @@
 #define LLVM_DEBUGINFO_PDB_RAW_PDBFILE_H
 
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/DebugInfo/CodeView/StreamArray.h"
 #include "llvm/DebugInfo/PDB/Raw/IPDBFile.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
@@ -48,11 +49,12 @@ class PDBFile : public IPDBFile {
 
   uint32_t getNumStreams() const override;
   uint32_t getStreamByteSize(uint32_t StreamIndex) const override;
-  ArrayRef<uint32_t> getStreamBlockList(uint32_t StreamIndex) const override;
+  ArrayRef<support::ulittle32_t>
+  getStreamBlockList(uint32_t StreamIndex) const override;
 
   StringRef getBlockData(uint32_t BlockIndex, uint32_t NumBytes) const override;
 
-  ArrayRef<support::ulittle32_t> getDirectoryBlockArray();
+  ArrayRef<support::ulittle32_t> getDirectoryBlockArray() const;
 
   Error parseFileHeaders();
   Error parseStreamData();
@@ -81,6 +83,7 @@ class PDBFile : public IPDBFile {
   std::unique_ptr<TpiStream> Ipi;
   std::unique_ptr<PublicsStream> Publics;
   std::unique_ptr<SymbolStream> Symbols;
+  std::unique_ptr<MappedBlockStream> DirectoryStream;
   std::unique_ptr<MappedBlockStream> StringTableStream;
   std::unique_ptr<NameHashTable> StringTable;
 };

llvm/include/llvm/DebugInfo/PDB/Raw/SymbolStream.h

@@ -22,7 +22,7 @@ class PDBFile;
 
 class SymbolStream {
 public:
-  SymbolStream(PDBFile &File, uint32_t StreamNum);
+  SymbolStream(const PDBFile &File, uint32_t StreamNum);
   ~SymbolStream();
   Error reload();
 

llvm/include/llvm/DebugInfo/PDB/Raw/TpiStream.h

@@ -31,7 +31,7 @@ class TpiStream {
   struct HeaderInfo;
 
 public:
-  TpiStream(PDBFile &File, uint32_t StreamIdx);
+  TpiStream(const PDBFile &File, uint32_t StreamIdx);
   ~TpiStream();
   Error reload();
 
@@ -50,7 +50,7 @@ class TpiStream {
   iterator_range<codeview::CVTypeArray::Iterator> types(bool *HadError) const;
 
 private:
-  PDBFile &Pdb;
+  const PDBFile &Pdb;
   MappedBlockStream Stream;
   HashFunctionType HashFunction;
 

llvm/lib/DebugInfo/PDB/CMakeLists.txt

@@ -30,6 +30,7 @@ endif()
 add_pdb_impl_folder(Raw
   Raw/DbiStream.cpp
   Raw/EnumTables.cpp
+  Raw/IndexedStreamData.cpp
   Raw/InfoStream.cpp
   Raw/MappedBlockStream.cpp
   Raw/ModInfo.cpp

llvm/lib/DebugInfo/PDB/Raw/DbiStream.cpp

@@ -12,6 +12,7 @@
 #include "llvm/DebugInfo/CodeView/StreamArray.h"
 #include "llvm/DebugInfo/CodeView/StreamReader.h"
 #include "llvm/DebugInfo/PDB/Raw/ISectionContribVisitor.h"
+#include "llvm/DebugInfo/PDB/Raw/IndexedStreamData.h"
 #include "llvm/DebugInfo/PDB/Raw/InfoStream.h"
 #include "llvm/DebugInfo/PDB/Raw/ModInfo.h"
 #include "llvm/DebugInfo/PDB/Raw/NameHashTable.h"
@@ -93,7 +94,9 @@ Error loadSectionContribs(FixedStreamArray<ContribType> &Output,
 }
 
 DbiStream::DbiStream(PDBFile &File)
-    : Pdb(File), Stream(StreamDBI, File), Header(nullptr) {
+    : Pdb(File),
+      Stream(llvm::make_unique<IndexedStreamData>(StreamDBI, File), File),
+      Header(nullptr) {
   static_assert(sizeof(HeaderInfo) == 64, "Invalid HeaderInfo size!");
 }
 
@@ -290,7 +293,8 @@ Error DbiStream::initializeSectionContributionData() {
 // Initializes this->SectionHeaders.
 Error DbiStream::initializeSectionHeadersData() {
   uint32_t StreamNum = getDebugStreamIndex(DbgHeaderType::SectionHdr);
-  SectionHeaderStream.reset(new MappedBlockStream(StreamNum, Pdb));
+  SectionHeaderStream.reset(new MappedBlockStream(
+      llvm::make_unique<IndexedStreamData>(StreamNum, Pdb), Pdb));
 
   size_t StreamLen = SectionHeaderStream->getLength();
   if (StreamLen % sizeof(object::coff_section))
@@ -308,7 +312,8 @@ Error DbiStream::initializeSectionHeadersData() {
 // Initializes this->Fpos.
 Error DbiStream::initializeFpoRecords() {
   uint32_t StreamNum = getDebugStreamIndex(DbgHeaderType::NewFPO);
-  FpoStream.reset(new MappedBlockStream(StreamNum, Pdb));
+  FpoStream.reset(new MappedBlockStream(
+      llvm::make_unique<IndexedStreamData>(StreamNum, Pdb), Pdb));
 
   size_t StreamLen = FpoStream->getLength();
   if (StreamLen % sizeof(object::FpoData))

llvm/lib/DebugInfo/PDB/Raw/IndexedStreamData.cpp

@@ -0,0 +1,25 @@
+//===- IndexedStreamData.cpp - Standard PDB Stream Data ---------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/DebugInfo/PDB/Raw/IndexedStreamData.h"
+#include "llvm/DebugInfo/PDB/Raw/IPDBFile.h"
+
+using namespace llvm;
+using namespace llvm::pdb;
+
+IndexedStreamData::IndexedStreamData(uint32_t StreamIdx, const IPDBFile &File)
+    : StreamIdx(StreamIdx), File(File) {}
+
+uint32_t IndexedStreamData::getLength() {
+  return File.getStreamByteSize(StreamIdx);
+}
+
+ArrayRef<support::ulittle32_t> IndexedStreamData::getStreamBlocks() {
+  return File.getStreamBlockList(StreamIdx);
+}

llvm/lib/DebugInfo/PDB/Raw/InfoStream.cpp

@@ -11,14 +11,16 @@
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/DebugInfo/CodeView/StreamReader.h"
+#include "llvm/DebugInfo/PDB/Raw/IndexedStreamData.h"
 #include "llvm/DebugInfo/PDB/Raw/PDBFile.h"
 #include "llvm/DebugInfo/PDB/Raw/RawConstants.h"
 #include "llvm/DebugInfo/PDB/Raw/RawError.h"
 
 using namespace llvm;
 using namespace llvm::pdb;
 
-InfoStream::InfoStream(PDBFile &File) : Pdb(File), Stream(StreamPDB, File) {}
+InfoStream::InfoStream(const PDBFile &File)
+    : Stream(llvm::make_unique<IndexedStreamData>(StreamPDB, File), File) {}
 
 Error InfoStream::reload() {
   codeview::StreamReader Reader(Stream);