[clang-doc] Setup clang-doc frontend framework

Setting up the mapper part of the frontend framework for a clang-doc
tool. It creates a series of relevant matchers for declarations, and
uses the ToolExecutor to traverse the AST and extract the matching
declarations and comments. The mapper serializes the extracted
information to individual records for reducing and eventually doc
generation.

For a more detailed overview of the tool, see the design document on the
mailing list: http://lists.llvm.org/pipermail/cfe-dev/2017-December/056203.html

Differential Revision: https://reviews.llvm.org/D41102

llvm-svn: 327102

clang-tools-extra/CMakeLists.txt

@@ -7,6 +7,7 @@ add_subdirectory(clang-tidy-vs)
 endif()
 
 add_subdirectory(change-namespace)
+add_subdirectory(clang-doc)
 add_subdirectory(clang-query)
 add_subdirectory(clang-move)
 add_subdirectory(clangd)

clang-tools-extra/clang-doc/BitcodeWriter.h

@@ -0,0 +1,201 @@
+//===--  BitcodeWriter.h - ClangDoc Bitcode Writer --------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a writer for serializing the clang-doc internal
+// representation to LLVM bitcode. The writer takes in a stream and emits the
+// generated bitcode to that stream.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_BITCODEWRITER_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_BITCODEWRITER_H
+
+#include "Representation.h"
+#include "clang/AST/AST.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Bitcode/BitstreamWriter.h"
+#include <initializer_list>
+#include <vector>
+
+namespace clang {
+namespace doc {
+
+// Current version number of clang-doc bitcode.
+// Should be bumped when removing or changing BlockIds, RecordIds, or
+// BitCodeConstants, though they can be added without breaking it.
+static const unsigned VersionNumber = 1;
+
+struct BitCodeConstants {
+  static constexpr unsigned RecordSize = 16U;
+  static constexpr unsigned SignatureBitSize = 8U;
+  static constexpr unsigned SubblockIDSize = 4U;
+  static constexpr unsigned BoolSize = 1U;
+  static constexpr unsigned IntSize = 16U;
+  static constexpr unsigned StringLengthSize = 16U;
+  static constexpr unsigned FilenameLengthSize = 16U;
+  static constexpr unsigned LineNumberSize = 16U;
+  static constexpr unsigned ReferenceTypeSize = 8U;
+  static constexpr unsigned USRLengthSize = 6U;
+  static constexpr unsigned USRBitLengthSize = 8U;
+};
+
+// New Ids need to be added to both the enum here and the relevant IdNameMap in
+// the implementation file.
+enum BlockId {
+  BI_VERSION_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID,
+  BI_NAMESPACE_BLOCK_ID,
+  BI_ENUM_BLOCK_ID,
+  BI_TYPE_BLOCK_ID,
+  BI_FIELD_TYPE_BLOCK_ID,
+  BI_MEMBER_TYPE_BLOCK_ID,
+  BI_RECORD_BLOCK_ID,
+  BI_FUNCTION_BLOCK_ID,
+  BI_COMMENT_BLOCK_ID,
+  BI_FIRST = BI_VERSION_BLOCK_ID,
+  BI_LAST = BI_COMMENT_BLOCK_ID
+};
+
+// New Ids need to be added to the enum here, and to the relevant IdNameMap and
+// initialization list in the implementation file.
+#define INFORECORDS(X) X##_USR, X##_NAME, X##_NAMESPACE
+
+enum RecordId {
+  VERSION = 1,
+  INFORECORDS(FUNCTION),
+  FUNCTION_DEFLOCATION,
+  FUNCTION_LOCATION,
+  FUNCTION_PARENT,
+  FUNCTION_ACCESS,
+  FUNCTION_IS_METHOD,
+  COMMENT_KIND,
+  COMMENT_TEXT,
+  COMMENT_NAME,
+  COMMENT_DIRECTION,
+  COMMENT_PARAMNAME,
+  COMMENT_CLOSENAME,
+  COMMENT_SELFCLOSING,
+  COMMENT_EXPLICIT,
+  COMMENT_ATTRKEY,
+  COMMENT_ATTRVAL,
+  COMMENT_ARG,
+  TYPE_REF,
+  FIELD_TYPE_REF,
+  FIELD_TYPE_NAME,
+  MEMBER_TYPE_REF,
+  MEMBER_TYPE_NAME,
+  MEMBER_TYPE_ACCESS,
+  INFORECORDS(NAMESPACE),
+  INFORECORDS(ENUM),
+  ENUM_DEFLOCATION,
+  ENUM_LOCATION,
+  ENUM_MEMBER,
+  ENUM_SCOPED,
+  INFORECORDS(RECORD),
+  RECORD_DEFLOCATION,
+  RECORD_LOCATION,
+  RECORD_TAG_TYPE,
+  RECORD_PARENT,
+  RECORD_VPARENT,
+  RI_FIRST = VERSION,
+  RI_LAST = RECORD_VPARENT
+};
+
+static constexpr unsigned BlockIdCount = BI_LAST - BI_FIRST + 1;
+static constexpr unsigned RecordIdCount = RI_LAST - RI_FIRST + 1;
+
+#undef INFORECORDS
+
+class ClangDocBitcodeWriter {
+public:
+  ClangDocBitcodeWriter(llvm::BitstreamWriter &Stream) : Stream(Stream) {
+    emitHeader();
+    emitBlockInfoBlock();
+    emitVersionBlock();
+  }
+
+#ifndef NDEBUG // Don't want explicit dtor unless needed.
+  ~ClangDocBitcodeWriter() {
+    // Check that the static size is large-enough.
+    assert(Record.capacity() > BitCodeConstants::RecordSize);
+  }
+#endif
+
+  // Block emission of different info types.
+  void emitBlock(const NamespaceInfo &I);
+  void emitBlock(const RecordInfo &I);
+  void emitBlock(const FunctionInfo &I);
+  void emitBlock(const EnumInfo &I);
+  void emitBlock(const TypeInfo &B);
+  void emitBlock(const FieldTypeInfo &B);
+  void emitBlock(const MemberTypeInfo &B);
+  void emitBlock(const CommentInfo &B);
+
+private:
+  class AbbreviationMap {
+    llvm::DenseMap<unsigned, unsigned> Abbrevs;
+
+  public:
+    AbbreviationMap() : Abbrevs(RecordIdCount) {}
+
+    void add(RecordId RID, unsigned AbbrevID);
+    unsigned get(RecordId RID) const;
+  };
+
+  class StreamSubBlockGuard {
+    llvm::BitstreamWriter &Stream;
+
+  public:
+    StreamSubBlockGuard(llvm::BitstreamWriter &Stream_, BlockId ID)
+        : Stream(Stream_) {
+      // NOTE: SubBlockIDSize could theoretically be calculated on the fly,
+      // based on the initialization list of records in each block.
+      Stream.EnterSubblock(ID, BitCodeConstants::SubblockIDSize);
+    }
+
+    StreamSubBlockGuard() = default;
+    StreamSubBlockGuard(const StreamSubBlockGuard &) = delete;
+    StreamSubBlockGuard &operator=(const StreamSubBlockGuard &) = delete;
+
+    ~StreamSubBlockGuard() { Stream.ExitBlock(); }
+  };
+
+  // Emission of validation and overview blocks.
+  void emitHeader();
+  void emitVersionBlock();
+  void emitRecordID(RecordId ID);
+  void emitBlockID(BlockId ID);
+  void emitBlockInfoBlock();
+  void emitBlockInfo(BlockId BID, const std::initializer_list<RecordId> &RIDs);
+
+  // Emission of individual record types.
+  void emitRecord(StringRef Str, RecordId ID);
+  void emitRecord(const SymbolID &Str, RecordId ID);
+  void emitRecord(const Location &Loc, RecordId ID);
+  void emitRecord(const Reference &Ref, RecordId ID);
+  void emitRecord(bool Value, RecordId ID);
+  void emitRecord(int Value, RecordId ID);
+  void emitRecord(unsigned Value, RecordId ID);
+  bool prepRecordData(RecordId ID, bool ShouldEmit = true);
+
+  // Emission of appropriate abbreviation type.
+  void emitAbbrev(RecordId ID, BlockId Block);
+
+  // Static size is the maximum length of the block/record names we're pushing
+  // to this + 1. Longest is currently `MemberTypeBlock` at 15 chars.
+  SmallVector<uint32_t, BitCodeConstants::RecordSize> Record;
+  llvm::BitstreamWriter &Stream;
+  AbbreviationMap Abbrevs;
+};
+
+} // namespace doc
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_BITCODEWRITER_H

clang-tools-extra/clang-doc/CMakeLists.txt

@@ -0,0 +1,23 @@
+set(LLVM_LINK_COMPONENTS
+  support
+  )
+
+add_clang_library(clangDoc
+  BitcodeWriter.cpp
+  ClangDoc.cpp
+  Mapper.cpp
+  Serialize.cpp
+
+  LINK_LIBS
+  clangAnalysis
+  clangAST
+  clangASTMatchers
+  clangBasic
+  clangFrontend
+  clangIndex
+  clangLex
+  clangTooling
+  clangToolingCore
+  )
+
+add_subdirectory(tool)

clang-tools-extra/clang-doc/ClangDoc.cpp

@@ -0,0 +1,61 @@
+//===-- ClangDoc.cpp - ClangDoc ---------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the main entry point for the clang-doc tool. It runs
+// the clang-doc mapper on a given set of source code files using a
+// FrontendActionFactory.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ClangDoc.h"
+#include "Mapper.h"
+#include "clang/AST/AST.h"
+#include "clang/AST/ASTConsumer.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/Frontend/ASTConsumers.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/FrontendActions.h"
+
+namespace clang {
+namespace doc {
+
+class MapperActionFactory : public tooling::FrontendActionFactory {
+public:
+  MapperActionFactory(tooling::ExecutionContext *ECtx) : ECtx(ECtx) {}
+  clang::FrontendAction *create() override;
+
+private:
+  tooling::ExecutionContext *ECtx;
+};
+
+clang::FrontendAction *MapperActionFactory::create() {
+  class ClangDocAction : public clang::ASTFrontendAction {
+  public:
+    ClangDocAction(ExecutionContext *ECtx) : ECtx(ECtx) {}
+
+    std::unique_ptr<clang::ASTConsumer>
+    CreateASTConsumer(clang::CompilerInstance &Compiler,
+                      llvm::StringRef InFile) override {
+      return llvm::make_unique<MapASTVisitor>(&Compiler.getASTContext(), ECtx);
+    }
+
+  private:
+    ExecutionContext *ECtx;
+  };
+  return new ClangDocAction(ECtx);
+}
+
+std::unique_ptr<tooling::FrontendActionFactory>
+newMapperActionFactory(tooling::ExecutionContext *ECtx) {
+  return llvm::make_unique<MapperActionFactory>(ECtx);
+}
+
+} // namespace doc
+} // namespace clang

clang-tools-extra/clang-doc/ClangDoc.h

@@ -0,0 +1,33 @@
+//===-- ClangDoc.h - ClangDoc -----------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file exposes a method to craete the FrontendActionFactory for the
+// clang-doc tool. The factory runs the clang-doc mapper on a given set of
+// source code files, storing the results key-value pairs in its
+// ExecutionContext.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_CLANGDOC_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_CLANGDOC_H
+
+#include "clang/Tooling/Execution.h"
+#include "clang/Tooling/StandaloneExecution.h"
+#include "clang/Tooling/Tooling.h"
+
+namespace clang {
+namespace doc {
+
+std::unique_ptr<tooling::FrontendActionFactory>
+newMapperActionFactory(tooling::ExecutionContext *ECtx);
+
+} // namespace doc
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_CLANGDOC_H