[clang-doc] Add support for explicitly typed enums

Add support for explicitly typed enums:
  enum Foo : unsigned { ... };
to the internal representation and to the YAML output.

Add support for getting the value of an enum constant, as well as accessing the original expression that produced it. This changes the YAML output of enums from an array of strings for the enum members to an array of dictionaries. These dictionaries now report the name, value, and original expression.

The markdown and HTML outputs are unchanged, they still output the name from the new enhanced internal schema.

Reviewed By: paulkirth

Differential Revision: https://reviews.llvm.org/D134055

clang-tools-extra/clang-doc/BitcodeReader.cpp

@@ -17,12 +17,19 @@ namespace doc {
 
 using Record = llvm::SmallVector<uint64_t, 1024>;
 
+// This implements decode for SmallString.
 llvm::Error decodeRecord(const Record &R, llvm::SmallVectorImpl<char> &Field,
                          llvm::StringRef Blob) {
   Field.assign(Blob.begin(), Blob.end());
   return llvm::Error::success();
 }
 
+llvm::Error decodeRecord(const Record &R, std::string &Field,
+                         llvm::StringRef Blob) {
+  Field.assign(Blob.begin(), Blob.end());
+  return llvm::Error::success();
+}
+
 llvm::Error decodeRecord(const Record &R, SymbolID &Field,
                          llvm::StringRef Blob) {
   if (R[0] != BitCodeConstants::USRHashSize)
@@ -218,8 +225,6 @@ llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
     return decodeRecord(R, I->DefLoc, Blob);
   case ENUM_LOCATION:
     return decodeRecord(R, I->Loc, Blob);
-  case ENUM_MEMBER:
-    return decodeRecord(R, I->Members, Blob);
   case ENUM_SCOPED:
     return decodeRecord(R, I->Scoped, Blob);
   default:
@@ -228,6 +233,21 @@ llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
   }
 }
 
+llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
+                        EnumValueInfo *I) {
+  switch (ID) {
+  case ENUM_VALUE_NAME:
+    return decodeRecord(R, I->Name, Blob);
+  case ENUM_VALUE_VALUE:
+    return decodeRecord(R, I->Value, Blob);
+  case ENUM_VALUE_EXPR:
+    return decodeRecord(R, I->ValueExpr, Blob);
+  default:
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "invalid field for EnumValueInfo");
+  }
+}
+
 llvm::Error parseRecord(const Record &R, unsigned ID, llvm::StringRef Blob,
                         FunctionInfo *I) {
   switch (ID) {
@@ -372,6 +392,9 @@ llvm::Expected<CommentInfo *> getCommentInfo(std::unique_ptr<CommentInfo> &I) {
   return getCommentInfo(I.get());
 }
 
+// When readSubBlock encounters a TypeInfo sub-block, it calls addTypeInfo on
+// the parent block to set it. The template specializations define what to do
+// for each supported parent block.
 template <typename T, typename TTypeInfo>
 llvm::Error addTypeInfo(T I, TTypeInfo &&TI) {
   return llvm::createStringError(llvm::inconvertibleErrorCode(),
@@ -398,6 +421,11 @@ template <> llvm::Error addTypeInfo(FunctionInfo *I, FieldTypeInfo &&T) {
   return llvm::Error::success();
 }
 
+template <> llvm::Error addTypeInfo(EnumInfo *I, TypeInfo &&T) {
+  I->BaseType = std::move(T);
+  return llvm::Error::success();
+}
+
 template <typename T> llvm::Error addReference(T I, Reference &&R, FieldId F) {
   return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                  "invalid type cannot contain Reference");
@@ -524,6 +552,10 @@ template <> void addChild(RecordInfo *I, EnumInfo &&R) {
   I->ChildEnums.emplace_back(std::move(R));
 }
 
+template <> void addChild(EnumInfo *I, EnumValueInfo &&R) {
+  I->Members.emplace_back(std::move(R));
+}
+
 template <> void addChild(RecordInfo *I, BaseRecordInfo &&R) {
   I->Bases.emplace_back(std::move(R));
 }
@@ -587,8 +619,7 @@ llvm::Error ClangDocBitcodeReader::readBlock(unsigned ID, T I) {
 template <typename T>
 llvm::Error ClangDocBitcodeReader::readSubBlock(unsigned ID, T I) {
   switch (ID) {
-  // Blocks can only have Comment, Reference, TypeInfo, FunctionInfo, or
-  // EnumInfo subblocks
+  // Blocks can only have certain types of sub blocks.
   case BI_COMMENT_BLOCK_ID: {
     auto Comment = getCommentInfo(I);
     if (!Comment)
@@ -650,6 +681,13 @@ llvm::Error ClangDocBitcodeReader::readSubBlock(unsigned ID, T I) {
     addChild(I, std::move(E));
     return llvm::Error::success();
   }
+  case BI_ENUM_VALUE_BLOCK_ID: {
+    EnumValueInfo EV;
+    if (auto Err = readBlock(ID, &EV))
+      return Err;
+    addChild(I, std::move(EV));
+    return llvm::Error::success();
+  }
   default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "invalid subblock type");

clang-tools-extra/clang-doc/BitcodeWriter.cpp

@@ -112,6 +112,7 @@ static const llvm::IndexedMap<llvm::StringRef, BlockIdToIndexFunctor>
           {BI_VERSION_BLOCK_ID, "VersionBlock"},
           {BI_NAMESPACE_BLOCK_ID, "NamespaceBlock"},
           {BI_ENUM_BLOCK_ID, "EnumBlock"},
+          {BI_ENUM_VALUE_BLOCK_ID, "EnumValueBlock"},
           {BI_TYPE_BLOCK_ID, "TypeBlock"},
           {BI_FIELD_TYPE_BLOCK_ID, "FieldTypeBlock"},
           {BI_MEMBER_TYPE_BLOCK_ID, "MemberTypeBlock"},
@@ -158,8 +159,10 @@ static const llvm::IndexedMap<RecordIdDsc, RecordIdToIndexFunctor>
           {ENUM_NAME, {"Name", &StringAbbrev}},
           {ENUM_DEFLOCATION, {"DefLocation", &LocationAbbrev}},
           {ENUM_LOCATION, {"Location", &LocationAbbrev}},
-          {ENUM_MEMBER, {"Member", &StringAbbrev}},
           {ENUM_SCOPED, {"Scoped", &BoolAbbrev}},
+          {ENUM_VALUE_NAME, {"Name", &StringAbbrev}},
+          {ENUM_VALUE_VALUE, {"Value", &StringAbbrev}},
+          {ENUM_VALUE_EXPR, {"Expr", &StringAbbrev}},
           {RECORD_USR, {"USR", &SymbolIDAbbrev}},
           {RECORD_NAME, {"Name", &StringAbbrev}},
           {RECORD_PATH, {"Path", &StringAbbrev}},
@@ -213,8 +216,10 @@ static const std::vector<std::pair<BlockId, std::vector<RecordId>>>
         {BI_MEMBER_TYPE_BLOCK_ID, {MEMBER_TYPE_NAME, MEMBER_TYPE_ACCESS}},
         // Enum Block
         {BI_ENUM_BLOCK_ID,
-         {ENUM_USR, ENUM_NAME, ENUM_DEFLOCATION, ENUM_LOCATION, ENUM_MEMBER,
-          ENUM_SCOPED}},
+         {ENUM_USR, ENUM_NAME, ENUM_DEFLOCATION, ENUM_LOCATION, ENUM_SCOPED}},
+        // Enum Value Block
+        {BI_ENUM_VALUE_BLOCK_ID,
+         {ENUM_VALUE_NAME, ENUM_VALUE_VALUE, ENUM_VALUE_EXPR}},
         // Namespace Block
         {BI_NAMESPACE_BLOCK_ID,
          {NAMESPACE_USR, NAMESPACE_NAME, NAMESPACE_PATH}},
@@ -486,8 +491,17 @@ void ClangDocBitcodeWriter::emitBlock(const EnumInfo &I) {
   for (const auto &L : I.Loc)
     emitRecord(L, ENUM_LOCATION);
   emitRecord(I.Scoped, ENUM_SCOPED);
+  if (I.BaseType)
+    emitBlock(*I.BaseType);
   for (const auto &N : I.Members)
-    emitRecord(N, ENUM_MEMBER);
+    emitBlock(N);
+}
+
+void ClangDocBitcodeWriter::emitBlock(const EnumValueInfo &I) {
+  StreamSubBlockGuard Block(Stream, BI_ENUM_VALUE_BLOCK_ID);
+  emitRecord(I.Name, ENUM_VALUE_NAME);
+  emitRecord(I.Value, ENUM_VALUE_VALUE);
+  emitRecord(I.ValueExpr, ENUM_VALUE_EXPR);
 }
 
 void ClangDocBitcodeWriter::emitBlock(const RecordInfo &I) {

clang-tools-extra/clang-doc/BitcodeWriter.h

@@ -17,6 +17,7 @@
 
 #include "Representation.h"
 #include "clang/AST/AST.h"
+#include "llvm/ADT/APSInt.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -54,6 +55,7 @@ enum BlockId {
   BI_VERSION_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID,
   BI_NAMESPACE_BLOCK_ID,
   BI_ENUM_BLOCK_ID,
+  BI_ENUM_VALUE_BLOCK_ID,
   BI_TYPE_BLOCK_ID,
   BI_FIELD_TYPE_BLOCK_ID,
   BI_MEMBER_TYPE_BLOCK_ID,
@@ -98,8 +100,10 @@ enum RecordId {
   ENUM_NAME,
   ENUM_DEFLOCATION,
   ENUM_LOCATION,
-  ENUM_MEMBER,
   ENUM_SCOPED,
+  ENUM_VALUE_NAME,
+  ENUM_VALUE_VALUE,
+  ENUM_VALUE_EXPR,
   RECORD_USR,
   RECORD_NAME,
   RECORD_PATH,
@@ -155,6 +159,7 @@ class ClangDocBitcodeWriter {
   void emitBlock(const BaseRecordInfo &I);
   void emitBlock(const FunctionInfo &I);
   void emitBlock(const EnumInfo &I);
+  void emitBlock(const EnumValueInfo &I);
   void emitBlock(const TypeInfo &B);
   void emitBlock(const FieldTypeInfo &B);
   void emitBlock(const MemberTypeInfo &B);
@@ -205,6 +210,7 @@ class ClangDocBitcodeWriter {
   void emitRecord(bool Value, RecordId ID);
   void emitRecord(int Value, RecordId ID);
   void emitRecord(unsigned Value, RecordId ID);
+  void emitRecord(llvm::APSInt Value, RecordId ID);
   bool prepRecordData(RecordId ID, bool ShouldEmit = true);
 
   // Emission of appropriate abbreviation type.

clang-tools-extra/clang-doc/HTMLGenerator.cpp

@@ -361,13 +361,14 @@ genEnumsBlock(const std::vector<EnumInfo> &Enums,
 }
 
 static std::unique_ptr<TagNode>
-genEnumMembersBlock(const llvm::SmallVector<SmallString<16>, 4> &Members) {
+genEnumMembersBlock(const llvm::SmallVector<EnumValueInfo, 4> &Members) {
   if (Members.empty())
     return nullptr;
 
   auto List = std::make_unique<TagNode>(HTMLTag::TAG_UL);
   for (const auto &M : Members)
-    List->Children.emplace_back(std::make_unique<TagNode>(HTMLTag::TAG_LI, M));
+    List->Children.emplace_back(
+        std::make_unique<TagNode>(HTMLTag::TAG_LI, M.Name));
   return List;
 }
 

clang-tools-extra/clang-doc/MDGenerator.cpp

@@ -136,7 +136,7 @@ static void genMarkdown(const ClangDocContext &CDCtx, const EnumInfo &I,
   llvm::raw_string_ostream Members(Buffer);
   if (!I.Members.empty())
     for (const auto &N : I.Members)
-      Members << "| " << N << " |\n";
+      Members << "| " << N.Name << " |\n";
   writeLine(Members.str(), OS);
   if (I.DefLoc)
     writeFileDefinition(CDCtx, *I.DefLoc, OS);

clang-tools-extra/clang-doc/Representation.h

@@ -17,6 +17,7 @@
 #include "clang/AST/Type.h"
 #include "clang/Basic/Specifiers.h"
 #include "clang/Tooling/StandaloneExecution.h"
+#include "llvm/ADT/APSInt.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
@@ -397,6 +398,30 @@ struct BaseRecordInfo : public RecordInfo {
   bool IsParent = false; // Indicates if this base is a direct parent
 };
 
+// Information for a single possible value of an enumeration.
+struct EnumValueInfo {
+  explicit EnumValueInfo(StringRef Name = StringRef(),
+                         StringRef Value = StringRef("0"),
+                         StringRef ValueExpr = StringRef())
+      : Name(Name), Value(Value), ValueExpr(ValueExpr) {}
+
+  bool operator==(const EnumValueInfo &Other) const {
+    return std::tie(Name, Value, ValueExpr) ==
+           std::tie(Other.Name, Other.Value, Other.ValueExpr);
+  }
+
+  SmallString<16> Name;
+
+  // The computed value of the enumeration constant. This could be the result of
+  // evaluating the ValueExpr, or it could be automatically generated according
+  // to C rules.
+  SmallString<16> Value;
+
+  // Stores the user-supplied initialization expression for this enumeration
+  // constant. This will be empty for implicit enumeration values.
+  SmallString<16> ValueExpr;
+};
+
 // TODO: Expand to allow for documenting templating.
 // Info for types.
 struct EnumInfo : public SymbolInfo {
@@ -405,9 +430,15 @@ struct EnumInfo : public SymbolInfo {
 
   void merge(EnumInfo &&I);
 
-  bool Scoped =
-      false; // Indicates whether this enum is scoped (e.g. enum class).
-  llvm::SmallVector<SmallString<16>, 4> Members; // List of enum members.
+  // Indicates whether this enum is scoped (e.g. enum class).
+  bool Scoped = false;
+
+  // Set to nonempty to the type when this is an explicitly typed enum. For
+  //   enum Foo : short { ... };
+  // this will be "short".
+  llvm::Optional<TypeInfo> BaseType;
+
+  llvm::SmallVector<EnumValueInfo, 4> Members; // List of enum members.
 };
 
 struct Index : public Reference {

clang-tools-extra/clang-doc/Serialize.cpp

@@ -182,6 +182,13 @@ std::string ClangDocCommentVisitor::getCommandName(unsigned CommandID) const {
 
 // Serializing functions.
 
+std::string getSourceCode(const Decl *D, const SourceRange &R) {
+  return Lexer::getSourceText(CharSourceRange::getTokenRange(R),
+                              D->getASTContext().getSourceManager(),
+                              D->getASTContext().getLangOpts())
+      .str();
+}
+
 template <typename T> static std::string serialize(T &I) {
   SmallString<2048> Buffer;
   llvm::BitstreamWriter Stream(Buffer);
@@ -305,8 +312,15 @@ static void parseFields(RecordInfo &I, const RecordDecl *D, bool PublicOnly,
 }
 
 static void parseEnumerators(EnumInfo &I, const EnumDecl *D) {
-  for (const EnumConstantDecl *E : D->enumerators())
-    I.Members.emplace_back(E->getNameAsString());
+  for (const EnumConstantDecl *E : D->enumerators()) {
+    std::string ValueExpr;
+    if (const Expr *InitExpr = E->getInitExpr())
+      ValueExpr = getSourceCode(D, InitExpr->getSourceRange());
+
+    SmallString<16> ValueStr;
+    E->getInitVal().toString(ValueStr);
+    I.Members.emplace_back(E->getNameAsString(), ValueStr, ValueExpr);
+  }
 }
 
 static void parseParameters(FunctionInfo &I, const FunctionDecl *D) {
@@ -331,10 +345,7 @@ static void parseParameters(FunctionInfo &I, const FunctionDecl *D) {
     }
 
     if (const Expr *DefaultArg = P->getDefaultArg()) {
-      FieldInfo->DefaultValue = Lexer::getSourceText(
-          CharSourceRange::getTokenRange(DefaultArg->getSourceRange()),
-          D->getASTContext().getSourceManager(),
-          D->getASTContext().getLangOpts());
+      FieldInfo->DefaultValue = getSourceCode(D, DefaultArg->getSourceRange());
     }
   }
 }
@@ -657,6 +668,8 @@ emitInfo(const EnumDecl *D, const FullComment *FC, int LineNumber,
     return {};
 
   Enum.Scoped = D->isScoped();
+  if (D->isFixed())
+    Enum.BaseType = TypeInfo(D->getIntegerType().getAsString());
   parseEnumerators(Enum, D);
 
   // Put in global namespace

clang-tools-extra/clang-doc/YAMLGenerator.cpp

@@ -14,13 +14,15 @@
 
 using namespace clang::doc;
 
+// These define YAML traits for decoding the listed values within a vector.
 LLVM_YAML_IS_SEQUENCE_VECTOR(FieldTypeInfo)
 LLVM_YAML_IS_SEQUENCE_VECTOR(MemberTypeInfo)
 LLVM_YAML_IS_SEQUENCE_VECTOR(Reference)
 LLVM_YAML_IS_SEQUENCE_VECTOR(Location)
 LLVM_YAML_IS_SEQUENCE_VECTOR(CommentInfo)
 LLVM_YAML_IS_SEQUENCE_VECTOR(FunctionInfo)
 LLVM_YAML_IS_SEQUENCE_VECTOR(EnumInfo)
+LLVM_YAML_IS_SEQUENCE_VECTOR(EnumValueInfo)
 LLVM_YAML_IS_SEQUENCE_VECTOR(BaseRecordInfo)
 LLVM_YAML_IS_SEQUENCE_VECTOR(std::unique_ptr<CommentInfo>)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::SmallString<16>)
@@ -226,10 +228,19 @@ template <> struct MappingTraits<BaseRecordInfo> {
   }
 };
 
+template <> struct MappingTraits<EnumValueInfo> {
+  static void mapping(IO &IO, EnumValueInfo &I) {
+    IO.mapOptional("Name", I.Name);
+    IO.mapOptional("Value", I.Value);
+    IO.mapOptional("Expr", I.ValueExpr, SmallString<16>());
+  }
+};
+
 template <> struct MappingTraits<EnumInfo> {
   static void mapping(IO &IO, EnumInfo &I) {
     SymbolInfoMapping(IO, I);
     IO.mapOptional("Scoped", I.Scoped, false);
+    IO.mapOptional("BaseType", I.BaseType);
     IO.mapOptional("Members", I.Members);
   }
 };

clang-tools-extra/unittests/clang-doc/SerializeTest.cpp

@@ -266,8 +266,8 @@ TEST(SerializeTest, emitEnumInfo) {
   EnumInfo E;
   E.Name = "E";
   E.DefLoc = Location(0, llvm::SmallString<16>{"test.cpp"});
-  E.Members.emplace_back("X");
-  E.Members.emplace_back("Y");
+  E.Members.emplace_back("X", "0");
+  E.Members.emplace_back("Y", "1");
   ExpectedNamespaceWithEnum.ChildEnums.emplace_back(std::move(E));
   CheckNamespaceInfo(&ExpectedNamespaceWithEnum, NamespaceWithEnum);
 
@@ -277,8 +277,8 @@ TEST(SerializeTest, emitEnumInfo) {
   G.Name = "G";
   G.Scoped = true;
   G.DefLoc = Location(0, llvm::SmallString<16>{"test.cpp"});
-  G.Members.emplace_back("A");
-  G.Members.emplace_back("B");
+  G.Members.emplace_back("A", "0");
+  G.Members.emplace_back("B", "1");
   ExpectedNamespaceWithScopedEnum.ChildEnums.emplace_back(std::move(G));
   CheckNamespaceInfo(&ExpectedNamespaceWithScopedEnum, NamespaceWithScopedEnum);
 }