[clang-format] Add a MacroExpander.

Summary:
The MacroExpander allows to expand simple (non-resursive) macro
definitions from a macro identifier token and macro arguments. It
annotates the tokens with a newly introduced MacroContext that keeps
track of the role a token played in expanding the macro in order to
be able to reconstruct the macro expansion from an expanded (formatted)
token stream.

Made Token explicitly copy-able to enable copying tokens from the parsed
macro definition.

Reviewers: sammccall

Subscribers: mgorny, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D83296

clang/lib/Format/CMakeLists.txt

@@ -7,6 +7,7 @@ add_clang_library(clangFormat
   Format.cpp
   FormatToken.cpp
   FormatTokenLexer.cpp
+  MacroExpander.cpp
   NamespaceEndCommentsFixer.cpp
   SortJavaScriptImports.cpp
   TokenAnalyzer.cpp

clang/lib/Format/FormatToken.h

@@ -136,6 +136,68 @@ enum ParameterPackingKind { PPK_BinPacked, PPK_OnePerLine, PPK_Inconclusive };
 
 enum FormatDecision { FD_Unformatted, FD_Continue, FD_Break };
 
+/// Roles a token can take in a configured macro expansion.
+enum MacroRole {
+  /// The token was expanded from a macro argument when formatting the expanded
+  /// token sequence.
+  MR_ExpandedArg,
+  /// The token is part of a macro argument that was previously formatted as
+  /// expansion when formatting the unexpanded macro call.
+  MR_UnexpandedArg,
+  /// The token was expanded from a macro definition, and is not visible as part
+  /// of the macro call.
+  MR_Hidden,
+};
+
+struct FormatToken;
+
+/// Contains information on the token's role in a macro expansion.
+///
+/// Given the following definitions:
+/// A(X) = [ X ]
+/// B(X) = < X >
+/// C(X) = X
+///
+/// Consider the macro call:
+/// A({B(C(C(x)))}) -> [{<x>}]
+///
+/// In this case, the tokens of the unexpanded macro call will have the
+/// following relevant entries in their macro context (note that formatting
+/// the unexpanded macro call happens *after* formatting the expanded macro
+/// call):
+///                   A( { B( C( C(x) ) ) } )
+/// Role:             NN U NN NN NNUN N N U N  (N=None, U=UnexpandedArg)
+///
+///                   [  { <       x    > } ]
+/// Role:             H  E H       E    H E H  (H=Hidden, E=ExpandedArg)
+/// ExpandedFrom[0]:  A  A A       A    A A A
+/// ExpandedFrom[1]:       B       B    B
+/// ExpandedFrom[2]:               C
+/// ExpandedFrom[3]:               C
+/// StartOfExpansion: 1  0 1       2    0 0 0
+/// EndOfExpansion:   0  0 0       2    1 0 1
+struct MacroExpansion {
+  MacroExpansion(MacroRole Role) : Role(Role) {}
+
+  /// The token's role in the macro expansion.
+  /// When formatting an expanded macro, all tokens that are part of macro
+  /// arguments will be MR_ExpandedArg, while all tokens that are not visible in
+  /// the macro call will be MR_Hidden.
+  /// When formatting an unexpanded macro call, all tokens that are part of
+  /// macro arguments will be MR_UnexpandedArg.
+  MacroRole Role;
+
+  /// The stack of macro call identifier tokens this token was expanded from.
+  llvm::SmallVector<FormatToken *, 1> ExpandedFrom;
+
+  /// The number of expansions of which this macro is the first entry.
+  unsigned StartOfExpansion = 0;
+
+  /// The number of currently open expansions in \c ExpandedFrom this macro is
+  /// the last token in.
+  unsigned EndOfExpansion = 0;
+};
+
 class TokenRole;
 class AnnotatedLine;
 
@@ -163,7 +225,9 @@ struct FormatToken {
 
   /// A token can have a special role that can carry extra information
   /// about the token's formatting.
-  std::unique_ptr<TokenRole> Role;
+  /// FIXME: Make FormatToken for parsing and AnnotatedToken two different
+  /// classes and make this a unique_ptr in the AnnotatedToken class.
+  std::shared_ptr<TokenRole> Role;
 
   /// The range of the whitespace immediately preceding the \c Token.
   SourceRange WhitespaceRange;
@@ -378,6 +442,10 @@ struct FormatToken {
   /// in it.
   SmallVector<AnnotatedLine *, 1> Children;
 
+  // Contains all attributes related to how this token takes part
+  // in a configured macro expansion.
+  llvm::Optional<MacroExpansion> MacroCtx;
+
   bool is(tok::TokenKind Kind) const { return Tok.is(Kind); }
   bool is(TokenType TT) const { return getType() == TT; }
   bool is(const IdentifierInfo *II) const {
@@ -631,10 +699,12 @@ struct FormatToken {
                : nullptr;
   }
 
+  void copyFrom(const FormatToken &Tok) { *this = Tok; }
+
 private:
-  // Disallow copying.
+  // Only allow copying via the explicit copyFrom method.
   FormatToken(const FormatToken &) = delete;
-  void operator=(const FormatToken &) = delete;
+  FormatToken &operator=(const FormatToken &) = default;
 
   template <typename A, typename... Ts>
   bool startsSequenceInternal(A K1, Ts... Tokens) const {

clang/lib/Format/MacroExpander.cpp

@@ -0,0 +1,225 @@
+//===--- MacroExpander.cpp - Format C++ code --------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the implementation of MacroExpander, which handles macro
+/// configuration and expansion while formatting.
+///
+//===----------------------------------------------------------------------===//
+
+#include "Macros.h"
+
+#include "Encoding.h"
+#include "FormatToken.h"
+#include "FormatTokenLexer.h"
+#include "clang/Basic/TokenKinds.h"
+#include "clang/Format/Format.h"
+#include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/HeaderSearchOptions.h"
+#include "clang/Lex/Lexer.h"
+#include "clang/Lex/ModuleLoader.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/PreprocessorOptions.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/Support/ErrorHandling.h"
+
+namespace clang {
+namespace format {
+
+struct MacroExpander::Definition {
+  StringRef Name;
+  SmallVector<FormatToken *, 8> Params;
+  SmallVector<FormatToken *, 8> Body;
+
+  // Map from each argument's name to its position in the argument list.
+  // With "M(x, y) x + y":
+  //   x -> 0
+  //   y -> 1
+  llvm::StringMap<size_t> ArgMap;
+
+  bool ObjectLike = true;
+};
+
+class MacroExpander::DefinitionParser {
+public:
+  DefinitionParser(ArrayRef<FormatToken *> Tokens) : Tokens(Tokens) {
+    assert(!Tokens.empty());
+    Current = Tokens[0];
+  }
+
+  // Parse the token stream and return the corresonding Definition object.
+  // Returns an empty definition object with a null-Name on error.
+  MacroExpander::Definition parse() {
+    if (!Current->is(tok::identifier))
+      return {};
+    Def.Name = Current->TokenText;
+    nextToken();
+    if (Current->is(tok::l_paren)) {
+      Def.ObjectLike = false;
+      if (!parseParams())
+        return {};
+    }
+    if (!parseExpansion())
+      return {};
+
+    return Def;
+  }
+
+private:
+  bool parseParams() {
+    assert(Current->is(tok::l_paren));
+    nextToken();
+    while (Current->is(tok::identifier)) {
+      Def.Params.push_back(Current);
+      Def.ArgMap[Def.Params.back()->TokenText] = Def.Params.size() - 1;
+      nextToken();
+      if (Current->isNot(tok::comma))
+        break;
+      nextToken();
+    }
+    if (Current->isNot(tok::r_paren))
+      return false;
+    nextToken();
+    return true;
+  }
+
+  bool parseExpansion() {
+    if (!Current->isOneOf(tok::equal, tok::eof))
+      return false;
+    if (Current->is(tok::equal))
+      nextToken();
+    parseTail();
+    return true;
+  }
+
+  void parseTail() {
+    while (Current->isNot(tok::eof)) {
+      Def.Body.push_back(Current);
+      nextToken();
+    }
+    Def.Body.push_back(Current);
+  }
+
+  void nextToken() {
+    if (Pos + 1 < Tokens.size())
+      ++Pos;
+    Current = Tokens[Pos];
+    Current->Finalized = true;
+  }
+
+  size_t Pos = 0;
+  FormatToken *Current = nullptr;
+  Definition Def;
+  ArrayRef<FormatToken *> Tokens;
+};
+
+MacroExpander::MacroExpander(
+    const std::vector<std::string> &Macros, clang::SourceManager &SourceMgr,
+    const FormatStyle &Style,
+    llvm::SpecificBumpPtrAllocator<FormatToken> &Allocator,
+    IdentifierTable &IdentTable)
+    : SourceMgr(SourceMgr), Style(Style), Allocator(Allocator),
+      IdentTable(IdentTable) {
+  for (const std::string &Macro : Macros) {
+    parseDefinition(Macro);
+  }
+}
+
+MacroExpander::~MacroExpander() = default;
+
+void MacroExpander::parseDefinition(const std::string &Macro) {
+  Buffers.push_back(
+      llvm::MemoryBuffer::getMemBufferCopy(Macro, "<scratch space>"));
+  clang::FileID FID =
+      SourceMgr.createFileID(SourceManager::Unowned, Buffers.back().get());
+  FormatTokenLexer Lex(SourceMgr, FID, 0, Style, encoding::Encoding_UTF8,
+                       Allocator, IdentTable);
+  const auto Tokens = Lex.lex();
+  if (!Tokens.empty()) {
+    DefinitionParser Parser(Tokens);
+    auto Definition = Parser.parse();
+    Definitions[Definition.Name] = std::move(Definition);
+  }
+}
+
+bool MacroExpander::defined(llvm::StringRef Name) const {
+  return Definitions.find(Name) != Definitions.end();
+}
+
+bool MacroExpander::objectLike(llvm::StringRef Name) const {
+  return Definitions.find(Name)->second.ObjectLike;
+}
+
+llvm::SmallVector<FormatToken *, 8> MacroExpander::expand(FormatToken *ID,
+                                                          ArgsList Args) const {
+  assert(defined(ID->TokenText));
+  SmallVector<FormatToken *, 8> Result;
+  const Definition &Def = Definitions.find(ID->TokenText)->second;
+
+  // Expand each argument at most once.
+  llvm::StringSet<> ExpandedArgs;
+
+  // Adds the given token to Result.
+  auto pushToken = [&](FormatToken *Tok) {
+    Tok->MacroCtx->ExpandedFrom.push_back(ID);
+    Result.push_back(Tok);
+  };
+
+  // If Tok references a parameter, adds the corresponding argument to Result.
+  // Returns false if Tok does not reference a parameter.
+  auto expandArgument = [&](FormatToken *Tok) -> bool {
+    // If the current token references a parameter, expand the corresponding
+    // argument.
+    if (!Tok->is(tok::identifier) || ExpandedArgs.contains(Tok->TokenText))
+      return false;
+    ExpandedArgs.insert(Tok->TokenText);
+    auto I = Def.ArgMap.find(Tok->TokenText);
+    if (I == Def.ArgMap.end())
+      return false;
+    // If there are fewer arguments than referenced parameters, treat the
+    // parameter as empty.
+    // FIXME: Potentially fully abort the expansion instead.
+    if (I->getValue() >= Args.size())
+      return true;
+    for (FormatToken *Arg : Args[I->getValue()]) {
+      // A token can be part of a macro argument at multiple levels.
+      // For example, with "ID(x) x":
+      // in ID(ID(x)), 'x' is expanded first as argument to the inner
+      // ID, then again as argument to the outer ID. We keep the macro
+      // role the token had from the inner expansion.
+      if (!Arg->MacroCtx)
+        Arg->MacroCtx = MacroExpansion(MR_ExpandedArg);
+      pushToken(Arg);
+    }
+    return true;
+  };
+
+  // Expand the definition into Result.
+  for (FormatToken *Tok : Def.Body) {
+    if (expandArgument(Tok))
+      continue;
+    // Create a copy of the tokens from the macro body, i.e. were not provided
+    // by user code.
+    FormatToken *New = new (Allocator.Allocate()) FormatToken;
+    New->copyFrom(*Tok);
+    assert(!New->MacroCtx);
+    // Tokens that are not part of the user code are not formatted.
+    New->MacroCtx = MacroExpansion(MR_Hidden);
+    pushToken(New);
+  }
+  assert(Result.size() >= 1 && Result.back()->is(tok::eof));
+  if (Result.size() > 1) {
+    ++Result[0]->MacroCtx->StartOfExpansion;
+    ++Result[Result.size() - 2]->MacroCtx->EndOfExpansion;
+  }
+  return Result;
+}
+
+} // namespace format
+} // namespace clang