[pseudo] Move pseudoparser from clang to clang-tools-extra

This should make clearer that:
 - it's not part of clang proper
 - there's no expectation to update it along with clang (beyond green tests)
 - clang should not depend on it

This is intended to be expose a library, so unlike other tools has a split
between include/ and lib/.

The main renames are:
  clang/lib/Tooling/Syntax/Pseudo/*           => clang-tools-extra/pseudo/lib/*
  clang/include/clang/Tooling/Syntax/Pseudo/* => clang-tools-extra/pseudo/include/clang-pseudo/*
  clang/tools/clang/pseudo/*                  => clang-tools-extra/pseudo/tool/*
  clang/test/Syntax/*                         => clang-tools-extra/pseudo/test/*
  clang/unittests/Tooling/Syntax/Pseudo/*     => clang-tools-extra/pseudo/unittests/*
  #include "clang/Tooling/Syntax/Pseudo/*"    => #include "clang-pseudo/*"
  namespace clang::syntax::pseudo             => namespace clang::pseudo
  check-clang                                 => check-clang-pseudo
  clangToolingSyntaxPseudo                    => clangPseudo
The clang-pseudo and ClangPseudoTests binaries are not renamed.

See discussion around:
https://discourse.llvm.org/t/rfc-a-c-pseudo-parser-for-tooling/59217/50

Differential Revision: https://reviews.llvm.org/D121233

clang-tools-extra/CMakeLists.txt

@@ -15,6 +15,7 @@ add_subdirectory(clang-include-fixer)
 add_subdirectory(clang-move)
 add_subdirectory(clang-query)
 add_subdirectory(pp-trace)
+add_subdirectory(pseudo)
 add_subdirectory(tool-template)
 
 # Add the common testsuite after all the tools.

clang-tools-extra/pseudo/CMakeLists.txt

@@ -0,0 +1,6 @@
+include_directories(include)
+include_directories(${CMAKE_CURRENT_BINARY_DIR}/include)
+add_subdirectory(lib)
+add_subdirectory(tool)
+add_subdirectory(unittests)
+add_subdirectory(test)

clang/include/clang/Tooling/Syntax/Pseudo/DirectiveMap.h → clang-tools-extra/pseudo/include/clang-pseudo/DirectiveMap.h

@@ -25,16 +25,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_TOOLING_SYNTAX_PREPROCESS_H
-#define LLVM_CLANG_TOOLING_SYNTAX_PREPROCESS_H
+#ifndef CLANG_PSEUDO_DIRECTIVEMAP_H
+#define CLANG_PSEUDO_DIRECTIVEMAP_H
 
+#include "clang-pseudo/Token.h"
 #include "clang/Basic/TokenKinds.h"
-#include "clang/Tooling/Syntax/Pseudo/Token.h"
 #include <vector>
 
 namespace clang {
 class LangOptions;
-namespace syntax {
 namespace pseudo {
 
 /// Describes the structure of a source file, as seen by the preprocessor.
@@ -142,7 +141,6 @@ class DirectiveMap::Chunk {
 };
 
 } // namespace pseudo
-} // namespace syntax
 } // namespace clang
 
 #endif

clang/include/clang/Tooling/Syntax/Pseudo/Grammar.h → clang-tools-extra/pseudo/include/clang-pseudo/Grammar.h

@@ -33,8 +33,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_TOOLING_SYNTAX_GRAMMAR_H
-#define LLVM_CLANG_TOOLING_SYNTAX_GRAMMAR_H
+#ifndef CLANG_PSEUDO_GRAMMAR_H
+#define CLANG_PSEUDO_GRAMMAR_H
 
 #include "clang/Basic/TokenKinds.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -44,7 +44,6 @@
 #include <vector>
 
 namespace clang {
-namespace syntax {
 namespace pseudo {
 // A SymbolID uniquely identifies a terminal/non-terminal symbol in a grammar.
 // Non-terminal IDs are indexes into a table of non-terminal symbols.
@@ -178,7 +177,6 @@ struct GrammarTable {
 };
 
 } // namespace pseudo
-} // namespace syntax
 } // namespace clang
 
 #endif // LLVM_CLANG_TOOLING_SYNTAX_GRAMMAR_H

clang/include/clang/Tooling/Syntax/Pseudo/LRGraph.h → clang-tools-extra/pseudo/include/clang-pseudo/LRGraph.h

@@ -28,15 +28,14 @@
 //  in making decisions.
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_TOOLING_SYNTAX_PSEUDO_LRGRAPH_H
-#define LLVM_CLANG_TOOLING_SYNTAX_PSEUDO_LRGRAPH_H
+#ifndef CLANG_PSEUDO_LRGRAPH_H
+#define CLANG_PSEUDO_LRGRAPH_H
 
-#include "clang/Tooling/Syntax/Pseudo/Grammar.h"
+#include "clang-pseudo/Grammar.h"
 #include "llvm/ADT/Hashing.h"
 #include <vector>
 
 namespace clang {
-namespace syntax {
 namespace pseudo {
 
 // An LR item -- a grammar rule with a dot at some position of the body.
@@ -152,23 +151,22 @@ class LRGraph {
 };
 
 } // namespace pseudo
-} // namespace syntax
 } // namespace clang
 
 namespace llvm {
-// Support clang::syntax::pseudo::Item as DenseMap keys.
-template <> struct DenseMapInfo<clang::syntax::pseudo::Item> {
-  static inline clang::syntax::pseudo::Item getEmptyKey() {
-    return clang::syntax::pseudo::Item::sentinel(-1);
+// Support clang::pseudo::Item as DenseMap keys.
+template <> struct DenseMapInfo<clang::pseudo::Item> {
+  static inline clang::pseudo::Item getEmptyKey() {
+    return clang::pseudo::Item::sentinel(-1);
   }
-  static inline clang::syntax::pseudo::Item getTombstoneKey() {
-    return clang::syntax::pseudo::Item::sentinel(-2);
+  static inline clang::pseudo::Item getTombstoneKey() {
+    return clang::pseudo::Item::sentinel(-2);
   }
-  static unsigned getHashValue(const clang::syntax::pseudo::Item &I) {
+  static unsigned getHashValue(const clang::pseudo::Item &I) {
     return hash_value(I);
   }
-  static bool isEqual(const clang::syntax::pseudo::Item &LHS,
-                      const clang::syntax::pseudo::Item &RHS) {
+  static bool isEqual(const clang::pseudo::Item &LHS,
+                      const clang::pseudo::Item &RHS) {
     return LHS == RHS;
   }
 };

clang/include/clang/Tooling/Syntax/Pseudo/LRTable.h → clang-tools-extra/pseudo/include/clang-pseudo/LRTable.h

@@ -34,16 +34,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_TOOLING_SYNTAX_PSEUDO_LRTABLE_H
-#define LLVM_CLANG_TOOLING_SYNTAX_PSEUDO_LRTABLE_H
+#ifndef CLANG_PSEUDO_LRTABLE_H
+#define CLANG_PSEUDO_LRTABLE_H
 
-#include "clang/Tooling/Syntax/Pseudo/Grammar.h"
+#include "clang-pseudo/Grammar.h"
 #include "llvm/ADT/ArrayRef.h"
 #include <cstdint>
 #include <vector>
 
 namespace clang {
-namespace syntax {
 namespace pseudo {
 
 // Represents the LR parsing table, which can efficiently the question "what is
@@ -176,7 +175,6 @@ class LRTable {
 llvm::raw_ostream &operator<<(llvm::raw_ostream &, const LRTable::Action &);
 
 } // namespace pseudo
-} // namespace syntax
 } // namespace clang
 
 #endif // LLVM_CLANG_TOOLING_SYNTAX_PSEUDO_LRTABLE_H

clang/include/clang/Tooling/Syntax/Pseudo/Token.h → clang-tools-extra/pseudo/include/clang-pseudo/Token.h

@@ -25,8 +25,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_TOOLING_SYNTAX_TOKEN_H
-#define LLVM_CLANG_TOOLING_SYNTAX_TOKEN_H
+#ifndef CLANG_PSEUDO_TOKEN_H
+#define CLANG_PSEUDO_TOKEN_H
 
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/TokenKinds.h"
@@ -39,7 +39,6 @@
 
 namespace clang {
 class LangOptions;
-namespace syntax {
 namespace pseudo {
 
 /// A single C++ or preprocessor token.
@@ -199,7 +198,6 @@ TokenStream cook(const TokenStream &, const clang::LangOptions &);
 TokenStream stripComments(const TokenStream &);
 
 } // namespace pseudo
-} // namespace syntax
 } // namespace clang
 
 #endif

clang/lib/Tooling/Syntax/Pseudo/CMakeLists.txt → clang-tools-extra/pseudo/lib/CMakeLists.txt

@@ -1,6 +1,6 @@
 set(LLVM_LINK_COMPONENTS Support)
 
-add_clang_library(clangToolingSyntaxPseudo
+add_clang_library(clangPseudo
   DirectiveMap.cpp
   Grammar.cpp
   GrammarBNF.cpp

clang/lib/Tooling/Syntax/Pseudo/DirectiveMap.cpp → clang-tools-extra/pseudo/lib/DirectiveMap.cpp

@@ -6,13 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Tooling/Syntax/Pseudo/DirectiveMap.h"
+#include "clang-pseudo/DirectiveMap.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/TokenKinds.h"
 #include "llvm/Support/FormatVariadic.h"
 
 namespace clang {
-namespace syntax {
 namespace pseudo {
 namespace {
 
@@ -151,8 +150,8 @@ DirectiveMap DirectiveMap::parse(const TokenStream &Code) {
 }
 
 static void dump(llvm::raw_ostream &OS, const DirectiveMap &, unsigned Indent);
-static void dump(llvm::raw_ostream &OS, const DirectiveMap::Directive &Directive,
-                 unsigned Indent) {
+static void dump(llvm::raw_ostream &OS,
+                 const DirectiveMap::Directive &Directive, unsigned Indent) {
   OS.indent(Indent) << llvm::formatv("#{0} ({1} tokens)\n",
                                      tok::getPPKeywordSpelling(Directive.Kind),
                                      Directive.Tokens.size());
@@ -205,5 +204,4 @@ OSTREAM_DUMP(DirectiveMap::Code)
 #undef OSTREAM_DUMP
 
 } // namespace pseudo
-} // namespace syntax
 } // namespace clang

clang/lib/Tooling/Syntax/Pseudo/Grammar.cpp → clang-tools-extra/pseudo/lib/Grammar.cpp

@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Tooling/Syntax/Pseudo/Grammar.h"
+#include "clang-pseudo/Grammar.h"
 #include "clang/Basic/TokenKinds.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
@@ -15,7 +15,6 @@
 #include "llvm/Support/raw_ostream.h"
 
 namespace clang {
-namespace syntax {
 namespace pseudo {
 
 Rule::Rule(SymbolID Target, llvm::ArrayRef<SymbolID> Sequence)
@@ -181,5 +180,4 @@ static llvm::ArrayRef<std::string> getTerminalNames() {
 GrammarTable::GrammarTable() : Terminals(getTerminalNames()) {}
 
 } // namespace pseudo
-} // namespace syntax
 } // namespace clang

clang/lib/Tooling/Syntax/Pseudo/GrammarBNF.cpp → clang-tools-extra/pseudo/lib/GrammarBNF.cpp

@@ -6,15 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang-pseudo/Grammar.h"
 #include "clang/Basic/TokenKinds.h"
-#include "clang/Tooling/Syntax/Pseudo/Grammar.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/FormatVariadic.h"
 #include <memory>
 
 namespace clang {
-namespace syntax {
 namespace pseudo {
 
 namespace {
@@ -244,5 +243,4 @@ Grammar::parseBNF(llvm::StringRef BNF, std::vector<std::string> &Diagnostics) {
 }
 
 } // namespace pseudo
-} // namespace syntax
 } // namespace clang

clang/lib/Tooling/Syntax/Pseudo/LRGraph.cpp → clang-tools-extra/pseudo/lib/LRGraph.cpp

@@ -6,25 +6,25 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Tooling/Syntax/Pseudo/LRGraph.h"
-#include "clang/Tooling/Syntax/Pseudo/Grammar.h"
+#include "clang-pseudo/LRGraph.h"
+#include "clang-pseudo/Grammar.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/raw_ostream.h"
 
-using ItemSet = std::vector<clang::syntax::pseudo::Item>;
+using ItemSet = std::vector<clang::pseudo::Item>;
 
 namespace llvm {
-// Support clang::syntax::pseudo::Item as DenseMap keys.
+// Support clang::pseudo::Item as DenseMap keys.
 template <> struct DenseMapInfo<ItemSet> {
   static inline ItemSet getEmptyKey() {
-    return {DenseMapInfo<clang::syntax::pseudo::Item>::getEmptyKey()};
+    return {DenseMapInfo<clang::pseudo::Item>::getEmptyKey()};
   }
   static inline ItemSet getTombstoneKey() {
-    return {DenseMapInfo<clang::syntax::pseudo::Item>::getTombstoneKey()};
+    return {DenseMapInfo<clang::pseudo::Item>::getTombstoneKey()};
   }
   static unsigned getHashValue(const ItemSet &I) {
     return llvm::hash_combine_range(I.begin(), I.end());
@@ -36,7 +36,6 @@ template <> struct DenseMapInfo<ItemSet> {
 } // namespace llvm
 
 namespace clang {
-namespace syntax {
 namespace pseudo {
 namespace {
 
@@ -227,5 +226,4 @@ LRGraph LRGraph::buildLR0(const Grammar &G) {
 }
 
 } // namespace pseudo
-} // namespace syntax
 } // namespace clang

clang/lib/Tooling/Syntax/Pseudo/LRTable.cpp → clang-tools-extra/pseudo/lib/LRTable.cpp

@@ -6,16 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Tooling/Syntax/Pseudo/LRTable.h"
-#include "clang/Tooling/Syntax/Pseudo/Grammar.h"
+#include "clang-pseudo/LRTable.h"
+#include "clang-pseudo/Grammar.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace clang {
-namespace syntax {
 namespace pseudo {
 
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const LRTable::Action &A) {
@@ -122,5 +121,4 @@ llvm::ArrayRef<LRTable::Action> LRTable::find(StateID Src, SymbolID ID) const {
 }
 
 } // namespace pseudo
-} // namespace syntax
 } // namespace clang

clang/lib/Tooling/Syntax/Pseudo/LRTableBuild.cpp → clang-tools-extra/pseudo/lib/LRTableBuild.cpp

@@ -6,23 +6,23 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang-pseudo/Grammar.h"
+#include "clang-pseudo/LRGraph.h"
+#include "clang-pseudo/LRTable.h"
 #include "clang/Basic/TokenKinds.h"
-#include "clang/Tooling/Syntax/Pseudo/Grammar.h"
-#include "clang/Tooling/Syntax/Pseudo/LRGraph.h"
-#include "clang/Tooling/Syntax/Pseudo/LRTable.h"
 #include <cstdint>
 
 namespace llvm {
-template <> struct DenseMapInfo<clang::syntax::pseudo::LRTable::Entry> {
-  using Entry = clang::syntax::pseudo::LRTable::Entry;
+template <> struct DenseMapInfo<clang::pseudo::LRTable::Entry> {
+  using Entry = clang::pseudo::LRTable::Entry;
   static inline Entry getEmptyKey() {
-    static Entry E{static_cast<clang::syntax::pseudo::SymbolID>(-1), 0,
-                   clang::syntax::pseudo::LRTable::Action::sentinel()};
+    static Entry E{static_cast<clang::pseudo::SymbolID>(-1), 0,
+                   clang::pseudo::LRTable::Action::sentinel()};
     return E;
   }
   static inline Entry getTombstoneKey() {
-    static Entry E{static_cast<clang::syntax::pseudo::SymbolID>(-2), 0,
-                   clang::syntax::pseudo::LRTable::Action::sentinel()};
+    static Entry E{static_cast<clang::pseudo::SymbolID>(-2), 0,
+                   clang::pseudo::LRTable::Action::sentinel()};
     return E;
   }
   static unsigned getHashValue(const Entry &I) {
@@ -36,7 +36,6 @@ template <> struct DenseMapInfo<clang::syntax::pseudo::LRTable::Entry> {
 } // namespace llvm
 
 namespace clang {
-namespace syntax {
 namespace pseudo {
 
 class LRTable::Builder {
@@ -139,5 +138,4 @@ LRTable LRTable::buildSLR(const Grammar &G) {
 }
 
 } // namespace pseudo
-} // namespace syntax
 } // namespace clang