[Parser] Decouple the parser from AST creation (part 1)

Instead of creating the AST directly in the parser (and libSyntax or
SwiftSyntax via SyntaxParsingContext), make Parser to explicitly create
a tree of ParsedSyntaxNodes. Their OpaqueSyntaxNodes can be either
libSyntax or SwiftSyntax. If AST is needed, it can be generated from the
libSyntax tree.

Author: Jan Svoboda

include/swift/Parse/HiddenLibSyntaxAction.h

@@ -0,0 +1,71 @@
+//===--- HiddenLibSyntaxAction.h ------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_PARSE_HIDDENLIBSYNTAXACTION_H
+#define SWIFT_PARSE_HIDDENLIBSYNTAXACTION_H
+
+#include "swift/Parse/SyntaxParseActions.h"
+#include "swift/SyntaxParse/SyntaxTreeCreator.h"
+#include "llvm/ADT/DenseMap.h"
+
+namespace swift {
+/// Holds an explicitly provided action and uses it to handle all function
+/// calls. Also hides an implicit SyntaxTreeCreator and ensures libSyntax nodes
+/// are always created. Provides an interface to map results of the explicitly
+/// provided action to the hidden libSyntax action.
+// todo [gsoc]: remove when possible
+class HiddenLibSyntaxAction : public SyntaxParseActions {
+  std::shared_ptr<SyntaxParseActions> ExplicitAction;
+  std::shared_ptr<SyntaxTreeCreator> LibSyntaxAction;
+  llvm::DenseMap<OpaqueSyntaxNode, OpaqueSyntaxNode> OpaqueNodeMap;
+
+  bool areBothLibSyntax() {
+    return ExplicitAction->getOpaqueKind() == OpaqueSyntaxNodeKind::LibSyntax;
+  }
+
+public:
+  HiddenLibSyntaxAction(
+      const std::shared_ptr<SyntaxParseActions> &SPActions,
+      const std::shared_ptr<SyntaxTreeCreator> &LibSyntaxAction)
+      : ExplicitAction(SPActions != nullptr ? SPActions : LibSyntaxAction),
+        LibSyntaxAction(LibSyntaxAction){};
+
+  OpaqueSyntaxNode recordToken(tok tokenKind,
+                               ArrayRef<ParsedTriviaPiece> leadingTrivia,
+                               ArrayRef<ParsedTriviaPiece> trailingTrivia,
+                               CharSourceRange range) override;
+
+  OpaqueSyntaxNode recordMissingToken(tok tokenKind, SourceLoc loc) override;
+
+  OpaqueSyntaxNode recordRawSyntax(syntax::SyntaxKind kind,
+                                   ArrayRef<OpaqueSyntaxNode> elements,
+                                   CharSourceRange range) override;
+
+  std::pair<size_t, OpaqueSyntaxNode>
+  lookupNode(size_t lexerOffset, syntax::SyntaxKind kind) override;
+
+  OpaqueSyntaxNodeKind getOpaqueKind() override {
+    return ExplicitAction->getOpaqueKind();
+  }
+
+  /// Returns the libSyntax node corresponding to the provided node that has
+  /// been created by the explicit action.
+  OpaqueSyntaxNode getLibSyntaxNodeFor(OpaqueSyntaxNode explicitNode);
+
+  /// Returns the underlying libSyntax SyntaxTreeCreator.
+  std::shared_ptr<SyntaxTreeCreator> getLibSyntaxAction() {
+    return LibSyntaxAction;
+  }
+};
+} // namespace swift
+
+#endif // SWIFT_PARSE_HIDDENLIBSYNTAXACTION_H

include/swift/Parse/LibSyntaxGenerator.h

@@ -0,0 +1,74 @@
+//===----------- LibSyntaxGenerator.h -------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_PARSE_LIBSYNTAXGENERATOR_H
+#define SWIFT_PARSE_LIBSYNTAXGENERATOR_H
+
+#include "swift/Parse/HiddenLibSyntaxAction.h"
+#include "swift/Parse/ParsedRawSyntaxNode.h"
+#include "swift/Parse/ParsedRawSyntaxRecorder.h"
+#include "swift/Parse/SyntaxParsingCache.h"
+#include "swift/Syntax/RawSyntax.h"
+#include "swift/SyntaxParse/SyntaxTreeCreator.h"
+
+namespace swift {
+/// Generates libSyntax nodes either by looking them up using
+/// HiddenLibSyntaxAction (based on provided OpaqueSyntaxNode) or by recording
+/// them with ParsedRawSyntaxRecorder.
+// todo [gsoc]: remove when possible
+class LibSyntaxGenerator {
+  std::shared_ptr<HiddenLibSyntaxAction> Actions;
+  ParsedRawSyntaxRecorder Recorder;
+
+public:
+  explicit LibSyntaxGenerator(std::shared_ptr<HiddenLibSyntaxAction> TwoActions)
+      : Actions(std::move(TwoActions)),
+        Recorder(Actions->getLibSyntaxAction()) {}
+
+  TokenSyntax createToken(ParsedRawSyntaxNode Node) {
+    assert(Node.isDeferredToken());
+
+    auto Kind = Node.getTokenKind();
+    auto Range = Node.getDeferredTokenRangeWithTrivia();
+    auto LeadingTriviaPieces = Node.getDeferredLeadingTriviaPieces();
+    auto TrailingTriviaPieces = Node.getDeferredTrailingTriviaPieces();
+
+    auto Recorded = Recorder.recordToken(Kind, Range, LeadingTriviaPieces,
+                                         TrailingTriviaPieces);
+    auto Raw = static_cast<RawSyntax *>(Recorded.getOpaqueNode());
+    return make<TokenSyntax>(Raw);
+  }
+
+  template <typename SyntaxNode>
+  SyntaxNode createNode(ParsedRawSyntaxNode Node) {
+    assert(Node.isDeferredLayout());
+    auto Kind = Node.getKind();
+    auto Children = Node.getDeferredChildren();
+
+    auto Recorded = Recorder.recordRawSyntax(Kind, Children);
+    auto Raw = static_cast<RawSyntax *>(Recorded.getOpaqueNode());
+    return make<SyntaxNode>(Raw);
+  }
+
+  TokenSyntax getLibSyntaxTokenFor(OpaqueSyntaxNode Node) {
+    return getLibSyntaxNodeFor<TokenSyntax>(Node);
+  }
+
+  template <typename SyntaxNode>
+  SyntaxNode getLibSyntaxNodeFor(OpaqueSyntaxNode Node) {
+    auto Raw = static_cast<RawSyntax *>(Actions->getLibSyntaxNodeFor(Node));
+    return make<SyntaxNode>(Raw);
+  }
+};
+} // namespace swift
+
+#endif // SWIFT_PARSE_LIBSYNTAXGENERATOR_H

include/swift/Parse/ParsedRawSyntaxNode.h

@@ -156,6 +156,19 @@ class ParsedRawSyntaxNode {
 
   // Deferred Token Data =====================================================//
 
+  CharSourceRange getDeferredTokenRangeWithTrivia() const {
+    assert(DK == DataKind::DeferredToken);
+    auto leadTriviaPieces = getDeferredLeadingTriviaPieces();
+    auto trailTriviaPieces = getDeferredTrailingTriviaPieces();
+
+    auto leadTriviaLen = ParsedTriviaPiece::getTotalLength(leadTriviaPieces);
+    auto trailTriviaLen = ParsedTriviaPiece::getTotalLength(trailTriviaPieces);
+
+    SourceLoc begin = DeferredToken.TokLoc.getAdvancedLoc(-leadTriviaLen);
+    unsigned len = leadTriviaLen + DeferredToken.TokLength + trailTriviaLen;
+
+    return CharSourceRange{begin, len};
+  }
   CharSourceRange getDeferredTokenRangeWithoutBackticks() const {
     assert(DK == DataKind::DeferredToken);
     return CharSourceRange{DeferredToken.TokLoc, DeferredToken.TokLength};

include/swift/Parse/ParsedSyntaxRecorder.h.gyb

@@ -66,11 +66,31 @@ public:
 
   static Parsed${node.name} makeBlank${node.syntax_kind}(SourceLoc loc,
       SyntaxParsingContext &SPCtx);
+%   elif node.is_unknown():
+private:
+  static Parsed${node.name} record${node.syntax_kind}(
+      ArrayRef<ParsedSyntax> elts,
+      ParsedRawSyntaxRecorder &rec);
+
+public:
+  static Parsed${node.name} defer${node.syntax_kind}(
+      ArrayRef<ParsedSyntax> elts,
+      SyntaxParsingContext &SPCtx);
+
+  static Parsed${node.name} make${node.syntax_kind}(
+      ArrayRef<ParsedSyntax> elts,
+      SyntaxParsingContext &SPCtx);
 %   end
 % end
 
 #pragma mark - Convenience APIs
 
+public:
+  static ParsedTokenSyntax makeToken(const Token &Tok,
+                                     const ParsedTrivia &LeadingTrivia,
+                                     const ParsedTrivia &TrailingTrivia,
+                                     SyntaxParsingContext &SPCtx);
+
   /// Records an unlabelled TupleTypeElementSyntax with the provided type and
   /// optional trailing comma.
   static ParsedTupleTypeElementSyntax

include/swift/Parse/Parser.h

@@ -29,10 +29,12 @@
 #include "swift/Parse/LocalContext.h"
 #include "swift/Parse/PersistentParserState.h"
 #include "swift/Parse/Token.h"
+#include "swift/Parse/ParsedSyntaxNodes.h"
 #include "swift/Parse/ParserPosition.h"
 #include "swift/Parse/ParserResult.h"
 #include "swift/Parse/SyntaxParserResult.h"
 #include "swift/Parse/SyntaxParsingContext.h"
+#include "swift/Parse/SyntaxTransformer.h"
 #include "swift/Syntax/References.h"
 #include "swift/Config.h"
 #include "llvm/ADT/SetVector.h"
@@ -376,6 +378,9 @@ class Parser {
   /// Current syntax parsing context where call backs should be directed to.
   SyntaxParsingContext *SyntaxContext;
 
+  /// The libSyntax to AST transformer.
+  SyntaxTransformer Transformer;
+
 public:
   Parser(unsigned BufferID, SourceFile &SF, DiagnosticEngine* LexerDiags,
          SILParserTUStateBase *SIL,
@@ -518,6 +523,12 @@ class Parser {
     assert(Tok.is(K) && "Consuming wrong token kind");
     return consumeToken();
   }
+  /// Consume a token without providing it to the SyntaxParsingContext.
+  ParsedTokenSyntax consumeTokenSyntax();
+  ParsedTokenSyntax consumeTokenSyntax(tok K) {
+    assert(Tok.is(K) && "Consuming wrong token kind");
+    return consumeTokenSyntax();
+  }
 
   SourceLoc consumeIdentifier(Identifier *Result = nullptr,
                               bool allowDollarIdentifier = false) {
@@ -1321,6 +1332,15 @@ class Parser {
   ParserResult<Expr> parseExprSuper();
   ParserResult<Expr> parseExprStringLiteral();
 
+  // todo [gsoc]: create new result type for ParsedSyntax
+  // todo [gsoc]: turn into proper non-templated methods later
+  template <typename SyntaxNode>
+  ParsedExprSyntax parseExprSyntax();
+
+  // todo [gsoc]: remove when possible
+  template <typename SyntaxNode>
+  ParserResult<Expr> parseExprAST();
+
   StringRef copyAndStripUnderscores(StringRef text);
 
   ParserStatus parseStringSegments(SmallVectorImpl<Lexer::StringSegment> &Segments,

include/swift/Parse/SyntaxParseActions.h

@@ -33,6 +33,12 @@ namespace syntax {
 
 typedef void *OpaqueSyntaxNode;
 
+// todo [gsoc]: remove when possible
+enum class OpaqueSyntaxNodeKind {
+  SwiftSyntax,
+  LibSyntax,
+};
+
 class SyntaxParseActions {
   virtual void _anchor();
 
@@ -60,6 +66,9 @@ class SyntaxParseActions {
   lookupNode(size_t lexerOffset, syntax::SyntaxKind kind) {
     return std::make_pair(0, nullptr);
   }
+
+  /// Returns what kind of OpaqueSyntaxNode is created by `recordXXX` methods.
+  virtual OpaqueSyntaxNodeKind getOpaqueKind() = 0;
 };
 
 } // end namespace swift

include/swift/Parse/SyntaxParsingContext.h

@@ -15,8 +15,11 @@
 
 #include "llvm/ADT/PointerUnion.h"
 #include "swift/Basic/SourceLoc.h"
+#include "swift/Parse/HiddenLibSyntaxAction.h"
+#include "swift/Parse/LibSyntaxGenerator.h"
 #include "swift/Parse/ParsedRawSyntaxNode.h"
 #include "swift/Parse/ParsedRawSyntaxRecorder.h"
+#include "swift/Parse/ParsedSyntaxNodes.h"
 
 namespace swift {
 
@@ -74,6 +77,7 @@ constexpr size_t SyntaxAlignInBits = 3;
 ///     // Now the context holds { '(' Expr ')' }.
 ///     // From these parts, it creates ParenExpr node and add it to the parent.
 ///   }
+// todo [gsoc]: remove when/if possible
 class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
 public:
   /// The shared data for all syntax parsing contexts with the same root.
@@ -94,13 +98,15 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
 
     ParsedRawSyntaxRecorder Recorder;
 
+    LibSyntaxGenerator LibSyntaxCreator;
+
     llvm::BumpPtrAllocator ScratchAlloc;
 
     RootContextData(SourceFile &SF, DiagnosticEngine &Diags,
                     SourceManager &SourceMgr, unsigned BufferID,
-                    std::shared_ptr<SyntaxParseActions> spActions)
+                    const std::shared_ptr<HiddenLibSyntaxAction>& TwoActions)
         : SF(SF), Diags(Diags), SourceMgr(SourceMgr), BufferID(BufferID),
-          Recorder(std::move(spActions)) {}
+          Recorder(TwoActions), LibSyntaxCreator(TwoActions) {}
   };
 
 private:
@@ -162,9 +168,6 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
   /// true if it's in backtracking context.
   bool IsBacktracking = false;
 
-  // If false, context does nothing.
-  bool Enabled;
-
   /// Create a syntax node using the tail \c N elements of collected parts and
   /// replace those parts with the single result.
   void createNodeInPlace(SyntaxKind Kind, size_t N,
@@ -182,18 +185,19 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
   Optional<ParsedRawSyntaxNode> bridgeAs(SyntaxContextKind Kind,
                               ArrayRef<ParsedRawSyntaxNode> Parts);
 
+  ParsedRawSyntaxNode finalizeSourceFile();
+
 public:
   /// Construct root context.
   SyntaxParsingContext(SyntaxParsingContext *&CtxtHolder, SourceFile &SF,
                        unsigned BufferID,
-                       std::shared_ptr<SyntaxParseActions> SPActions);
+                       std::shared_ptr<HiddenLibSyntaxAction> SPActions);
 
   /// Designated constructor for child context.
   SyntaxParsingContext(SyntaxParsingContext *&CtxtHolder)
       : RootDataOrParent(CtxtHolder), CtxtHolder(CtxtHolder),
         RootData(CtxtHolder->RootData), Offset(RootData->Storage.size()),
-        IsBacktracking(CtxtHolder->IsBacktracking),
-        Enabled(CtxtHolder->isEnabled()) {
+        IsBacktracking(CtxtHolder->IsBacktracking) {
     assert(CtxtHolder->isTopOfContextStack() &&
            "SyntaxParsingContext cannot have multiple children");
     assert(CtxtHolder->Mode != AccumulationMode::SkippedForIncrementalUpdate &&
@@ -221,8 +225,6 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
   /// offset. If nothing is found \c 0 is returned.
   size_t lookupNode(size_t LexerOffset, SourceLoc Loc);
 
-  void disable() { Enabled = false; }
-  bool isEnabled() const { return Enabled; }
   bool isRoot() const { return RootDataOrParent.is<RootContextData*>(); }
   bool isTopOfContextStack() const { return this == CtxtHolder; }
 
@@ -240,6 +242,10 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
     return getRootData()->Storage;
   }
 
+  LibSyntaxGenerator &getSyntaxCreator() {
+    return getRootData()->LibSyntaxCreator;
+  }
+
   const SyntaxParsingContext *getRoot() const;
 
   ParsedRawSyntaxRecorder &getRecorder() { return getRootData()->Recorder; }
@@ -258,6 +264,21 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
   /// Add Syntax to the parts.
   void addSyntax(ParsedSyntax Node);
 
+  template <SyntaxKind Kind>
+  bool isTopNode() {
+    return getStorage().back().getKind() == Kind;
+  }
+
+  /// Returns the topmost Syntax node.
+  template <typename SyntaxNode> SyntaxNode topNode() {
+    ParsedRawSyntaxNode TopNode = getStorage().back();
+
+    if (IsBacktracking)
+      return getSyntaxCreator().createNode<SyntaxNode>(TopNode);
+
+    OpaqueSyntaxNode OpaqueNode = TopNode.getOpaqueNode();
+    return getSyntaxCreator().getLibSyntaxNodeFor<SyntaxNode>(OpaqueNode);
+  }
 
   template<typename SyntaxNode>
   llvm::Optional<SyntaxNode> popIf() {
@@ -300,6 +321,11 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
     SynKind = Kind;
   }
 
+  /// On destruction, do not attempt to finalize the root node.
+  void setDiscard() {
+    Mode = AccumulationMode::Discard;
+  }
+
   /// On destruction, if the parts size is 1 and it's kind of \c Kind, just
   /// append it to the parent context. Otherwise, create Unknown{Kind} node from
   /// the collected parts.