[Syntax] Swift libSyntax API (#11320)

* Create Swift libSyntax API

This patch is an initial implementation of the Swift libSyntax API. It
aims to provide all features of the C++ API but exposed to Swift.

It currently resides in SwiftExperimental and will likely exist in a
molten state for a while.

* Only build SwiftSyntax on macOS

AtomicCache.swift

@@ -0,0 +1,63 @@
+//===----------- AtomicCache.swift - Atomically Initialized Cache ---------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+
+/// AtomicCache is a wrapper class around an uninitialized value.
+/// It takes a closure that it will use to create the value atomically. The
+/// value is guaranteed to be set exactly one time, but the provided closure
+/// may be called multiple times by threads racing to initialize the value.
+/// Do not rely on the closure being called only one time.
+class AtomicCache<Value: AnyObject> {
+  /// The cached pointer that will be filled in the first time `value` is
+  /// accessed.
+  private var _cachedValue: AnyObject?
+
+  /// The value inside this cache. If the value has not been initialized when
+  /// this value is requested, then the closure will be called and its resulting
+  /// value will be atomically compare-exchanged into the cache.
+  /// If multiple threads access the value before initialization, they will all
+  /// end up returning the correct, initialized value.
+  /// - Parameter create: The closure that will return the fully realized value
+  ///                     inside the cache.
+  func value(_ create: () -> Value) -> Value {
+    return withUnsafeMutablePointer(to: &_cachedValue) { ptr in
+      // Perform an atomic load -- if we get a value, then return it.
+      if let _cached = _stdlib_atomicLoadARCRef(object: ptr) {
+        return _cached as! Value
+      }
+
+      // Otherwise, create the value...
+      let value = create()
+
+      // ...and attempt to initialize the pointer with that value.
+      if _stdlib_atomicInitializeARCRef(object: ptr, desired: value) {
+        // If we won the race, just return the value we made.
+        return value
+      }
+
+      // Otherwise, perform _another_ load to get the up-to-date value,
+      // and let the one we just made die.
+      return _stdlib_atomicLoadARCRef(object: ptr) as! Value
+    }
+  }
+
+  /// Unsafely attempts to load the value and cast it to the appropriate
+  /// type.
+  /// - note: Only for use in the debugger!
+  @available(*, deprecated, message: "Only for use in the debugger.")
+  var unsafeValue: Value? {
+    return withUnsafeMutablePointer(to: &_cachedValue) {
+      return _stdlib_atomicLoadARCRef(object: $0) as? Value
+    }
+  }
+}

CMakeLists.txt

@@ -0,0 +1,24 @@
+add_swift_library(swiftSwiftSyntax SHARED
+  # This file should be listed the first.  Module name is inferred from the
+  # filename.
+  SwiftSyntax.swift
+  AtomicCache.swift
+  RawSyntax.swift
+  SourcePresence.swift
+  SwiftcInvocation.swift
+  Syntax.swift
+  SyntaxData.swift
+  SyntaxChildren.swift
+  SyntaxCollection.swift
+  SyntaxBuilders.swift.gyb
+  SyntaxFactory.swift.gyb
+  SyntaxKind.swift.gyb
+  SyntaxNodes.swift.gyb
+  SyntaxRewriter.swift.gyb
+  TokenKind.swift.gyb
+  Trivia.swift
+
+  SWIFT_MODULE_DEPENDS Foundation
+  INSTALL_IN_COMPONENT swift-syntax
+  TARGET_SDKS OSX
+  IS_STDLIB)

README.md

@@ -0,0 +1,4 @@
+# SwiftLanguage
+
+This is an in-progress implementation of a Swift API for the
+[libSyntax](https://github.com/apple/swift/tree/master/lib/Syntax) library.

RawSyntax.swift

@@ -0,0 +1,194 @@
+//===------------------ RawSyntax.swift - Raw Syntax nodes ----------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+
+/// Represents the raw tree structure underlying the syntax tree. These nodes
+/// have no notion of identity and only provide structure to the tree. They
+/// are immutable and can be freely shared between syntax nodes.
+indirect enum RawSyntax: Codable {
+  /// A tree node with a kind, an array of children, and a source presence.
+  case node(SyntaxKind, [RawSyntax], SourcePresence)
+
+  /// A token with a token kind, leading trivia, trailing trivia, and a source
+  /// presence.
+  case token(TokenKind, Trivia, Trivia, SourcePresence)
+
+  /// The syntax kind of this raw syntax.
+  var kind: SyntaxKind {
+    switch self {
+    case .node(let kind, _, _): return kind
+    case .token(_, _, _, _): return .token
+    }
+  }
+
+  var tokenKind: TokenKind? {
+    switch self {
+    case .node(_, _, _): return nil
+    case .token(let kind, _, _, _): return kind
+    }
+  }
+
+  /// The layout of the children of this Raw syntax node.
+  var layout: [RawSyntax] {
+    switch self {
+    case .node(_, let layout, _): return layout
+    case .token(_, _, _, _): return []
+    }
+  }
+
+  /// The source presence of this node.
+  var presence: SourcePresence {
+    switch self {
+      case .node(_, _, let presence),
+           .token(_, _, _, let presence): return presence
+    }
+  }
+
+  /// Whether this node is present in the original source.
+  var isPresent: Bool {
+    return presence == .present
+  }
+
+  /// Whether this node is missing from the original source.
+  var isMissing: Bool {
+    return presence == .missing
+  }
+
+  /// Keys for serializing RawSyntax nodes.
+  enum CodingKeys: String, CodingKey {
+    // Keys for the `node` case
+    case kind, layout, presence
+
+    // Keys for the `token` case
+    case tokenKind, leadingTrivia, trailingTrivia
+  }
+
+  /// Creates a RawSyntax from the provided Foundation Decoder.
+  init(from decoder: Decoder) throws {
+    let container = try decoder.container(keyedBy: CodingKeys.self)
+    let presence = try container.decode(SourcePresence.self, forKey: .presence)
+    if let kind = try container.decodeIfPresent(SyntaxKind.self, forKey: .kind) {
+      let layout = try container.decode([RawSyntax].self, forKey: .layout)
+      self = .node(kind, layout, presence)
+    } else {
+      let kind = try container.decode(TokenKind.self, forKey: .tokenKind)
+      let leadingTrivia = try container.decode(Trivia.self, forKey: .leadingTrivia)
+      let trailingTrivia = try container.decode(Trivia.self, forKey: .trailingTrivia)
+      self = .token(kind, leadingTrivia, trailingTrivia, presence)
+    }
+  }
+
+  /// Encodes the RawSyntax to the provided Foundation Encoder.
+  func encode(to encoder: Encoder) throws {
+    var container = encoder.container(keyedBy: CodingKeys.self)
+    switch self {
+    case let .node(kind, layout, presence):
+      try container.encode(kind, forKey: .kind)
+      try container.encode(layout, forKey: .layout)
+      try container.encode(presence, forKey: .presence)
+    case let .token(kind, leadingTrivia, trailingTrivia, presence):
+      try container.encode(kind, forKey: .tokenKind)
+      try container.encode(leadingTrivia, forKey: .leadingTrivia)
+      try container.encode(trailingTrivia, forKey: .trailingTrivia)
+      try container.encode(presence, forKey: .presence)
+    }
+  }
+
+  /// Creates a RawSyntax node that's marked missing in the source with the
+  /// provided kind and layout.
+  /// - Parameters:
+  ///   - kind: The syntax kind underlying this node.
+  ///   - layout: The children of this node.
+  /// - Returns: A new RawSyntax `.node` with the provided kind and layout, with
+  ///            `.missing` source presence.
+  static func missing(_ kind: SyntaxKind) -> RawSyntax {
+    return .node(kind, [], .missing)
+  }
+
+  /// Creates a RawSyntax token that's marked missing in the source with the
+  /// provided kind and no leading/trailing trivia.
+  /// - Parameter kind: The token kind.
+  /// - Returns: A new RawSyntax `.token` with the provided kind, no
+  ///            leading/trailing trivia, and `.missing` source presence.
+  static func missingToken(_ kind: TokenKind) -> RawSyntax {
+    return .token(kind, [], [], .missing)
+  }
+
+  /// Returns a new RawSyntax node with the provided layout instead of the
+  /// existing layout.
+  /// - Note: This function does nothing with `.token` nodes --- the same token
+  ///         is returned.
+  /// - Parameter newLayout: The children of the new node you're creating.
+  func replacingLayout(_ newLayout: [RawSyntax]) -> RawSyntax {
+    switch self {
+    case let .node(kind, _, presence): return .node(kind, newLayout, presence)
+    case .token(_, _, _, _): return self
+    }
+  }
+
+  /// Creates a new RawSyntax with the provided child appended to its layout.
+  /// - Parameter child: The child to append
+  /// - Note: This function does nothing with `.token` nodes --- the same token
+  ///         is returned.
+  /// - Return: A new RawSyntax node with the provided child at the end.
+  func appending(_ child: RawSyntax) -> RawSyntax {
+    var newLayout = layout
+    newLayout.append(child)
+    return replacingLayout(newLayout)
+  }
+
+  /// Returns the child at the provided cursor in the layout.
+  /// - Parameter index: The index of the child you're accessing.
+  /// - Returns: The child at the provided index.
+  subscript<CursorType: RawRepresentable>(_ index: CursorType) -> RawSyntax
+    where CursorType.RawValue == Int {
+      return layout[index.rawValue]
+  }
+
+  /// Replaces the child at the provided index in this node with the provided
+  /// child.
+  /// - Parameters:
+  ///   - index: The index of the child to replace.
+  ///   - newChild: The new child that should occupy that index in the node.
+  func replacingChild(_ index: Int,
+                      with newChild: RawSyntax) -> RawSyntax {
+    precondition(index < layout.count, "cursor \(index) reached past layout")
+    var newLayout = layout
+    newLayout[index] = newChild
+    return replacingLayout(newLayout)
+  }
+}
+
+extension RawSyntax: TextOutputStreamable {
+  /// Prints the RawSyntax node, and all of its children, to the provided
+  /// stream. This implementation must be source-accurate.
+  /// - Parameter stream: The stream on which to output this node.
+  func write<Target>(to target: inout Target)
+    where Target: TextOutputStream {
+    switch self {
+    case .node(_, let layout, _):
+      for child in layout {
+        child.write(to: &target)
+      }
+    case let .token(kind, leadingTrivia, trailingTrivia, presence):
+      guard case .present = presence else { return }
+      for piece in leadingTrivia {
+        piece.write(to: &target)
+      }
+      target.write(kind.text)
+      for piece in trailingTrivia {
+        piece.write(to: &target)
+      }
+    }
+  }
+}

SourcePresence.swift

@@ -0,0 +1,25 @@
+//===---------------- SourcePresence.swift - Source Presence --------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+
+/// An indicator of whether a Syntax node was found or written in the source.
+///
+/// A `missing` node does not mean, necessarily, that the source item is
+/// considered "implicit", but rather that it was not found in the source.
+public enum SourcePresence: String, Codable {
+  /// The syntax was authored by a human and found, or was generated.
+  case present = "Present"
+
+  /// The syntax was expected or optional, but not found in the source.
+  case missing = "Missing"
+}

SwiftSyntax.swift

@@ -0,0 +1,48 @@
+//===--------------- SwiftLanguage.swift - Swift Syntax Library -----------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+// This file provides main entry point into the Syntax library.
+//===----------------------------------------------------------------------===//
+
+import Foundation
+
+/// A list of possible errors that could be encountered while parsing a
+/// Syntax tree.
+public enum ParserError: Error {
+  case swiftcFailed(Int, String)
+  case invalidFile
+}
+
+extension Syntax {
+  /// Parses the Swift file at the provided URL into a full-fidelity `Syntax`
+  /// tree.
+  /// - Parameter url: The URL you wish to parse.
+  /// - Returns: A top-level Syntax node representing the contents of the tree,
+  ///            if the parse was successful.
+  /// - Throws: `ParseError.couldNotFindSwiftc` if `swiftc` could not be
+  ///           located, `ParseError.invalidFile` if the file is invalid.
+  ///           FIXME: Fill this out with all error cases.
+  public static func parse(_ url: URL) throws -> SourceFileSyntax {
+    let swiftcRunner = try SwiftcRunner(sourceFile: url)
+    let result = try swiftcRunner.invoke()
+    guard result.wasSuccessful else {
+      throw ParserError.swiftcFailed(result.exitCode, result.stderr)
+    }
+    let decoder = JSONDecoder()
+    let raw = try decoder.decode([RawSyntax].self, from: result.stdoutData)
+    let topLevelNodes = raw.map { Syntax.fromRaw($0) }
+    let eof = topLevelNodes.last! as! TokenSyntax
+    let decls = Array(topLevelNodes.dropLast()) as! [DeclSyntax]
+    let declList = SyntaxFactory.makeDeclList(decls)
+    return SyntaxFactory.makeSourceFile(topLevelDecls: declList,
+                                        eofToken: eof)
+  }
+}