[Syntax] Swift libSyntax API (#11320)

* Create Swift libSyntax API

This patch is an initial implementation of the Swift libSyntax API. It
aims to provide all features of the C++ API but exposed to Swift.

It currently resides in SwiftExperimental and will likely exist in a
molten state for a while.

* Only build SwiftSyntax on macOS

Author: Harlan

cmake/modules/SwiftComponents.cmake

@@ -57,6 +57,7 @@
 # * stdlib -- the Swift standard library.
 # * stdlib-experimental -- the Swift standard library module for experimental
 #   APIs.
+# * swift-syntax -- the Swift module for the libSyntax Swift API
 # * sdk-overlay -- the Swift SDK overlay.
 # * editor-integration -- scripts for Swift integration in IDEs other than
 #   Xcode;
@@ -66,7 +67,7 @@
 # * toolchain-dev-tools -- install development tools useful in a shared toolchain
 # * dev -- headers and libraries required to use Swift compiler as a library.
 set(_SWIFT_DEFINED_COMPONENTS
-  "autolink-driver;compiler;clang-builtin-headers;clang-resource-dir-symlink;llvm-resource-dir-symlink;clang-builtin-headers-in-clang-resource-dir;stdlib;stdlib-experimental;sdk-overlay;editor-integration;tools;testsuite-tools;toolchain-dev-tools;dev;license;sourcekit-xpc-service;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers")
+  "autolink-driver;compiler;clang-builtin-headers;clang-resource-dir-symlink;llvm-resource-dir-symlink;clang-builtin-headers-in-clang-resource-dir;stdlib;stdlib-experimental;swift-syntax;sdk-overlay;editor-integration;tools;testsuite-tools;toolchain-dev-tools;dev;license;sourcekit-xpc-service;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers")
 
 macro(swift_configure_components)
   # Set the SWIFT_INSTALL_COMPONENTS variable to the default value if it is not passed in via -D

lib/AST/LegacyASTTransformer.cpp

@@ -225,7 +225,10 @@ RC<SyntaxData>
 LegacyASTTransformer::visitTopLevelCodeDecl(TopLevelCodeDecl *D,
                                             const SyntaxData *Parent,
                                             const CursorIndex IndexInParent) {
-  return visitBraceStmt(D->getBody(), Parent, IndexInParent);
+  auto body = visitBraceStmt(D->getBody(), Parent, IndexInParent);
+  return SyntaxData::make(RawSyntax::make(SyntaxKind::TopLevelCodeDecl,
+                                          {body->getRaw()},
+                                          SourcePresence::Present));
 }
 
 RC<SyntaxData>

stdlib/private/StdlibUnittest/StdlibUnittest.swift.gyb

@@ -532,6 +532,27 @@ public func expectFalse(_ actual: Bool, ${TRACE}) {
   }
 }
 
+public func expectThrows<ErrorType: Error & Equatable>(
+  _ expectedError: ErrorType? = nil, _ test: () throws -> Void, ${TRACE}) {
+  do {
+    try test()
+  } catch let error as ErrorType {
+    if let expectedError = expectedError {
+      expectEqual(expectedError, error)
+    }
+  } catch {
+    expectationFailure("unexpected error thrown: \"\(error)\"", trace: ${trace})
+  }
+}
+
+public func expectDoesNotThrow(_ test: () throws -> Void, ${TRACE}) {
+  do {
+    try test()
+  } catch {
+    expectationFailure("unexpected error thrown: \"\(error)\"", trace: ${trace})
+  }
+}
+
 public func expectNil<T>(_ value: T?, ${TRACE}) {
   if value != nil {
     expectationFailure(

test/SwiftSyntax/LazyCaching.swift

@@ -0,0 +1,48 @@
+// RUN: %target-run-simple-swift
+// REQUIRES: executable_test
+// REQUIRES: OS=macosx
+
+import StdlibUnittest
+import Foundation
+import Dispatch
+
+import SwiftSyntax
+
+var LazyCaching = TestSuite("LazyCaching")
+
+LazyCaching.test("Pathological") {
+  let tuple = SyntaxFactory.makeVoidTupleType()
+
+  DispatchQueue.concurrentPerform(iterations: 100) { _ in
+    expectEqual(tuple.leftParen, tuple.leftParen)
+  }
+}
+
+LazyCaching.test("TwoAccesses") {
+  let tuple = SyntaxFactory.makeVoidTupleType()
+
+  let queue1 = DispatchQueue(label: "queue1")
+  let queue2 = DispatchQueue(label: "queue2")
+
+  var node1: TokenSyntax?
+  var node2: TokenSyntax?
+
+  let group = DispatchGroup()
+  queue1.async(group: group) {
+    node1 = tuple.leftParen
+  }
+  queue2.async(group: group) {
+    node2 = tuple.leftParen
+  }
+  group.wait()
+
+  let final = tuple.leftParen
+
+  expectNotNil(node1)
+  expectNotNil(node2)
+  expectEqual(node1, node2)
+  expectEqual(node1, final)
+  expectEqual(node2, final)
+}
+
+runAllTests()

test/SwiftSyntax/ParseFile.swift

@@ -0,0 +1,20 @@
+// RUN: %target-run-simple-swift
+// REQUIRES: executable_test
+// REQUIRES: OS=macosx
+
+import Foundation
+import StdlibUnittest
+import SwiftSyntax
+
+var ParseFile = TestSuite("ParseFile")
+
+ParseFile.test("ParseSingleFile") {
+  let currentFile = URL(fileURLWithPath: #file)
+  expectDoesNotThrow({
+    let currentFileContents = try String(contentsOf: currentFile)
+    let parsed = try Syntax.parse(currentFile)
+    expectEqual("\(parsed)", currentFileContents)
+  })
+}
+
+runAllTests()

test/SwiftSyntax/SyntaxFactory.swift

@@ -0,0 +1,113 @@
+// RUN: %target-run-simple-swift
+// REQUIRES: executable_test
+// REQUIRES: OS=macosx
+
+import Foundation
+import StdlibUnittest
+import SwiftSyntax
+
+func cannedStructDecl() -> StructDeclSyntax {
+  let fooID = SyntaxFactory.makeIdentifier("Foo", trailingTrivia: .spaces(1))
+  let structKW = SyntaxFactory.makeStructKeyword(trailingTrivia: .spaces(1))
+  let rBrace = SyntaxFactory.makeRightBraceToken(leadingTrivia: .newlines(1))
+  return StructDeclSyntax {
+    $0.useStructKeyword(structKW)
+    $0.useIdentifier(fooID)
+    $0.useLeftBrace(SyntaxFactory.makeLeftBraceToken())
+    $0.useRightBrace(rBrace)
+  }
+}
+
+var SyntaxFactoryAPI = TestSuite("SyntaxFactoryAPI")
+
+SyntaxFactoryAPI.test("Generated") {
+
+  let structDecl = cannedStructDecl()
+
+  expectEqual("\(structDecl)",
+              """
+              struct Foo {
+              }
+              """)
+
+  let forType = SyntaxFactory.makeIdentifier("for", 
+                                             leadingTrivia: .backticks(1),
+                                             trailingTrivia: [
+                                               .backticks(1), .spaces(1)
+                                             ])
+  let newBrace = SyntaxFactory.makeRightBraceToken(leadingTrivia: .newlines(2))
+
+  let renamed = structDecl.withIdentifier(forType)
+                          .withRightBrace(newBrace)
+
+  expectEqual("\(renamed)",
+              """
+              struct `for` {
+
+              }
+              """)
+
+  expectNotEqual(structDecl.leftBrace, renamed.leftBrace)
+  expectEqual(structDecl, structDecl.root)
+  expectNil(structDecl.parent)
+  expectNotNil(structDecl.leftBrace.parent)
+  expectEqual(structDecl.leftBrace.parent, structDecl)
+
+  // Ensure that accessing children via named identifiers is exactly the
+  // same as accessing them as their underlying data.
+  expectEqual(structDecl.leftBrace, structDecl.child(at: 7))
+  
+  expectEqual("\(structDecl.rightBrace)",
+              """
+
+              }
+              """)
+}
+
+SyntaxFactoryAPI.test("TokenSyntax") {
+  let tok = SyntaxFactory.makeStructKeyword()
+  expectEqual("\(tok)", "struct")
+  expectTrue(tok.isPresent)
+
+  let preSpacedTok = tok.withLeadingTrivia(.spaces(3))
+  expectEqual("\(preSpacedTok)", "   struct")
+
+  let postSpacedTok = tok.withTrailingTrivia(.spaces(6))
+  expectEqual("\(postSpacedTok)", "struct      ")
+
+  let prePostSpacedTok = preSpacedTok.withTrailingTrivia(.spaces(4))
+  expectEqual("\(prePostSpacedTok)", "   struct    ")
+}
+
+SyntaxFactoryAPI.test("FunctionCallSyntaxBuilder") {
+  let string = SyntaxFactory.makeStringLiteralExpr("Hello, world!")
+  let printID = SyntaxFactory.makeVariableExpr("print")
+  let arg = FunctionCallArgumentSyntax {
+    $0.useExpression(string)
+  }
+  let call = FunctionCallExprSyntax {
+    $0.useCalledExpression(printID)
+    $0.useLeftParen(SyntaxFactory.makeLeftParenToken())
+    $0.addFunctionCallArgument(arg)
+    $0.useRightParen(SyntaxFactory.makeRightParenToken())
+  }
+  expectEqual("\(call)", "print(\"Hello, world!\")")
+
+  let terminatorArg = FunctionCallArgumentSyntax {
+    $0.useLabel(SyntaxFactory.makeIdentifier("terminator"))
+    $0.useColon(SyntaxFactory.makeColonToken(trailingTrivia: .spaces(1)))
+    $0.useExpression(SyntaxFactory.makeStringLiteralExpr(" "))
+  }
+  let callWithTerminator = call.withArgumentList(
+    SyntaxFactory.makeFunctionCallArgumentList([
+      arg.withTrailingComma(
+        SyntaxFactory.makeCommaToken(trailingTrivia: .spaces(1))),
+      terminatorArg
+    ])
+  )
+
+  expectEqual("\(callWithTerminator)",
+              "print(\"Hello, world!\", terminator: \" \")")
+}
+
+runAllTests()

tools/CMakeLists.txt

@@ -16,9 +16,14 @@ if(SWIFT_BUILD_SOURCEKIT)
   add_swift_tool_subdirectory(SourceKit)
 endif()
 
+
 if(SWIFT_HOST_VARIANT STREQUAL "macosx")
   # Only build Darwin-specific tools when deploying to OS X.
   add_swift_tool_subdirectory(swift-stdlib-tool)
+
+  if(SWIFT_BUILD_STDLIB)
+    add_subdirectory(SwiftSyntax)
+  endif()
 endif()
 
 

tools/SwiftSyntax/AtomicCache.swift

@@ -0,0 +1,63 @@
+//===----------- AtomicCache.swift - Atomically Initialized Cache ---------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+
+/// AtomicCache is a wrapper class around an uninitialized value.
+/// It takes a closure that it will use to create the value atomically. The
+/// value is guaranteed to be set exactly one time, but the provided closure
+/// may be called multiple times by threads racing to initialize the value.
+/// Do not rely on the closure being called only one time.
+class AtomicCache<Value: AnyObject> {
+  /// The cached pointer that will be filled in the first time `value` is
+  /// accessed.
+  private var _cachedValue: AnyObject?
+
+  /// The value inside this cache. If the value has not been initialized when
+  /// this value is requested, then the closure will be called and its resulting
+  /// value will be atomically compare-exchanged into the cache.
+  /// If multiple threads access the value before initialization, they will all
+  /// end up returning the correct, initialized value.
+  /// - Parameter create: The closure that will return the fully realized value
+  ///                     inside the cache.
+  func value(_ create: () -> Value) -> Value {
+    return withUnsafeMutablePointer(to: &_cachedValue) { ptr in
+      // Perform an atomic load -- if we get a value, then return it.
+      if let _cached = _stdlib_atomicLoadARCRef(object: ptr) {
+        return _cached as! Value
+      }
+
+      // Otherwise, create the value...
+      let value = create()
+
+      // ...and attempt to initialize the pointer with that value.
+      if _stdlib_atomicInitializeARCRef(object: ptr, desired: value) {
+        // If we won the race, just return the value we made.
+        return value
+      }
+
+      // Otherwise, perform _another_ load to get the up-to-date value,
+      // and let the one we just made die.
+      return _stdlib_atomicLoadARCRef(object: ptr) as! Value
+    }
+  }
+
+  /// Unsafely attempts to load the value and cast it to the appropriate
+  /// type.
+  /// - note: Only for use in the debugger!
+  @available(*, deprecated, message: "Only for use in the debugger.")
+  var unsafeValue: Value? {
+    return withUnsafeMutablePointer(to: &_cachedValue) {
+      return _stdlib_atomicLoadARCRef(object: $0) as? Value
+    }
+  }
+}

tools/SwiftSyntax/CMakeLists.txt

@@ -0,0 +1,24 @@
+add_swift_library(swiftSwiftSyntax SHARED
+  # This file should be listed the first.  Module name is inferred from the
+  # filename.
+  SwiftSyntax.swift
+  AtomicCache.swift
+  RawSyntax.swift
+  SourcePresence.swift
+  SwiftcInvocation.swift
+  Syntax.swift
+  SyntaxData.swift
+  SyntaxChildren.swift
+  SyntaxCollection.swift
+  SyntaxBuilders.swift.gyb
+  SyntaxFactory.swift.gyb
+  SyntaxKind.swift.gyb
+  SyntaxNodes.swift.gyb
+  SyntaxRewriter.swift.gyb
+  TokenKind.swift.gyb
+  Trivia.swift
+
+  SWIFT_MODULE_DEPENDS Foundation
+  INSTALL_IN_COMPONENT swift-syntax
+  TARGET_SDKS OSX
+  IS_STDLIB)

tools/SwiftSyntax/README.md

@@ -0,0 +1,4 @@
+# SwiftLanguage
+
+This is an in-progress implementation of a Swift API for the
+[libSyntax](https://github.com/apple/swift/tree/master/lib/Syntax) library.